minor (#3137)

### What problem does this PR solve?

_Briefly describe what this PR aims to solve. Include background context
that will help reviewers understand the purpose of the PR._

### Type of change


- [x] Documentation Update

.github/workflows/tests.yml

@@ -0,0 +1,86 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - 'main'
+      - '*.*.*'
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '*.mdx'
+  pull_request:
+    types: [ opened, synchronize, reopened, labeled ]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '*.mdx'
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ragflow_tests:
+    name: ragflow_tests
+    # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
+    # https://github.com/orgs/community/discussions/26261
+    if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci') }}
+    runs-on: [ "self-hosted", "debug" ]
+    steps:
+      # https://github.com/hmarr/debug-action
+      #- uses: hmarr/debug-action@v2
+
+      - name: Show PR labels
+        run: |
+          echo "Workflow triggered by ${{ github.event_name }}"
+          if [[ ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}"
+          fi
+
+      - name: Ensure workspace ownership
+        run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
+
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Build ragflow:dev-slim
+        run: |
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
+          cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb .
+          sudo docker pull ubuntu:24.04
+          sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+
+      - name: Build ragflow:dev
+        run: |
+          sudo docker build -f Dockerfile -t infiniflow/ragflow:dev .
+
+      - name: Start ragflow:dev-slim
+        run: |
+          sudo docker compose -f docker/docker-compose.yml up -d
+
+      - name: Stop ragflow:dev-slim
+        if: always()  # always run this step even if previous steps failed
+        run: |
+          sudo docker compose -f docker/docker-compose.yml down -v
+
+      - name: Start ragflow:dev
+        run: |
+          echo "RAGFLOW_IMAGE=infiniflow/ragflow:dev" >> docker/.env
+          sudo docker compose -f docker/docker-compose.yml up -d
+
+      - name: Run tests
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          export HOST_ADDRESS=http://host.docker.internal:9380
+          until sudo docker exec ragflow-server curl -s --connect-timeout 5 ${HOST_ADDRESS} > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          cd sdk/python && poetry install && source .venv/bin/activate && cd test && pytest t_dataset.py t_chat.py t_session.py
+
+      - name: Stop ragflow:dev
+        if: always()  # always run this step even if previous steps failed
+        run: |
+          sudo docker compose -f docker/docker-compose.yml down -v

Dockerfile

@@ -2,6 +2,7 @@
 FROM ubuntu:24.04 AS base
 USER root
 
+ARG ARCH=amd64
 ENV LIGHTEN=0
 
 WORKDIR /ragflow
@@ -9,18 +10,24 @@ WORKDIR /ragflow
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt-get --no-install-recommends install -y ca-certificates
 
-# if you located in China, you can use tsinghua mirror to speed up apt
-RUN  sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources
+# If you download Python modules too slow, you can use a pip mirror site to speed up apt and poetry
+RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources
+ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus \
-    && rm -rf /var/lib/apt/lists/* \
-    && curl -sSL https://install.python-poetry.org | python3 -
+RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
+    apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-pip python3-poetry \
+    && pip3 install --user --break-system-packages poetry-plugin-pypi-mirror --index-url https://pypi.tuna.tsinghua.edu.cn/simple/ \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN curl -o libssl1.deb http://archive.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5_amd64.deb && dpkg -i libssl1.deb && rm -f libssl1.deb
+# https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
+# aspose-slides on linux/arm64 is unavailable
+RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+    if [ "${ARCH}" = "amd64" ]; then \
+        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
+    fi
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
 
@@ -36,21 +43,23 @@ USER root
 
 WORKDIR /ragflow
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_builder_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt install -y nodejs npm cargo && \
     rm -rf /var/lib/apt/lists/*
 
 COPY web web
-RUN cd web && npm i --force && npm run build
+COPY docs docs
+RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
+    cd web && npm i --force && npm run build
 
 # install dependencies from poetry.lock file
 COPY pyproject.toml poetry.toml poetry.lock ./
 
-RUN --mount=type=cache,target=/root/.cache/pypoetry,sharing=locked \
+RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
     if [ "$LIGHTEN" -eq 0 ]; then \
-        /root/.local/bin/poetry install --sync --no-cache --no-root --with=full; \
+        poetry install --sync --no-root --with=full; \
     else \
-        /root/.local/bin/poetry install --sync --no-cache --no-root; \
+        poetry install --sync --no-root; \
     fi
 
 # production stage
@@ -61,7 +70,7 @@ WORKDIR /ragflow
 
 # Install python packages' dependencies
 # cv2 requires libGL.so.1
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt install -y --no-install-recommends nginx libgl1 vim less && \
     rm -rf /var/lib/apt/lists/*
 
@@ -89,16 +98,16 @@ RUN --mount=type=bind,source=huggingface.co,target=/huggingface.co \
         /huggingface.co/maidalun1020/bce-reranker-base_v1 \
         | tar -xf - --strip-components=2 -C /root/.ragflow
 
+# Copy nltk data downloaded via download_deps.py
+COPY nltk_data /root/nltk_data
+
 # Copy compiled web pages
 COPY --from=builder /ragflow/web/dist /ragflow/web/dist
 
 # Copy Python environment and packages
 ENV VIRTUAL_ENV=/ragflow/.venv
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENV PATH="${VIRTUAL_ENV}/bin:/root/.local/bin:${PATH}"
-
-# Download nltk data
-RUN python3 -m nltk.downloader wordnet punkt punkt_tab
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 ENV PYTHONPATH=/ragflow/
 

Dockerfile.scratch.oc9

@@ -26,6 +26,7 @@ RUN dnf install -y nginx
 
 ADD ./web ./web
 ADD ./api ./api
+ADD ./docs ./docs
 ADD ./conf ./conf
 ADD ./deepdoc ./deepdoc
 ADD ./rag ./rag
@@ -37,7 +38,7 @@ RUN dnf install -y openmpi openmpi-devel python3-openmpi
 ENV C_INCLUDE_PATH /usr/include/openmpi-x86_64:$C_INCLUDE_PATH
 ENV LD_LIBRARY_PATH /usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
 RUN rm /root/miniconda3/envs/py11/compiler_compat/ld
-RUN cd ./web && npm i --force && npm run build
+RUN cd ./web && npm i && npm run build
 RUN conda run -n py11 pip install $(grep -ivE "mpi4py" ./requirements.txt) # without mpi4py==3.1.5
 RUN conda run -n py11 pip install redis
 

Dockerfile.slim

@@ -2,6 +2,7 @@
 FROM ubuntu:24.04 AS base
 USER root
 
+ARG ARCH=amd64
 ENV LIGHTEN=1
 
 WORKDIR /ragflow
@@ -9,18 +10,23 @@ WORKDIR /ragflow
 RUN rm -f /etc/apt/apt.conf.d/docker-clean \
     && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt-get --no-install-recommends install -y ca-certificates
 
-# if you located in China, you can use tsinghua mirror to speed up apt
-RUN  sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources
+# If you download Python modules too slow, you can use a pip mirror site to speed up apt and poetry
+RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources
+ENV POETRY_PYPI_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple/
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus \
-    && rm -rf /var/lib/apt/lists/* \
-    && curl -sSL https://install.python-poetry.org | python3 -
+RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
+    apt update && apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus python3-pip python3-poetry \
+    && pip3 install --user --break-system-packages poetry-plugin-pypi-mirror --index-url https://pypi.tuna.tsinghua.edu.cn/simple/ \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN curl -o libssl1.deb http://archive.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5_amd64.deb && dpkg -i libssl1.deb && rm -f libssl1.deb
+# https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
+# aspose-slides on linux/arm64 is unavailable
+RUN if [ "${ARCH}" = "amd64" ]; then \
+        curl -o libssl1.deb http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb && dpkg -i libssl1.deb && rm -f libssl1.deb; \
+    fi
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
 
@@ -36,21 +42,23 @@ USER root
 
 WORKDIR /ragflow
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_builder_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt install -y nodejs npm cargo && \
     rm -rf /var/lib/apt/lists/*
 
 COPY web web
-RUN cd web && npm i --force && npm run build
+COPY docs docs
+RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
+    cd web && npm i && npm run build
 
 # install dependencies from poetry.lock file
 COPY pyproject.toml poetry.toml poetry.lock ./
 
-RUN --mount=type=cache,target=/root/.cache/pypoetry,sharing=locked \
+RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
     if [ "$LIGHTEN" -eq 0 ]; then \
-        /root/.local/bin/poetry install --sync --no-cache --no-root --with=full; \
+        poetry install --sync --no-root --with=full; \
     else \
-        /root/.local/bin/poetry install --sync --no-cache --no-root; \
+        poetry install --sync --no-root; \
     fi
 
 # production stage
@@ -61,7 +69,7 @@ WORKDIR /ragflow
 
 # Install python packages' dependencies
 # cv2 requires libGL.so.1
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
     apt update && apt install -y --no-install-recommends nginx libgl1 vim less && \
     rm -rf /var/lib/apt/lists/*
 
@@ -82,16 +90,16 @@ RUN --mount=type=bind,source=huggingface.co,target=/huggingface.co \
         /huggingface.co/InfiniFlow/deepdoc \
         | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc
 
+# Copy nltk data downloaded via download_deps.py
+COPY nltk_data /root/nltk_data
+
 # Copy compiled web pages
 COPY --from=builder /ragflow/web/dist /ragflow/web/dist
 
 # Copy Python environment and packages
 ENV VIRTUAL_ENV=/ragflow/.venv
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENV PATH="${VIRTUAL_ENV}/bin:/root/.local/bin:${PATH}"
-
-# Download nltk data
-RUN python3 -m nltk.downloader wordnet punkt punkt_tab
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 ENV PYTHONPATH=/ragflow/
 

README.md

@@ -12,16 +12,21 @@
 </p>
 
 <p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.13.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.13.0">
+    </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
     </a>
-    <a href="https://demo.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
-    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.12.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.12.0"></a>
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
-  </a>
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
+    </a>
 </p>
 
 <h4 align="center">
@@ -34,7 +39,7 @@
 
 <details open>
 <summary></b>📕 Table of Contents</b></summary>
-  
+
 - 💡 [What is RAGFlow?](#-what-is-ragflow)
 - 🎮 [Demo](#-demo)
 - 📌 [Latest Updates](#-latest-updates)
@@ -42,8 +47,8 @@
 - 🔎 [System Architecture](#-system-architecture)
 - 🎬 [Get Started](#-get-started)
 - 🔧 [Configurations](#-configurations)
-- 🪛 [Build the docker image without embedding models](#-build-the-docker-image-without-embedding-models)
-- 🪚 [Build the docker image including embedding models](#-build-the-docker-image-including-embedding-models)
+- 🔧 [Build a docker image without embedding models](#-build-a-docker-image-without-embedding-models)
+- 🔧 [Build a docker image including embedding models](#-build-a-docker-image-including-embedding-models)
 - 🔨 [Launch service from source for development](#-launch-service-from-source-for-development)
 - 📚 [Documentation](#-documentation)
 - 📜 [Roadmap](#-roadmap)
@@ -54,7 +59,10 @@
 
 ## 💡 What is RAGFlow?
 
-[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data.
+[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document
+understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models)
+to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted
+data.
 
 ## 🎮 Demo
 
@@ -64,7 +72,6 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 <img src="https://github.com/infiniflow/ragflow/assets/12318111/b083d173-dadc-4ea9-bdeb-180d7df514eb" width="1200"/>
 </div>
 
-
 ## 🔥 Latest Updates
 
 - 2024-09-29 Optimizes multi-round conversations.
@@ -72,17 +79,21 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 - 2024-09-09 Adds a medical consultant agent template.
 - 2024-08-22 Support text to SQL statements through RAG.
 - 2024-08-02 Supports GraphRAG inspired by [graphrag](https://github.com/microsoft/graphrag) and mind map.
-- 2024-07-23 Supports audio file parsing.
-- 2024-07-08 Supports workflow based on [Graph](./agent/README.md).
-- 2024-06-27 Supports Markdown and Docx in the Q&A parsing method, extracting images from Docx files, extracting tables from Markdown files.
-- 2024-05-23 Supports [RAPTOR](https://arxiv.org/html/2401.18059v1) for better text retrieval.
 
+## 🎉 Stay Tuned
+
+⭐️ Star our repository to stay up-to-date with exciting new features and improvements! Get instant notifications for new
+releases! 🌟
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
 
 ## 🌟 Key Features
 
 ### 🍭 **"Quality in, quality out"**
 
-- [Deep document understanding](./deepdoc/README.md)-based knowledge extraction from unstructured data with complicated formats.
+- [Deep document understanding](./deepdoc/README.md)-based knowledge extraction from unstructured data with complicated
+  formats.
 - Finds "needle in a data haystack" of literally unlimited tokens.
 
 ### 🍱 **Template-based chunking**
@@ -120,7 +131,8 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
-  > If you have not installed Docker on your local machine (Windows, Mac, or Linux), see [Install Docker Engine](https://docs.docker.com/engine/install/).
+  > If you have not installed Docker on your local machine (Windows, Mac, or Linux),
+  see [Install Docker Engine](https://docs.docker.com/engine/install/).
 
 ### 🚀 Start up the server
 
@@ -139,7 +151,8 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
    > $ sudo sysctl -w vm.max_map_count=262144
    > ```
    >
-   > This change will be reset after a system reboot. To ensure your change remains permanent, add or update the `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
+   > This change will be reset after a system reboot. To ensure your change remains permanent, add or update the
+   `vm.max_map_count` value in **/etc/sysctl.conf** accordingly:
    >
    > ```bash
    > vm.max_map_count=262144
@@ -152,14 +165,28 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
    ```
 
 3. Build the pre-built Docker images and start up the server:
-   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_IMAGE` in **docker/.env** to the intended version, for example `RAGFLOW_IMAGE=infiniflow/ragflow:v0.12.0`, before running the following commands.
+
+   > The command below downloads the dev version Docker image for RAGFlow slim (`dev-slim`). Note that RAGFlow slim
+   Docker images do not include embedding models or Python libraries and hence are approximately 1GB in size.
 
    ```bash
    $ cd ragflow/docker
-   $ docker compose up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
-   > The core image is about 9 GB in size and may take a while to load.
+   > - To download a RAGFlow slim Docker image of a specific version, update the `RAGFlow_IMAGE` variable in *
+       *docker/.env** to your desired version. For example, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0-slim`. After
+       making this change, rerun the command above to initiate the download.
+   > - To download the dev version of RAGFlow Docker image *including* embedding models and Python libraries, update the
+       `RAGFlow_IMAGE` variable in **docker/.env** to `RAGFLOW_IMAGE=infiniflow/ragflow:dev`. After making this change,
+       rerun the command above to initiate the download.
+   > - To download a specific version of RAGFlow Docker image *including* embedding models and Python libraries, update
+       the `RAGFlow_IMAGE` variable in **docker/.env** to your desired version. For example,
+       `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0`. After making this change, rerun the command above to initiate the
+       download.
+
+   > **NOTE:** A RAGFlow Docker image that includes embedding models and Python libraries is approximately 9GB in size
+   and may take significantly longer time to load.
 
 4. Check the server status after having the server up and running:
 
@@ -170,23 +197,26 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
    _The following output confirms a successful launch of the system:_
 
    ```bash
-        ____   ___    ______ ______ __               
-       / __ \ /   |  / ____// ____// /____  _      __
-      / /_/ // /| | / / __ / /_   / // __ \| | /| / /
-     / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
-    /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/  
- 
+
+         ____   ___    ______ ______ __               
+        / __ \ /   |  / ____// ____// /____  _      __
+       / /_/ // /| | / / __ / /_   / // __ \| | /| / /
+      / _, _// ___ |/ /_/ // __/  / // /_/ /| |/ |/ / 
+     /_/ |_|/_/  |_|\____//_/    /_/ \____/ |__/|__/ 
 
     * Running on all addresses (0.0.0.0)
     * Running on http://127.0.0.1:9380
     * Running on http://x.x.x.x:9380
     INFO:werkzeug:Press CTRL+C to quit
    ```
-   > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network abnormal` error because, at that moment, your RAGFlow may not be fully initialized.  
+   > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network abnormal`
+   error because, at that moment, your RAGFlow may not be fully initialized.
 
 5. In your web browser, enter the IP address of your server and log in to RAGFlow.
-   > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default HTTP serving port `80` can be omitted when using the default configurations.
-6. In [service_conf.yaml](./docker/service_conf.yaml), select the desired LLM factory in `user_default_llm` and update the `API_KEY` field with the corresponding API key.
+   > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default
+   HTTP serving port `80` can be omitted when using the default configurations.
+6. In [service_conf.yaml](./docker/service_conf.yaml), select the desired LLM factory in `user_default_llm` and update
+   the `API_KEY` field with the corresponding API key.
 
    > See [llm_api_key_setup](https://ragflow.io/docs/dev/llm_api_key_setup) for more information.
 
@@ -196,54 +226,61 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
 
 When it comes to system configurations, you will need to manage the following files:
 
-- [.env](./docker/.env): Keeps the fundamental setups for the system, such as `SVR_HTTP_PORT`, `MYSQL_PASSWORD`, and `MINIO_PASSWORD`.
+- [.env](./docker/.env): Keeps the fundamental setups for the system, such as `SVR_HTTP_PORT`, `MYSQL_PASSWORD`, and
+  `MINIO_PASSWORD`.
 - [service_conf.yaml](./docker/service_conf.yaml): Configures the back-end services.
-- [docker-compose.yml](./docker/docker-compose.yml): The system relies on [docker-compose.yml](./docker/docker-compose.yml) to start up.
+- [docker-compose.yml](./docker/docker-compose.yml): The system relies
+  on [docker-compose.yml](./docker/docker-compose.yml) to start up.
 
-You must ensure that changes to the [.env](./docker/.env) file are in line with what are in the [service_conf.yaml](./docker/service_conf.yaml) file.
+You must ensure that changes to the [.env](./docker/.env) file are in line with what are in
+the [service_conf.yaml](./docker/service_conf.yaml) file.
 
-> The [./docker/README](./docker/README.md) file provides a detailed description of the environment settings and service configurations, and you are REQUIRED to ensure that all environment settings listed in the [./docker/README](./docker/README.md) file are aligned with the corresponding configurations in the [service_conf.yaml](./docker/service_conf.yaml) file.
+> The [./docker/README](./docker/README.md) file provides a detailed description of the environment settings and service
+> configurations, and you are REQUIRED to ensure that all environment settings listed in
+> the [./docker/README](./docker/README.md) file are aligned with the corresponding configurations in
+> the [service_conf.yaml](./docker/service_conf.yaml) file.
 
-To update the default HTTP serving port (80), go to [docker-compose.yml](./docker/docker-compose.yml) and change `80:80` to `<YOUR_SERVING_PORT>:80`.
+To update the default HTTP serving port (80), go to [docker-compose.yml](./docker/docker-compose.yml) and change `80:80`
+to `<YOUR_SERVING_PORT>:80`.
 
 Updates to the above configurations require a reboot of all containers to take effect:
 
 > ```bash
-> $ docker-compose -f docker/docker-compose.yml up -d
+> $ docker compose -f docker/docker-compose.yml up -d
 > ```
 
-## 🪛 Build the Docker image without embedding models
+## 🔧 Build a Docker image without embedding models
 
 This image is approximately 1 GB in size and relies on external LLM and embedding services.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
-## 🪚 Build the Docker image including embedding models
+## 🔧 Build a Docker image including embedding models
 
-This image is approximately 9 GB in size. As it includes embedding models, it relies on external LLM services only.  
+This image is approximately 9 GB in size. As it includes embedding models, it relies on external LLM services only.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 Launch service from source for development
 
-1. Install Poetry, or skip this step if it is already installed:  
+1. Install Poetry, or skip this step if it is already installed:
    ```bash
    curl -sSL https://install.python-poetry.org | python3 -
    ```
 
-2. Clone the source code and install Python dependencies:  
+2. Clone the source code and install Python dependencies:
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
@@ -251,49 +288,49 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
    ~/.local/bin/poetry install --sync --no-root # install RAGFlow dependent python modules
    ```
 
-3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:  
+3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
    ```bash
    docker compose -f docker/docker-compose-base.yml up -d
    ```
 
-   Add the following line to `/etc/hosts` to resolve all hosts specified in **docker/service_conf.yaml** to `127.0.0.1`:  
+   Add the following line to `/etc/hosts` to resolve all hosts specified in **docker/service_conf.yaml** to `127.0.0.1`:
    ```
    127.0.0.1       es01 mysql minio redis
    ```  
    In **docker/service_conf.yaml**, update mysql port to `5455` and es port to `1200`, as specified in **docker/.env**.
 
-4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:  
- 
+4. If you cannot access HuggingFace, set the `HF_ENDPOINT` environment variable to use a mirror site:
+
    ```bash
    export HF_ENDPOINT=https://hf-mirror.com
    ```
 
-5. Launch backend service:  
+5. Launch backend service:
    ```bash
    source .venv/bin/activate
    export PYTHONPATH=$(pwd)
    bash docker/launch_backend_service.sh
    ```
 
-6. Install frontend dependencies:  
+6. Install frontend dependencies:
    ```bash
    cd web
    npm install --force
    ```  
 7. Configure frontend to update `proxy.target` in **.umirc.ts** to `http://127.0.0.1:9380`:
-8. Launch frontend service:  
+8. Launch frontend service:
    ```bash
    npm run dev 
    ```  
 
-   _The following output confirms a successful launch of the system:_  
+   _The following output confirms a successful launch of the system:_
 
    ![](https://github.com/user-attachments/assets/0daf462c-a24d-4496-a66f-92533534e187)
 
 ## 📚 Documentation
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/user-guides)
+- [User guide](https://ragflow.io/docs/dev/category/guides)
 - [References](https://ragflow.io/docs/dev/category/references)
 - [FAQ](https://ragflow.io/docs/dev/faq)
 
@@ -309,4 +346,5 @@ See the [RAGFlow Roadmap 2024](https://github.com/infiniflow/ragflow/issues/162)
 
 ## 🙌 Contributing
 
-RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community. If you would like to be a part, review our [Contribution Guidelines](./CONTRIBUTING.md) first.
+RAGFlow flourishes via open-source collaboration. In this spirit, we embrace diverse contributions from the community.
+If you would like to be a part, review our [Contribution Guidelines](./CONTRIBUTING.md) first.

README_ja.md

@@ -12,19 +12,24 @@
 </p>
 
 <p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.13.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.13.0">
+    </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
     </a>
-    <a href="https://demo.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
-    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.12.0-brightgreen"
-            alt="docker pull infiniflow/ragflow:v0.12.0"></a>
-      <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
-  </a>
+    <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
+    </a>
 </p>
 
+
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
@@ -53,11 +58,12 @@
 - 2024-09-09 エージェントに医療相談テンプレートを追加しました。
 - 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
 - 2024-08-02 [graphrag](https://github.com/microsoft/graphrag) からインスピレーションを得た GraphRAG とマインド マップをサポートします。
-- 2024-07-23 音声ファイルの解析をサポートしました。
-- 2024-07-08 [Graph](./agent/README.md) ベースのワークフローをサポート
-- 2024-06-27 Q&A 解析メソッドで Markdown と Docx をサポートし、Docx ファイルから画像を抽出し、Markdown ファイルからテーブルを抽出します。
-- 2024-05-23 より良いテキスト検索のために [RAPTOR](https://arxiv.org/html/2401.18059v1) をサポート。
 
+## 🎉 続きを楽しみに
+⭐️ リポジトリをスター登録して、エキサイティングな新機能やアップデートを最新の状態に保ちましょう！すべての新しいリリースに関する即時通知を受け取れます！ 🌟
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
 
 ## 🌟 主な特徴
 
@@ -134,15 +140,18 @@
 
 3. ビルド済みの Docker イメージをビルドし、サーバーを起動する:
 
+   > 以下のコマンドは、RAGFlow slim（`dev-slim`）の開発版Dockerイメージをダウンロードします。RAGFlow slimのDockerイメージには、埋め込みモデルやPythonライブラリが含まれていないため、サイズは約1GBです。
+
    ```bash
    $ cd ragflow/docker
-   $ chmod +x ./entrypoint.sh
-   $ docker compose up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
-   > 上記のコマンドを実行すると、RAGFlowの開発版dockerイメージが自動的にダウンロードされます。 特定のバージョンのDockerイメージをダウンロードして実行したい場合は、docker/.envファイルのRAGFLOW_IMAGE変数を見つけて、対応するバージョンに変更してください。 例えば、`RAGFLOW_IMAGE=infiniflow/ragflow:v0.12.0`として、上記のコマンドを実行してください。
-
-   > コアイメージのサイズは約 9 GB で、ロードに時間がかかる場合があります。
+   > - 特定のバージョンのRAGFlow slim Dockerイメージをダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を希望のバージョンに更新します。例えば、`RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0`とします。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。
+   > - RAGFlowの埋め込みモデルとPythonライブラリを含む開発版Dockerイメージをダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を`RAGFLOW_IMAGE=infiniflow/ragflow:dev`に更新します。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。
+   > - 特定のバージョンのRAGFlow Dockerイメージ（埋め込みモデルとPythonライブラリを含む）をダウンロードするには、**docker/.env**内の`RAGFlow_IMAGE`変数を希望のバージョンに更新します。例えば、`RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0`とします。この変更を行った後、上記のコマンドを再実行してダウンロードを開始してください。  
+   
+   > **NOTE:** 埋め込みモデルとPythonライブラリを含むRAGFlow Dockerイメージのサイズは約9GBであり、読み込みにかなりの時間がかかる場合があります。
 
 4. サーバーを立ち上げた後、サーバーの状態を確認する:
 
@@ -191,29 +200,29 @@
 > すべてのシステム設定のアップデートを有効にするには、システムの再起動が必要です:
 >
 > ```bash
-> $ docker-compose up -d
+> $ docker compose -f docker/docker-compose.yml up -d
 > ```
 
-## 🪛 ソースコードでDockerイメージを作成（埋め込みモデルなし）
+## 🔧 ソースコードでDockerイメージを作成（埋め込みモデルなし）
 
 この Docker イメージのサイズは約 1GB で、外部の大モデルと埋め込みサービスに依存しています。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
-## 🪚 ソースコードをコンパイルしたDockerイメージ（埋め込みモデルを含む）
+## 🔧 ソースコードをコンパイルしたDockerイメージ（埋め込みモデルを含む）
 
 この Docker のサイズは約 9GB で、埋め込みモデルを含むため、外部の大モデルサービスのみが必要です。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
@@ -275,7 +284,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ## 📚 ドキュメンテーション
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/user-guides)
+- [User guide](https://ragflow.io/docs/dev/category/guides)
 - [References](https://ragflow.io/docs/dev/category/references)
 - [FAQ](https://ragflow.io/docs/dev/faq)
 

README_ko.md

@@ -12,18 +12,24 @@
 </p>
 
 <p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.13.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.13.0">
+    </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
     </a>
-    <a href="https://demo.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
-    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.12.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.12.0"></a>
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
-  </a>
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
+    </a>
 </p>
 
+
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
@@ -59,14 +65,12 @@
   
 - 2024-08-02: [graphrag](https://github.com/microsoft/graphrag)와 마인드맵에서 영감을 받은 GraphRAG를 지원합니다.
 
-- 2024-07-23: 오디오 파일 분석을 지원합니다.
-
-- 2024-07-08: [Graph](./agent/README.md)를 기반으로 한 워크플로우를 지원합니다.
-
-- 2024-06-27 Q&A 구문 분석 방식에서 Markdown 및 Docx를 지원하고, Docx 파일에서 이미지 추출, Markdown 파일에서 테이블 추출을 지원합니다.
-
-- 2024-05-23: 더 나은 텍스트 검색을 위해 [RAPTOR](https://arxiv.org/html/2401.18059v1)를 지원합니다.
 
+## 🎉 계속 지켜봐 주세요
+⭐️우리의 저장소를 즐겨찾기에 등록하여 흥미로운 새로운 기능과 업데이트를 최신 상태로 유지하세요! 모든 새로운 릴리스에 대한 즉시 알림을 받으세요! 🌟
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
 
 
 ## 🌟 주요 기능
@@ -140,14 +144,18 @@
 
 3. 미리 빌드된 Docker 이미지를 생성하고 서버를 시작하세요:
 
-   > 다음 명령어를 실행하면 *dev* 버전의 RAGFlow Docker 이미지가 자동으로 다운로드됩니다. 특정 Docker 버전을 다운로드하고 실행하려면, **docker/.env** 파일에서 `RAGFLOW_IMAGE`을 원하는 버전으로 업데이트한 후, 예를 들어 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.12.0`로 업데이트 한 뒤, 다음 명령어를 실행하세요.
+   > 아래의 명령은 RAGFlow slim(dev-slim)의 개발 버전 Docker 이미지를 다운로드합니다. RAGFlow slim Docker 이미지에는 임베딩 모델이나 Python 라이브러리가 포함되어 있지 않으므로 크기는 약 1GB입니다.
+
    ```bash
    $ cd ragflow/docker
-   $ chmod +x ./entrypoint.sh
-   $ docker compose up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
    
-   > 기본 이미지는 약 9GB 크기이며 로드하는 데 시간이 걸릴 수 있습니다.
+   > - 특정 버전의 RAGFlow slim Docker 이미지를 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 원하는 버전으로 업데이트하세요. 예를 들어, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0-slim`으로 설정합니다. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.
+   > - RAGFlow의 임베딩 모델과 Python 라이브러리를 포함한 개발 버전 Docker 이미지를 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 `RAGFLOW_IMAGE=infiniflow/ragflow:dev`로 업데이트하세요. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.
+   > - 특정 버전의 RAGFlow Docker 이미지를 임베딩 모델과 Python 라이브러리를 포함하여 다운로드하려면, **docker/.env**에서 `RAGFlow_IMAGE` 변수를 원하는 버전으로 업데이트하세요. 예를 들어, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0` 로 설정합니다. 이 변경을 완료한 후, 위의 명령을 다시 실행하여 다운로드를 시작하세요.  
+   
+   > **NOTE:** 임베딩 모델과 Python 라이브러리를 포함한 RAGFlow Docker 이미지의 크기는 약 9GB이며, 로드하는 데 상당히 오랜 시간이 걸릴 수 있습니다.
 
 
 4. 서버가 시작된 후 서버 상태를 확인하세요:
@@ -196,29 +204,29 @@
 > 모든 시스템 구성 업데이트는 적용되기 위해 시스템 재부팅이 필요합니다.
 >
 > ```bash
-> $ docker-compose up -d
+> $ docker compose -f docker/docker-compose.yml up -d
 > ```
 
-## 🪛 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함하지 않음)
+## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함하지 않음)
 
 이 Docker 이미지의 크기는 약 1GB이며, 외부 대형 모델과 임베딩 서비스에 의존합니다.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
-## 🪚 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
+## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
 
 이 Docker의 크기는 약 9GB이며, 이미 임베딩 모델을 포함하고 있으므로 외부 대형 모델 서비스에만 의존하면 됩니다.
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
@@ -280,7 +288,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ## 📚 문서
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/user-guides)
+- [User guide](https://ragflow.io/docs/dev/category/guides)
 - [References](https://ragflow.io/docs/dev/category/references)
 - [FAQ](https://ragflow.io/docs/dev/faq)
 

README_zh.md

@@ -12,18 +12,24 @@
 </p>
 
 <p align="center">
+    <a href="https://x.com/intent/follow?screen_name=infiniflowai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
+    </a>
+    <a href="https://demo.ragflow.io" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+    </a>
+    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
+        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.13.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.13.0">
+    </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
     </a>
-    <a href="https://demo.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"></a>
-    <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/badge/docker_pull-ragflow:v0.12.0-brightgreen" alt="docker pull infiniflow/ragflow:v0.12.0"></a>
     <a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
-  </a>
+        <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="license">
+    </a>
 </p>
 
+
 <h4 align="center">
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/162">Roadmap</a> |
@@ -52,10 +58,13 @@
 - 2024-09-09 在 Agent 中加入医疗问诊模板。
 - 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
 - 2024-08-02 支持 GraphRAG 启发于 [graphrag](https://github.com/microsoft/graphrag) 和思维导图。
-- 2024-07-23 支持解析音频文件。
-- 2024-07-08 支持 Agentic RAG: 基于 [Graph](./agent/README.md) 的工作流。
-- 2024-06-27 Q&A 解析方式支持 Markdown 文件和 Docx 文件，支持提取出 Docx 文件中的图片和 Markdown 文件中的表格。
-- 2024-05-23 实现 [RAPTOR](https://arxiv.org/html/2401.18059v1) 提供更好的文本检索。
+
+## 🎉 关注项目
+⭐️点击右上角的 Star 关注RAGFlow，可以获取最新发布的实时通知 !🌟
+<div align="center" style="margin-top:20px;margin-bottom:20px;">
+<img src="https://github.com/user-attachments/assets/18c9707e-b8aa-4caf-a154-037089c105ba" width="1200"/>
+</div>
+
 
 ## 🌟 主要功能
 
@@ -132,16 +141,18 @@
 
 3. 进入 **docker** 文件夹，利用提前编译好的 Docker 镜像启动服务器：
 
+   > 运行以下命令会自动下载 dev 版的 RAGFlow slim Docker 镜像（`dev-slim`），该镜像并不包含 embedding 模型以及一些 Python 库，因此镜像大小约 1GB。
+
    ```bash
    $ cd ragflow/docker
-   $ chmod +x ./entrypoint.sh
    $ docker compose -f docker-compose.yml up -d
    ```
 
-   > 请注意，运行上述命令会自动下载 RAGFlow 的开发版本 docker 镜像。如果你想下载并运行特定版本的 docker 镜像，请在 docker/.env 文件中找到 RAGFLOW_IMAGE 变量，将其改为对应版本。例如 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.12.0`，然后运行上述命令。
-
-   > 核心镜像下载大小为 9 GB，可能需要一定时间拉取。请耐心等待。
-
+   > - 如果你想下载并运行特定版本的 RAGFlow slim Docker 镜像，请在 **docker/.env** 文件中找到 `RAGFLOW_IMAGE` 变量，将其改为对应版本。例如 `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0-slim`，然后再运行上述命令。
+   > - 如果您想安装内置 embedding 模型和 Python 库的 dev 版本的 Docker 镜像，需要将 **docker/.env** 文件中的 `RAGFLOW_IMAGE` 变量修改为： `RAGFLOW_IMAGE=infiniflow/ragflow:dev`。
+   > - 如果您想安装内置 embedding 模型和 Python 库的指定版本的 RAGFlow Docker 镜像，需要将 **docker/.env** 文件中的 `RAGFLOW_IMAGE` 变量修改为： `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0`。修改后，再运行上面的命令。
+   > **注意：** 安装内置 embedding 模型和 Python 库的指定版本的 RAGFlow Docker 镜像大小约 9 GB，可能需要更长时间下载，请耐心等待。
+   
 4. 服务器启动成功后再次确认服务器状态：
 
    ```bash
@@ -194,26 +205,26 @@
 > $ docker compose -f docker-compose.yml up -d
 > ```
 
-## 🪛 源码编译 Docker 镜像（不含 embedding 模型）
+## 🔧 源码编译 Docker 镜像（不含 embedding 模型）
 
 本 Docker 镜像大小约 1 GB 左右并且依赖外部的大模型和 embedding 服务。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
-## 🪚 源码编译 Docker 镜像（包含 embedding 模型）
+## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
 
 本 Docker 大小约 9 GB 左右。由于已包含 embedding 模型，所以只需依赖外部的大模型服务即可。
 
 ```bash
 git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
-pip3 install huggingface-hub
+pip3 install huggingface-hub nltk
 python3 download_deps.py
 docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
@@ -275,7 +286,7 @@ docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ## 📚 技术文档
 
 - [Quickstart](https://ragflow.io/docs/dev/)
-- [User guide](https://ragflow.io/docs/dev/category/user-guides)
+- [User guide](https://ragflow.io/docs/dev/category/guides)
 - [References](https://ragflow.io/docs/dev/category/references)
 - [FAQ](https://ragflow.io/docs/dev/faq)
 

agent/canvas.py

@@ -260,9 +260,9 @@ class Canvas(ABC):
 
     def get_history(self, window_size):
         convs = []
-        for role, obj in self.history[(window_size + 1) * -1:]:
+        for role, obj in self.history[window_size * -1:]:
             convs.append({"role": role, "content": (obj if role == "user" else
-                                                    '\n'.join(pd.DataFrame(obj)['content']))})
+                    '\n'.join([str(s) for s in pd.DataFrame(obj)['content']]))})
         return convs
 
     def add_user_input(self, question):

agent/component/__init__.py

@@ -28,6 +28,8 @@ from .wencai import WenCai, WenCaiParam
 from .jin10 import Jin10, Jin10Param
 from .tushare import TuShare, TuShareParam
 from .akshare import AkShare, AkShareParam
+from .crawler import Crawler, CrawlerParam
+from .invoke import Invoke, InvokeParam
 
 
 def component_class(class_name):

agent/component/baidufanyi.py

@@ -36,7 +36,6 @@ class BaiduFanyiParam(ComponentParamBase):
         self.domain = 'finance'
 
     def check(self):
-        self.check_positive_integer(self.top_n, "Top N")
         self.check_empty(self.appid, "BaiduFanyi APPID")
         self.check_empty(self.secret_key, "BaiduFanyi Secret Key")
         self.check_valid_value(self.trans_type, "Translate type", ['translate', 'fieldtranslate'])

agent/component/categorize.py

@@ -73,7 +73,7 @@ class Categorize(Generate, ABC):
 
     def _run(self, history, **kwargs):
         input = self.get_input()
-        input = "Question: " + ("; ".join(input["content"]) if "content" in input else "") + "Category: "
+        input = "Question: " + (list(input["content"])[-1] if "content" in input else "") + "\tCategory: "
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
         ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": input}],
                             self._param.gen_conf())

agent/component/crawler.py

@@ -0,0 +1,70 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from abc import ABC
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class CrawlerParam(ComponentParamBase):
+    """
+    Define the Crawler component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.proxy = None
+        self.extract_type = "markdown"
+    
+    def check(self):
+        self.check_valid_value(self.extract_type, "Type of content from the crawler", ['html', 'markdown', 'content'])
+
+
+class Crawler(ComponentBase, ABC):
+    component_name = "Crawler"
+
+    def _run(self, history, **kwargs):
+        ans = self.get_input()
+        ans = " - ".join(ans["content"]) if "content" in ans else ""
+        if not ans:
+            return Crawler.be_output("")
+        try:
+            result = asyncio.run(self.get_web(ans))
+
+            return Crawler.be_output(result)
+            
+        except Exception as e:
+            return Crawler.be_output(f"An unexpected error occurred: {str(e)}")
+
+    async def get_web(self, url):
+        proxy = self._param.proxy if self._param.proxy else None
+        async with AsyncWebCrawler(verbose=True, proxy=proxy) as crawler:
+            result = await crawler.arun(
+                url=url,
+                bypass_cache=True
+            )
+            
+            if self._param.extract_type == 'html':
+                return result.cleaned_html
+            elif self._param.extract_type == 'markdown':
+                return result.markdown
+            elif self._param.extract_type == 'content':
+                result.extracted_content
+            return result.markdown
+            
+
+
+    

agent/component/exesql.py

@@ -16,7 +16,8 @@
 from abc import ABC
 import re
 import pandas as pd
-from peewee import MySQLDatabase, PostgresqlDatabase
+import pymysql
+import psycopg2
 from agent.component.base import ComponentBase, ComponentParamBase
 
 
@@ -44,6 +45,9 @@ class ExeSQLParam(ComponentParamBase):
         self.check_positive_integer(self.port, "IP Port")
         self.check_empty(self.password, "Database password")
         self.check_positive_integer(self.top_n, "Number of records")
+        if self.database == "rag_flow":
+            if self.host == "ragflow-mysql": raise ValueError("The host is not accessible.")
+            if self.password == "infini_rag_flow": raise ValueError("The host is not accessible.")
 
 
 class ExeSQL(ComponentBase, ABC):
@@ -66,14 +70,14 @@ class ExeSQL(ComponentBase, ABC):
             raise Exception("SQL statement not found!")
 
         if self._param.db_type in ["mysql", "mariadb"]:
-            db = MySQLDatabase(self._param.database, user=self._param.username, host=self._param.host,
-                               port=self._param.port, password=self._param.password)
+            db = pymysql.connect(db=self._param.database, user=self._param.username, host=self._param.host,
+                                 port=self._param.port, password=self._param.password)
         elif self._param.db_type == 'postgresql':
-            db = PostgresqlDatabase(self._param.database, user=self._param.username, host=self._param.host,
-                                    port=self._param.port, password=self._param.password)
+            db = psycopg2.connect(dbname=self._param.database, user=self._param.username, host=self._param.host,
+                                  port=self._param.port, password=self._param.password)
 
         try:
-            db.connect()
+            cursor = db.cursor()
         except Exception as e:
             raise Exception("Database Connection Failed! \n" + str(e))
         sql_res = []
@@ -81,13 +85,13 @@ class ExeSQL(ComponentBase, ABC):
             if not single_sql:
                 continue
             try:
-                query = db.execute_sql(single_sql)
-                if query.rowcount == 0:
-                    sql_res.append({"content": "\nTotal: " + str(query.rowcount) + "\n No record in the database!"})
+                cursor.execute(single_sql)
+                if cursor.rowcount == 0:
+                    sql_res.append({"content": "\nTotal: 0\n No record in the database!"})
                     continue
-                single_res = pd.DataFrame([i for i in query.fetchmany(size=self._param.top_n)])
-                single_res.columns = [i[0] for i in query.description]
-                sql_res.append({"content": "\nTotal: " + str(query.rowcount) + "\n" + single_res.to_markdown()})
+                single_res = pd.DataFrame([i for i in cursor.fetchmany(size=self._param.top_n)])
+                single_res.columns = [i[0] for i in cursor.description]
+                sql_res.append({"content": "\nTotal: " + str(cursor.rowcount) + "\n" + single_res.to_markdown()})
             except Exception as e:
                 sql_res.append({"content": "**Error**:" + str(e) + "\nError SQL Statement:" + single_sql})
                 pass

agent/component/generate.py

@@ -17,6 +17,7 @@ import re
 from functools import partial
 import pandas as pd
 from api.db import LLMType
+from api.db.services.dialog_service import message_fit_in
 from api.db.services.llm_service import LLMBundle
 from api.settings import retrievaler
 from agent.component.base import ComponentBase, ComponentParamBase
@@ -101,18 +102,21 @@ class Generate(ComponentBase):
         prompt = self._param.prompt
 
         retrieval_res = self.get_input()
-        input = ("  - " + "\n  - ".join(retrieval_res["content"])) if "content" in retrieval_res else ""
+        input = ("  - "+"\n  - ".join([c for c in retrieval_res["content"] if isinstance(c, str)])) if "content" in retrieval_res else ""
         for para in self._param.parameters:
             cpn = self._canvas.get_component(para["component_id"])["obj"]
+            if cpn.component_name.lower() == "answer":
+                kwargs[para["key"]] = self._canvas.get_history(1)[0]["content"]
+                continue
             _, out = cpn.output(allow_partial=False)
             if "content" not in out.columns:
                 kwargs[para["key"]] = "Nothing"
             else:
-                kwargs[para["key"]] = "  - " + "\n  - ".join(out["content"])
+                kwargs[para["key"]] = "  - "+"\n - ".join([o if isinstance(o, str) else str(o) for o in out["content"]])
 
         kwargs["input"] = input
         for n, v in kwargs.items():
-            prompt = re.sub(r"\{%s\}" % n, re.escape(str(v)), prompt)
+            prompt = re.sub(r"\{%s\}" % re.escape(n), re.escape(str(v)), prompt)
 
         downstreams = self._canvas.get_component(self._id)["downstream"]
         if kwargs.get("stream") and len(downstreams) == 1 and self._canvas.get_component(downstreams[0])[
@@ -124,8 +128,10 @@ class Generate(ComponentBase):
                 retrieval_res["empty_response"]) else "Nothing found in knowledgebase!", "reference": []}
             return pd.DataFrame([res])
 
-        ans = chat_mdl.chat(prompt, self._canvas.get_history(self._param.message_history_window_size),
-                            self._param.gen_conf())
+        msg = self._canvas.get_history(self._param.message_history_window_size)
+        _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
+        ans = chat_mdl.chat(msg[0]["content"], msg[1:], self._param.gen_conf())
+
         if self._param.cite and "content_ltks" in retrieval_res.columns and "vector" in retrieval_res.columns:
             res = self.set_cite(retrieval_res, ans)
             return pd.DataFrame([res])
@@ -141,9 +147,10 @@ class Generate(ComponentBase):
             self.set_output(res)
             return
 
+        msg = self._canvas.get_history(self._param.message_history_window_size)
+        _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(chat_mdl.max_length * 0.97))
         answer = ""
-        for ans in chat_mdl.chat_streamly(prompt, self._canvas.get_history(self._param.message_history_window_size),
-                                          self._param.gen_conf()):
+        for ans in chat_mdl.chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf()):
             res = {"content": ans, "reference": []}
             answer = ans
             yield res

agent/component/invoke.py

@@ -0,0 +1,103 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import json
+import re
+from abc import ABC
+import requests
+from deepdoc.parser import HtmlParser
+from agent.component.base import ComponentBase, ComponentParamBase
+
+
+class InvokeParam(ComponentParamBase):
+    """
+    Define the Crawler component parameters.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.proxy = None
+        self.headers = ""
+        self.method = "get"
+        self.variables = []
+        self.url = ""
+        self.timeout = 60
+        self.clean_html = False
+
+    def check(self):
+        self.check_valid_value(self.method.lower(), "Type of content from the crawler", ['get', 'post', 'put'])
+        self.check_empty(self.url, "End point URL")
+        self.check_positive_integer(self.timeout, "Timeout time in second")
+        self.check_boolean(self.clean_html, "Clean HTML")
+
+
+class Invoke(ComponentBase, ABC):
+    component_name = "Invoke"
+
+    def _run(self, history, **kwargs):
+        args = {}
+        for para in self._param.variables:
+            if para.get("component_id"):
+                cpn = self._canvas.get_component(para["component_id"])["obj"]
+                _, out = cpn.output(allow_partial=False)
+                args[para["key"]] = "\n".join(out["content"])
+            else:
+                args[para["key"]] = "\n".join(para["value"])
+
+        url = self._param.url.strip()
+        if url.find("http") != 0:
+            url = "http://" + url
+
+        method = self._param.method.lower()
+        headers = {}
+        if self._param.headers:
+            headers = json.loads(self._param.headers)
+        proxies = None
+        if re.sub(r"https?:?/?/?", "", self._param.proxy):
+            proxies = {"http": self._param.proxy, "https": self._param.proxy}
+
+        if method == 'get':
+            response = requests.get(url=url,
+                                    params=args,
+                                    headers=headers,
+                                    proxies=proxies,
+                                    timeout=self._param.timeout)
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
+
+            return Invoke.be_output(response.text)
+
+        if method == 'put':
+            response = requests.put(url=url,
+                                    data=args,
+                                    headers=headers,
+                                    proxies=proxies,
+                                    timeout=self._param.timeout)
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
+            return Invoke.be_output(response.text)
+
+        if method == 'post':
+            response = requests.post(url=url,
+                                    json=args,
+                                    headers=headers,
+                                    proxies=proxies,
+                                    timeout=self._param.timeout)
+            if self._param.clean_html:
+                sections = HtmlParser()(None, response.content)
+                return Invoke.be_output("\n".join(sections))
+            return Invoke.be_output(response.text)

agent/component/retrieval.py

@@ -43,22 +43,19 @@ class RetrievalParam(ComponentParamBase):
         self.check_decimal_float(self.similarity_threshold, "[Retrieval] Similarity threshold")
         self.check_decimal_float(self.keywords_similarity_weight, "[Retrieval] Keywords similarity weight")
         self.check_positive_number(self.top_n, "[Retrieval] Top N")
-        self.check_empty(self.kb_ids, "[Retrieval] Knowledge bases")
 
 
 class Retrieval(ComponentBase, ABC):
     component_name = "Retrieval"
 
     def _run(self, history, **kwargs):
-        query = []
-        for role, cnt in history[::-1][:self._param.message_history_window_size]:
-            if role != "user":continue
-            query.append(cnt)
-        # query = "\n".join(query)
-        query = query[0]
+        query = self.get_input()
+        query = str(query["content"][0]) if "content" in query else ""
+
         kbs = KnowledgebaseService.get_by_ids(self._param.kb_ids)
         if not kbs:
-            raise ValueError("Can't find knowledgebases by {}".format(self._param.kb_ids))
+            return Retrieval.be_output("")
+
         embd_nms = list(set([kb.embd_id for kb in kbs]))
         assert len(embd_nms) == 1, "Knowledge bases use different embedding models."
 

agent/component/rewrite.py

@@ -33,7 +33,7 @@ class RewriteQuestionParam(GenerateParam):
     def check(self):
         super().check()
 
-    def get_prompt(self):
+    def get_prompt(self, conv):
         self.prompt = """
         You are an expert at query expansion to generate a paraphrasing of a question.
         I can't retrieval relevant information from the knowledge base by using user's question directly.     
@@ -43,6 +43,40 @@ class RewriteQuestionParam(GenerateParam):
         And return 5 versions of question and one is from translation.
         Just list the question. No other words are needed.
         """
+        return f"""
+Role: A helpful assistant
+Task: Generate a full user question that would follow the conversation.
+Requirements & Restrictions:
+  - Text generated MUST be in the same language of the original user's question.
+  - If the user's latest question is completely, don't do anything, just return the original question.
+  - DON'T generate anything except a refined question.
+
+######################
+-Examples-
+######################
+# Example 1
+## Conversation
+USER: What is the name of Donald Trump's father?
+ASSISTANT:  Fred Trump.
+USER: And his mother?
+###############
+Output: What's the name of Donald Trump's mother?
+------------
+# Example 2
+## Conversation
+USER: What is the name of Donald Trump's father?
+ASSISTANT:  Fred Trump.
+USER: And his mother?
+ASSISTANT:  Mary Trump.
+User: What's her full name?
+###############
+Output: What's the full name of Donald Trump's mother Mary Trump?
+######################
+# Real Data
+## Conversation
+{conv}
+###############
+    """
         return self.prompt
 
 
@@ -56,14 +90,16 @@ class RewriteQuestion(Generate, ABC):
             self._loop = 0
             raise Exception("Sorry! Nothing relevant found.")
         self._loop += 1
-        q = "Question: "
-        for r, c in self._canvas.history[::-1]:
-            if r == "user":
-                q += c
-                break
+
+        hist = self._canvas.get_history(4)
+        conv = []
+        for m in hist:
+            if m["role"] not in ["user", "assistant"]: continue
+            conv.append("{}: {}".format(m["role"].upper(), m["content"]))
+        conv = "\n".join(conv)
 
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
-        ans = chat_mdl.chat(self._param.get_prompt(), [{"role": "user", "content": q}],
+        ans = chat_mdl.chat(self._param.get_prompt(conv), [{"role": "user", "content": "Output: "}],
                             self._param.gen_conf())
         self._canvas.history.pop()
         self._canvas.history.append(("user", ans))

api/apps/__init__.py

@@ -83,7 +83,7 @@ def register_page(page_path):
     sys.modules[module_name] = page
     spec.loader.exec_module(page)
     page_name = getattr(page, 'page_name', page_name)
-    url_prefix = f'/api/{API_VERSION}/{page_name}' if "/sdk/" in path else f'/{API_VERSION}/{page_name}'
+    url_prefix = f'/api/{API_VERSION}' if "/sdk/" in path else f'/{API_VERSION}/{page_name}'
 
     app.register_blueprint(page.manager, url_prefix=url_prefix)
     return url_prefix

api/apps/api_app.py

@@ -22,10 +22,10 @@ from api.db.services.llm_service import TenantLLMService
 from flask_login import login_required, current_user
 
 from api.db import FileType, LLMType, ParserType, FileSource
-from api.db.db_models import APIToken, API4Conversation, Task, File
+from api.db.db_models import APIToken, Task, File
 from api.db.services import duplicate_name
 from api.db.services.api_service import APITokenService, API4ConversationService
-from api.db.services.dialog_service import DialogService, chat
+from api.db.services.dialog_service import DialogService, chat, keyword_extraction
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
@@ -34,23 +34,17 @@ from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
 from api.settings import RetCode, retrievaler
 from api.utils import get_uuid, current_timestamp, datetime_format
-from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request
-from itsdangerous import URLSafeTimedSerializer
+from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
+    generate_confirmation_token
 
 from api.utils.file_utils import filename_type, thumbnail
-from rag.nlp import keyword_extraction
 from rag.utils.storage_factory import STORAGE_IMPL
 
-from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
+from api.db.services.canvas_service import UserCanvasService
 from agent.canvas import Canvas
 from functools import partial
 
 
-def generate_confirmation_token(tenent_id):
-    serializer = URLSafeTimedSerializer(tenent_id)
-    return "ragflow-" + serializer.dumps(get_uuid(), salt=tenent_id)[2:34]
-
-
 @manager.route('/new_token', methods=['POST'])
 @login_required
 def new_token():

api/apps/canvas_app.py

@@ -18,8 +18,6 @@ from functools import partial
 from flask import request, Response
 from flask_login import login_required, current_user
 from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService
-from api.db.services.dialog_service import full_question
-from api.db.services.user_service import TenantService
 from api.settings import RetCode
 from api.utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
@@ -111,8 +109,9 @@ def run():
         if "message" in req:
             canvas.messages.append({"role": "user", "content": req["message"], "id": message_id})
             if len([m for m in canvas.messages if m["role"] == "user"]) > 1:
-                ten = TenantService.get_by_user_id(current_user.id)[0]
-                req["message"] = full_question(ten["tenant_id"], ten["llm_id"], canvas.messages)
+                #ten = TenantService.get_info_by(current_user.id)[0]
+                #req["message"] = full_question(ten["tenant_id"], ten["llm_id"], canvas.messages)
+                pass
             canvas.add_user_input(req["message"])
         answer = canvas.run(stream=stream)
         print(canvas)

api/apps/chunk_app.py

@@ -21,8 +21,9 @@ from flask import request
 from flask_login import login_required, current_user
 from elasticsearch_dsl import Q
 
+from api.db.services.dialog_service import keyword_extraction
 from rag.app.qa import rmPrefix, beAdoc
-from rag.nlp import search, rag_tokenizer, keyword_extraction
+from rag.nlp import search, rag_tokenizer
 from rag.utils.es_conn import ELASTICSEARCH
 from rag.utils import rmSpace
 from api.db import LLMType, ParserType
@@ -319,9 +320,28 @@ def knowledge_graph():
     for id in sres.ids[:2]:
         ty = sres.field[id]["knowledge_graph_kwd"]
         try:
-            obj[ty] = json.loads(sres.field[id]["content_with_weight"])
+            content_json = json.loads(sres.field[id]["content_with_weight"])
         except Exception as e:
-            print(traceback.format_exc(), flush=True)
+            continue
+
+        if ty == 'mind_map':
+            node_dict = {}
+
+            def repeat_deal(content_json, node_dict):
+                if 'id' in content_json:
+                    if content_json['id'] in node_dict:
+                        node_name = content_json['id']
+                        content_json['id'] += f"({node_dict[content_json['id']]})"
+                        node_dict[node_name] += 1
+                    else:
+                        node_dict[content_json['id']] = 1
+                if 'children' in content_json and content_json['children']:
+                    for item in content_json['children']:
+                        repeat_deal(item, node_dict)
+
+            repeat_deal(content_json, node_dict)
+
+        obj[ty] = content_json
 
     return get_json_result(data=obj)
 

api/apps/conversation_app.py

@@ -26,7 +26,6 @@ from api.db.services.dialog_service import DialogService, ConversationService, c
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle, TenantService, TenantLLMService
 from api.settings import RetCode, retrievaler
-from api.utils import get_uuid
 from api.utils.api_utils import get_json_result
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
 from graphrag.mind_map_extractor import MindMapExtractor
@@ -142,9 +141,6 @@ def list_convsersation():
 @validate_request("conversation_id", "messages")
 def completion():
     req = request.json
-    # req = {"conversation_id": "9aaaca4c11d311efa461fa163e197198", "messages": [
-    #    {"role": "user", "content": "上海有吗？"}
-    # ]}
     msg = []
     for m in req["messages"]:
         if m["role"] == "system":
@@ -187,6 +183,7 @@ def completion():
                     yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": ans}, ensure_ascii=False) + "\n\n"
                 ConversationService.update_by_id(conv.id, conv.to_dict())
             except Exception as e:
+                traceback.print_exc()
                 yield "data:" + json.dumps({"retcode": 500, "retmsg": str(e),
                                             "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
                                            ensure_ascii=False) + "\n\n"
@@ -218,7 +215,7 @@ def tts():
     req = request.json
     text = req["text"]
 
-    tenants = TenantService.get_by_user_id(current_user.id)
+    tenants = TenantService.get_info_by(current_user.id)
     if not tenants:
         return get_data_error_result(retmsg="Tenant not found!")
 

api/apps/dataset_api.py

@@ -1,880 +0,0 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-import os
-import pathlib
-import re
-import warnings
-from functools import partial
-from io import BytesIO
-
-from elasticsearch_dsl import Q
-from flask import request, send_file
-from flask_login import login_required, current_user
-from httpx import HTTPError
-
-from api.contants import NAME_LENGTH_LIMIT
-from api.db import FileType, ParserType, FileSource, TaskStatus
-from api.db import StatusEnum
-from api.db.db_models import File
-from api.db.services import duplicate_name
-from api.db.services.document_service import DocumentService
-from api.db.services.file2document_service import File2DocumentService
-from api.db.services.file_service import FileService
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.user_service import TenantService
-from api.settings import RetCode
-from api.utils import get_uuid
-from api.utils.api_utils import construct_json_result, construct_error_response
-from api.utils.api_utils import construct_result, validate_request
-from api.utils.file_utils import filename_type, thumbnail
-from rag.app import book, laws, manual, naive, one, paper, presentation, qa, resume, table, picture, audio, email
-from rag.nlp import search
-from rag.utils.es_conn import ELASTICSEARCH
-from rag.utils.storage_factory import STORAGE_IMPL
-
-MAXIMUM_OF_UPLOADING_FILES = 256
-
-
-# ------------------------------ create a dataset ---------------------------------------
-
-@manager.route("/", methods=["POST"])
-@login_required  # use login
-@validate_request("name")  # check name key
-def create_dataset():
-    # Check if Authorization header is present
-    authorization_token = request.headers.get("Authorization")
-    if not authorization_token:
-        return construct_json_result(code=RetCode.AUTHENTICATION_ERROR, message="Authorization header is missing.")
-
-    # TODO: Login or API key
-    # objs = APIToken.query(token=authorization_token)
-    #
-    # # Authorization error
-    # if not objs:
-    #     return construct_json_result(code=RetCode.AUTHENTICATION_ERROR, message="Token is invalid.")
-    #
-    # tenant_id = objs[0].tenant_id
-
-    tenant_id = current_user.id
-    request_body = request.json
-
-    # In case that there's no name
-    if "name" not in request_body:
-        return construct_json_result(code=RetCode.DATA_ERROR, message="Expected 'name' field in request body")
-
-    dataset_name = request_body["name"]
-
-    # empty dataset_name
-    if not dataset_name:
-        return construct_json_result(code=RetCode.DATA_ERROR, message="Empty dataset name")
-
-    # In case that there's space in the head or the tail
-    dataset_name = dataset_name.strip()
-
-    # In case that the length of the name exceeds the limit
-    dataset_name_length = len(dataset_name)
-    if dataset_name_length > NAME_LENGTH_LIMIT:
-        return construct_json_result(
-            code=RetCode.DATA_ERROR,
-            message=f"Dataset name: {dataset_name} with length {dataset_name_length} exceeds {NAME_LENGTH_LIMIT}!")
-
-    # In case that there are other fields in the data-binary
-    if len(request_body.keys()) > 1:
-        name_list = []
-        for key_name in request_body.keys():
-            if key_name != "name":
-                name_list.append(key_name)
-        return construct_json_result(code=RetCode.DATA_ERROR,
-                                     message=f"fields: {name_list}, are not allowed in request body.")
-
-    # If there is a duplicate name, it will modify it to make it unique
-    request_body["name"] = duplicate_name(
-        KnowledgebaseService.query,
-        name=dataset_name,
-        tenant_id=tenant_id,
-        status=StatusEnum.VALID.value)
-    try:
-        request_body["id"] = get_uuid()
-        request_body["tenant_id"] = tenant_id
-        request_body["created_by"] = tenant_id
-        exist, t = TenantService.get_by_id(tenant_id)
-        if not exist:
-            return construct_result(code=RetCode.AUTHENTICATION_ERROR, message="Tenant not found.")
-        request_body["embd_id"] = t.embd_id
-        if not KnowledgebaseService.save(**request_body):
-            # failed to create new dataset
-            return construct_result()
-        return construct_json_result(code=RetCode.SUCCESS,
-                                     data={"dataset_name": request_body["name"], "dataset_id": request_body["id"]})
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# -----------------------------list datasets-------------------------------------------------------
-
-@manager.route("/", methods=["GET"])
-@login_required
-def list_datasets():
-    offset = request.args.get("offset", 0)
-    count = request.args.get("count", -1)
-    orderby = request.args.get("orderby", "create_time")
-    desc = request.args.get("desc", True)
-    try:
-        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        datasets = KnowledgebaseService.get_by_tenant_ids_by_offset(
-            [m["tenant_id"] for m in tenants], current_user.id, int(offset), int(count), orderby, desc)
-        return construct_json_result(data=datasets, code=RetCode.SUCCESS, message=f"List datasets successfully!")
-    except Exception as e:
-        return construct_error_response(e)
-    except HTTPError as http_err:
-        return construct_json_result(http_err)
-
-
-# ---------------------------------delete a dataset ----------------------------
-
-@manager.route("/<dataset_id>", methods=["DELETE"])
-@login_required
-def remove_dataset(dataset_id):
-    try:
-        datasets = KnowledgebaseService.query(created_by=current_user.id, id=dataset_id)
-
-        # according to the id, searching for the dataset
-        if not datasets:
-            return construct_json_result(message=f"The dataset cannot be found for your current account.",
-                                         code=RetCode.OPERATING_ERROR)
-
-        # Iterating the documents inside the dataset
-        for doc in DocumentService.query(kb_id=dataset_id):
-            if not DocumentService.remove_document(doc, datasets[0].tenant_id):
-                # the process of deleting failed
-                return construct_json_result(code=RetCode.DATA_ERROR,
-                                             message="There was an error during the document removal process. "
-                                                     "Please check the status of the RAGFlow server and try the removal again.")
-            # delete the other files
-            f2d = File2DocumentService.get_by_document_id(doc.id)
-            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
-            File2DocumentService.delete_by_document_id(doc.id)
-
-        # delete the dataset
-        if not KnowledgebaseService.delete_by_id(dataset_id):
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message="There was an error during the dataset removal process. "
-                                                 "Please check the status of the RAGFlow server and try the removal again.")
-        # success
-        return construct_json_result(code=RetCode.SUCCESS, message=f"Remove dataset: {dataset_id} successfully")
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ------------------------------ get details of a dataset ----------------------------------------
-
-@manager.route("/<dataset_id>", methods=["GET"])
-@login_required
-def get_dataset(dataset_id):
-    try:
-        dataset = KnowledgebaseService.get_detail(dataset_id)
-        if not dataset:
-            return construct_json_result(code=RetCode.DATA_ERROR, message="Can't find this dataset!")
-        return construct_json_result(data=dataset, code=RetCode.SUCCESS)
-    except Exception as e:
-        return construct_json_result(e)
-
-
-# ------------------------------ update a dataset --------------------------------------------
-
-@manager.route("/<dataset_id>", methods=["PUT"])
-@login_required
-def update_dataset(dataset_id):
-    req = request.json
-    try:
-        # the request cannot be empty
-        if not req:
-            return construct_json_result(code=RetCode.DATA_ERROR, message="Please input at least one parameter that "
-                                                                          "you want to update!")
-        # check whether the dataset can be found
-        if not KnowledgebaseService.query(created_by=current_user.id, id=dataset_id):
-            return construct_json_result(message=f"Only the owner of knowledgebase is authorized for this operation!",
-                                         code=RetCode.OPERATING_ERROR)
-
-        exist, dataset = KnowledgebaseService.get_by_id(dataset_id)
-        # check whether there is this dataset
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR, message="This dataset cannot be found!")
-
-        if "name" in req:
-            name = req["name"].strip()
-            # check whether there is duplicate name
-            if name.lower() != dataset.name.lower() \
-                    and len(KnowledgebaseService.query(name=name, tenant_id=current_user.id,
-                                                       status=StatusEnum.VALID.value)) > 1:
-                return construct_json_result(code=RetCode.DATA_ERROR,
-                                             message=f"The name: {name.lower()} is already used by other "
-                                                     f"datasets. Please choose a different name.")
-
-        dataset_updating_data = {}
-        chunk_num = req.get("chunk_num")
-        # modify the value of 11 parameters
-
-        # 2 parameters: embedding id and chunk method
-        # only if chunk_num is 0, the user can update the embedding id
-        if req.get("embedding_model_id"):
-            if chunk_num == 0:
-                dataset_updating_data["embd_id"] = req["embedding_model_id"]
-            else:
-                return construct_json_result(code=RetCode.DATA_ERROR,
-                                             message="You have already parsed the document in this "
-                                                     "dataset, so you cannot change the embedding "
-                                                     "model.")
-        # only if chunk_num is 0, the user can update the chunk_method
-        if "chunk_method" in req:
-            type_value = req["chunk_method"]
-            if is_illegal_value_for_enum(type_value, ParserType):
-                return construct_json_result(message=f"Illegal value {type_value} for 'chunk_method' field.",
-                                             code=RetCode.DATA_ERROR)
-            if chunk_num != 0:
-                construct_json_result(code=RetCode.DATA_ERROR, message="You have already parsed the document "
-                                                                       "in this dataset, so you cannot "
-                                                                       "change the chunk method.")
-            dataset_updating_data["parser_id"] = req["template_type"]
-
-        # convert the photo parameter to avatar
-        if req.get("photo"):
-            dataset_updating_data["avatar"] = req["photo"]
-
-        # layout_recognize
-        if "layout_recognize" in req:
-            if "parser_config" not in dataset_updating_data:
-                dataset_updating_data['parser_config'] = {}
-            dataset_updating_data['parser_config']['layout_recognize'] = req['layout_recognize']
-
-        # TODO: updating use_raptor needs to construct a class
-
-        # 6 parameters
-        for key in ["name", "language", "description", "permission", "id", "token_num"]:
-            if key in req:
-                dataset_updating_data[key] = req.get(key)
-
-        # update
-        if not KnowledgebaseService.update_by_id(dataset.id, dataset_updating_data):
-            return construct_json_result(code=RetCode.OPERATING_ERROR, message="Failed to update! "
-                                                                               "Please check the status of RAGFlow "
-                                                                               "server and try again!")
-
-        exist, dataset = KnowledgebaseService.get_by_id(dataset.id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR, message="Failed to get the dataset "
-                                                                          "using the dataset ID.")
-
-        return construct_json_result(data=dataset.to_json(), code=RetCode.SUCCESS)
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# --------------------------------content management ----------------------------------------------
-
-# ----------------------------upload files-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/", methods=["POST"])
-@login_required
-def upload_documents(dataset_id):
-    # no files
-    if not request.files:
-        return construct_json_result(
-            message="There is no file!", code=RetCode.ARGUMENT_ERROR)
-
-    # the number of uploading files exceeds the limit
-    file_objs = request.files.getlist("file")
-    num_file_objs = len(file_objs)
-
-    if num_file_objs > MAXIMUM_OF_UPLOADING_FILES:
-        return construct_json_result(code=RetCode.DATA_ERROR, message=f"You try to upload {num_file_objs} files, "
-                                                                      f"which exceeds the maximum number of uploading files: {MAXIMUM_OF_UPLOADING_FILES}")
-
-    # no dataset
-    exist, dataset = KnowledgebaseService.get_by_id(dataset_id)
-    if not exist:
-        return construct_json_result(message="Can't find this dataset", code=RetCode.DATA_ERROR)
-
-    for file_obj in file_objs:
-        file_name = file_obj.filename
-        # no name
-        if not file_name:
-            return construct_json_result(
-                message="There is a file without name!", code=RetCode.ARGUMENT_ERROR)
-
-        # TODO: support the remote files
-        if 'http' in file_name:
-            return construct_json_result(code=RetCode.ARGUMENT_ERROR, message="Remote files have not unsupported.")
-
-    # get the root_folder
-    root_folder = FileService.get_root_folder(current_user.id)
-    # get the id of the root_folder
-    parent_file_id = root_folder["id"]  # document id
-    # this is for the new user, create '.knowledgebase' file
-    FileService.init_knowledgebase_docs(parent_file_id, current_user.id)
-    # go inside this folder, get the kb_root_folder
-    kb_root_folder = FileService.get_kb_folder(current_user.id)
-    # link the file management to the kb_folder
-    kb_folder = FileService.new_a_file_from_kb(dataset.tenant_id, dataset.name, kb_root_folder["id"])
-
-    # grab all the errs
-    err = []
-    MAX_FILE_NUM_PER_USER = int(os.environ.get("MAX_FILE_NUM_PER_USER", 0))
-    uploaded_docs_json = []
-    for file in file_objs:
-        try:
-            # TODO: get this value from the database as some tenants have this limit while others don't
-            if MAX_FILE_NUM_PER_USER > 0 and DocumentService.get_doc_count(dataset.tenant_id) >= MAX_FILE_NUM_PER_USER:
-                return construct_json_result(code=RetCode.DATA_ERROR,
-                                             message="Exceed the maximum file number of a free user!")
-            # deal with the duplicate name
-            filename = duplicate_name(
-                DocumentService.query,
-                name=file.filename,
-                kb_id=dataset.id)
-
-            # deal with the unsupported type
-            filetype = filename_type(filename)
-            if filetype == FileType.OTHER.value:
-                return construct_json_result(code=RetCode.DATA_ERROR,
-                                             message="This type of file has not been supported yet!")
-
-            # upload to the minio
-            location = filename
-            while STORAGE_IMPL.obj_exist(dataset_id, location):
-                location += "_"
-
-            blob = file.read()
-
-            # the content is empty, raising a warning
-            if blob == b'':
-                warnings.warn(f"[WARNING]: The content of the file {filename} is empty.")
-
-            STORAGE_IMPL.put(dataset_id, location, blob)
-
-            doc = {
-                "id": get_uuid(),
-                "kb_id": dataset.id,
-                "parser_id": dataset.parser_id,
-                "parser_config": dataset.parser_config,
-                "created_by": current_user.id,
-                "type": filetype,
-                "name": filename,
-                "location": location,
-                "size": len(blob),
-                "thumbnail": thumbnail(filename, blob)
-            }
-            if doc["type"] == FileType.VISUAL:
-                doc["parser_id"] = ParserType.PICTURE.value
-            if doc["type"] == FileType.AURAL:
-                doc["parser_id"] = ParserType.AUDIO.value
-            if re.search(r"\.(ppt|pptx|pages)$", filename):
-                doc["parser_id"] = ParserType.PRESENTATION.value
-            if re.search(r"\.(eml)$", filename):
-                doc["parser_id"] = ParserType.EMAIL.value
-            DocumentService.insert(doc)
-
-            FileService.add_file_from_kb(doc, kb_folder["id"], dataset.tenant_id)
-            uploaded_docs_json.append(doc)
-        except Exception as e:
-            err.append(file.filename + ": " + str(e))
-
-    if err:
-        # return all the errors
-        return construct_json_result(message="\n".join(err), code=RetCode.SERVER_ERROR)
-    # success
-    return construct_json_result(data=uploaded_docs_json, code=RetCode.SUCCESS)
-
-
-# ----------------------------delete a file-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/<document_id>", methods=["DELETE"])
-@login_required
-def delete_document(document_id, dataset_id):  # string
-    # get the root folder
-    root_folder = FileService.get_root_folder(current_user.id)
-    # parent file's id
-    parent_file_id = root_folder["id"]
-    # consider the new user
-    FileService.init_knowledgebase_docs(parent_file_id, current_user.id)
-    # store all the errors that may have
-    errors = ""
-    try:
-        # whether there is this document
-        exist, doc = DocumentService.get_by_id(document_id)
-        if not exist:
-            return construct_json_result(message=f"Document {document_id} not found!", code=RetCode.DATA_ERROR)
-        # whether this doc is authorized by this tenant
-        tenant_id = DocumentService.get_tenant_id(document_id)
-        if not tenant_id:
-            return construct_json_result(
-                message=f"You cannot delete this document {document_id} due to the authorization"
-                        f" reason!", code=RetCode.AUTHENTICATION_ERROR)
-
-        # get the doc's id and location
-        real_dataset_id, location = File2DocumentService.get_storage_address(doc_id=document_id)
-
-        if real_dataset_id != dataset_id:
-            return construct_json_result(message=f"The document {document_id} is not in the dataset: {dataset_id}, "
-                                                 f"but in the dataset: {real_dataset_id}.", code=RetCode.ARGUMENT_ERROR)
-
-        # there is an issue when removing
-        if not DocumentService.remove_document(doc, tenant_id):
-            return construct_json_result(
-                message="There was an error during the document removal process. Please check the status of the "
-                        "RAGFlow server and try the removal again.", code=RetCode.OPERATING_ERROR)
-
-        # fetch the File2Document record associated with the provided document ID.
-        file_to_doc = File2DocumentService.get_by_document_id(document_id)
-        # delete the associated File record.
-        FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == file_to_doc[0].file_id])
-        # delete the File2Document record itself using the document ID. This removes the
-        # association between the document and the file after the File record has been deleted.
-        File2DocumentService.delete_by_document_id(document_id)
-
-        # delete it from minio
-        STORAGE_IMPL.rm(dataset_id, location)
-    except Exception as e:
-        errors += str(e)
-    if errors:
-        return construct_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
-
-    return construct_json_result(data=True, code=RetCode.SUCCESS)
-
-
-# ----------------------------list files-----------------------------------------------------
-@manager.route('/<dataset_id>/documents/', methods=['GET'])
-@login_required
-def list_documents(dataset_id):
-    if not dataset_id:
-        return construct_json_result(
-            data=False, message="Lack of 'dataset_id'", code=RetCode.ARGUMENT_ERROR)
-
-    # searching keywords
-    keywords = request.args.get("keywords", "")
-
-    offset = request.args.get("offset", 0)
-    count = request.args.get("count", -1)
-    order_by = request.args.get("order_by", "create_time")
-    descend = request.args.get("descend", True)
-    try:
-        docs, total = DocumentService.list_documents_in_dataset(dataset_id, int(offset), int(count), order_by,
-                                                                descend, keywords)
-
-        return construct_json_result(data={"total": total, "docs": docs}, message=RetCode.SUCCESS)
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------update: enable rename-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/<document_id>", methods=["PUT"])
-@login_required
-def update_document(dataset_id, document_id):
-    req = request.json
-    try:
-        legal_parameters = set()
-        legal_parameters.add("name")
-        legal_parameters.add("enable")
-        legal_parameters.add("template_type")
-
-        for key in req.keys():
-            if key not in legal_parameters:
-                return construct_json_result(code=RetCode.ARGUMENT_ERROR, message=f"{key} is an illegal parameter.")
-
-        # The request body cannot be empty
-        if not req:
-            return construct_json_result(
-                code=RetCode.DATA_ERROR,
-                message="Please input at least one parameter that you want to update!")
-
-        # Check whether there is this dataset
-        exist, dataset = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR, message=f"This dataset {dataset_id} cannot be found!")
-
-        # The document does not exist
-        exist, document = DocumentService.get_by_id(document_id)
-        if not exist:
-            return construct_json_result(message=f"This document {document_id} cannot be found!",
-                                         code=RetCode.ARGUMENT_ERROR)
-
-        # Deal with the different keys
-        updating_data = {}
-        if "name" in req:
-            new_name = req["name"]
-            updating_data["name"] = new_name
-            # Check whether the new_name is suitable
-            # 1. no name value
-            if not new_name:
-                return construct_json_result(code=RetCode.DATA_ERROR, message="There is no new name.")
-
-            # 2. In case that there's space in the head or the tail
-            new_name = new_name.strip()
-
-            # 3. Check whether the new_name has the same extension of file as before
-            if pathlib.Path(new_name.lower()).suffix != pathlib.Path(
-                    document.name.lower()).suffix:
-                return construct_json_result(
-                    data=False,
-                    message="The extension of file cannot be changed",
-                    code=RetCode.ARGUMENT_ERROR)
-
-            # 4. Check whether the new name has already been occupied by other file
-            for d in DocumentService.query(name=new_name, kb_id=document.kb_id):
-                if d.name == new_name:
-                    return construct_json_result(
-                        message="Duplicated document name in the same dataset.",
-                        code=RetCode.ARGUMENT_ERROR)
-
-        if "enable" in req:
-            enable_value = req["enable"]
-            if is_illegal_value_for_enum(enable_value, StatusEnum):
-                return construct_json_result(message=f"Illegal value {enable_value} for 'enable' field.",
-                                             code=RetCode.DATA_ERROR)
-            updating_data["status"] = enable_value
-
-        # TODO: Chunk-method - update parameters inside the json object parser_config
-        if "template_type" in req:
-            type_value = req["template_type"]
-            if is_illegal_value_for_enum(type_value, ParserType):
-                return construct_json_result(message=f"Illegal value {type_value} for 'template_type' field.",
-                                             code=RetCode.DATA_ERROR)
-            updating_data["parser_id"] = req["template_type"]
-
-        # The process of updating
-        if not DocumentService.update_by_id(document_id, updating_data):
-            return construct_json_result(
-                code=RetCode.OPERATING_ERROR,
-                message="Failed to update document in the database! "
-                        "Please check the status of RAGFlow server and try again!")
-
-        # name part: file service
-        if "name" in req:
-            # Get file by document id
-            file_information = File2DocumentService.get_by_document_id(document_id)
-            if file_information:
-                exist, file = FileService.get_by_id(file_information[0].file_id)
-                FileService.update_by_id(file.id, {"name": req["name"]})
-
-        exist, document = DocumentService.get_by_id(document_id)
-
-        # Success
-        return construct_json_result(data=document.to_json(), message="Success", code=RetCode.SUCCESS)
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# Helper method to judge whether it's an illegal value
-def is_illegal_value_for_enum(value, enum_class):
-    return value not in enum_class.__members__.values()
-
-
-# ----------------------------download a file-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/<document_id>", methods=["GET"])
-@login_required
-def download_document(dataset_id, document_id):
-    try:
-        # Check whether there is this dataset
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset '{dataset_id}' cannot be found!")
-
-        # Check whether there is this document
-        exist, document = DocumentService.get_by_id(document_id)
-        if not exist:
-            return construct_json_result(message=f"This document '{document_id}' cannot be found!",
-                                         code=RetCode.ARGUMENT_ERROR)
-
-        # The process of downloading
-        doc_id, doc_location = File2DocumentService.get_storage_address(doc_id=document_id)  # minio address
-        file_stream = STORAGE_IMPL.get(doc_id, doc_location)
-        if not file_stream:
-            return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
-
-        file = BytesIO(file_stream)
-
-        # Use send_file with a proper filename and MIME type
-        return send_file(
-            file,
-            as_attachment=True,
-            download_name=document.name,
-            mimetype='application/octet-stream'  # Set a default MIME type
-        )
-
-    # Error
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------start parsing a document-----------------------------------------------------
-# helper method for parsing
-# callback method
-def doc_parse_callback(doc_id, prog=None, msg=""):
-    cancel = DocumentService.do_cancel(doc_id)
-    if cancel:
-        raise Exception("The parsing process has been cancelled!")
-
-"""
-def doc_parse(binary, doc_name, parser_name, tenant_id, doc_id):
-    match parser_name:
-        case "book":
-            book.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "laws":
-            laws.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "manual":
-            manual.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "naive":
-            # It's the mode by default, which is general in the front-end
-            naive.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "one":
-            one.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "paper":
-            paper.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "picture":
-            picture.chunk(doc_name, binary=binary, tenant_id=tenant_id, lang="Chinese",
-                          callback=partial(doc_parse_callback, doc_id))
-        case "presentation":
-            presentation.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "qa":
-            qa.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "resume":
-            resume.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "table":
-            table.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "audio":
-            audio.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case "email":
-            email.chunk(doc_name, binary=binary, callback=partial(doc_parse_callback, doc_id))
-        case _:
-            return False
-
-    return True
-    """
-
-
-@manager.route("/<dataset_id>/documents/<document_id>/status", methods=["POST"])
-@login_required
-def parse_document(dataset_id, document_id):
-    try:
-        # valid dataset
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset '{dataset_id}' cannot be found!")
-
-        return parsing_document_internal(document_id)
-
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------start parsing documents-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/status", methods=["POST"])
-@login_required
-def parse_documents(dataset_id):
-    doc_ids = request.json["doc_ids"]
-    try:
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset '{dataset_id}' cannot be found!")
-        # two conditions
-        if not doc_ids:
-            # documents inside the dataset
-            docs, total = DocumentService.list_documents_in_dataset(dataset_id, 0, -1, "create_time",
-                                                                    True, "")
-            doc_ids = [doc["id"] for doc in docs]
-
-        message = ""
-        # for loop
-        for id in doc_ids:
-            res = parsing_document_internal(id)
-            res_body = res.json
-            if res_body["code"] == RetCode.SUCCESS:
-                message += res_body["message"]
-            else:
-                return res
-        return construct_json_result(data=True, code=RetCode.SUCCESS, message=message)
-
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# helper method for parsing the document
-def parsing_document_internal(id):
-    message = ""
-    try:
-        # Check whether there is this document
-        exist, document = DocumentService.get_by_id(id)
-        if not exist:
-            return construct_json_result(message=f"This document '{id}' cannot be found!",
-                                         code=RetCode.ARGUMENT_ERROR)
-
-        tenant_id = DocumentService.get_tenant_id(id)
-        if not tenant_id:
-            return construct_json_result(message="Tenant not found!", code=RetCode.AUTHENTICATION_ERROR)
-
-        info = {"run": "1", "progress": 0}
-        info["progress_msg"] = ""
-        info["chunk_num"] = 0
-        info["token_num"] = 0
-
-        DocumentService.update_by_id(id, info)
-
-        ELASTICSEARCH.deleteByQuery(Q("match", doc_id=id), idxnm=search.index_name(tenant_id))
-
-        _, doc_attributes = DocumentService.get_by_id(id)
-        doc_attributes = doc_attributes.to_dict()
-        doc_id = doc_attributes["id"]
-
-        bucket, doc_name = File2DocumentService.get_storage_address(doc_id=doc_id)
-        binary = STORAGE_IMPL.get(bucket, doc_name)
-        parser_name = doc_attributes["parser_id"]
-        if binary:
-            res = doc_parse(binary, doc_name, parser_name, tenant_id, doc_id)
-            if res is False:
-                message += f"The parser id: {parser_name} of the document {doc_id} is not supported; "
-        else:
-            message += f"Empty data in the document: {doc_name}; "
-        # failed in parsing
-        if doc_attributes["status"] == TaskStatus.FAIL.value:
-            message += f"Failed in parsing the document: {doc_id}; "
-        return construct_json_result(code=RetCode.SUCCESS, message=message)
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------stop parsing a doc-----------------------------------------------------
-@manager.route("<dataset_id>/documents/<document_id>/status", methods=["DELETE"])
-@login_required
-def stop_parsing_document(dataset_id, document_id):
-    try:
-        # valid dataset
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset '{dataset_id}' cannot be found!")
-
-        return stop_parsing_document_internal(document_id)
-
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------stop parsing docs-----------------------------------------------------
-@manager.route("<dataset_id>/documents/status", methods=["DELETE"])
-@login_required
-def stop_parsing_documents(dataset_id):
-    doc_ids = request.json["doc_ids"]
-    try:
-        # valid dataset?
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset '{dataset_id}' cannot be found!")
-        if not doc_ids:
-            # documents inside the dataset
-            docs, total = DocumentService.list_documents_in_dataset(dataset_id, 0, -1, "create_time",
-                                                                        True, "")
-            doc_ids = [doc["id"] for doc in docs]
-
-        message = ""
-        # for loop
-        for id in doc_ids:
-            res = stop_parsing_document_internal(id)
-            res_body = res.json
-            if res_body["code"] == RetCode.SUCCESS:
-                message += res_body["message"]
-            else:
-                return res
-        return construct_json_result(data=True, code=RetCode.SUCCESS, message=message)
-
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# Helper method
-def stop_parsing_document_internal(document_id):
-    try:
-        # valid doc?
-        exist, doc = DocumentService.get_by_id(document_id)
-        if not exist:
-            return construct_json_result(message=f"This document '{document_id}' cannot be found!",
-                                         code=RetCode.ARGUMENT_ERROR)
-        doc_attributes = doc.to_dict()
-
-        # only when the status is parsing, we need to stop it
-        if doc_attributes["status"] == TaskStatus.RUNNING.value:
-            tenant_id = DocumentService.get_tenant_id(document_id)
-            if not tenant_id:
-                return construct_json_result(message="Tenant not found!", code=RetCode.AUTHENTICATION_ERROR)
-
-            # update successfully?
-            if not DocumentService.update_by_id(document_id, {"status": "2"}):  # cancel
-                return construct_json_result(
-                    code=RetCode.OPERATING_ERROR,
-                    message="There was an error during the stopping parsing the document process. "
-                            "Please check the status of the RAGFlow server and try the update again."
-                )
-
-            _, doc_attributes = DocumentService.get_by_id(document_id)
-            doc_attributes = doc_attributes.to_dict()
-
-            # failed in stop parsing
-            if doc_attributes["status"] == TaskStatus.RUNNING.value:
-                return construct_json_result(message=f"Failed in parsing the document: {document_id}; ", code=RetCode.SUCCESS)
-        return construct_json_result(code=RetCode.SUCCESS, message="")
-    except Exception as e:
-        return construct_error_response(e)
-
-
-# ----------------------------show the status of the file-----------------------------------------------------
-@manager.route("/<dataset_id>/documents/<document_id>/status", methods=["GET"])
-@login_required
-def show_parsing_status(dataset_id, document_id):
-    try:
-        # valid dataset
-        exist, _ = KnowledgebaseService.get_by_id(dataset_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This dataset: '{dataset_id}' cannot be found!")
-        # valid document
-        exist, _ = DocumentService.get_by_id(document_id)
-        if not exist:
-            return construct_json_result(code=RetCode.DATA_ERROR,
-                                         message=f"This document: '{document_id}' is not a valid document.")
-
-        _, doc = DocumentService.get_by_id(document_id)  # get doc object
-        doc_attributes = doc.to_dict()
-
-        return construct_json_result(
-            data={"progress": doc_attributes["progress"], "status": TaskStatus(doc_attributes["status"]).name},
-            code=RetCode.SUCCESS
-        )
-    except Exception as e:
-        return construct_error_response(e)
-
-# ----------------------------list the chunks of the file-----------------------------------------------------
-
-# -- --------------------------delete the chunk-----------------------------------------------------
-
-# ----------------------------edit the status of the chunk-----------------------------------------------------
-
-# ----------------------------insert a new chunk-----------------------------------------------------
-
-# ----------------------------upload a file-----------------------------------------------------
-
-# ----------------------------get a specific chunk-----------------------------------------------------
-
-# ----------------------------retrieval test-----------------------------------------------------

api/apps/document_app.py

@@ -13,16 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License
 #
-import datetime
-import hashlib
-import json
-import os
 import pathlib
 import re
-import traceback
-from concurrent.futures import ThreadPoolExecutor
-from copy import deepcopy
-from io import BytesIO
 
 import flask
 from elasticsearch_dsl import Q
@@ -30,27 +22,24 @@ from flask import request
 from flask_login import login_required, current_user
 
 from api.db.db_models import Task, File
-from api.db.services.dialog_service import DialogService, ConversationService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
-from api.db.services.llm_service import LLMBundle
 from api.db.services.task_service import TaskService, queue_tasks
-from api.db.services.user_service import TenantService, UserTenantService
-from graphrag.mind_map_extractor import MindMapExtractor
-from rag.app import naive
+from api.db.services.user_service import UserTenantService
 from rag.nlp import search
 from rag.utils.es_conn import ELASTICSEARCH
 from api.db.services import duplicate_name
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
 from api.utils import get_uuid
-from api.db import FileType, TaskStatus, ParserType, FileSource, LLMType
+from api.db import FileType, TaskStatus, ParserType, FileSource
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
-from api.settings import RetCode, stat_logger
+from api.settings import RetCode
 from api.utils.api_utils import get_json_result
 from rag.utils.storage_factory import STORAGE_IMPL
-from api.utils.file_utils import filename_type, thumbnail, get_project_base_directory
+from api.utils.file_utils import filename_type, thumbnail
 from api.utils.web_utils import html2pdf, is_valid_url
+from api.contants import IMG_BASE64_PREFIX
 
 
 @manager.route('/upload', methods=['POST'])
@@ -209,15 +198,28 @@ def list_docs():
     try:
         docs, tol = DocumentService.get_by_kb_id(
             kb_id, page_number, items_per_page, orderby, desc, keywords)
+
+        for doc_item in docs:
+            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
+                doc_item['thumbnail'] = f"/v1/document/image/{kb_id}-{doc_item['thumbnail']}"
+
         return get_json_result(data={"total": tol, "docs": docs})
     except Exception as e:
         return server_error_response(e)
 
 
 @manager.route('/infos', methods=['POST'])
+@login_required
 def docinfos():
     req = request.json
     doc_ids = req["doc_ids"]
+    for doc_id in doc_ids:
+        if not DocumentService.accessible(doc_id, current_user.id):
+            return get_json_result(
+                data=False,
+                retmsg='No authorization.',
+                retcode=RetCode.AUTHENTICATION_ERROR
+            )
     docs = DocumentService.get_by_ids(doc_ids)
     return get_json_result(data=list(docs.dicts()))
 
@@ -232,6 +234,11 @@ def thumbnails():
 
     try:
         docs = DocumentService.get_thumbnails(doc_ids)
+
+        for doc_item in docs:
+            if doc_item['thumbnail'] and not doc_item['thumbnail'].startswith(IMG_BASE64_PREFIX):
+                doc_item['thumbnail'] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
+
         return get_json_result(data={d["id"]: d["thumbnail"] for d in docs})
     except Exception as e:
         return server_error_response(e)
@@ -243,11 +250,17 @@ def thumbnails():
 def change_status():
     req = request.json
     if str(req["status"]) not in ["0", "1"]:
-        get_json_result(
+        return get_json_result(
             data=False,
             retmsg='"Status" must be either 0 or 1!',
             retcode=RetCode.ARGUMENT_ERROR)
 
+    if not DocumentService.accessible(req["doc_id"], current_user.id):
+        return get_json_result(
+            data=False,
+            retmsg='No authorization.',
+            retcode=RetCode.AUTHENTICATION_ERROR)
+
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
@@ -286,6 +299,15 @@ def rm():
     req = request.json
     doc_ids = req["doc_id"]
     if isinstance(doc_ids, str): doc_ids = [doc_ids]
+
+    for doc_id in doc_ids:
+        if not DocumentService.accessible4deletion(doc_id, current_user.id):
+            return get_json_result(
+                data=False,
+                retmsg='No authorization.',
+                retcode=RetCode.AUTHENTICATION_ERROR
+            )
+
     root_folder = FileService.get_root_folder(current_user.id)
     pf_id = root_folder["id"]
     FileService.init_knowledgebase_docs(pf_id, current_user.id)
@@ -324,6 +346,13 @@ def rm():
 @validate_request("doc_ids", "run")
 def run():
     req = request.json
+    for doc_id in req["doc_ids"]:
+        if not DocumentService.accessible(doc_id, current_user.id):
+            return get_json_result(
+                data=False,
+                retmsg='No authorization.',
+                retcode=RetCode.AUTHENTICATION_ERROR
+            )
     try:
         for id in req["doc_ids"]:
             info = {"run": str(req["run"]), "progress": 0}
@@ -357,6 +386,12 @@ def run():
 @validate_request("doc_id", "name")
 def rename():
     req = request.json
+    if not DocumentService.accessible(req["doc_id"], current_user.id):
+        return get_json_result(
+            data=False,
+            retmsg='No authorization.',
+            retcode=RetCode.AUTHENTICATION_ERROR
+        )
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
@@ -417,6 +452,13 @@ def get(doc_id):
 @validate_request("doc_id", "parser_id")
 def change_parser():
     req = request.json
+
+    if not DocumentService.accessible(req["doc_id"], current_user.id):
+        return get_json_result(
+            data=False,
+            retmsg='No authorization.',
+            retcode=RetCode.AUTHENTICATION_ERROR
+        )
     try:
         e, doc = DocumentService.get_by_id(req["doc_id"])
         if not e:
@@ -428,8 +470,9 @@ def change_parser():
             else:
                 return get_json_result(data=True)
 
-        if doc.type == FileType.VISUAL or re.search(
-                r"\.(ppt|pptx|pages)$", doc.name):
+        if ((doc.type == FileType.VISUAL and req["parser_id"] != "picture")
+                or (re.search(
+                    r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation")):
             return get_data_error_result(retmsg="Not supported yet!")
 
         e = DocumentService.update_by_id(doc.id,

api/apps/kb_app.py

@@ -13,7 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-from elasticsearch_dsl import Q
 from flask import request
 from flask_login import login_required, current_user
 
@@ -23,14 +22,12 @@ from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid, get_format_time
-from api.db import StatusEnum, UserTenantRole, FileSource
+from api.utils import get_uuid
+from api.db import StatusEnum, FileSource
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.db_models import Knowledgebase, File
-from api.settings import stat_logger, RetCode
+from api.db.db_models import File
+from api.settings import RetCode
 from api.utils.api_utils import get_json_result
-from rag.nlp import search
-from rag.utils.es_conn import ELASTICSEARCH
 
 
 @manager.route('/create', methods=['post'])
@@ -65,6 +62,12 @@ def create():
 def update():
     req = request.json
     req["name"] = req["name"].strip()
+    if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id):
+        return get_json_result(
+            data=False,
+            retmsg='No authorization.',
+            retcode=RetCode.AUTHENTICATION_ERROR
+        )
     try:
         if not KnowledgebaseService.query(
                 created_by=current_user.id, id=req["kb_id"]):
@@ -139,6 +142,12 @@ def list_kbs():
 @validate_request("kb_id")
 def rm():
     req = request.json
+    if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id):
+        return get_json_result(
+            data=False,
+            retmsg='No authorization.',
+            retcode=RetCode.AUTHENTICATION_ERROR
+        )
     try:
         kbs = KnowledgebaseService.query(
                 created_by=current_user.id, id=req["kb_id"])

api/apps/llm_app.py

@@ -58,7 +58,7 @@ def set_api_key():
     chat_passed, embd_passed, rerank_passed = False, False, False
     factory = req["llm_factory"]
     msg = ""
-    for llm in LLMService.query(fid=factory)[:3]:
+    for llm in LLMService.query(fid=factory):
         if not embd_passed and llm.model_type == LLMType.EMBEDDING.value:
             mdl = EmbeddingModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
@@ -77,10 +77,10 @@ def set_api_key():
                                  {"temperature": 0.9,'max_tokens':50})
                 if m.find("**ERROR**") >=0:
                     raise Exception(m)
+                chat_passed = True
             except Exception as e:
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)
-            chat_passed = True
         elif not rerank_passed and llm.model_type == LLMType.RERANK:
             mdl = RerankModel[factory](
                 req["api_key"], llm.llm_name, base_url=req.get("base_url"))
@@ -88,10 +88,14 @@ def set_api_key():
                 arr, tc = mdl.similarity("What's the weather?", ["Is it sunny today?"])
                 if len(arr) == 0 or tc == 0:
                     raise Exception("Fail")
+                rerank_passed = True
+                print(f'passed model rerank{llm.llm_name}',flush=True)
             except Exception as e:
                 msg += f"\nFail to access model({llm.llm_name}) using this api key." + str(
                     e)
-            rerank_passed = True
+        if any([embd_passed, chat_passed, rerank_passed]):
+            msg = ''
+            break
 
     if msg:
         return get_data_error_result(retmsg=msg)
@@ -183,6 +187,10 @@ def add_llm():
         llm_name = req["llm_name"]
         api_key = apikey_json(["google_project_id", "google_region", "google_service_account_key"])
 
+    elif factory == "Azure-OpenAI":
+        llm_name = req["llm_name"]
+        api_key = apikey_json(["api_key", "api_version"])
+
     else:
         llm_name = req["llm_name"]
         api_key = req.get("api_key", "xxxxxxxxxxxxxxx")
@@ -324,7 +332,7 @@ def my_llms():
 @login_required
 def list_app():
     self_deploied = ["Youdao","FastEmbed", "BAAI", "Ollama", "Xinference", "LocalAI", "LM-Studio"]
-    weighted = ["Youdao","FastEmbed", "BAAI"] if LIGHTEN else []
+    weighted = ["Youdao","FastEmbed", "BAAI"] if LIGHTEN != 0 else []
     model_type = request.args.get("model_type")
     try:
         objs = TenantLLMService.query(tenant_id=current_user.id)
@@ -335,10 +343,10 @@ def list_app():
         for m in llms:
             m["available"] = m["fid"] in facts or m["llm_name"].lower() == "flag-embedding" or m["fid"] in self_deploied
 
-        llm_set = set([m["llm_name"] for m in llms])
+        llm_set = set([m["llm_name"]+"@"+m["fid"] for m in llms])
         for o in objs:
             if not o.api_key:continue
-            if o.llm_name in llm_set:continue
+            if o.llm_name+"@"+o.llm_factory in llm_set:continue
             llms.append({"llm_name": o.llm_name, "model_type": o.model_type, "fid": o.llm_factory, "available": True})
 
         res = {}

api/apps/sdk/assistant.py

@@ -1,304 +0,0 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-from flask import request
-
-from api.db import StatusEnum
-from api.db.db_models import TenantLLM
-from api.db.services.dialog_service import DialogService
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import LLMService, TenantLLMService
-from api.db.services.user_service import TenantService
-from api.settings import RetCode
-from api.utils import get_uuid
-from api.utils.api_utils import get_data_error_result, token_required
-from api.utils.api_utils import get_json_result
-
-
-@manager.route('/save', methods=['POST'])
-@token_required
-def save(tenant_id):
-    req = request.json
-    # dataset
-    if req.get("knowledgebases") == []:
-        return get_data_error_result(retmsg="knowledgebases can not be empty list")
-    kb_list = []
-    if req.get("knowledgebases"):
-        for kb in req.get("knowledgebases"):
-            if not kb["id"]:
-                return get_data_error_result(retmsg="knowledgebase needs id")
-            if not KnowledgebaseService.query(id=kb["id"], tenant_id=tenant_id):
-                return get_data_error_result(retmsg="you do not own the knowledgebase")
-            # if not DocumentService.query(kb_id=kb["id"]):
-            #  return get_data_error_result(retmsg="There is a invalid knowledgebase")
-            kb_list.append(kb["id"])
-    req["kb_ids"] = kb_list
-    # llm
-    llm = req.get("llm")
-    if llm:
-        if "model_name" in llm:
-            req["llm_id"] = llm.pop("model_name")
-        req["llm_setting"] = req.pop("llm")
-    e, tenant = TenantService.get_by_id(tenant_id)
-    if not e:
-        return get_data_error_result(retmsg="Tenant not found!")
-    # prompt
-    prompt = req.get("prompt")
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
-    if prompt:
-        for new_key, old_key in key_mapping.items():
-            if old_key in prompt:
-                prompt[new_key] = prompt.pop(old_key)
-        for key in key_list:
-            if key in prompt:
-                req[key] = prompt.pop(key)
-        req["prompt_config"] = req.pop("prompt")
-    # create
-    if "id" not in req:
-        # dataset
-        if not kb_list:
-            return get_data_error_result(retmsg="knowledgebases are required!")
-        # init
-        req["id"] = get_uuid()
-        req["description"] = req.get("description", "A helpful Assistant")
-        req["icon"] = req.get("avatar", "")
-        req["top_n"] = req.get("top_n", 6)
-        req["top_k"] = req.get("top_k", 1024)
-        req["rerank_id"] = req.get("rerank_id", "")
-        if req.get("llm_id"):
-            if not TenantLLMService.query(llm_name=req["llm_id"]):
-                return get_data_error_result(retmsg="the model_name does not exist.")
-        else:
-            req["llm_id"] = tenant.llm_id
-        if not req.get("name"):
-            return get_data_error_result(retmsg="name is required.")
-        if DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
-            return get_data_error_result(retmsg="Duplicated assistant name in creating dataset.")
-        # tenant_id
-        if req.get("tenant_id"):
-            return get_data_error_result(retmsg="tenant_id must not be provided.")
-        req["tenant_id"] = tenant_id
-        # prompt more parameter
-        default_prompt = {
-            "system": """你是一个智能助手，请总结知识库的内容来回答问题，请列举知识库中的数据详细回答。当所有知识库内容都与问题无关时，你的回答必须包括“知识库中未找到您要的答案！”这句话。回答需要考虑聊天历史。
-                以下是知识库：
-                {knowledge}
-                以上是知识库。""",
-            "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
-            "parameters": [
-                {"key": "knowledge", "optional": False}
-            ],
-            "empty_response": "Sorry! 知识库中未找到相关内容！"
-        }
-        key_list_2 = ["system", "prologue", "parameters", "empty_response"]
-        if "prompt_config" not in req:
-            req['prompt_config'] = {}
-        for key in key_list_2:
-            temp = req['prompt_config'].get(key)
-            if not temp:
-                req['prompt_config'][key] = default_prompt[key]
-        for p in req['prompt_config']["parameters"]:
-            if p["optional"]:
-                continue
-            if req['prompt_config']["system"].find("{%s}" % p["key"]) < 0:
-                return get_data_error_result(
-                    retmsg="Parameter '{}' is not used".format(p["key"]))
-        # save
-        if not DialogService.save(**req):
-            return get_data_error_result(retmsg="Fail to new an assistant!")
-        # response
-        e, res = DialogService.get_by_id(req["id"])
-        if not e:
-            return get_data_error_result(retmsg="Fail to new an assistant!")
-        res = res.to_json()
-        renamed_dict = {}
-        for key, value in res["prompt_config"].items():
-            new_key = key_mapping.get(key, key)
-            renamed_dict[new_key] = value
-        res["prompt"] = renamed_dict
-        del res["prompt_config"]
-        new_dict = {"similarity_threshold": res["similarity_threshold"],
-                    "keywords_similarity_weight": res["vector_similarity_weight"],
-                    "top_n": res["top_n"],
-                    "rerank_model": res['rerank_id']}
-        res["prompt"].update(new_dict)
-        for key in key_list:
-            del res[key]
-        res["llm"] = res.pop("llm_setting")
-        res["llm"]["model_name"] = res.pop("llm_id")
-        del res["kb_ids"]
-        res["knowledgebases"] = req["knowledgebases"]
-        res["avatar"] = res.pop("icon")
-        return get_json_result(data=res)
-    else:
-        # authorization
-        if not DialogService.query(tenant_id=tenant_id, id=req["id"], status=StatusEnum.VALID.value):
-            return get_json_result(data=False, retmsg='You do not own the assistant', retcode=RetCode.OPERATING_ERROR)
-        # prompt
-        if not req["id"]:
-            return get_data_error_result(retmsg="id can not be empty")
-        e, res = DialogService.get_by_id(req["id"])
-        res = res.to_json()
-        if "llm_id" in req:
-            if not TenantLLMService.query(llm_name=req["llm_id"]):
-                return get_data_error_result(retmsg="the model_name does not exist.")
-        if "name" in req:
-            if not req.get("name"):
-                return get_data_error_result(retmsg="name is not empty.")
-            if req["name"].lower() != res["name"].lower() \
-                    and len(
-                DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
-                return get_data_error_result(retmsg="Duplicated assistant name in updating dataset.")
-        if "prompt_config" in req:
-            res["prompt_config"].update(req["prompt_config"])
-            for p in res["prompt_config"]["parameters"]:
-                if p["optional"]:
-                    continue
-                if res["prompt_config"]["system"].find("{%s}" % p["key"]) < 0:
-                    return get_data_error_result(retmsg="Parameter '{}' is not used".format(p["key"]))
-        if "llm_setting" in req:
-            res["llm_setting"].update(req["llm_setting"])
-        req["prompt_config"] = res["prompt_config"]
-        req["llm_setting"] = res["llm_setting"]
-        # avatar
-        if "avatar" in req:
-            req["icon"] = req.pop("avatar")
-        assistant_id = req.pop("id")
-        if "knowledgebases" in req:
-            req.pop("knowledgebases")
-        if not DialogService.update_by_id(assistant_id, req):
-            return get_data_error_result(retmsg="Assistant not found!")
-        return get_json_result(data=True)
-
-
-@manager.route('/delete', methods=['DELETE'])
-@token_required
-def delete(tenant_id):
-    req = request.args
-    if "id" not in req:
-        return get_data_error_result(retmsg="id is required")
-    id = req['id']
-    if not DialogService.query(tenant_id=tenant_id, id=id, status=StatusEnum.VALID.value):
-        return get_json_result(data=False, retmsg='you do not own the assistant.', retcode=RetCode.OPERATING_ERROR)
-
-    temp_dict = {"status": StatusEnum.INVALID.value}
-    DialogService.update_by_id(req["id"], temp_dict)
-    return get_json_result(data=True)
-
-
-@manager.route('/get', methods=['GET'])
-@token_required
-def get(tenant_id):
-    req = request.args
-    if "id" in req:
-        id = req["id"]
-        ass = DialogService.query(tenant_id=tenant_id, id=id, status=StatusEnum.VALID.value)
-        if not ass:
-            return get_json_result(data=False, retmsg='You do not own the assistant.', retcode=RetCode.OPERATING_ERROR)
-        if "name" in req:
-            name = req["name"]
-            if ass[0].name != name:
-                return get_json_result(data=False, retmsg='name does not match id.', retcode=RetCode.OPERATING_ERROR)
-        res = ass[0].to_json()
-    else:
-        if "name" in req:
-            name = req["name"]
-            ass = DialogService.query(name=name, tenant_id=tenant_id, status=StatusEnum.VALID.value)
-            if not ass:
-                return get_json_result(data=False, retmsg='You do not own the assistant.',
-                                       retcode=RetCode.OPERATING_ERROR)
-            res = ass[0].to_json()
-        else:
-            return get_data_error_result(retmsg="At least one of `id` or `name` must be provided.")
-    renamed_dict = {}
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
-    for key, value in res["prompt_config"].items():
-        new_key = key_mapping.get(key, key)
-        renamed_dict[new_key] = value
-    res["prompt"] = renamed_dict
-    del res["prompt_config"]
-    new_dict = {"similarity_threshold": res["similarity_threshold"],
-                "keywords_similarity_weight": res["vector_similarity_weight"],
-                "top_n": res["top_n"],
-                "rerank_model": res['rerank_id']}
-    res["prompt"].update(new_dict)
-    for key in key_list:
-        del res[key]
-    res["llm"] = res.pop("llm_setting")
-    res["llm"]["model_name"] = res.pop("llm_id")
-    kb_list = []
-    for kb_id in res["kb_ids"]:
-        kb = KnowledgebaseService.query(id=kb_id)
-        kb_list.append(kb[0].to_json())
-    del res["kb_ids"]
-    res["knowledgebases"] = kb_list
-    res["avatar"] = res.pop("icon")
-    return get_json_result(data=res)
-
-
-@manager.route('/list', methods=['GET'])
-@token_required
-def list_assistants(tenant_id):
-    assts = DialogService.query(
-        tenant_id=tenant_id,
-        status=StatusEnum.VALID.value,
-        reverse=True,
-        order_by=DialogService.model.create_time)
-    assts = [d.to_dict() for d in assts]
-    list_assts = []
-    renamed_dict = {}
-    key_mapping = {"parameters": "variables",
-                   "prologue": "opener",
-                   "quote": "show_quote",
-                   "system": "prompt",
-                   "rerank_id": "rerank_model",
-                   "vector_similarity_weight": "keywords_similarity_weight"}
-    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
-    for res in assts:
-        for key, value in res["prompt_config"].items():
-            new_key = key_mapping.get(key, key)
-            renamed_dict[new_key] = value
-        res["prompt"] = renamed_dict
-        del res["prompt_config"]
-        new_dict = {"similarity_threshold": res["similarity_threshold"],
-                    "keywords_similarity_weight": res["vector_similarity_weight"],
-                    "top_n": res["top_n"],
-                    "rerank_model": res['rerank_id']}
-        res["prompt"].update(new_dict)
-        for key in key_list:
-            del res[key]
-        res["llm"] = res.pop("llm_setting")
-        res["llm"]["model_name"] = res.pop("llm_id")
-        kb_list = []
-        for kb_id in res["kb_ids"]:
-            kb = KnowledgebaseService.query(id=kb_id)
-            kb_list.append(kb[0].to_json())
-        del res["kb_ids"]
-        res["knowledgebases"] = kb_list
-        res["avatar"] = res.pop("icon")
-        list_assts.append(res)
-    return get_json_result(data=list_assts)

api/apps/sdk/chat.py

@@ -0,0 +1,311 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from flask import request
+from api.settings import RetCode
+from api.db import StatusEnum
+from api.db.services.dialog_service import DialogService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import  TenantLLMService
+from api.db.services.user_service import TenantService
+from api.utils import get_uuid
+from api.utils.api_utils import get_error_data_result, token_required
+from api.utils.api_utils import get_result
+
+
+
+@manager.route('/chats', methods=['POST'])
+@token_required
+def create(tenant_id):
+    req=request.json
+    ids= req.get("dataset_ids")
+    if not ids:
+        return get_error_data_result(retmsg="`dataset_ids` is required")
+    for kb_id in ids:
+        kbs = KnowledgebaseService.query(id=kb_id,tenant_id=tenant_id)
+        if not kbs:
+            return get_error_data_result(f"You don't own the dataset {kb_id}")
+        kb=kbs[0]
+        if kb.chunk_num == 0:
+            return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
+    kbs = KnowledgebaseService.get_by_ids(ids)
+    embd_count = list(set([kb.embd_id for kb in kbs]))
+    if len(embd_count) != 1:
+        return get_result(retmsg='Datasets use different embedding models."',retcode=RetCode.AUTHENTICATION_ERROR)
+    req["kb_ids"] = ids
+    # llm
+    llm = req.get("llm")
+    if llm:
+        if "model_name" in llm:
+            req["llm_id"] = llm.pop("model_name")
+            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
+                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+        req["llm_setting"] = req.pop("llm")
+    e, tenant = TenantService.get_by_id(tenant_id)
+    if not e:
+        return get_error_data_result(retmsg="Tenant not found!")
+    # prompt
+    prompt = req.get("prompt")
+    key_mapping = {"parameters": "variables",
+                   "prologue": "opener",
+                   "quote": "show_quote",
+                   "system": "prompt",
+                   "rerank_id": "rerank_model",
+                   "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
+    if prompt:
+        for new_key, old_key in key_mapping.items():
+            if old_key in prompt:
+                prompt[new_key] = prompt.pop(old_key)
+        for key in key_list:
+            if key in prompt:
+                req[key] = prompt.pop(key)
+        req["prompt_config"] = req.pop("prompt")
+    # init
+    req["id"] = get_uuid()
+    req["description"] = req.get("description", "A helpful Assistant")
+    req["icon"] = req.get("avatar", "")
+    req["top_n"] = req.get("top_n", 6)
+    req["top_k"] = req.get("top_k", 1024)
+    req["rerank_id"] = req.get("rerank_id", "")
+    if req.get("rerank_id"):
+        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_id"),model_type="rerank"):
+            return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
+    if not req.get("llm_id"):
+        req["llm_id"] = tenant.llm_id
+    if not req.get("name"):
+        return get_error_data_result(retmsg="`name` is required.")
+    if DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg="Duplicated chat name in creating chat.")
+    # tenant_id
+    if req.get("tenant_id"):
+        return get_error_data_result(retmsg="`tenant_id` must not be provided.")
+    req["tenant_id"] = tenant_id
+    # prompt more parameter
+    default_prompt = {
+        "system": """You are an intelligent assistant. Please summarize the content of the knowledge base to answer the question. Please list the data in the knowledge base and answer in detail. When all knowledge base content is irrelevant to the question, your answer must include the sentence "The answer you are looking for is not found in the knowledge base!" Answers need to consider chat history.
+      Here is the knowledge base:
+      {knowledge}
+      The above is the knowledge base.""",
+        "prologue": "Hi! I'm your assistant, what can I do for you?",
+        "parameters": [
+            {"key": "knowledge", "optional": False}
+        ],
+        "empty_response": "Sorry! No relevant content was found in the knowledge base!"
+    }
+    key_list_2 = ["system", "prologue", "parameters", "empty_response"]
+    if "prompt_config" not in req:
+        req['prompt_config'] = {}
+    for key in key_list_2:
+        temp = req['prompt_config'].get(key)
+        if not temp:
+            req['prompt_config'][key] = default_prompt[key]
+    for p in req['prompt_config']["parameters"]:
+        if p["optional"]:
+            continue
+        if req['prompt_config']["system"].find("{%s}" % p["key"]) < 0:
+            return get_error_data_result(
+                retmsg="Parameter '{}' is not used".format(p["key"]))
+    # save
+    if not DialogService.save(**req):
+        return get_error_data_result(retmsg="Fail to new a chat!")
+    # response
+    e, res = DialogService.get_by_id(req["id"])
+    if not e:
+        return get_error_data_result(retmsg="Fail to new a chat!")
+    res = res.to_json()
+    renamed_dict = {}
+    for key, value in res["prompt_config"].items():
+        new_key = key_mapping.get(key, key)
+        renamed_dict[new_key] = value
+    res["prompt"] = renamed_dict
+    del res["prompt_config"]
+    new_dict = {"similarity_threshold": res["similarity_threshold"],
+                "keywords_similarity_weight": res["vector_similarity_weight"],
+                "top_n": res["top_n"],
+                "rerank_model": res['rerank_id']}
+    res["prompt"].update(new_dict)
+    for key in key_list:
+        del res[key]
+    res["llm"] = res.pop("llm_setting")
+    res["llm"]["model_name"] = res.pop("llm_id")
+    del res["kb_ids"]
+    res["dataset_ids"] = req["dataset_ids"]
+    res["avatar"] = res.pop("icon")
+    return get_result(data=res)
+
+@manager.route('/chats/<chat_id>', methods=['PUT'])
+@token_required
+def update(tenant_id,chat_id):
+    if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg='You do not own the chat')
+    req =request.json
+    ids = req.get("dataset_ids")
+    if "show_quotation" in req:
+        req["do_refer"]=req.pop("show_quotation")
+    if "dataset_ids" in req:
+        if not ids:
+            return get_error_data_result("`datasets` can't be empty")
+        if ids:
+            for kb_id in ids:
+                kbs = KnowledgebaseService.query(id=kb_id, tenant_id=tenant_id)
+                if not kbs:
+                    return get_error_data_result(f"You don't own the dataset {kb_id}")
+                kb = kbs[0]
+                if kb.chunk_num == 0:
+                    return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
+            kbs = KnowledgebaseService.get_by_ids(ids)
+            embd_count=list(set([kb.embd_id for kb in kbs]))
+            if len(embd_count) != 1 :
+                return get_result(
+                    retmsg='Datasets use different embedding models."',
+                    retcode=RetCode.AUTHENTICATION_ERROR)
+            req["kb_ids"] = ids
+    llm = req.get("llm")
+    if llm:
+        if "model_name" in llm:
+            req["llm_id"] = llm.pop("model_name")
+            if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
+                return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
+        req["llm_setting"] = req.pop("llm")
+    e, tenant = TenantService.get_by_id(tenant_id)
+    if not e:
+        return get_error_data_result(retmsg="Tenant not found!")
+    if req.get("rerank_model"):
+        if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_model"),model_type="rerank"):
+            return get_error_data_result(f"`rerank_model` {req.get('rerank_model')} doesn't exist")
+    # prompt
+    prompt = req.get("prompt")
+    key_mapping = {"parameters": "variables",
+                   "prologue": "opener",
+                   "quote": "show_quote",
+                   "system": "prompt",
+                   "rerank_id": "rerank_model",
+                   "vector_similarity_weight": "keywords_similarity_weight"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
+    if prompt:
+        for new_key, old_key in key_mapping.items():
+            if old_key in prompt:
+                prompt[new_key] = prompt.pop(old_key)
+        for key in key_list:
+            if key in prompt:
+                req[key] = prompt.pop(key)
+        req["prompt_config"] = req.pop("prompt")
+    e, res = DialogService.get_by_id(chat_id)
+    res = res.to_json()
+    if "name" in req:
+        if not req.get("name"):
+            return get_error_data_result(retmsg="`name` is not empty.")
+        if req["name"].lower() != res["name"].lower() \
+                and len(
+            DialogService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value)) > 0:
+            return get_error_data_result(retmsg="Duplicated chat name in updating dataset.")
+    if "prompt_config" in req:
+        res["prompt_config"].update(req["prompt_config"])
+        for p in res["prompt_config"]["parameters"]:
+            if p["optional"]:
+                continue
+            if res["prompt_config"]["system"].find("{%s}" % p["key"]) < 0:
+                return get_error_data_result(retmsg="Parameter '{}' is not used".format(p["key"]))
+    if "llm_setting" in req:
+        res["llm_setting"].update(req["llm_setting"])
+    req["prompt_config"] = res["prompt_config"]
+    req["llm_setting"] = res["llm_setting"]
+    # avatar
+    if "avatar" in req:
+        req["icon"] = req.pop("avatar")
+    if "dataset_ids" in req:
+        req.pop("dataset_ids")
+    if not DialogService.update_by_id(chat_id, req):
+        return get_error_data_result(retmsg="Chat not found!")
+    return get_result()
+
+
+@manager.route('/chats', methods=['DELETE'])
+@token_required
+def delete(tenant_id):
+    req = request.json
+    if not req:
+        ids=None
+    else:
+        ids=req.get("ids")
+    if not ids:
+        id_list = []
+        dias=DialogService.query(tenant_id=tenant_id,status=StatusEnum.VALID.value)
+        for dia in dias:
+            id_list.append(dia.id)
+    else:
+        id_list=ids
+    for id in id_list:
+        if not DialogService.query(tenant_id=tenant_id, id=id, status=StatusEnum.VALID.value):
+            return get_error_data_result(retmsg=f"You don't own the chat {id}")
+        temp_dict = {"status": StatusEnum.INVALID.value}
+        DialogService.update_by_id(id, temp_dict)
+    return get_result()
+
+@manager.route('/chats', methods=['GET'])
+@token_required
+def list_chat(tenant_id):
+    id = request.args.get("id")
+    name = request.args.get("name")
+    chat = DialogService.query(id=id,name=name,status=StatusEnum.VALID.value)
+    if not chat:
+        return get_error_data_result(retmsg="The chat doesn't exist")
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 1024))
+    orderby = request.args.get("orderby", "create_time")
+    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
+        desc = False
+    else:
+        desc = True
+    chats = DialogService.get_list(tenant_id,page_number,items_per_page,orderby,desc,id,name)
+    if not chats:
+        return get_result(data=[])
+    list_assts = []
+    renamed_dict = {}
+    key_mapping = {"parameters": "variables",
+                   "prologue": "opener",
+                   "quote": "show_quote",
+                   "system": "prompt",
+                   "rerank_id": "rerank_model",
+                   "vector_similarity_weight": "keywords_similarity_weight",
+                   "do_refer":"show_quotation"}
+    key_list = ["similarity_threshold", "vector_similarity_weight", "top_n", "rerank_id"]
+    for res in chats:
+        for key, value in res["prompt_config"].items():
+            new_key = key_mapping.get(key, key)
+            renamed_dict[new_key] = value
+        res["prompt"] = renamed_dict
+        del res["prompt_config"]
+        new_dict = {"similarity_threshold": res["similarity_threshold"],
+                    "keywords_similarity_weight": res["vector_similarity_weight"],
+                    "top_n": res["top_n"],
+                    "rerank_model": res['rerank_id']}
+        res["prompt"].update(new_dict)
+        for key in key_list:
+            del res[key]
+        res["llm"] = res.pop("llm_setting")
+        res["llm"]["model_name"] = res.pop("llm_id")
+        kb_list = []
+        for kb_id in res["kb_ids"]:
+            kb = KnowledgebaseService.query(id=kb_id)
+            if not kb :
+                return get_error_data_result(retmsg=f"Don't exist the kb {kb_id}")
+            kb_list.append(kb[0].to_json())
+        del res["kb_ids"]
+        res["datasets"] = kb_list
+        res["avatar"] = res.pop("icon")
+        list_assts.append(res)
+    return get_result(data=list_assts)

api/apps/sdk/dataset.py

@@ -15,159 +15,213 @@
 #
 
 from flask import request
-
 from api.db import StatusEnum, FileSource
 from api.db.db_models import File
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import TenantLLMService,LLMService
 from api.db.services.user_service import TenantService
 from api.settings import RetCode
 from api.utils import get_uuid
-from api.utils.api_utils import get_json_result, token_required, get_data_error_result
+from api.utils.api_utils import get_result, token_required, get_error_data_result, valid,get_parser_config
 
 
-@manager.route('/save', methods=['POST'])
+@manager.route('/datasets', methods=['POST'])
 @token_required
-def save(tenant_id):
+def create(tenant_id):
     req = request.json
     e, t = TenantService.get_by_id(tenant_id)
-    if "id" not in req:
-        if "tenant_id" in req or "embedding_model" in req:
-            return get_data_error_result(
-                retmsg="Tenant_id or embedding_model must not be provided")
-        if "name" not in req:
-            return get_data_error_result(
-                retmsg="Name is not empty!")
-        req['id'] = get_uuid()
-        req["name"] = req["name"].strip()
-        if req["name"] == "":
-            return get_data_error_result(
-                retmsg="Name is not empty string!")
-        if KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
-            return get_data_error_result(
-                retmsg="Duplicated knowledgebase name in creating dataset.")
-        req["tenant_id"] = req['created_by'] = tenant_id
+    permission = req.get("permission")
+    language = req.get("language")
+    chunk_method = req.get("chunk_method")
+    parser_config = req.get("parser_config")
+    valid_permission = ["me", "team"]
+    valid_language =["Chinese", "English"]
+    valid_chunk_method = ["naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email"]
+    check_validation=valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method)
+    if check_validation:
+        return check_validation
+    req["parser_config"]=get_parser_config(chunk_method,parser_config)
+    if "tenant_id" in req:
+        return get_error_data_result(
+            retmsg="`tenant_id` must not be provided")
+    if "chunk_count" in req or "document_count" in req:
+        return get_error_data_result(retmsg="`chunk_count` or `document_count` must not be provided")
+    if "name" not in req:
+        return get_error_data_result(
+            retmsg="`name` is not empty!")
+    req['id'] = get_uuid()
+    req["name"] = req["name"].strip()
+    if req["name"] == "":
+        return get_error_data_result(
+            retmsg="`name` is not empty string!")
+    if KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(
+            retmsg="Duplicated dataset name in creating dataset.")
+    req["tenant_id"] = req['created_by'] = tenant_id
+    if not req.get("embedding_model"):
         req['embedding_model'] = t.embd_id
-        key_mapping = {
-            "chunk_num": "chunk_count",
-            "doc_num": "document_count",
-            "parser_id": "parse_method",
-            "embd_id": "embedding_model"
-        }
-        mapped_keys = {new_key: req[old_key] for new_key, old_key in key_mapping.items() if old_key in req}
-        req.update(mapped_keys)
-        if not KnowledgebaseService.save(**req):
-            return get_data_error_result(retmsg="Create dataset error.(Database error)")
-        renamed_data = {}
-        e, k = KnowledgebaseService.get_by_id(req["id"])
-        for key, value in k.to_dict().items():
-            new_key = key_mapping.get(key, key)
-            renamed_data[new_key] = value
-        return get_json_result(data=renamed_data)
     else:
-        invalid_keys = {"embd_id", "chunk_num", "doc_num", "parser_id"}
-        if any(key in req for key in invalid_keys):
-            return get_data_error_result(retmsg="The input parameters are invalid.")
+        valid_embedding_models=["BAAI/bge-large-zh-v1.5","BAAI/bge-base-en-v1.5","BAAI/bge-large-en-v1.5","BAAI/bge-small-en-v1.5",
+                                "BAAI/bge-small-zh-v1.5","jinaai/jina-embeddings-v2-base-en","jinaai/jina-embeddings-v2-small-en",
+                                "nomic-ai/nomic-embed-text-v1.5","sentence-transformers/all-MiniLM-L6-v2","text-embedding-v2",
+                                "text-embedding-v3","maidalun1020/bce-embedding-base_v1"]
+        embd_model=LLMService.query(llm_name=req["embedding_model"],model_type="embedding")
+        if not embd_model:
+            return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
+        if embd_model:
+            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
+                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
+    key_mapping = {
+        "chunk_num": "chunk_count",
+        "doc_num": "document_count",
+        "parser_id": "chunk_method",
+        "embd_id": "embedding_model"
+    }
+    mapped_keys = {new_key: req[old_key] for new_key, old_key in key_mapping.items() if old_key in req}
+    req.update(mapped_keys)
+    if not KnowledgebaseService.save(**req):
+        return get_error_data_result(retmsg="Create dataset error.(Database error)")
+    renamed_data = {}
+    e, k = KnowledgebaseService.get_by_id(req["id"])
+    for key, value in k.to_dict().items():
+        new_key = key_mapping.get(key, key)
+        renamed_data[new_key] = value
+    return get_result(data=renamed_data)
 
-        if "tenant_id" in req:
-            if req["tenant_id"] != tenant_id:
-                return get_data_error_result(
-                    retmsg="Can't change tenant_id.")
-
-        if "embedding_model" in req:
-            if req["embedding_model"] != t.embd_id:
-                return get_data_error_result(
-                    retmsg="Can't change embedding_model.")
-            req.pop("embedding_model")
-
-        if not KnowledgebaseService.query(
-                created_by=tenant_id, id=req["id"]):
-            return get_json_result(
-                data=False, retmsg='You do not own the dataset.',
-                retcode=RetCode.OPERATING_ERROR)
-
-        if not req["id"]:
-            return get_data_error_result(
-                retmsg="id can not be empty.")
-        e, kb = KnowledgebaseService.get_by_id(req["id"])
-
-        if "chunk_count" in req:
-            if req["chunk_count"] != kb.chunk_num:
-                return get_data_error_result(
-                    retmsg="Can't change chunk_count.")
-            req.pop("chunk_count")
-
-        if "document_count" in req:
-            if req['document_count'] != kb.doc_num:
-                return get_data_error_result(
-                    retmsg="Can't change document_count.")
-            req.pop("document_count")
-
-        if "parse_method" in req:
-            if kb.chunk_num != 0 and req['parse_method'] != kb.parser_id:
-                return get_data_error_result(
-                    retmsg="If chunk count is not 0, parse method is not changable.")
-            req['parser_id'] = req.pop('parse_method')
-        if "name" in req:
-            req["name"] = req["name"].strip()
-            if req["name"].lower() != kb.name.lower() \
-                    and len(KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id,
-                                                       status=StatusEnum.VALID.value)) > 0:
-                return get_data_error_result(
-                    retmsg="Duplicated knowledgebase name in updating dataset.")
-
-        del req["id"]
-        if not KnowledgebaseService.update_by_id(kb.id, req):
-            return get_data_error_result(retmsg="Update dataset error.(Database error)")
-        return get_json_result(data=True)
-
-
-@manager.route('/delete', methods=['DELETE'])
+@manager.route('/datasets', methods=['DELETE'])
 @token_required
 def delete(tenant_id):
-    req = request.args
-    if "id" not in req:
-        return get_data_error_result(
-            retmsg="id is required")
-    kbs = KnowledgebaseService.query(
-        created_by=tenant_id, id=req["id"])
-    if not kbs:
-        return get_json_result(
-            data=False, retmsg='You do not own the dataset',
-            retcode=RetCode.OPERATING_ERROR)
+    req = request.json
+    if not req:
+        ids=None
+    else:
+        ids=req.get("ids")
+    if not ids:
+        id_list = []
+        kbs=KnowledgebaseService.query(tenant_id=tenant_id)
+        for kb in kbs:
+            id_list.append(kb.id)
+    else:
+        id_list=ids
+    for id in id_list:
+        kbs = KnowledgebaseService.query(id=id, tenant_id=tenant_id)
+        if not kbs:
+            return get_error_data_result(retmsg=f"You don't own the dataset {id}")
+        for doc in DocumentService.query(kb_id=id):
+            if not DocumentService.remove_document(doc, tenant_id):
+                return get_error_data_result(
+                    retmsg="Remove document error.(Database error)")
+            f2d = File2DocumentService.get_by_document_id(doc.id)
+            FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
+            File2DocumentService.delete_by_document_id(doc.id)
+        if not KnowledgebaseService.delete_by_id(id):
+            return get_error_data_result(
+                retmsg="Delete dataset error.(Database error)")
+    return get_result(retcode=RetCode.SUCCESS)
 
-    for doc in DocumentService.query(kb_id=req["id"]):
-        if not DocumentService.remove_document(doc, kbs[0].tenant_id):
-            return get_data_error_result(
-                retmsg="Remove document error.(Database error)")
-        f2d = File2DocumentService.get_by_document_id(doc.id)
-        FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
-        File2DocumentService.delete_by_document_id(doc.id)
-
-    if not KnowledgebaseService.delete_by_id(req["id"]):
-        return get_data_error_result(
-            retmsg="Delete dataset error.(Database serror)")
-    return get_json_result(data=True)
-
-
-@manager.route('/list', methods=['GET'])
+@manager.route('/datasets/<dataset_id>', methods=['PUT'])
 @token_required
-def list_datasets(tenant_id):
+def update(tenant_id,dataset_id):
+    if not KnowledgebaseService.query(id=dataset_id,tenant_id=tenant_id):
+        return get_error_data_result(retmsg="You don't own the dataset")
+    req = request.json
+    e, t = TenantService.get_by_id(tenant_id)
+    invalid_keys = {"id", "embd_id", "chunk_num", "doc_num", "parser_id"}
+    if any(key in req for key in invalid_keys):
+        return get_error_data_result(retmsg="The input parameters are invalid.")
+    permission = req.get("permission")
+    language = req.get("language")
+    chunk_method = req.get("chunk_method")
+    parser_config = req.get("parser_config")
+    valid_permission = ["me", "team"]
+    valid_language = ["Chinese", "English"]
+    valid_chunk_method = ["naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one",
+                          "knowledge_graph", "email"]
+    check_validation = valid(permission, valid_permission, language, valid_language, chunk_method, valid_chunk_method)
+    if check_validation:
+        return check_validation
+    if "tenant_id" in req:
+        if req["tenant_id"] != tenant_id:
+            return get_error_data_result(
+                retmsg="Can't change `tenant_id`.")
+    e, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if "parser_config" in req:
+        temp_dict=kb.parser_config
+        temp_dict.update(req["parser_config"])
+        req["parser_config"] = temp_dict
+    if "chunk_count" in req:
+        if req["chunk_count"] != kb.chunk_num:
+            return get_error_data_result(
+                retmsg="Can't change `chunk_count`.")
+        req.pop("chunk_count")
+    if "document_count" in req:
+        if req['document_count'] != kb.doc_num:
+            return get_error_data_result(
+                retmsg="Can't change `document_count`.")
+        req.pop("document_count")
+    if "chunk_method" in req:
+        if kb.chunk_num != 0 and req['chunk_method'] != kb.parser_id:
+            return get_error_data_result(
+                retmsg="If `chunk_count` is not 0, `chunk_method` is not changeable.")
+        req['parser_id'] = req.pop('chunk_method')
+        if req['parser_id'] != kb.parser_id:
+            if not req.get("parser_config"):
+                req["parser_config"] = get_parser_config(chunk_method, parser_config)
+    if "embedding_model" in req:
+        if kb.chunk_num != 0 and req['embedding_model'] != kb.embd_id:
+            return get_error_data_result(
+                retmsg="If `chunk_count` is not 0, `embedding_model` is not changeable.")
+        if not req.get("embedding_model"):
+            return get_error_data_result("`embedding_model` can't be empty")
+        valid_embedding_models=["BAAI/bge-large-zh-v1.5","BAAI/bge-base-en-v1.5","BAAI/bge-large-en-v1.5","BAAI/bge-small-en-v1.5",
+                                "BAAI/bge-small-zh-v1.5","jinaai/jina-embeddings-v2-base-en","jinaai/jina-embeddings-v2-small-en",
+                                "nomic-ai/nomic-embed-text-v1.5","sentence-transformers/all-MiniLM-L6-v2","text-embedding-v2",
+                                "text-embedding-v3","maidalun1020/bce-embedding-base_v1"]
+        embd_model=LLMService.query(llm_name=req["embedding_model"],model_type="embedding")
+        if not embd_model:
+            return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
+        if embd_model:
+            if req["embedding_model"] not in valid_embedding_models and not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
+                return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
+        req['embd_id'] = req.pop('embedding_model')
+    if "name" in req:
+        req["name"] = req["name"].strip()
+        if req["name"].lower() != kb.name.lower() \
+                and len(KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id,
+                                                   status=StatusEnum.VALID.value)) > 0:
+            return get_error_data_result(
+                retmsg="Duplicated dataset name in updating dataset.")
+    if not KnowledgebaseService.update_by_id(kb.id, req):
+        return get_error_data_result(retmsg="Update dataset error.(Database error)")
+    return get_result(retcode=RetCode.SUCCESS)
+
+@manager.route('/datasets', methods=['GET'])
+@token_required
+def list(tenant_id):
+    id = request.args.get("id")
+    name = request.args.get("name")
+    kbs = KnowledgebaseService.query(id=id,name=name,status=1)
+    if not kbs:
+        return get_error_data_result(retmsg="The dataset doesn't exist")
     page_number = int(request.args.get("page", 1))
     items_per_page = int(request.args.get("page_size", 1024))
     orderby = request.args.get("orderby", "create_time")
-    desc = bool(request.args.get("desc", True))
+    if request.args.get("desc") == "False" or request.args.get("desc") == "false" :
+        desc = False
+    else:
+        desc = True
     tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
-    kbs = KnowledgebaseService.get_by_tenant_ids(
-        [m["tenant_id"] for m in tenants], tenant_id, page_number, items_per_page, orderby, desc)
+    kbs = KnowledgebaseService.get_list(
+        [m["tenant_id"] for m in tenants], tenant_id, page_number, items_per_page, orderby, desc, id, name)
     renamed_list = []
     for kb in kbs:
         key_mapping = {
             "chunk_num": "chunk_count",
             "doc_num": "document_count",
-            "parser_id": "parse_method",
+            "parser_id": "chunk_method",
             "embd_id": "embedding_model"
         }
         renamed_data = {}
@@ -175,50 +229,4 @@ def list_datasets(tenant_id):
             new_key = key_mapping.get(key, key)
             renamed_data[new_key] = value
         renamed_list.append(renamed_data)
-    return get_json_result(data=renamed_list)
-
-
-@manager.route('/detail', methods=['GET'])
-@token_required
-def detail(tenant_id):
-    req = request.args
-    key_mapping = {
-        "chunk_num": "chunk_count",
-        "doc_num": "document_count",
-        "parser_id": "parse_method",
-        "embd_id": "embedding_model"
-    }
-    renamed_data = {}
-    if "id" in req:
-        id = req["id"]
-        kb = KnowledgebaseService.query(created_by=tenant_id, id=req["id"])
-        if not kb:
-            return get_json_result(
-                data=False, retmsg='You do not own the dataset.',
-                retcode=RetCode.OPERATING_ERROR)
-        if "name" in req:
-            name = req["name"]
-            if kb[0].name != name:
-                return get_json_result(
-                    data=False, retmsg='You do not own the dataset.',
-                    retcode=RetCode.OPERATING_ERROR)
-        e, k = KnowledgebaseService.get_by_id(id)
-        for key, value in k.to_dict().items():
-            new_key = key_mapping.get(key, key)
-            renamed_data[new_key] = value
-        return get_json_result(data=renamed_data)
-    else:
-        if "name" in req:
-            name = req["name"]
-            e, k = KnowledgebaseService.get_by_name(kb_name=name, tenant_id=tenant_id)
-            if not e:
-                return get_json_result(
-                    data=False, retmsg='You do not own the dataset.',
-                    retcode=RetCode.OPERATING_ERROR)
-            for key, value in k.to_dict().items():
-                new_key = key_mapping.get(key, key)
-                renamed_data[new_key] = value
-            return get_json_result(data=renamed_data)
-        else:
-            return get_data_error_result(
-                retmsg="At least one of `id` or `name` must be provided.")
+    return get_result(data=renamed_list)

api/apps/sdk/dify_retrieval.py

@@ -0,0 +1,77 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from flask import request, jsonify
+
+from api.db import LLMType, ParserType
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import LLMBundle
+from api.settings import retrievaler, kg_retrievaler, RetCode
+from api.utils.api_utils import validate_request, build_error_result, apikey_required
+
+
+@manager.route('/dify/retrieval', methods=['POST'])
+@apikey_required
+@validate_request("knowledge_id", "query")
+def retrieval(tenant_id):
+    req = request.json
+    question = req["query"]
+    kb_id = req["knowledge_id"]
+    retrieval_setting = req.get("retrieval_setting", {})
+    similarity_threshold = float(retrieval_setting.get("score_threshold", 0.0))
+    top = int(retrieval_setting.get("top_k", 1024))
+
+    try:
+
+        e, kb = KnowledgebaseService.get_by_id(kb_id)
+        if not e:
+            return build_error_result(error_msg="Knowledgebase not found!", retcode=RetCode.NOT_FOUND)
+
+        if kb.tenant_id != tenant_id:
+            return build_error_result(error_msg="Knowledgebase not found!", retcode=RetCode.NOT_FOUND)
+
+        embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
+
+        retr = retrievaler if kb.parser_id != ParserType.KG else kg_retrievaler
+        ranks = retr.retrieval(
+            question,
+            embd_mdl,
+            kb.tenant_id,
+            [kb_id],
+            page=1,
+            page_size=top,
+            similarity_threshold=similarity_threshold,
+            vector_similarity_weight=0.3,
+            top=top
+        )
+        records = []
+        for c in ranks["chunks"]:
+            if "vector" in c:
+                del c["vector"]
+            records.append({
+                "content": c["content_ltks"],
+                "score": c["similarity"],
+                "title": c["docnm_kwd"],
+                "metadata": {}
+            })
+
+        return jsonify({"records": records})
+    except Exception as e:
+        if str(e).find("not_found") > 0:
+            return build_error_result(
+                error_msg=f'No chunk found! Check the chunk status please!',
+                retcode=RetCode.NOT_FOUND
+            )
+        return build_error_result(error_msg=str(e), retcode=RetCode.SERVER_ERROR)

api/apps/sdk/session.py

@@ -20,47 +20,18 @@ from flask import request, Response
 
 from api.db import StatusEnum
 from api.db.services.dialog_service import DialogService, ConversationService, chat
-from api.settings import RetCode
 from api.utils import get_uuid
-from api.utils.api_utils import get_data_error_result
-from api.utils.api_utils import get_json_result, token_required
+from api.utils.api_utils import get_error_data_result
+from api.utils.api_utils import get_result, token_required
 
-
-@manager.route('/save', methods=['POST'])
+@manager.route('/chats/<chat_id>/sessions', methods=['POST'])
 @token_required
-def set_conversation(tenant_id):
+def create(tenant_id,chat_id):
     req = request.json
-    conv_id = req.get("id")
-    if "assistant_id" in req:
-        req["dialog_id"] = req.pop("assistant_id")
-    if "id" in req:
-        del req["id"]
-        conv = ConversationService.query(id=conv_id)
-        if not conv:
-            return get_data_error_result(retmsg="Session does not exist")
-        if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
-            return get_data_error_result(retmsg="You do not own the session")
-        if req.get("dialog_id"):
-            dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
-            if not dia:
-                return get_data_error_result(retmsg="You do not own the assistant")
-        if "dialog_id" in req and not req.get("dialog_id"):
-            return get_data_error_result(retmsg="assistant_id can not be empty.")
-        if "message" in req:
-            return get_data_error_result(retmsg="message can not be change")
-        if "reference" in req:
-            return get_data_error_result(retmsg="reference can not be change")
-        if "name" in req and not req.get("name"):
-            return get_data_error_result(retmsg="name can not be empty.")
-        if not ConversationService.update_by_id(conv_id, req):
-            return get_data_error_result(retmsg="Session updates error")
-        return get_json_result(data=True)
-
-    if not req.get("dialog_id"):
-        return get_data_error_result(retmsg="assistant_id is required.")
+    req["dialog_id"] = chat_id
     dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
     if not dia:
-        return get_data_error_result(retmsg="You do not own the assistant")
+        return get_error_data_result(retmsg="You do not own the assistant")
     conv = {
         "id": get_uuid(),
         "dialog_id": req["dialog_id"],
@@ -68,33 +39,65 @@ def set_conversation(tenant_id):
         "message": [{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
     }
     if not conv.get("name"):
-        return get_data_error_result(retmsg="name can not be empty.")
+        return get_error_data_result(retmsg="`name` can not be empty.")
     ConversationService.save(**conv)
     e, conv = ConversationService.get_by_id(conv["id"])
     if not e:
-        return get_data_error_result(retmsg="Fail to new session!")
+        return get_error_data_result(retmsg="Fail to create a session!")
     conv = conv.to_dict()
     conv['messages'] = conv.pop("message")
-    conv["assistant_id"] = conv.pop("dialog_id")
+    conv["chat_id"] = conv.pop("dialog_id")
     del conv["reference"]
-    return get_json_result(data=conv)
+    return get_result(data=conv)
 
-
-@manager.route('/completion', methods=['POST'])
+@manager.route('/chats/<chat_id>/sessions/<session_id>', methods=['PUT'])
 @token_required
-def completion(tenant_id):
+def update(tenant_id,chat_id,session_id):
     req = request.json
-    # req = {"conversation_id": "9aaaca4c11d311efa461fa163e197198", "messages": [
-    #    {"role": "user", "content": "上海有吗？"}
-    # ]}
-    if "session_id" not in req:
-        return get_data_error_result(retmsg="session_id is required")
-    conv = ConversationService.query(id=req["session_id"])
+    req["dialog_id"] = chat_id
+    conv_id = session_id
+    conv = ConversationService.query(id=conv_id,dialog_id=chat_id)
     if not conv:
-        return get_data_error_result(retmsg="Session does not exist")
+        return get_error_data_result(retmsg="Session does not exist")
+    if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg="You do not own the session")
+    if "message" in req or "messages" in req:
+        return get_error_data_result(retmsg="`message` can not be change")
+    if "reference" in req:
+        return get_error_data_result(retmsg="`reference` can not be change")
+    if "name" in req and not req.get("name"):
+        return get_error_data_result(retmsg="`name` can not be empty.")
+    if not ConversationService.update_by_id(conv_id, req):
+        return get_error_data_result(retmsg="Session updates error")
+    return get_result()
+
+
+@manager.route('/chats/<chat_id>/completions', methods=['POST'])
+@token_required
+def completion(tenant_id,chat_id):
+    req = request.json
+    if not req.get("session_id"):
+        conv = {
+            "id": get_uuid(),
+            "dialog_id": chat_id,
+            "name": req.get("name", "New session"),
+            "message": [{"role": "assistant", "content": "Hi! I am your assistant，can I help you?"}]
+        }
+        if not conv.get("name"):
+            return get_error_data_result(retmsg="`name` can not be empty.")
+        ConversationService.save(**conv)
+        e, conv = ConversationService.get_by_id(conv["id"])
+        session_id=conv.id
+    else:
+        session_id = req.get("session_id")
+    if not req.get("question"):
+        return get_error_data_result(retmsg="Please input your question.")
+    conv = ConversationService.query(id=session_id,dialog_id=chat_id)
+    if not conv:
+        return get_error_data_result(retmsg="Session does not exist")
     conv = conv[0]
-    if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
-        return get_data_error_result(retmsg="You do not own the session")
+    if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg="You do not own the chat")
     msg = []
     question = {
         "content": req.get("question"),
@@ -108,7 +111,6 @@ def completion(tenant_id):
         msg.append(m)
     message_id = msg[-1].get("id")
     e, dia = DialogService.get_by_id(conv.dialog_id)
-    del req["session_id"]
 
     if not conv.reference:
         conv.reference = []
@@ -124,19 +126,20 @@ def completion(tenant_id):
         conv.message[-1] = {"role": "assistant", "content": ans["answer"],
                             "id": message_id, "prompt": ans.get("prompt", "")}
         ans["id"] = message_id
+        ans["session_id"]=session_id
 
     def stream():
         nonlocal dia, msg, req, conv
         try:
             for ans in chat(dia, msg, **req):
                 fillin_conv(ans)
-                yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": ans}, ensure_ascii=False) + "\n\n"
+                yield "data:" + json.dumps({"code": 0,  "data": ans}, ensure_ascii=False) + "\n\n"
             ConversationService.update_by_id(conv.id, conv.to_dict())
         except Exception as e:
-            yield "data:" + json.dumps({"retcode": 500, "retmsg": str(e),
-                                        "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
+            yield "data:" + json.dumps({"code": 500, "message": str(e),
+                                        "data": {"answer": "**ERROR**: " + str(e),"reference": []}},
                                        ensure_ascii=False) + "\n\n"
-        yield "data:" + json.dumps({"retcode": 0, "retmsg": "", "data": True}, ensure_ascii=False) + "\n\n"
+        yield "data:" + json.dumps({"code": 0, "data": True}, ensure_ascii=False) + "\n\n"
 
     if req.get("stream", True):
         resp = Response(stream(), mimetype="text/event-stream")
@@ -153,73 +156,32 @@ def completion(tenant_id):
             fillin_conv(ans)
             ConversationService.update_by_id(conv.id, conv.to_dict())
             break
-        return get_json_result(data=answer)
+        return get_result(data=answer)
 
-
-@manager.route('/get', methods=['GET'])
+@manager.route('/chats/<chat_id>/sessions', methods=['GET'])
 @token_required
-def get(tenant_id):
-    req = request.args
-    if "id" not in req:
-        return get_data_error_result(retmsg="id is required")
-    conv_id = req["id"]
-    conv = ConversationService.query(id=conv_id)
-    if not conv:
-        return get_data_error_result(retmsg="Session does not exist")
-    if not DialogService.query(id=conv[0].dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
-        return get_data_error_result(retmsg="You do not own the session")
-    if "assistant_id" in req:
-        if req["assistant_id"] != conv[0].dialog_id:
-            return get_data_error_result(retmsg="The session doesn't belong to the assistant")
-    conv = conv[0].to_dict()
-    conv['messages'] = conv.pop("message")
-    conv["assistant_id"] = conv.pop("dialog_id")
-    if conv["reference"]:
-        messages = conv["messages"]
-        message_num = 0
-        chunk_num = 0
-        while message_num < len(messages):
-            if message_num != 0 and messages[message_num]["role"] != "user":
-                chunk_list = []
-                if "chunks" in conv["reference"][chunk_num]:
-                    chunks = conv["reference"][chunk_num]["chunks"]
-                    for chunk in chunks:
-                        new_chunk = {
-                            "id": chunk["chunk_id"],
-                            "content": chunk["content_with_weight"],
-                            "document_id": chunk["doc_id"],
-                            "document_name": chunk["docnm_kwd"],
-                            "knowledgebase_id": chunk["kb_id"],
-                            "image_id": chunk["img_id"],
-                            "similarity": chunk["similarity"],
-                            "vector_similarity": chunk["vector_similarity"],
-                            "term_similarity": chunk["term_similarity"],
-                            "positions": chunk["positions"],
-                        }
-                        chunk_list.append(new_chunk)
-                chunk_num += 1
-                messages[message_num]["reference"] = chunk_list
-            message_num += 1
-    del conv["reference"]
-    return get_json_result(data=conv)
-
-
-@manager.route('/list', methods=["GET"])
-@token_required
-def list(tenant_id):
-    assistant_id = request.args["assistant_id"]
-    if not DialogService.query(tenant_id=tenant_id, id=assistant_id, status=StatusEnum.VALID.value):
-        return get_json_result(
-            data=False, retmsg=f"You don't own the assistant.",
-            retcode=RetCode.OPERATING_ERROR)
-    convs = ConversationService.query(
-        dialog_id=assistant_id,
-        order_by=ConversationService.model.create_time,
-        reverse=True)
-    convs = [d.to_dict() for d in convs]
+def list(chat_id,tenant_id):
+    if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg=f"You don't own the assistant {chat_id}.")
+    id = request.args.get("id")
+    name = request.args.get("name")
+    page_number = int(request.args.get("page", 1))
+    items_per_page = int(request.args.get("page_size", 1024))
+    orderby = request.args.get("orderby", "create_time")
+    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
+        desc = False
+    else:
+        desc = True
+    convs = ConversationService.get_list(chat_id,page_number,items_per_page,orderby,desc,id,name)
+    if not convs:
+        return get_result(data=[])
     for conv in convs:
         conv['messages'] = conv.pop("message")
-        conv["assistant_id"] = conv.pop("dialog_id")
+        infos = conv["messages"]
+        for info in infos:
+            if "prompt" in info:
+                info.pop("prompt")
+        conv["chat"] = conv.pop("dialog_id")
         if conv["reference"]:
             messages = conv["messages"]
             message_num = 0
@@ -235,7 +197,7 @@ def list(tenant_id):
                                 "content": chunk["content_with_weight"],
                                 "document_id": chunk["doc_id"],
                                 "document_name": chunk["docnm_kwd"],
-                                "knowledgebase_id": chunk["kb_id"],
+                                "dataset_id": chunk["kb_id"],
                                 "image_id": chunk["img_id"],
                                 "similarity": chunk["similarity"],
                                 "vector_similarity": chunk["vector_similarity"],
@@ -247,20 +209,29 @@ def list(tenant_id):
                     messages[message_num]["reference"] = chunk_list
                 message_num += 1
         del conv["reference"]
-    return get_json_result(data=convs)
+    return get_result(data=convs)
 
-
-@manager.route('/delete', methods=["DELETE"])
+@manager.route('/chats/<chat_id>/sessions', methods=["DELETE"])
 @token_required
-def delete(tenant_id):
-    id = request.args.get("id")
-    if not id:
-        return get_data_error_result(retmsg="`id` is required in deleting operation")
-    conv = ConversationService.query(id=id)
-    if not conv:
-        return get_data_error_result(retmsg="Session doesn't exist")
-    conv = conv[0]
-    if not DialogService.query(id=conv.dialog_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
-        return get_data_error_result(retmsg="You don't own the session")
-    ConversationService.delete_by_id(id)
-    return get_json_result(data=True)
+def delete(tenant_id,chat_id):
+    if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
+        return get_error_data_result(retmsg="You don't own the chat")
+    req = request.json
+    convs = ConversationService.query(dialog_id=chat_id)
+    if not req:
+        ids = None
+    else:
+        ids=req.get("ids")
+
+    if not ids:
+        conv_list = []
+        for conv in convs:
+            conv_list.append(conv.id)
+    else:
+        conv_list=ids
+    for id in conv_list:
+        conv = ConversationService.query(id=id,dialog_id=chat_id)
+        if not conv:
+            return get_error_data_result(retmsg="The chat doesn't own the session")
+        ConversationService.delete_by_id(id)
+    return get_result()

api/apps/system_app.py

@@ -14,14 +14,19 @@
 #  limitations under the License
 #
 import json
+from datetime import datetime
 
-from flask_login import login_required
+from flask_login import login_required, current_user
 
+from api.db.db_models import APIToken
+from api.db.services.api_service import APITokenService
 from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.user_service import UserTenantService
 from api.settings import DATABASE_TYPE
-from api.utils.api_utils import get_json_result
+from api.utils import current_timestamp, datetime_format
+from api.utils.api_utils import get_json_result, get_data_error_result, server_error_response, \
+    generate_confirmation_token, request, validate_request
 from api.versions import get_rag_version
-from rag.settings import SVR_QUEUE_NAME
 from rag.utils.es_conn import ELASTICSEARCH
 from rag.utils.storage_factory import STORAGE_IMPL, STORAGE_IMPL_TYPE
 from timeit import default_timer as timer
@@ -88,3 +93,49 @@ def status():
         res["task_executor"] = {"status": "red", "error": str(e)}
 
     return get_json_result(data=res)
+
+
+@manager.route('/new_token', methods=['POST'])
+@login_required
+def new_token():
+    try:
+        tenants = UserTenantService.query(user_id=current_user.id)
+        if not tenants:
+            return get_data_error_result(retmsg="Tenant not found!")
+
+        tenant_id = tenants[0].tenant_id
+        obj = {"tenant_id": tenant_id, "token": generate_confirmation_token(tenant_id),
+               "create_time": current_timestamp(),
+               "create_date": datetime_format(datetime.now()),
+               "update_time": None,
+               "update_date": None
+               }
+
+        if not APITokenService.save(**obj):
+            return get_data_error_result(retmsg="Fail to new a dialog!")
+
+        return get_json_result(data=obj)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/token_list', methods=['GET'])
+@login_required
+def token_list():
+    try:
+        tenants = UserTenantService.query(user_id=current_user.id)
+        if not tenants:
+            return get_data_error_result(retmsg="Tenant not found!")
+
+        objs = APITokenService.query(tenant_id=tenants[0].tenant_id)
+        return get_json_result(data=[o.to_dict() for o in objs])
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/token/<token>', methods=['DELETE'])
+@login_required
+def rm(token):
+    APITokenService.filter_delete(
+                [APIToken.tenant_id == current_user.id, APIToken.token == token])
+    return get_json_result(data=True)

api/apps/tenant_app.py

@@ -15,25 +15,14 @@
 #
 
 from flask import request
-from flask_login import current_user, login_required
+from flask_login import login_required, current_user
 
 from api.db import UserTenantRole, StatusEnum
 from api.db.db_models import UserTenant
-from api.db.services.user_service import TenantService, UserTenantService
-from api.settings import RetCode
+from api.db.services.user_service import UserTenantService, UserService
 
-from api.utils import get_uuid
-from api.utils.api_utils import get_json_result, validate_request, server_error_response
-
-
-@manager.route("/list", methods=["GET"])
-@login_required
-def tenant_list():
-    try:
-        tenants = TenantService.get_by_user_id(current_user.id)
-        return get_json_result(data=tenants)
-    except Exception as e:
-        return server_error_response(e)
+from api.utils import get_uuid, delta_seconds
+from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 
 
 @manager.route("/<tenant_id>/user/list", methods=["GET"])
@@ -41,6 +30,8 @@ def tenant_list():
 def user_list(tenant_id):
     try:
         users = UserTenantService.get_by_tenant_id(tenant_id)
+        for u in users:
+            u["delta_seconds"] = delta_seconds(str(u["update_date"]))
         return get_json_result(data=users)
     except Exception as e:
         return server_error_response(e)
@@ -48,30 +39,32 @@ def user_list(tenant_id):
 
 @manager.route('/<tenant_id>/user', methods=['POST'])
 @login_required
-@validate_request("user_id")
+@validate_request("email")
 def create(tenant_id):
-    user_id = request.json.get("user_id")
-    if not user_id:
-        return get_json_result(
-            data=False, retmsg='Lack of "USER ID"', retcode=RetCode.ARGUMENT_ERROR)
+    req = request.json
+    usrs = UserService.query(email=req["email"])
+    if not usrs:
+        return get_data_error_result(retmsg="User not found.")
 
-    try:
-        user_tenants = UserTenantService.query(user_id=user_id, tenant_id=tenant_id)
-        if user_tenants:
-            uuid = user_tenants[0].id
-            return get_json_result(data={"id": uuid})
+    user_id = usrs[0].id
+    user_tenants = UserTenantService.query(user_id=user_id, tenant_id=tenant_id)
+    if user_tenants:
+        if user_tenants[0].status == UserTenantRole.NORMAL.value:
+            return get_data_error_result(retmsg="This user is in the team already.")
+        return get_data_error_result(retmsg="Invitation notification is sent.")
 
-        uuid = get_uuid()
-        UserTenantService.save(
-            id = uuid,
-            user_id = user_id,
-            tenant_id = tenant_id,
-            role = UserTenantRole.NORMAL.value,
-            status = StatusEnum.VALID.value)
+    UserTenantService.save(
+        id=get_uuid(),
+        user_id=user_id,
+        tenant_id=tenant_id,
+        invited_by=current_user.id,
+        role=UserTenantRole.INVITE,
+        status=StatusEnum.VALID.value)
 
-        return get_json_result(data={"id": uuid})
-    except Exception as e:
-        return server_error_response(e)
+    usr = usrs[0].to_dict()
+    usr = {k: v for k, v in usr.items() if k in ["id", "avatar", "email", "nickname"]}
+
+    return get_json_result(data=usr)
 
 
 @manager.route('/<tenant_id>/user/<user_id>', methods=['DELETE'])
@@ -82,4 +75,25 @@ def rm(tenant_id, user_id):
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)
-    
+
+
+@manager.route("/list", methods=["GET"])
+@login_required
+def tenant_list():
+    try:
+        users = UserTenantService.get_tenants_by_user_id(current_user.id)
+        for u in users:
+            u["delta_seconds"] = delta_seconds(str(u["update_date"]))
+        return get_json_result(data=users)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/agree/<tenant_id>", methods=["PUT"])
+@login_required
+def agree(tenant_id):
+    try:
+        UserTenantService.filter_update([UserTenant.tenant_id == tenant_id, UserTenant.user_id == current_user.id], {"role": UserTenantRole.NORMAL})
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)

api/apps/user_app.py

@@ -23,7 +23,7 @@ from flask_login import login_required, current_user, login_user, logout_user
 
 from api.db.db_models import TenantLLM
 from api.db.services.llm_service import TenantLLMService, LLMService
-from api.utils.api_utils import server_error_response, validate_request
+from api.utils.api_utils import server_error_response, validate_request, get_data_error_result
 from api.utils import get_uuid, get_format_time, decrypt, download_img, current_timestamp, datetime_format
 from api.db import UserTenantRole, LLMType, FileType
 from api.settings import RetCode, GITHUB_OAUTH, FEISHU_OAUTH, CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, \
@@ -260,7 +260,8 @@ def setting_user():
             update_dict["password"] = generate_password_hash(decrypt(new_password))
 
     for k in request_data.keys():
-        if k in ["password", "new_password"]:
+        if k in ["password", "new_password", "email", "status", "is_superuser", "login_channel", "is_anonymous",
+                 "is_active", "is_authenticated", "last_login_time"]:
             continue
         update_dict[k] = request_data[k]
 
@@ -354,7 +355,7 @@ def user_add():
     email_address = req["email"]
 
     # Validate the email address
-    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,4}$", email_address):
+    if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,5}$", email_address):
         return get_json_result(data=False,
                                retmsg=f'Invalid email address: {email_address}!',
                                retcode=RetCode.OPERATING_ERROR)
@@ -402,8 +403,10 @@ def user_add():
 @login_required
 def tenant_info():
     try:
-        tenants = TenantService.get_by_user_id(current_user.id)[0]
-        return get_json_result(data=tenants)
+        tenants = TenantService.get_info_by(current_user.id)
+        if not tenants:
+            return get_data_error_result(retmsg="Tenant not found!")
+        return get_json_result(data=tenants[0])
     except Exception as e:
         return server_error_response(e)
 

api/contants.py

@@ -13,4 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-NAME_LENGTH_LIMIT = 2 ** 10
+NAME_LENGTH_LIMIT = 2 ** 10
+
+IMG_BASE64_PREFIX = 'data:image/png;base64,'

api/db/__init__.py

@@ -27,6 +27,7 @@ class UserTenantRole(StrEnum):
     OWNER = 'owner'
     ADMIN = 'admin'
     NORMAL = 'normal'
+    INVITE = 'invite'
 
 
 class TenantPermission(StrEnum):

api/db/db_models.py

@@ -879,8 +879,8 @@ class Dialog(DataBaseModel):
         default="simple",
         help_text="simple|advanced",
         index=True)
-    prompt_config = JSONField(null=False, default={"system": "", "prologue": "您好，我是您的助手小樱，长得可爱又善良，can I help you?",
-                                                   "parameters": [], "empty_response": "Sorry! 知识库中未找到相关内容！"})
+    prompt_config = JSONField(null=False, default={"system": "", "prologue": "Hi! I'm your assistant, what can I do for you?",
+                                                   "parameters": [], "empty_response": "Sorry! No relevant content was found in the knowledge base!"})
 
     similarity_threshold = FloatField(default=0.2)
     vector_similarity_weight = FloatField(default=0.3)
@@ -1052,4 +1052,11 @@ def migrate_db():
             )
         except Exception as e:
             pass
+        try:
+            migrate(
+                migrator.alter_column_type('api_token', 'dialog_id',
+                                           CharField(max_length=32, null=True, index=True))
+            )
+        except Exception as e:
+            pass
 

api/db/init_data.py

@@ -132,7 +132,7 @@ def init_llm_factory():
     TenantService.filter_update([1 == 1], {
         "parser_ids": "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,knowledge_graph:Knowledge Graph,email:Email"})
     ## insert openai two embedding models to the current openai user.
-    print("Start to insert 2 OpenAI embedding models...")
+    # print("Start to insert 2 OpenAI embedding models...")
     tenant_ids = set([row["tenant_id"] for row in TenantLLMService.get_openai_models()])
     for tid in tenant_ids:
         for row in TenantLLMService.query(llm_factory="OpenAI", tenant_id=tid):

api/db/services/dialog_service.py

@@ -19,14 +19,15 @@ import json
 import re
 from copy import deepcopy
 from timeit import default_timer as timer
-from api.db import LLMType, ParserType
-from api.db.db_models import Dialog, Conversation
+
+
+from api.db import LLMType, ParserType,StatusEnum
+from api.db.db_models import Dialog, Conversation,DB
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMService, TenantLLMService, LLMBundle
 from api.settings import chat_logger, retrievaler, kg_retrievaler
 from rag.app.resume import forbidden_select_fields4resume
-from rag.nlp import keyword_extraction
 from rag.nlp.search import index_name
 from rag.utils import rmSpace, num_tokens_from_string, encoder
 from api.utils.file_utils import get_project_base_directory
@@ -35,10 +36,49 @@ from api.utils.file_utils import get_project_base_directory
 class DialogService(CommonService):
     model = Dialog
 
+    @classmethod
+    @DB.connection_context()
+    def get_list(cls, tenant_id,
+                 page_number, items_per_page, orderby, desc, id , name):
+        chats = cls.model.select()
+        if id:
+            chats = chats.where(cls.model.id == id)
+        if name:
+            chats = chats.where(cls.model.name == name)
+        chats = chats.where(
+              (cls.model.tenant_id == tenant_id)
+            & (cls.model.status == StatusEnum.VALID.value)
+        )
+        if desc:
+            chats = chats.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            chats = chats.order_by(cls.model.getter_by(orderby).asc())
+
+        chats = chats.paginate(page_number, items_per_page)
+
+        return list(chats.dicts())
+
 
 class ConversationService(CommonService):
     model = Conversation
 
+    @classmethod
+    @DB.connection_context()
+    def get_list(cls,dialog_id,page_number, items_per_page, orderby, desc, id , name):
+        sessions = cls.model.select().where(cls.model.dialog_id ==dialog_id)
+        if id:
+            sessions = sessions.where(cls.model.id == id)
+        if name:
+            sessions = sessions.where(cls.model.name == name)
+        if desc:
+            sessions = sessions.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            sessions = sessions.order_by(cls.model.getter_by(orderby).asc())
+
+        sessions = sessions.paginate(page_number, items_per_page)
+
+        return list(sessions.dicts())
+
 
 def message_fit_in(msg, max_length=4000):
     def count():
@@ -85,7 +125,7 @@ def llm_id2llm_type(llm_id):
         for llm in llm_factory["llm"]:
             if llm_id == llm["llm_name"]:
                 return llm["model_type"].strip(",")[-1]
-                
+
 
 def chat(dialog, messages, stream=True, **kwargs):
     assert messages[-1]["role"] == "user", "The last content of this conversation is not from user."
@@ -165,7 +205,9 @@ def chat(dialog, messages, stream=True, **kwargs):
     else:
         if prompt_config.get("keyword", False):
             questions[-1] += keyword_extraction(chat_mdl, questions[-1])
-        kbinfos = retr.retrieval(" ".join(questions), embd_mdl, dialog.tenant_id, dialog.kb_ids, 1, dialog.top_n,
+
+        tenant_ids = list(set([kb.tenant_id for kb in kbs]))
+        kbinfos = retr.retrieval(" ".join(questions), embd_mdl, tenant_ids, dialog.kb_ids, 1, dialog.top_n,
                                         dialog.similarity_threshold,
                                         dialog.vector_similarity_weight,
                                         doc_ids=attachments,
@@ -189,6 +231,7 @@ def chat(dialog, messages, stream=True, **kwargs):
     used_token_count, msg = message_fit_in(msg, int(max_tokens * 0.97))
     assert len(msg) >= 2, f"message_fit_in has bug: {msg}"
     prompt = msg[0]["content"]
+    prompt += "\n\n### Query:\n%s" % " ".join(questions)
 
     if "max_tokens" in gen_conf:
         gen_conf["max_tokens"] = min(
@@ -415,6 +458,58 @@ def rewrite(tenant_id, llm_id, question):
     return ans
 
 
+def keyword_extraction(chat_mdl, content, topn=3):
+    prompt = f"""
+Role: You're a text analyzer. 
+Task: extract the most important keywords/phrases of a given piece of text content.
+Requirements: 
+  - Summarize the text content, and give top {topn} important keywords/phrases.
+  - The keywords MUST be in language of the given piece of text content.
+  - The keywords are delimited by ENGLISH COMMA.
+  - Keywords ONLY in output.
+
+### Text Content 
+{content}
+
+"""
+    msg = [
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": "Output: "}
+    ]
+    _, msg = message_fit_in(msg, chat_mdl.max_length)
+    kwd = chat_mdl.chat(prompt, msg[1:], {"temperature": 0.2})
+    if isinstance(kwd, tuple): kwd = kwd[0]
+    if kwd.find("**ERROR**") >=0: return ""
+    return kwd
+
+
+def question_proposal(chat_mdl, content, topn=3):
+    prompt = f"""
+Role: You're a text analyzer. 
+Task:  propose {topn} questions about a given piece of text content.
+Requirements: 
+  - Understand and summarize the text content, and propose top {topn} important questions.
+  - The questions SHOULD NOT have overlapping meanings.
+  - The questions SHOULD cover the main content of the text as much as possible.
+  - The questions MUST be in language of the given piece of text content.
+  - One question per line.
+  - Question ONLY in output.
+
+### Text Content 
+{content}
+
+"""
+    msg = [
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": "Output: "}
+    ]
+    _, msg = message_fit_in(msg, chat_mdl.max_length)
+    kwd = chat_mdl.chat(prompt, msg[1:], {"temperature": 0.2})
+    if isinstance(kwd, tuple): kwd = kwd[0]
+    if kwd.find("**ERROR**") >= 0: return ""
+    return kwd
+
+
 def full_question(tenant_id, llm_id, messages):
     if llm_id2llm_type(llm_id) == "image2text":
         chat_mdl = LLMBundle(tenant_id, LLMType.IMAGE2TEXT, llm_id)

api/db/services/document_service.py

@@ -38,7 +38,7 @@ from rag.utils.storage_factory import STORAGE_IMPL
 from rag.nlp import search, rag_tokenizer
 
 from api.db import FileType, TaskStatus, ParserType, LLMType
-from api.db.db_models import DB, Knowledgebase, Tenant, Task
+from api.db.db_models import DB, Knowledgebase, Tenant, Task, UserTenant
 from api.db.db_models import Document
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -49,6 +49,28 @@ from rag.utils.redis_conn import REDIS_CONN
 class DocumentService(CommonService):
     model = Document
 
+    @classmethod
+    @DB.connection_context()
+    def get_list(cls, kb_id, page_number, items_per_page,
+                     orderby, desc, keywords, id):
+        docs =cls.model.select().where(cls.model.kb_id==kb_id)
+        if id:
+            docs = docs.where(
+                cls.model.id== id )
+        if keywords:
+            docs = docs.where(
+                fn.LOWER(cls.model.name).contains(keywords.lower())
+            )
+        if desc:
+            docs = docs.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            docs = docs.order_by(cls.model.getter_by(orderby).asc())
+
+        docs = docs.paginate(page_number, items_per_page)
+        count = docs.count()
+        return list(docs.dicts()), count
+
+
     @classmethod
     @DB.connection_context()
     def get_by_kb_id(cls, kb_id, page_number, items_per_page,
@@ -241,6 +263,33 @@ class DocumentService(CommonService):
             return
         return docs[0]["tenant_id"]
 
+    @classmethod
+    @DB.connection_context()
+    def accessible(cls, doc_id, user_id):
+        docs = cls.model.select(
+            cls.model.id).join(
+            Knowledgebase, on=(
+                    Knowledgebase.id == cls.model.kb_id)
+            ).join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
+            ).where(cls.model.id == doc_id, UserTenant.user_id == user_id).paginate(0, 1)
+        docs = docs.dicts()
+        if not docs:
+            return False
+        return True
+
+    @classmethod
+    @DB.connection_context()
+    def accessible4deletion(cls, doc_id, user_id):
+        docs = cls.model.select(
+            cls.model.id).join(
+            Knowledgebase, on=(
+                    Knowledgebase.id == cls.model.kb_id)
+            ).where(cls.model.id == doc_id, Knowledgebase.created_by == user_id).paginate(0, 1)
+        docs = docs.dicts()
+        if not docs:
+            return False
+        return True
+
     @classmethod
     @DB.connection_context()
     def get_embd_id(cls, doc_id):
@@ -268,7 +317,7 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_thumbnails(cls, docids):
-        fields = [cls.model.id, cls.model.thumbnail]
+        fields = [cls.model.id, cls.model.kb_id, cls.model.thumbnail]
         return list(cls.model.select(
             *fields).where(cls.model.id.in_(docids)).dicts())
 
@@ -339,7 +388,7 @@ class DocumentService(CommonService):
                 elif finished:
                     if d["parser_config"].get("raptor", {}).get("use_raptor") and d["progress_msg"].lower().find(" raptor")<0:
                         queue_raptor_tasks(d)
-                        prg *= 0.98
+                        prg = 0.98 * len(tsks)/(len(tsks)+1)
                         msg.append("------ RAPTOR -------")
                     else:
                         status = TaskStatus.DONE.value
@@ -356,7 +405,8 @@ class DocumentService(CommonService):
                     info["progress_msg"] = msg
                 cls.update_by_id(d["id"], info)
             except Exception as e:
-                stat_logger.error("fetch task exception:" + str(e))
+                if str(e).find("'0'") < 0:
+                    stat_logger.error("fetch task exception:" + str(e))
 
     @classmethod
     @DB.connection_context()

api/db/services/file_service.py

@@ -26,7 +26,7 @@ from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.utils import get_uuid
-from api.utils.file_utils import filename_type, thumbnail
+from api.utils.file_utils import filename_type, thumbnail_img
 from rag.utils.storage_factory import STORAGE_IMPL
 
 
@@ -354,8 +354,17 @@ class FileService(CommonService):
                     location += "_"
                 blob = file.read()
                 STORAGE_IMPL.put(kb.id, location, blob)
+
+                doc_id = get_uuid()
+
+                img = thumbnail_img(filename, blob)
+                thumbnail_location = ''
+                if img is not None:
+                    thumbnail_location = f'thumbnail_{doc_id}.png'
+                    STORAGE_IMPL.put(kb.id, thumbnail_location, img)
+
                 doc = {
-                    "id": get_uuid(),
+                    "id": doc_id,
                     "kb_id": kb.id,
                     "parser_id": self.get_parser(filetype, filename, kb.parser_id),
                     "parser_config": kb.parser_config,
@@ -364,7 +373,7 @@ class FileService(CommonService):
                     "name": filename,
                     "location": location,
                     "size": len(blob),
-                    "thumbnail": thumbnail(filename, blob)
+                    "thumbnail": thumbnail_location
                 }
                 DocumentService.insert(doc)
 

api/db/services/knowledgebase_service.py

@@ -14,21 +14,47 @@
 #  limitations under the License.
 #
 from api.db import StatusEnum, TenantPermission
-from api.db.db_models import Knowledgebase, DB, Tenant
+from api.db.db_models import Knowledgebase, DB, Tenant, User, UserTenant,Document
 from api.db.services.common_service import CommonService
 
 
 class KnowledgebaseService(CommonService):
     model = Knowledgebase
 
+    @classmethod
+    @DB.connection_context()
+    def list_documents_by_ids(cls,kb_ids):
+        doc_ids=cls.model.select(Document.id.alias("document_id")).join(Document,on=(cls.model.id == Document.kb_id)).where(
+            cls.model.id.in_(kb_ids)
+        )
+        doc_ids =list(doc_ids.dicts())
+        doc_ids = [doc["document_id"] for doc in doc_ids]
+        return doc_ids
+
     @classmethod
     @DB.connection_context()
     def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
                           page_number, items_per_page, orderby, desc):
-        kbs = cls.model.select().where(
+        fields = [
+            cls.model.id,
+            cls.model.avatar,
+            cls.model.name,
+            cls.model.language,
+            cls.model.description,
+            cls.model.permission,
+            cls.model.doc_num,
+            cls.model.token_num,
+            cls.model.chunk_num,
+            cls.model.parser_id,
+            cls.model.embd_id,
+            User.nickname,
+            User.avatar.alias('tenant_avatar'),
+            cls.model.update_time
+        ]
+        kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
             ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
                                                             TenantPermission.TEAM.value)) | (
-                         cls.model.tenant_id == user_id))
+                     cls.model.tenant_id == user_id))
             & (cls.model.status == StatusEnum.VALID.value)
         )
         if desc:
@@ -63,14 +89,14 @@ class KnowledgebaseService(CommonService):
         if count == -1:
             return kbs[offset:]
 
-        return kbs[offset:offset+count]
+        return kbs[offset:offset + count]
 
     @classmethod
     @DB.connection_context()
     def get_detail(cls, kb_id):
         fields = [
             cls.model.id,
-            #Tenant.embd_id,
+            # Tenant.embd_id,
             cls.model.embd_id,
             cls.model.avatar,
             cls.model.name,
@@ -83,14 +109,14 @@ class KnowledgebaseService(CommonService):
             cls.model.parser_id,
             cls.model.parser_config]
         kbs = cls.model.select(*fields).join(Tenant, on=(
-                    (Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
+                (Tenant.id == cls.model.tenant_id) & (Tenant.status == StatusEnum.VALID.value))).where(
             (cls.model.id == kb_id),
             (cls.model.status == StatusEnum.VALID.value)
         )
         if not kbs:
             return
         d = kbs[0].to_dict()
-        #d["embd_id"] = kbs[0].tenant.embd_id
+        # d["embd_id"] = kbs[0].tenant.embd_id
         return d
 
     @classmethod
@@ -142,3 +168,49 @@ class KnowledgebaseService(CommonService):
     @DB.connection_context()
     def get_all_ids(cls):
         return [m["id"] for m in cls.model.select(cls.model.id).dicts()]
+
+    @classmethod
+    @DB.connection_context()
+    def get_list(cls, joined_tenant_ids, user_id,
+                 page_number, items_per_page, orderby, desc, id, name):
+        kbs = cls.model.select()
+        if id:
+            kbs = kbs.where(cls.model.id == id)
+        if name:
+            kbs = kbs.where(cls.model.name == name)
+        kbs = kbs.where(
+            ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                            TenantPermission.TEAM.value)) | (
+                     cls.model.tenant_id == user_id))
+            & (cls.model.status == StatusEnum.VALID.value)
+        )
+        if desc:
+            kbs = kbs.order_by(cls.model.getter_by(orderby).desc())
+        else:
+            kbs = kbs.order_by(cls.model.getter_by(orderby).asc())
+
+        kbs = kbs.paginate(page_number, items_per_page)
+
+        return list(kbs.dicts())
+
+    @classmethod
+    @DB.connection_context()
+    def accessible(cls, kb_id, user_id):
+        docs = cls.model.select(
+            cls.model.id).join(UserTenant, on=(UserTenant.tenant_id == Knowledgebase.tenant_id)
+            ).where(cls.model.id == kb_id, UserTenant.user_id == user_id).paginate(0, 1)
+        docs = docs.dicts()
+        if not docs:
+            return False
+        return True
+
+    @classmethod
+    @DB.connection_context()
+    def accessible4deletion(cls, kb_id, user_id):
+        docs = cls.model.select(
+            cls.model.id).where(cls.model.id == kb_id, cls.model.created_by == user_id).paginate(0, 1)
+        docs = docs.dicts()
+        if not docs:
+            return False
+        return True
+

api/db/services/llm_service.py

@@ -133,7 +133,8 @@ class TenantLLMService(CommonService):
             if model_config["llm_factory"] not in Seq2txtModel:
                 return
             return Seq2txtModel[model_config["llm_factory"]](
-                model_config["api_key"], model_config["llm_name"], lang,
+                key=model_config["api_key"], model_name=model_config["llm_name"],
+                lang=lang,
                 base_url=model_config["api_base"]
             )
         if llm_type == LLMType.TTS:
@@ -167,11 +168,13 @@ class TenantLLMService(CommonService):
         else:
             assert False, "LLM type error"
 
+        llm_name = mdlnm.split("@")[0] if "@" in mdlnm else mdlnm
+
         num = 0
         try:
-            for u in cls.query(tenant_id=tenant_id, llm_name=mdlnm):
+            for u in cls.query(tenant_id=tenant_id, llm_name=llm_name):
                 num += cls.model.update(used_tokens=u.used_tokens + used_tokens)\
-                    .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == mdlnm)\
+                    .where(cls.model.tenant_id == tenant_id, cls.model.llm_name == llm_name)\
                     .execute()
         except Exception as e:
             pass
@@ -195,7 +198,7 @@ class LLMBundle(object):
         self.llm_name = llm_name
         self.mdl = TenantLLMService.model_instance(
             tenant_id, llm_type, llm_name, lang=lang)
-        assert self.mdl, "Can't find mole for {}/{}/{}".format(
+        assert self.mdl, "Can't find model for {}/{}/{}".format(
             tenant_id, llm_type, llm_name)
         self.max_length = 8192
         for lm in LLMService.query(llm_name=llm_name):
@@ -207,7 +210,7 @@ class LLMBundle(object):
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
             database_logger.error(
-                "Can't update token usage for {}/EMBEDDING".format(self.tenant_id))
+                "Can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
         return emd, used_tokens
 
     def encode_queries(self, query: str):
@@ -215,7 +218,7 @@ class LLMBundle(object):
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
             database_logger.error(
-                "Can't update token usage for {}/EMBEDDING".format(self.tenant_id))
+                "Can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
         return emd, used_tokens
 
     def similarity(self, query: str, texts: list):
@@ -223,7 +226,7 @@ class LLMBundle(object):
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
             database_logger.error(
-                "Can't update token usage for {}/RERANK".format(self.tenant_id))
+                "Can't update token usage for {}/RERANK used_tokens: {}".format(self.tenant_id, used_tokens))
         return sim, used_tokens
 
     def describe(self, image, max_tokens=300):
@@ -231,7 +234,7 @@ class LLMBundle(object):
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
             database_logger.error(
-                "Can't update token usage for {}/IMAGE2TEXT".format(self.tenant_id))
+                "Can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
         return txt
 
     def transcription(self, audio):
@@ -239,7 +242,7 @@ class LLMBundle(object):
         if not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens):
             database_logger.error(
-                "Can't update token usage for {}/SEQUENCE2TXT".format(self.tenant_id))
+                "Can't update token usage for {}/SEQUENCE2TXT used_tokens: {}".format(self.tenant_id, used_tokens))
         return txt
 
     def tts(self, text):
@@ -254,10 +257,10 @@ class LLMBundle(object):
 
     def chat(self, system, history, gen_conf):
         txt, used_tokens = self.mdl.chat(system, history, gen_conf)
-        if not TenantLLMService.increase_usage(
+        if isinstance(txt, int) and not TenantLLMService.increase_usage(
                 self.tenant_id, self.llm_type, used_tokens, self.llm_name):
             database_logger.error(
-                "Can't update token usage for {}/CHAT".format(self.tenant_id))
+                "Can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
         return txt
 
     def chat_streamly(self, system, history, gen_conf):
@@ -266,6 +269,6 @@ class LLMBundle(object):
                 if not TenantLLMService.increase_usage(
                         self.tenant_id, self.llm_type, txt, self.llm_name):
                     database_logger.error(
-                        "Can't update token usage for {}/CHAT".format(self.tenant_id))
+                        "Can't update token usage for {}/CHAT llm_name: {}, content: {}".format(self.tenant_id, self.llm_name, txt))
                 return
             yield txt

api/db/services/user_service.py

@@ -87,7 +87,7 @@ class TenantService(CommonService):
 
     @classmethod
     @DB.connection_context()
-    def get_by_user_id(cls, user_id):
+    def get_info_by(cls, user_id):
         fields = [
             cls.model.id.alias("tenant_id"),
             cls.model.name,
@@ -100,7 +100,7 @@ class TenantService(CommonService):
             cls.model.parser_ids,
             UserTenant.role]
         return list(cls.model.select(*fields)
-                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value)))
+                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.OWNER)))
                     .where(cls.model.status == StatusEnum.VALID.value).dicts())
 
     @classmethod
@@ -115,7 +115,7 @@ class TenantService(CommonService):
             cls.model.img2txt_id,
             UserTenant.role]
         return list(cls.model.select(*fields)
-                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.NORMAL.value)))
+                    .join(UserTenant, on=((cls.model.id == UserTenant.tenant_id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value) & (UserTenant.role == UserTenantRole.NORMAL)))
                     .where(cls.model.status == StatusEnum.VALID.value).dicts())
 
     @classmethod
@@ -143,9 +143,8 @@ class UserTenantService(CommonService):
     def get_by_tenant_id(cls, tenant_id):
         fields = [
             cls.model.user_id,
-            cls.model.tenant_id,
-            cls.model.role,
             cls.model.status,
+            cls.model.role,
             User.nickname,
             User.email,
             User.avatar,
@@ -153,8 +152,24 @@ class UserTenantService(CommonService):
             User.is_active,
             User.is_anonymous,
             User.status,
+            User.update_date,
             User.is_superuser]
         return list(cls.model.select(*fields)
-                    .join(User, on=((cls.model.user_id == User.id) & (cls.model.status == StatusEnum.VALID.value)))
+                    .join(User, on=((cls.model.user_id == User.id) & (cls.model.status == StatusEnum.VALID.value) & (cls.model.role != UserTenantRole.OWNER)))
                     .where(cls.model.tenant_id == tenant_id)
-                    .dicts())
+                    .dicts())
+
+    @classmethod
+    @DB.connection_context()
+    def get_tenants_by_user_id(cls, user_id):
+        fields = [
+            cls.model.tenant_id,
+            cls.model.role,
+            User.nickname,
+            User.email,
+            User.avatar,
+            User.update_date
+        ]
+        return list(cls.model.select(*fields)
+                    .join(User, on=((cls.model.tenant_id == User.id) & (UserTenant.user_id == user_id) & (UserTenant.status == StatusEnum.VALID.value)))
+                    .where(cls.model.status == StatusEnum.VALID.value).dicts())

api/ragflow_server.py

@@ -38,7 +38,7 @@ from api.versions import get_versions
 
 def update_progress():
     while True:
-        time.sleep(1)
+        time.sleep(3)
         try:
             DocumentService.update_progress()
         except Exception as e:

api/settings.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 import os
+from datetime import date
 from enum import IntEnum, Enum
 from api.utils.file_utils import get_project_base_directory
 from api.utils.log_utils import LoggerFactory, getLogger
@@ -42,7 +43,7 @@ RAG_FLOW_SERVICE_NAME = "ragflow"
 SERVER_MODULE = "rag_flow_server.py"
 TEMP_DIRECTORY = os.path.join(get_project_base_directory(), "temp")
 RAG_FLOW_CONF_PATH = os.path.join(get_project_base_directory(), "conf")
-LIGHTEN = os.environ.get('LIGHTEN')
+LIGHTEN = int(os.environ.get('LIGHTEN', "0"))
 
 SUBPROCESS_STD_LOG_NAME = "std.log"
 
@@ -123,7 +124,7 @@ if not LIGHTEN:
 
     CHAT_MDL = default_llm[LLM_FACTORY]["chat_model"]
     EMBEDDING_MDL = default_llm["BAAI"]["embedding_model"]
-    RERANK_MDL = default_llm["BAAI"]["rerank_model"] if not LIGHTEN else ""
+    RERANK_MDL = default_llm["BAAI"]["rerank_model"]
     ASR_MDL = default_llm[LLM_FACTORY]["asr_model"]
     IMAGE2TEXT_MDL = default_llm[LLM_FACTORY]["image2text_model"]
 else:
@@ -143,9 +144,8 @@ HTTP_PORT = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("http_port")
 
 SECRET_KEY = get_base_config(
     RAG_FLOW_SERVICE_NAME,
-    {}).get(
-        "secret_key",
-    "infiniflow")
+    {}).get("secret_key", str(date.today()))
+
 TOKEN_EXPIRE_IN = get_base_config(
     RAG_FLOW_SERVICE_NAME, {}).get(
         "token_expires_in", 3600)
@@ -250,3 +250,5 @@ class RetCode(IntEnum, CustomEnum):
     AUTHENTICATION_ERROR = 109
     UNAUTHORIZED = 401
     SERVER_ERROR = 500
+    FORBIDDEN = 403
+    NOT_FOUND = 404

api/utils/__init__.py

@@ -344,3 +344,8 @@ def download_img(url):
     return "data:" + \
            response.headers.get('Content-Type', 'image/jpg') + ";" + \
            "base64," + base64.b64encode(response.content).decode("utf-8")
+
+
+def delta_seconds(date_string: str):
+    dt = datetime.datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
+    return (datetime.datetime.now() - dt).total_seconds()

api/utils/api_utils.py

@@ -29,6 +29,7 @@ from flask import (
     Response, jsonify, send_file, make_response,
     request as flask_request,
 )
+from itsdangerous import URLSafeTimedSerializer
 from werkzeug.http import HTTP_STATUS_CODES
 
 from api.db.db_models import APIToken
@@ -37,7 +38,7 @@ from api.settings import (
     stat_logger, CLIENT_AUTHENTICATION, HTTP_APP_KEY, SECRET_KEY
 )
 from api.settings import RetCode
-from api.utils import CustomJSONEncoder
+from api.utils import CustomJSONEncoder, get_uuid
 from api.utils import json_dumps
 
 requests.models.complexjson.dumps = functools.partial(
@@ -52,7 +53,7 @@ def request(**kwargs):
         k.replace(
             '_',
             '-').upper(): v for k,
-        v in kwargs.get(
+                                v in kwargs.get(
             'headers',
             {}).items()}
     prepped = requests.Request(**kwargs).prepare()
@@ -96,26 +97,6 @@ def get_exponential_backoff_interval(retries, full_jitter=False):
     return max(0, countdown)
 
 
-def get_json_result(retcode=RetCode.SUCCESS, retmsg='success',
-                    data=None, job_id=None, meta=None):
-    result_dict = {
-        "retcode": retcode,
-        "retmsg": retmsg,
-        # "retmsg": re.sub(r"rag", "seceum", retmsg, flags=re.IGNORECASE),
-        "data": data,
-        "jobId": job_id,
-        "meta": meta,
-    }
-
-    response = {}
-    for key, value in result_dict.items():
-        if value is None and key != "retcode":
-            continue
-        else:
-            response[key] = value
-    return jsonify(response)
-
-
 def get_data_error_result(retcode=RetCode.DATA_ERROR,
                           retmsg='Sorry! Data missing!'):
     import re
@@ -219,6 +200,27 @@ def get_json_result(retcode=RetCode.SUCCESS, retmsg='success', data=None):
     response = {"retcode": retcode, "retmsg": retmsg, "data": data}
     return jsonify(response)
 
+def apikey_required(func):
+    @wraps(func)
+    def decorated_function(*args, **kwargs):
+        token = flask_request.headers.get('Authorization').split()[1]
+        objs = APIToken.query(token=token)
+        if not objs:
+            return build_error_result(
+                error_msg='API-KEY is invalid!', retcode=RetCode.FORBIDDEN
+            )
+        kwargs['tenant_id'] = objs[0].tenant_id
+        return func(*args, **kwargs)
+
+    return decorated_function
+
+
+def build_error_result(retcode=RetCode.FORBIDDEN, error_msg='success'):
+    response = {"error_code": retcode, "error_msg": error_msg}
+    response = jsonify(response)
+    response.status_code = retcode
+    return response
+
 
 def construct_response(retcode=RetCode.SUCCESS,
                        retmsg='success', data=None, auth=None):
@@ -288,3 +290,72 @@ def token_required(func):
         return func(*args, **kwargs)
 
     return decorated_function
+
+
+def get_result(retcode=RetCode.SUCCESS, retmsg='error', data=None):
+    if retcode == 0:
+        if data is not None:
+            response = {"code": retcode, "data": data}
+        else:
+            response = {"code": retcode}
+    else:
+        response = {"code": retcode, "message": retmsg}
+    return jsonify(response)
+
+
+def get_error_data_result(retmsg='Sorry! Data missing!', retcode=RetCode.DATA_ERROR,
+                          ):
+    import re
+    result_dict = {
+        "code": retcode,
+        "message": re.sub(
+            r"rag",
+            "seceum",
+            retmsg,
+            flags=re.IGNORECASE)}
+    response = {}
+    for key, value in result_dict.items():
+        if value is None and key != "code":
+            continue
+        else:
+            response[key] = value
+    return jsonify(response)
+
+
+def generate_confirmation_token(tenent_id):
+    serializer = URLSafeTimedSerializer(tenent_id)
+    return "ragflow-" + serializer.dumps(get_uuid(), salt=tenent_id)[2:34]
+
+
+def valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method):
+    if valid_parameter(permission,valid_permission):
+        return valid_parameter(permission,valid_permission)
+    if valid_parameter(language,valid_language):
+        return valid_parameter(language,valid_language)
+    if valid_parameter(chunk_method,valid_chunk_method):
+        return valid_parameter(chunk_method,valid_chunk_method)
+
+def valid_parameter(parameter,valid_values):
+    if parameter and parameter not in valid_values:
+       return get_error_data_result(f"'{parameter}' is not in {valid_values}")
+
+def get_parser_config(chunk_method,parser_config):
+    if parser_config:
+        return parser_config
+    if not chunk_method:
+        chunk_method = "naive"
+    key_mapping={"naive":{"chunk_token_num": 128, "delimiter": "\\n!?;。；！？", "html4excel": False,"layout_recognize": True, "raptor": {"use_raptor": False}},
+                 "qa":{"raptor":{"use_raptor":False}},
+                 "resume":None,
+                 "manual":{"raptor":{"use_raptor":False}},
+                 "table":None,
+                 "paper":{"raptor":{"use_raptor":False}},
+                 "book":{"raptor":{"use_raptor":False}},
+                 "laws":{"raptor":{"use_raptor":False}},
+                 "presentation":{"raptor":{"use_raptor":False}},
+                 "one":None,
+                 "knowledge_graph":{"chunk_token_num":8192,"delimiter":"\\n!?;。；！？","entity_types":["organization","person","location","event","time"]},
+                 "email":None,
+                 "picture":None}
+    parser_config=key_mapping[chunk_method]
+    return parser_config

api/utils/file_utils.py

@@ -25,6 +25,7 @@ from cachetools import LRUCache, cached
 from ruamel.yaml import YAML
 
 from api.db import FileType
+from api.contants import IMG_BASE64_PREFIX
 
 PROJECT_BASE = os.getenv("RAG_PROJECT_BASE") or os.getenv("RAG_DEPLOY_BASE")
 RAG_BASE = os.getenv("RAG_BASE")
@@ -168,23 +169,20 @@ def filename_type(filename):
 
     return FileType.OTHER.value
 
-
-def thumbnail(filename, blob):
+def thumbnail_img(filename, blob):
     filename = filename.lower()
     if re.match(r".*\.pdf$", filename):
         pdf = pdfplumber.open(BytesIO(blob))
         buffered = BytesIO()
         pdf.pages[0].to_image(resolution=32).annotated.save(buffered, format="png")
-        return "data:image/png;base64," + \
-            base64.b64encode(buffered.getvalue()).decode("utf-8")
+        return buffered.getvalue()
 
     if re.match(r".*\.(jpg|jpeg|png|tif|gif|icon|ico|webp)$", filename):
         image = Image.open(BytesIO(blob))
         image.thumbnail((30, 30))
         buffered = BytesIO()
         image.save(buffered, format="png")
-        return "data:image/png;base64," + \
-            base64.b64encode(buffered.getvalue()).decode("utf-8")
+        return buffered.getvalue()
 
     if re.match(r".*\.(ppt|pptx)$", filename):
         import aspose.slides as slides
@@ -194,10 +192,19 @@ def thumbnail(filename, blob):
                 buffered = BytesIO()
                 presentation.slides[0].get_thumbnail(0.03, 0.03).save(
                     buffered, drawing.imaging.ImageFormat.png)
-                return "data:image/png;base64," + \
-                    base64.b64encode(buffered.getvalue()).decode("utf-8")
+                return buffered.getvalue()
         except Exception as e:
             pass
+    return None
+
+
+def thumbnail(filename, blob):
+    img = thumbnail_img(filename, blob)
+    if img is not None:
+        return IMG_BASE64_PREFIX + \
+            base64.b64encode(img).decode("utf-8")
+    else:
+        return ''
 
 
 def traversal_files(base):

conf/llm_factories.json

@@ -89,9 +89,15 @@
         {
             "name": "Tongyi-Qianwen",
             "logo": "",
-            "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION",
+            "tags": "LLM,TEXT EMBEDDING,TEXT RE-RANK,SPEECH2TEXT,MODERATION",
             "status": "1",
             "llm": [
+                {
+                    "llm_name": "qwen-long",
+                    "tags": "LLM,CHAT,10000K",
+                    "max_tokens": 1000000,
+                    "model_type": "chat"
+                },
                 {
                     "llm_name": "qwen-turbo",
                     "tags": "LLM,CHAT,8K",
@@ -139,6 +145,12 @@
                     "tags": "LLM,CHAT,IMAGE2TEXT",
                     "max_tokens": 765,
                     "model_type": "image2text"
+                },
+                {
+                    "llm_name": "gte-rerank",
+                    "tags": "RE-RANK,4k",
+                    "max_tokens": 4000,
+                    "model_type": "rerank"
                 }
             ]
         },
@@ -190,6 +202,12 @@
                     "max_tokens": 2000,
                     "model_type": "image2text"
                 },
+                {
+                    "llm_name": "glm-4-9b",
+                    "tags": "LLM,CHAT,",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
                 {
                     "llm_name": "embedding-2",
                     "tags": "TEXT EMBEDDING",
@@ -248,6 +266,12 @@
                     "tags": "LLM,CHAT",
                     "max_tokens": 128000,
                     "model_type": "chat"
+                },
+                {
+                    "llm_name": "moonshot-v1-auto",
+                    "tags": "LLM,CHAT,",
+                    "max_tokens": 128000,
+                    "model_type": "chat"
                 }
             ]
         },
@@ -619,13 +643,13 @@
                     "model_type": "chat,image2text"
                 },
                 {
-                    "llm_name": "gpt-35-turbo",
+                    "llm_name": "gpt-3.5-turbo",
                     "tags": "LLM,CHAT,4K",
                     "max_tokens": 4096,
                     "model_type": "chat"
                 },
                 {
-                    "llm_name": "gpt-35-turbo-16k",
+                    "llm_name": "gpt-3.5-turbo-16k",
                     "tags": "LLM,CHAT,16k",
                     "max_tokens": 16385,
                     "model_type": "chat"
@@ -2097,6 +2121,12 @@
             "tags": "LLM,IMAGE2TEXT",
             "status": "1",
             "llm": [
+                {
+                    "llm_name": "yi-lightning",
+                    "tags": "LLM,CHAT,16k",
+                    "max_tokens": 16384,
+                    "model_type": "chat"
+                },
                 {
                     "llm_name": "yi-large",
                     "tags": "LLM,CHAT,32k",

deepdoc/parser/html_parser.py

@@ -16,11 +16,13 @@ import readability
 import html_text
 import chardet
 
+
 def get_encoding(file):
     with open(file,'rb') as f:
         tmp = chardet.detect(f.read())
         return tmp['encoding']
 
+
 class RAGFlowHtmlParser:
     def __call__(self, fnm, binary=None):
         txt = ""

deepdoc/parser/pdf_parser.py

@@ -45,9 +45,12 @@ class RAGFlowPdfParser:
 
         self.updown_cnt_mdl = xgb.Booster()
         if not LIGHTEN:
-            import torch
-            if torch.cuda.is_available():
-                self.updown_cnt_mdl.set_param({"device": "cuda"})
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    self.updown_cnt_mdl.set_param({"device": "cuda"})
+            except Exception as e:
+                logging.error(str(e))
         try:
             model_dir = os.path.join(
                 get_project_base_directory(),
@@ -954,6 +957,8 @@ class RAGFlowPdfParser:
                 fnm, str) else pdfplumber.open(BytesIO(fnm))
             self.page_images = [p.to_image(resolution=72 * zoomin).annotated for i, p in
                                 enumerate(self.pdf.pages[page_from:page_to])]
+            self.page_images_x2 = [p.to_image(resolution=72 * zoomin * 2).annotated for i, p in
+                                enumerate(self.pdf.pages[page_from:page_to])]
             self.page_chars = [[{**c, 'top': c['top'], 'bottom': c['bottom']} for c in page.dedupe_chars().chars if self._has_color(c)] for page in
                                self.pdf.pages[page_from:page_to]]
             self.total_page = len(self.pdf.pages)
@@ -989,7 +994,7 @@ class RAGFlowPdfParser:
             self.is_english = False
 
         st = timer()
-        for i, img in enumerate(self.page_images):
+        for i, img in enumerate(self.page_images_x2):
             chars = self.page_chars[i] if not self.is_english else []
             self.mean_height.append(
                 np.median(sorted([c["height"] for c in chars])) if chars else 0
@@ -997,7 +1002,7 @@ class RAGFlowPdfParser:
             self.mean_width.append(
                 np.median(sorted([c["width"] for c in chars])) if chars else 8
             )
-            self.page_cum_height.append(img.size[1] / zoomin)
+            self.page_cum_height.append(img.size[1] / zoomin/2)
             j = 0
             while j + 1 < len(chars):
                 if chars[j]["text"] and chars[j + 1]["text"] \
@@ -1007,7 +1012,7 @@ class RAGFlowPdfParser:
                     chars[j]["text"] += " "
                 j += 1
 
-            self.__ocr(i + 1, img, chars, zoomin)
+            self.__ocr(i + 1, img, chars, zoomin*2)
             if callback and i % 6 == 5:
                 callback(prog=(i + 1) * 0.6 / len(self.page_images), msg="")
         # print("OCR:", timer()-st)

deepdoc/vision/operators.py

@@ -102,7 +102,7 @@ class StandardizeImage(object):
 
 
 class NormalizeImage(object):
-    """ normalize image such as substract mean, divide std
+    """ normalize image such as subtract mean, divide std
     """
 
     def __init__(self, scale=None, mean=None, std=None, order='chw', **kwargs):

docker/.env

@@ -1,7 +1,6 @@
 # Version of Elastic products
 STACK_VERSION=8.11.3
 
-
 # Port to expose Elasticsearch HTTP API to the host
 ES_PORT=1200
 
@@ -13,11 +12,10 @@ KIBANA_PORT=6601
 KIBANA_USER=rag_flow
 KIBANA_PASSWORD=infini_rag_flow
 
-# Increase or decrease based on the available host memory (in bytes)
+# Update according to the available host memory (in bytes)
 
 MEM_LIMIT=8073741824
 
-
 MYSQL_PASSWORD=infini_rag_flow
 MYSQL_PORT=5455
 
@@ -33,21 +31,45 @@ REDIS_PASSWORD=infini_rag_flow
 
 SVR_HTTP_PORT=9380
 
+# the Docker image for the slim version
 RAGFLOW_IMAGE=infiniflow/ragflow:dev-slim
 
-# If inside mainland China, decomment either of the following hub.docker.com mirrors:
+# If you cannot download the RAGFlow Docker image, try uncommenting either of the following hub.docker.com mirrors:
 # RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:dev-slim
 # RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:dev-slim
 
+# To download the RAGFlow Docker image with embedding models, modify the line above as follows:
+# RAGFLOW_IMAGE=infiniflow/ragflow:dev
+
+# This Docker image includes the following four models:
+# - BAAI/bge-large-zh-v1.5
+# - BAAI/bge-reranker-v2-m3
+# - maidalun1020/bce-embedding-base_v1
+# - maidalun1020/bce-reranker-base_v1
+
+# And the following models will be downloaded if you select them in the RAGFlow UI.
+# - BAAI/bge-base-en-v1.5
+# - BAAI/bge-large-en-v1.5
+# - BAAI/bge-small-en-v1.5
+# - BAAI/bge-small-zh-v1.5
+# - jinaai/jina-embeddings-v2-base-en
+# - jinaai/jina-embeddings-v2-small-en
+# - nomic-ai/nomic-embed-text-v1.5
+# - sentence-transformers/all-MiniLM-L6-v2
+
+# If you cannot download the RAGFlow Docker image, try uncommenting either of the following hub.docker.com mirrors:
+# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:dev
+# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:dev
+
 TIMEZONE='Asia/Shanghai'
 
-# If inside mainland China, decomment the following huggingface.co mirror:
+# If you cannot download the RAGFlow Docker image, try uncommenting the following huggingface.co mirror:
 # HF_ENDPOINT=https://hf-mirror.com
 
 ######## OS setup for ES ###########
 # sysctl vm.max_map_count
 # sudo sysctl -w vm.max_map_count=262144
-# However, this change is not persistent and will be reset after a system reboot.
-# To make the change permanent, you need to update the /etc/sysctl.conf file.
-# Add or update the following line in the file:
+# Note that this change is not permanent and will be reset after a system reboot.
+# To make your change permanent, update /etc/sysctl.conf by:
+# Adding or modifying the following line:
 # vm.max_map_count=262144

docker/docker-compose.yml

@@ -29,3 +29,5 @@ services:
     networks:
       - ragflow
     restart: always
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

docker/nginx/ragflow.conf

@@ -10,7 +10,7 @@ server {
     gzip_vary on;
     gzip_disable "MSIE [1-6]\.";
 
-    location /v1 {
+    location ~ ^/(v1|api) {
         proxy_pass http://ragflow:9380;
         include proxy.conf;
     }

docs/guides/configure_knowledge_base.md

@@ -58,7 +58,7 @@ You can also change the chunk template for a particular file on the **Datasets**
 
 ### Select embedding model
 
-An embedding model builds vector index on file chunks. Once you have chosen an embedding model and used it to parse a file, you are no longer allowed to change it. To switch to a different embedding model, you *must* deletes all completed file chunks in the knowledge base. The obvious reason is that we must *ensure* that all files in a specific knowledge base are parsed using the *same* embedding model (ensure that they are compared in the same embedding space). 
+An embedding model builds vector index on file chunks. Once you have chosen an embedding model and used it to parse a file, you are no longer allowed to change it. To switch to a different embedding model, you *must* delete all completed file chunks in the knowledge base. The obvious reason is that we must *ensure* that all files in a specific knowledge base are parsed using the *same* embedding model (ensure that they are compared in the same embedding space). 
 
 The following embedding models can be deployed locally:
 
@@ -128,7 +128,7 @@ RAGFlow uses multiple recall of both full-text search and vector search in its c
 
 ## Search for knowledge base
 
-As of RAGFlow v0.12.0, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
+As of RAGFlow v0.13.0, the search feature is still in a rudimentary form, supporting only knowledge base search by name.
 
 ![search knowledge base](https://github.com/infiniflow/ragflow/assets/93570324/836ae94c-2438-42be-879e-c7ad2a59693e)
 

docs/guides/develop/acquire_ragflow_api_key.md

@@ -0,0 +1,18 @@
+---
+sidebar_position: 3
+slug: /acquire_ragflow_api_key
+---
+
+# Acquire a RAGFlow API key
+
+A key is required for the RAGFlow server to authenticate your requests via HTTP or a Python API. This documents provides instructions on obtaining a RAGFlow API key.
+
+1. Click your avatar on the top right corner of the RAGFlow UI to access the configuration page.
+2. Click **API** to switch to the **API** page.
+3. Obtain a RAGFlow API key:
+
+![ragflow_api_key](https://github.com/user-attachments/assets/f461ed61-04c6-4faf-b3d8-6b5fa56be4e7)
+
+:::tip NOTE
+See the [RAGFlow HTTP API reference](../../references/http_api_reference.md) or the [RAGFlow Python API reference](../../references/python_api_reference.md) for a complete reference of RAGFlow's HTTP or Python APIs.
+:::

docs/guides/develop/build_docker_image.md

@@ -1,87 +0,0 @@
----
-sidebar_position: 1
-slug: /build_docker_image
----
-
-# Build a RAGFlow Docker Image
-
-A guide explaining how to build a RAGFlow Docker image from its source code. By following this guide, you'll be able to create a local Docker image that can be used for development, debugging, or testing purposes.
-
-## Target Audience
-
-- Developers who have added new features or modified the existing code and require a Docker image to view and debug their changes.
-- Testers looking to explore the latest features of RAGFlow in a Docker image.
-
-## Prerequisites
-
-- CPU ≥ 4 cores
-- RAM ≥ 16 GB
-- Disk ≥ 50 GB
-- Docker ≥ 24.0.0 & Docker Compose ≥ v2.26.1
-
-:::tip NOTE
-If you have not installed Docker on your local machine (Windows, Mac, or Linux), see the [Install Docker Engine](https://docs.docker.com/engine/install/) guide.
-:::
-
-## Build a RAGFlow Docker Image
-
-To build a RAGFlow Docker image from source code:
-
-### Git Clone the Repository
-
-```bash
-git clone https://github.com/infiniflow/ragflow.git
-cd ragflow
-```
-
-### Build the Docker Image
-
-Navigate to the `ragflow` directory where the Dockerfile and other necessary files are located. Now you can build the Docker image using the provided Dockerfile. The command below specifies which Dockerfile to use and tags the image with a name for reference purpose.
-
-#### Build and push multi-arch image `infiniflow/ragflow:dev-slim`
-
-On a `linux/amd64` host:
-```bash
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim-amd64 .
-docker push infiniflow/ragflow:dev-slim-amd64
-```
-
-On a `linux/arm64` host:
-```bash
-docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim-arm64 .
-docker push infiniflow/ragflow:dev-slim-arm64
-```
-
-On a Linux host:
-```bash
-docker manifest create infiniflow/ragflow:dev-slim --amend infiniflow/ragflow:dev-slim-amd64 --amend infiniflow/ragflow:dev-slim-arm64
-docker manifest push infiniflow/ragflow:dev-slim
-```
-
-This image is approximately 1 GB in size and relies on external LLM services, as it does not include deepdoc, embedding, or chat models.
-
-#### Build and push multi-arch image `infiniflow/ragflow:dev`
-
-On a `linux/amd64` host:
-```bash
-pip3 install huggingface-hub
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev-amd64 .
-docker push infiniflow/ragflow:dev-amd64
-```
-
-On a `linux/arm64` host:
-```bash
-pip3 install huggingface-hub
-python3 download_deps.py
-docker build -f Dockerfile -t infiniflow/ragflow:dev-arm64 .
-docker push infiniflow/ragflow:dev-arm64
-```
-
-On any linux host:
-```bash
-docker manifest create infiniflow/ragflow:dev --amend infiniflow/ragflow:dev-amd64 --amend infiniflow/ragflow:dev-arm64
-docker manifest push infiniflow/ragflow:dev
-```
-
-This image's size is approximately 9 GB in size and can reference via either local CPU/GPU or an external LLM, as it includes deepdoc, embedding, and chat models.

docs/guides/develop/build_docker_image.mdx

@@ -0,0 +1,64 @@
+---
+sidebar_position: 1
+slug: /build_docker_image
+---
+
+# Build a RAGFlow Docker Image
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+A guide explaining how to build a RAGFlow Docker image from its source code. By following this guide, you'll be able to create a local Docker image that can be used for development, debugging, or testing purposes.
+
+## Target Audience
+
+- Developers who have added new features or modified the existing code and require a Docker image to view and debug their changes.
+- Testers looking to explore the latest features of RAGFlow in a Docker image.
+
+## Prerequisites
+
+- CPU ≥ 4 cores
+- RAM ≥ 16 GB
+- Disk ≥ 50 GB
+- Docker ≥ 24.0.0 & Docker Compose ≥ v2.26.1
+
+:::tip NOTE
+If you have not installed Docker on your local machine (Windows, Mac, or Linux), see the [Install Docker Engine](https://docs.docker.com/engine/install/) guide.
+:::
+
+## Build a Docker image
+
+<Tabs
+  defaultValue="without"
+  values={[
+    {label: 'Build a Docker image without embedding models', value: 'without'},
+    {label: 'Build a Docker image including embedding models', value: 'including'}
+  ]}>
+  <TabItem value="without">
+
+This image is approximately 1 GB in size and relies on external LLM and embedding services.
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+pip3 install huggingface-hub nltk
+python3 download_deps.py
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+```
+
+  </TabItem>
+  <TabItem value="including">
+
+## Build a Docker image including embedding models
+
+This image is approximately 9 GB in size. As it includes embedding models, it relies on external LLM services only.  
+
+```bash
+git clone https://github.com/infiniflow/ragflow.git
+cd ragflow/
+pip3 install huggingface-hub nltk
+python3 download_deps.py
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
+```
+
+  </TabItem>
+</Tabs>

docs/guides/llm_api_key_setup.md

@@ -3,13 +3,13 @@ sidebar_position: 5
 slug: /llm_api_key_setup
 ---
 
-# Configure your API key
+# Configure model API key
 
-An API key is required for RAGFlow to interact with an online AI model. This guide provides information about setting your API key in RAGFlow.
+An API key is required for RAGFlow to interact with an online AI model. This guide provides information about setting your model API key in RAGFlow.
 
-## Get your API key
+## Get model API key
 
-For now, RAGFlow supports the following online LLMs. Click the corresponding link to apply for your API key. Most LLM providers grant newly-created accounts trial credit, which will expire in a couple of months, or a promotional amount of free quota.
+For now, RAGFlow supports the following online LLMs. Click the corresponding link to apply for your model API key. Most LLM providers grant newly-created accounts trial credit, which will expire in a couple of months, or a promotional amount of free quota.
 
 - [OpenAI](https://platform.openai.com/login?launch)
 - [Azure-OpenAI](https://ai.azure.com/)
@@ -29,17 +29,17 @@ For now, RAGFlow supports the following online LLMs. Click the corresponding lin
 - [StepFun](https://platform.stepfun.com/)
 
 :::note
-If you find your online LLM is not on the list, don't feel disheartened. The list is expanding, and you can [file a feature request](https://github.com/infiniflow/ragflow/issues/new?assignees=&labels=feature+request&projects=&template=feature_request.yml&title=%5BFeature+Request%5D%3A+) with us! Alternatively, if you have customized or locally-deployed models, you can [bind them to RAGFlow using Ollama, Xinferenc, or LocalAI](./deploy_local_llm.mdx).
+If you find your online LLM is not on the list, don't feel disheartened. The list is expanding, and you can [file a feature request](https://github.com/infiniflow/ragflow/issues/new?assignees=&labels=feature+request&projects=&template=feature_request.yml&title=%5BFeature+Request%5D%3A+) with us! Alternatively, if you have customized or locally-deployed models, you can [bind them to RAGFlow using Ollama, Xinference, or LocalAI](./deploy_local_llm.mdx).
 :::
 
-## Configure your API key
+## Configure model API key
 
-You have two options for configuring your API key:
+You have two options for configuring your model API key:
 
 - Configure it in **service_conf.yaml** before starting RAGFlow.
 - Configure it on the **Model Providers** page after logging into RAGFlow.
 
-### Configure API key before starting up RAGFlow
+### Configure model API key before starting up RAGFlow
 
 1. Navigate to **./docker/ragflow**.
 2. Find entry **user_default_llm**:
@@ -51,10 +51,10 @@ You have two options for configuring your API key:
    
    *After logging into RAGFlow, you will find your chosen model appears under **Added models** on the **Model Providers** page.*
 
-### Configure API key after logging into RAGFlow
+### Configure model API key after logging into RAGFlow
 
 :::caution WARNING
-After logging into RAGFlow, configuring API key through the **service_conf.yaml** file will no longer take effect.
+After logging into RAGFlow, configuring your model API key through the **service_conf.yaml** file will no longer take effect.
 :::
 
 After logging into RAGFlow, you can *only* configure API Key on the **Model Providers** page:
@@ -62,11 +62,11 @@ After logging into RAGFlow, you can *only* configure API Key on the **Model Prov
 1. Click on your logo on the top right of the page **>** **Model Providers**.
 2. Find your model card under **Models to be added** and click **Add the model**:
    ![add model](https://github.com/infiniflow/ragflow/assets/93570324/07e43f63-367c-4c9c-8ed3-8a3a24703f4e)
-3. Paste your API key.
+3. Paste your model API key.
 4. Fill in your base URL if you use a proxy to connect to the remote service.
 5. Click **OK** to confirm your changes.
 
 :::note
-If you wish to update an existing API key at a later point:
+To update an existing model API key at a later point:
 ![update api key](https://github.com/infiniflow/ragflow/assets/93570324/0bfba679-33f7-4f6b-9ed6-f0e6e4b228ad)
 :::

docs/guides/manage_files.md

@@ -49,7 +49,7 @@ You can link your file to one knowledge base or multiple knowledge bases at one
 
 ## Search files or folders
 
-As of RAGFlow v0.12.0, the search feature is still in a rudimentary form, supporting only file and folder search in the current directory by name (files or folders in the child directory will not be retrieved).
+As of RAGFlow v0.13.0, the search feature is still in a rudimentary form, supporting only file and folder search in the current directory by name (files or folders in the child directory will not be retrieved).
 
 ![search file](https://github.com/infiniflow/ragflow/assets/93570324/77ffc2e5-bd80-4ed1-841f-068e664efffe)
 
@@ -81,4 +81,4 @@ RAGFlow's file management allows you to download an uploaded file:
 
 ![download_file](https://github.com/infiniflow/ragflow/assets/93570324/cf3b297f-7d9b-4522-bf5f-4f45743e4ed5)
 
-> As of RAGFlow v0.12.0, bulk download is not supported, nor can you download an entire folder. 
+> As of RAGFlow v0.13.0, bulk download is not supported, nor can you download an entire folder. 

docs/guides/upgrade_ragflow.md

@@ -0,0 +1,41 @@
+---
+sidebar_position: 7
+slug: /upgrade_ragflow
+---
+
+# Upgrade RAGFlow
+
+You can upgrade RAGFlow to dev version or the latest version:
+
+- A Dev version (Development version) is the latest, tested image.
+- The latest version is the most recent, officially published release.
+
+## Upgrade RAGFlow to the dev version
+
+1. Update **ragflow/docker/.env** as follows:
+
+   ```bash
+   RAGFLOW_IMAGE=infiniflow/ragflow:dev
+   ```
+
+2. Update RAGFlow image and restart RAGFlow:
+
+   ```bash
+   docker compose -f docker/docker-compose.yml pull
+   docker compose -f docker/docker-compose.yml up -d
+   ```
+
+## Upgrade RAGFlow to the latest version
+
+1. Update **ragflow/docker/.env** as follows:
+
+   ```bash
+   RAGFLOW_IMAGE=infiniflow/ragflow:latest
+   ```
+
+2. Update the RAGFlow image and restart RAGFlow:
+
+   ```bash
+   docker compose -f docker/docker-compose.yml pull
+   docker compose -f docker/docker-compose.yml up -d
+   ```

docs/quickstart.mdx

@@ -34,7 +34,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
 
    `vm.max_map_count`. This value sets the maximum number of memory map areas a process may have. Its default value is 65530. While most applications require fewer than a thousand maps, reducing this value can result in abmornal behaviors, and the system will throw out-of-memory errors when a process reaches the limitation. 
 
-   RAGFlow v0.12.0 uses Elasticsearch for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
+   RAGFlow v0.13.0 uses Elasticsearch for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
 
 <Tabs
   defaultValue="linux"
@@ -177,15 +177,20 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
 
 3. Build the pre-built Docker images and start up the server:
 
-   > Running the following commands automatically downloads the *dev* version RAGFlow Docker image. To download and run a specified Docker version, update `RAGFLOW_IMAGE` in **docker/.env** to the intended version, for example `RAGFLOW_IMAGE=infiniflow/ragflow:v0.12.0`, before running the following commands.
+   > The command below downloads the dev version Docker image for RAGFlow slim (`dev-slim`). Note that RAGFlow slim Docker images do not include embedding models or Python libraries and hence are approximately 1GB in size.
 
    ```bash
    $ cd ragflow/docker
-   $ chmod +x ./entrypoint.sh
-   $ docker compose up -d
+   $ docker compose -f docker-compose.yml up -d
    ```
 
-> The core image is about 9 GB in size and may take a while to load.
+   > - To download a RAGFlow slim Docker image of a specific version, update the `RAGFlow_IMAGE` variable in **docker/.env** to your desired version. For example, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0-slim`. After making this change, rerun the command above to initiate the download.
+   > - To download the dev version of RAGFlow Docker image *including* embedding models and Python libraries, update the `RAGFlow_IMAGE` variable in **docker/.env** to `RAGFLOW_IMAGE=infiniflow/ragflow:dev`. After making this change, rerun the command above to initiate the download.
+   > - To download a specific version of RAGFlow Docker image *including* embedding models and Python libraries, update the `RAGFlow_IMAGE` variable in **docker/.env** to your desired version. For example, `RAGFLOW_IMAGE=infiniflow/ragflow:v0.13.0`. After making this change, rerun the command above to initiate the download.
+
+:::tip NOTE
+A RAGFlow Docker image that includes embedding models and Python libraries is approximately 9GB in size and may take significantly longer time to load.
+:::
 
 4. Check the server status after having the server up and running:
 
@@ -346,13 +351,17 @@ Conversations in RAGFlow are based on a particular knowledge base or multiple kn
 
 4. Update **Model Setting**.
 
-5. RAGFlow also offers conversation APIs. Hover over your dialogue **>** **Chat Bot API** to integrate RAGFlow's chat capabilities into your applications:
-
-   ![chatbot api](https://github.com/infiniflow/ragflow/assets/93570324/fec23715-f9af-4ac2-81e5-942c5035c5e6)
-
-6. Now, let's start the show:
+5. Now, let's start the show:
 
    ![question1](https://github.com/infiniflow/ragflow/assets/93570324/bb72dd67-b35e-4b2a-87e9-4e4edbd6e677)
 
    ![question2](https://github.com/infiniflow/ragflow/assets/93570324/7cc585ae-88d0-4aa2-817d-0370b2ad7230)
 
+
+:::tip NOTE
+RAGFlow also offers HTTP and Python APIs for you to integrate RAGFlow's capabilities into your applications. Read the following documents for more information:
+
+- [Acquire a RAGFlow API key](./guides/develop/acquire_ragflow_api_key.md)
+- [HTTP API reference](./references/http_api_reference.md)
+- [Python API reference](./references/python_api_reference.md)
+:::

docs/references/api.md

@@ -1,553 +0,0 @@
----
-sidebar_position: 1
-slug: /api
----
-
-# API reference
-
-RAGFlow offers RESTful APIs for you to integrate its capabilities into third-party applications. 
-
-## Base URL
-```
-https://demo.ragflow.io/v1/
-```
-
-## Authorization
-
-All of RAGFlow's RESTful APIs use API key for authorization, so keep it safe and do not expose it to the front end. 
-Put your API key in the request header. 
-
-```buildoutcfg
-Authorization: Bearer {API_KEY}
-```
-
-:::note
-In the current design, the RESTful API key you get from RAGFlow does not expire.
-:::
-
-To get your Chat API key or Agent API key:
-
-For Chat API key:
-   1. In RAGFlow, click **Chat** tab in the middle top of the page.
-   2. Hover over the corresponding dialogue **>** **Chat Bot API** to show the chatbot API configuration page.
-   3. Click **API Key** **>** **Create new key** to create your API key.
-   4. Copy and keep your API key safe. 
-
-For Agent API key:
-   1. In RAGFlow, click **Agent** tab in the middle top of the page.
-   2. Click your agent **>** ** Chat Bot API** to show the chatbot API configuration page.
-   3. Click **API Key** **>** **Create new key** to create your API key.
-   4. Copy and keep your API key safe. 
-
-## Create conversation
-
-This method creates (news) a conversation for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| GET      | `/api/new_conversation`                                     |
-
-:::note
-You are *required* to save the `data.id` value returned in the response data, which is the session ID for all upcoming conversations.
-:::
-
-#### Request parameter
-
-| Name     |  Type  | Required |        Description                                          |
-|----------|--------|----------|-------------------------------------------------------------|
-| `user_id`| string |   Yes    | The unique identifier assigned to each user. `user_id` must be less than 32 characters and cannot be empty. The following character sets are supported: <br />- 26 lowercase English letters (a-z)<br />- 26 uppercase English letters (A-Z)<br />- 10 digits (0-9)<br />- "_", "-", "." |
-
-### Response 
-
-```json
-{
-    "data": {
-        "create_date": "Fri, 12 Apr 2024 17:26:21 GMT",
-        "create_time": 1712913981857,
-        "dialog_id": "4f0a2e4cb9af11ee9ba20aef05f5e94f",
-        "duration": 0.0,
-        "id": "b9b2e098f8ae11ee9f45fa163e197198",
-        "message": [
-            {
-                "content": "Hi, I'm your assistant, what can I do for you?",
-                "role": "assistant"
-            }
-        ],
-        "reference": [],
-        "tokens": 0,
-        "update_date": "Fri, 12 Apr 2024 17:26:21 GMT",
-        "update_time": 1712913981857,
-        "user_id": "<USER_ID_SET_BY_THE_CALLER>"
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-
-## Get conversation history
-
-This method retrieves the history of a specified conversation session. 
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| GET      | `/api/conversation/<id>`                                    |
-
-#### Request parameter
-
-| Name     |  Type  | Required |        Description                                          |
-|----------|--------|----------|-------------------------------------------------------------|
-| `id`     | string |   Yes    | The unique identifier assigned to a conversation session. `id` must be less than 32 characters and cannot be empty. The following character sets are supported: <br />- 26 lowercase English letters (a-z)<br />- 26 uppercase English letters (A-Z)<br />- 10 digits (0-9)<br />- "_", "-", "." |
-
-### Response 
-
-#### Response parameter
-
-- `message`: All conversations in the specified conversation session.
-    - `role`: `"user"` or `"assistant"`.
-    - `content`: The text content of user or assistant. The citations are in a format like `##0$$`. The number in the middle, 0 in this case, indicates which part in data.reference.chunks it refers to.
-    
-- `user_id`: This is set by the caller.
-- `reference`: Each reference corresponds to one of the assistant's answers in `data.message`.
-    - `chunks`
-        - `content_with_weight`: Content of the chunk.
-        - `doc_name`: Name of the *hit* document.
-        - `img_id`: The image ID of the chunk. It is an optional field only for PDF, PPTX, and images. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the image.
-        - `positions`: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
-        - `similarity`: The hybrid similarity.
-        - `term_similarity`: The keyword simimlarity.
-        - `vector_similarity`: The embedding similarity.
-    - `doc_aggs`:
-        - `doc_id`: ID of the *hit* document. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the document.
-        - `doc_name`: Name of the *hit* document.
-        - `count`: The number of *hit* chunks in this document.
-
-```json
-{
-    "data": {
-        "create_date": "Mon, 01 Apr 2024 09:28:42 GMT",
-        "create_time": 1711934922220,
-        "dialog_id": "df4a4916d7bd11eeaa650242ac180006",
-        "id": "2cae30fcefc711ee94140242ac180006",
-        "message": [
-            {
-                "content": "Hi! I'm your assistant, what can I do for you?",
-                "role": "assistant"
-            },
-            {
-                "content": "What's the vit score for GPT-4?",
-                "role": "user"
-            },
-            {
-                "content": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
-                "role": "assistant"
-            }
-        ],
-        "user_id": "<USER_ID_SET_BY_THE_CALLER>",
-        "reference": [
-            {
-                "chunks": [
-                    {
-                        "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
-                        "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
-                        "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
-                        "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
-                        "important_kwd": [],
-                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
-                        "positions": [
-                            [
-                                9.0,
-                                159.9383341471354,
-                                472.1773274739583,
-                                223.58013916015625,
-                                307.86692301432294
-                            ]
-                        ],
-                        "similarity": 0.7310340654129031,
-                        "term_similarity": 0.7671974387781668,
-                        "vector_similarity": 0.40556370512552886
-                    },
-                    {
-                        "chunk_id": "7e2345d440383b756670e1b0f43a7007",
-                        "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . ",
-                        "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores.",
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
-                        "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
-                        "important_kwd": [],
-                        "kb_id": "afab9fdad6e511eebdb20242ac180006",
-                        "positions": [
-                            [
-                                8.0,
-                                107.3,
-                                508.90000000000003,
-                                686.3,
-                                697.0
-                            ],
-                        ],
-                        "similarity": 0.6691508616357027,
-                        "term_similarity": 0.6999011754270821,
-                        "vector_similarity": 0.39239803751328806
-                    },
-                ],
-                "doc_aggs": [
-                    {
-                        "count": 8,
-                        "doc_id": "c790da40ea8911ee928e0242ac180005",
-                        "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf"
-                    }
-                ],
-                "total": 8
-            },
-        ],
-        "update_date": "Tue, 02 Apr 2024 09:07:49 GMT",
-        "update_time": 1712020069421
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-    
-## Get answer
-
-This method retrieves from RAGFlow Chat or RAGFlow Agent the answer to the user's latest question.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| POST     | `/api/completion`                                           |
-
-#### Request parameter
-
-|   Name           |  Type  | Required | Description   |
-|------------------|--------|----------|---------------|
-| `conversation_id`| string | Yes      | The ID of the conversation session. Call ['GET' /new_conversation](#create-conversation) to retrieve the ID.|
-| `messages`       |  json  | Yes      | The latest question in a JSON form, such as `[{"role": "user", "content": "How are you doing!"}]`|
-| `quote`          |  bool  |  No      | Default: false|
-| `stream`         |  bool  |  No      | Default: true |
-| `doc_ids`        | string |  No      | Document IDs delimited by comma, like `c790da40ea8911ee928e0242ac180005,23dsf34ree928e0242ac180005`. The retrieved contents will be confined to these documents. |
-
-### Response 
-
-- `answer`: The answer to the user's latest question.
-- `reference`: 
-    - `chunks`: The retrieved chunks that contribute to the answer.  
-        - `content_with_weight`: Content of the chunk.
-        - `doc_name`: Name of the *hit* document.
-        - `img_id`: The image ID of the chunk. It is an optional field only for PDF, PPTX, and images. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the image.
-        - `positions`: [page_number, [upleft corner(x, y)], [right bottom(x, y)]], the chunk position, only for PDF.
-        - `similarity`: The hybrid similarity.
-        - `term_similarity`: The keyword simimlarity.
-        - `vector_similarity`: The embedding similarity.
-    - `doc_aggs`:
-        - `doc_id`: ID of the *hit* document. Call ['GET' /document/get/\<id\>](#get-document-content) to retrieve the document.
-        - `doc_name`: Name of the *hit* document. 
-        - `count`: The number of *hit* chunks in this document.
-
-```json
-{
-    "data": {
-      "answer": "The ViT Score for GPT-4 in the zero-shot scenario is 0.5058, and in the few-shot scenario, it is 0.6480. ##0$$",
-      "reference": {
-        "chunks": [
-          {
-            "chunk_id": "d0bc7892c3ec4aeac071544fd56730a8",
-            "content_ltks": "tabl 1:openagi task-solv perform under differ set for three closed-sourc llm . boldfac denot the highest score under each learn schema . metric gpt-3.5-turbo claude-2 gpt-4 zero few zero few zero few clip score 0.0 0.0 0.0 0.2543 0.0 0.3055 bert score 0.1914 0.3820 0.2111 0.5038 0.2076 0.6307 vit score 0.2437 0.7497 0.4082 0.5416 0.5058 0.6480 overal 0.1450 0.3772 0.2064 0.4332 0.2378 0.5281",
-            "content_with_weight": "<table><caption>Table 1: OpenAGI task-solving performances under different settings for three closed-source LLMs. Boldface denotes the highest score under each learning schema.</caption>\n<tr><th  rowspan=2 >Metrics</th><th  >GPT-3.5-turbo</th><th></th><th  >Claude-2</th><th  >GPT-4</th></tr>\n<tr><th  >Zero</th><th  >Few</th><th  >Zero Few</th><th  >Zero Few</th></tr>\n<tr><td  >CLIP Score</td><td  >0.0</td><td  >0.0</td><td  >0.0 0.2543</td><td  >0.0 0.3055</td></tr>\n<tr><td  >BERT Score</td><td  >0.1914</td><td  >0.3820</td><td  >0.2111 0.5038</td><td  >0.2076 0.6307</td></tr>\n<tr><td  >ViT Score</td><td  >0.2437</td><td  >0.7497</td><td  >0.4082 0.5416</td><td  >0.5058 0.6480</td></tr>\n<tr><td  >Overall</td><td  >0.1450</td><td  >0.3772</td><td  >0.2064 0.4332</td><td  >0.2378 0.5281</td></tr>\n</table>",
-            "doc_id": "c790da40ea8911ee928e0242ac180005",
-            "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
-            "img_id": "afab9fdad6e511eebdb20242ac180006-d0bc7892c3ec4aeac071544fd56730a8",
-            "important_kwd": [],
-            "kb_id": "afab9fdad6e511eebdb20242ac180006",
-            "positions": [
-              [
-                9.0,
-                159.9383341471354,
-                472.1773274739583,
-                223.58013916015625,
-                307.86692301432294
-              ]
-            ],
-            "similarity": 0.7310340654129031,
-            "term_similarity": 0.7671974387781668,
-            "vector_similarity": 0.40556370512552886
-          },
-          {
-            "chunk_id": "7e2345d440383b756670e1b0f43a7007",
-            "content_ltks": "5.5 experiment analysi the main experiment result are tabul in tab . 1 and 2 , showcas the result for closed-sourc and open-sourc llm , respect . the overal perform is calcul a the averag of cllp 8 bert and vit score . here , onli the task descript of the benchmark task are fed into llm(addit inform , such a the input prompt and llm\u2019output , is provid in fig . a.4 and a.5 in supplementari). broadli speak , closed-sourc llm demonstr superior perform on openagi task , with gpt-4 lead the pack under both zero-and few-shot scenario . in the open-sourc categori , llama-2-13b take the lead , consist post top result across variou learn schema--the perform possibl influenc by it larger model size . notabl , open-sourc llm significantli benefit from the tune method , particularli fine-tun and\u2019rltf . these method mark notic enhanc for flan-t5-larg , vicuna-7b , and llama-2-13b when compar with zero-shot and few-shot learn schema . in fact , each of these open-sourc model hit it pinnacl under the rltf approach . conclus , with rltf tune , the perform of llama-2-13b approach that of gpt-3.5 , illustr it potenti .",
-            "content_with_weight": "5.5 Experimental Analysis\nThe main experimental results are tabulated in Tab. 1 and 2, showcasing the results for closed-source and open-source LLMs, respectively. The overall performance is calculated as the average of CLlP\n8\nBERT and ViT scores. Here, only the task descriptions of the benchmark tasks are fed into LLMs (additional information, such as the input prompt and LLMs\u2019 outputs, is provided in Fig. A.4 and A.5 in supplementary). Broadly speaking, closed-source LLMs demonstrate superior performance on OpenAGI tasks, with GPT-4 leading the pack under both zero- and few-shot scenarios. In the open-source category, LLaMA-2-13B takes the lead, consistently posting top results across various learning schema--the performance possibly influenced by its larger model size. Notably, open-source LLMs significantly benefit from the tuning methods, particularly Fine-tuning and\u2019 RLTF. These methods mark noticeable enhancements for Flan-T5-Large, Vicuna-7B, and LLaMA-2-13B when compared with zero-shot and few-shot learning schema. In fact, each of these open-source models hits its pinnacle under the RLTF approach. Conclusively, with RLTF tuning, the performance of LLaMA-2-13B approaches that of GPT-3.5, illustrating its potential.",
-            "doc_id": "c790da40ea8911ee928e0242ac180005",
-            "doc_name": "OpenAGI When LLM Meets Domain Experts.pdf",
-            "img_id": "afab9fdad6e511eebdb20242ac180006-7e2345d440383b756670e1b0f43a7007",
-            "important_kwd": [],
-            "kb_id": "afab9fdad6e511eebdb20242ac180006",
-            "positions": [
-              [
-                8.0,
-                107.3,
-                508.90000000000003,
-                686.3,
-                697.0
-              ]
-            ],
-            "similarity": 0.6691508616357027,
-            "term_similarity": 0.6999011754270821,
-            "vector_similarity": 0.39239803751328806
-          }
-        ],
-        "doc_aggs": {
-          "OpenAGI When LLM Meets Domain Experts.pdf": 4
-        },
-        "total": 8
-      }
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```  
-
-## Get document content
-
-This method retrieves the content of a document.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| GET      | `/document/get/<id>`                                        |
-
-### Response
-
-A binary file. 
-
-## Upload file
-
-This method uploads a specific file to a specified knowledge base.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| POST     | `/api/document/upload`                                      |
-
-#### Response parameter
-
-|   Name      | Type   | Required | Description                                             |
-|-------------|--------|----------|---------------------------------------------------------|
-| `file`      | file   | Yes      | The file to upload.                                     |
-| `kb_name`   | string | Yes      | The name of the knowledge base to upload the file to.   |
-| `parser_id` | string |  No      | The parsing method (chunk template) to use. <br />- "naive": General;<br />- "qa": Q&A;<br />- "manual": Manual;<br />- "table": Table;<br />- "paper": Paper;<br />- "laws": Laws;<br />- "presentation": Presentation;<br />- "picture": Picture;<br />- "one": One. |
-| `run`       | string |  No      | 1: Automatically start file parsing. If `parser_id` is not set, RAGFlow uses the general template by default. |
-
-
-### Response 
-
-```json
-{
-    "data": {
-        "chunk_num": 0,
-        "create_date": "Thu, 25 Apr 2024 14:30:06 GMT",
-        "create_time": 1714026606921,
-        "created_by": "553ec818fd5711ee8ea63043d7ed348e",
-        "id": "41e9324602cd11ef9f5f3043d7ed348e",
-        "kb_id": "06802686c0a311ee85d6246e9694c130",
-        "location": "readme.txt",
-        "name": "readme.txt",
-        "parser_config": {
-            "field_map": {
-            },
-            "pages": [
-                [
-                    0,
-                    1000000
-                ]
-            ]
-        },
-        "parser_id": "general",
-        "process_begin_at": null,
-        "process_duation": 0.0,
-        "progress": 0.0,
-        "progress_msg": "",
-        "run": "0",
-        "size": 929,
-        "source_type": "local",
-        "status": "1",
-        "thumbnail": null,
-        "token_num": 0,
-        "type": "doc",
-        "update_date": "Thu, 25 Apr 2024 14:30:06 GMT",
-        "update_time": 1714026606921
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-### Demo for Upload File(Python) 
-
-```python
-# upload_to_kb.py  
-import requests
-
-
-def upload_file_to_kb(file_path, kb_name, token='ragflow-xxxxxxxxxxxxx', parser_id='naive'):
-    """  
-    Uploads a file to a knowledge base.  
-
-    Args:  
-    - file_path: Path to the file to upload.  
-    - kb_name: Name of the target knowledge base.  
-    - parser_id: ID of the chosen file parser (defaults to 'naive').  
-    - token: API token for authentication.  
-    """
-    url = 'http://127.0.0.1/v1/api/document/upload'  # Replace with your actual API URL  
-    files = {'file': open(file_path, 'rb')}  # The file to upload  
-    data = {'kb_name': kb_name, 'parser_id': parser_id, 'run': '1'}  # Additional form data  
-    headers = {'Authorization': f'Bearer {token}'}  # Replace with your actual Bearer token  
-
-    response = requests.post(url, files=files, data=data, headers=headers)
-
-    if response.status_code == 200:
-        print("File uploaded successfully:", response.json())
-    else:
-        print("Failed to upload file:", response.status_code, response.text)
-
-file_to_upload = './ai_intro.pdf'  # For example: './documents/report.pdf'  
-knowledge_base_name = 'AI_knowledge_base'
-# Assume you have already obtained your token and set it here  
-token = 'ragflow-xxxxxxxxxxxxx'
-
-# Call the function to upload the file  
-upload_file_to_kb(file_to_upload, knowledge_base_name, token=token)
-```
-## Get document chunks
-
-This method retrieves the chunks of a specific document by `doc_name` or `doc_id`.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| GET      | `/api/list_chunks`                                          |
-
-#### Request parameter
-
-|   Name     | Type   | Required |                        Description                                                          |
-|------------|--------|----------|---------------------------------------------------------------------------------------------|
-| `doc_name` | string |  No      | The name of the document in the knowledge base. It must not be empty if `doc_id` is not set.|
-| `doc_id`   | string |  No      | The ID of the document in the knowledge base. It must not be empty if `doc_name` is not set.|
-
-
-### Response
-
-```json
-{
-    "data": [
-        {
-            "content": "Figure 14: Per-request neural-net processingof RL-Cache.\n103\n(sn)\nCPU\n 102\nGPU\n8101\n100\n8\n16 64 256 1K\n4K",
-            "doc_name": "RL-Cache.pdf",
-            "img_id": "0335167613f011ef91240242ac120006-b46c3524952f82dbe061ce9b123f2211"
-        },
-        {
-            "content": "4.3 ProcessingOverheadof RL-CacheACKNOWLEDGMENTSThis section evaluates how effectively our RL-Cache implemen-tation leverages modern multi-core CPUs and GPUs to keep the per-request neural-net processing overhead low. Figure 14 depictsThis researchwas supported inpart by the Regional Government of Madrid (grant P2018/TCS-4499, EdgeData-CM)andU.S. National Science Foundation (grants CNS-1763617 andCNS-1717179).REFERENCES",
-            "doc_name": "RL-Cache.pdf",
-            "img_id": "0335167613f011ef91240242ac120006-d4c12c43938eb55d2d8278eea0d7e6d7"
-        }
-    ],
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-
-## Get document list
-
-This method retrieves a list of documents from a specified knowledge base.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| POST     | `/api/list_kb_docs`                                         |
-
-#### Request parameter
-
-| Name        | Type   | Required |  Description                                                          |
-|-------------|--------|----------|-----------------------------------------------------------------------|
-| `kb_name`   | string | Yes      | The name of the knowledge base, from which you get the document list. |
-| `page`      | int    |  No      | The number of pages, default:1.                                       |
-| `page_size` | int    |  No      | The number of docs for each page, default:15.                         |
-| `orderby`   | string |  No      | `chunk_num`, `create_time`, or `size`, default:`create_time`          |
-| `desc`      | bool   |  No      | Default:True.                                                         |
-| `keywords`  | string |  No      | Keyword of the document name.                                         |
-
-
-### Response 
-
-```json
-{
-    "data": {
-        "docs": [
-            {
-                "doc_id": "bad89a84168c11ef9ce40242ac120006",
-                "doc_name": "test.xlsx"
-            },
-            {
-                "doc_id": "641a9b4013f111efb53f0242ac120006",
-                "doc_name": "1111.pdf"
-            }
-        ],
-        "total": 2
-    },
-    "retcode": 0,
-    "retmsg": "success"
-}
-```
-
-## Delete documents 
-
-This method deletes documents by document ID or name.
-
-### Request
-
-#### Request URI
-
-| Method   |        Request URI                                          |
-|----------|-------------------------------------------------------------|
-| DELETE   | `/api/document`                                             |
-
-#### Request parameter
-
-| Name        | Type   | Required | Description                |
-|-------------|--------|----------|----------------------------|
-| `doc_names` | List   |  No      | A list of document names. It must not be empty if `doc_ids` is not set.  |
-| `doc_ids`   | List   |  No      | A list of document IDs. It must not be empty if `doc_names` is not set.  |
-
-
-### Response
-
-```json
-{
-    "data": true,
-    "retcode": 0,
-    "retmsg": "success"
-}
-```

docs/references/faq.md

@@ -49,7 +49,7 @@ Currently, we only support x86 CPU and Nvidia GPU.
 
 ### 2. Do you offer an API for integration with third-party applications?
 
-The corresponding APIs are now available. See the [RAGFlow API Reference](./api.md) for more information.
+The corresponding APIs are now available. See the [RAGFlow HTTP API Reference](./http_api_reference.md) or the [RAGFlow Python API Reference](./python_api_reference.md) for more information.
 
 ### 3. Do you support stream output?
 
@@ -75,7 +75,6 @@ $ git clone https://github.com/infiniflow/ragflow.git
 $ cd ragflow
 $ docker build -t infiniflow/ragflow:latest .
 $ cd ragflow/docker
-$ chmod +x ./entrypoint.sh
 $ docker compose up -d
 ```
 
@@ -398,34 +397,5 @@ This error occurs because there are too many chunks matching your search criteri
 ![topn](https://github.com/infiniflow/ragflow/assets/93570324/7ec72ab3-0dd2-4cff-af44-e2663b67b2fc)
 
 ### 9. How to upgrade RAGFlow?
-   
-You can upgrade RAGFlow to either the dev version or the latest version:
 
-- Dev versions are for developers and contributors. They are published on a nightly basis and may crash because they are not fully tested. We cannot guarantee their validity and you are at your own risk trying out latest, untested features.
-- The latest version refers to the most recent, officially published release. It is stable and works best with regular users.
-
-
-To upgrade RAGFlow to the dev version:
-
-Update the RAGFlow image and restart RAGFlow:
-1. Update **ragflow/docker/.env** as follows:
-   ```bash
-   RAGFLOW_IMAGE=infiniflow/ragflow:dev
-   ```
-2. Update ragflow image and restart ragflow:
-   ```bash
-   docker compose -f docker/docker-compose.yml pull
-   docker compose -f docker/docker-compose.yml up -d
-   ```
-   
-To upgrade RAGFlow to the latest version:
-
-1. Update **ragflow/docker/.env** as follows:
-   ```bash
-   RAGFLOW_IMAGE=infiniflow/ragflow:latest
-   ```
-2. Update the RAGFlow image and restart RAGFlow:
-   ```bash
-   docker compose -f docker/docker-compose.yml pull
-   docker compose -f docker/docker-compose.yml up -d
-   ```   
+See [Upgrade RAGFlow](../guides/upgrade_ragflow.md) for more information.

docs/references/ragflow_api.md

@@ -1,881 +0,0 @@
----
-sidebar_class_name: hidden
----
-
-# API reference
-
-RAGFlow offers RESTful APIs for you to integrate its capabilities into third-party applications. 
-
-## Base URL
-```
-http://<host_address>/v1/api/
-```
-
-## Dataset URL
-```
-http://<host_address>/api/v1/dataset
-```
-
-## Authorization
-
-All of RAGFlow's RESTFul APIs use API key for authorization, so keep it safe and do not expose it to the front end. 
-Put your API key in the request header. 
-
-```buildoutcfg
-Authorization: Bearer {API_KEY}
-```
-
-To get your API key:
-
-1. In RAGFlow, click **Chat** tab in the middle top of the page.
-2. Hover over the corresponding dialogue **>** **Chat Bot API** to show the chatbot API configuration page.
-3. Click **API Key** **>** **Create new key** to create your API key.
-4. Copy and keep your API key safe. 
-
-## Create dataset
-
-This method creates (news) a dataset for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI |
-|--------|-------------|
-| POST   | `/dataset`  |
-
-:::note
-You are *required* to save the `data.dataset_id` value returned in the response data, which is the session ID for all upcoming conversations.
-:::
-
-#### Request parameter
-
-| Name           |  Type  | Required | Description                                                                                                                                                                                                                                                                                                         |
-|----------------|--------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `dataset_name` | string |   Yes    | The unique identifier assigned to each newly created dataset. `dataset_name` must be less than 2 ** 10 characters and cannot be empty. The following character sets are supported: <br />- 26 lowercase English letters (a-z)<br />- 26 uppercase English letters (A-Z)<br />- 10 digits (0-9)<br />- "_", "-", "." |
-
-### Response 
-
-```json
-{
-  "code": 0, 
-  "data": {
-    "dataset_name": "kb1",
-    "dataset_id": "375e8ada2d3c11ef98f93043d7ee537e"
-  }, 
-  "message": "success"
-}
-```
-
-## Get dataset list
-
-This method lists the created datasets for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method   | Request URI |
-|----------|-------------|
-| GET      | `/dataset`  |
-
-### Response 
-
-#### Response parameter
-
-```json
-{
-    "code": 0,
-    "data": [
-        {
-            "avatar": null,
-            "chunk_num": 0,
-            "create_date": "Mon, 17 Jun 2024 16:00:05 GMT",
-            "create_time": 1718611205876,
-            "created_by": "b48110a0286411ef994a3043d7ee537e",
-            "description": null,
-            "doc_num": 0,
-            "embd_id": "BAAI/bge-large-zh-v1.5",
-            "id": "9bd6424a2c7f11ef81b83043d7ee537e",
-            "language": "Chinese",
-            "name": "dataset3(23)",
-            "parser_config": {
-                "pages": [
-                    [
-                        1,
-                        1000000
-                    ]
-                ]
-            },
-            "parser_id": "naive",
-            "permission": "me",
-            "similarity_threshold": 0.2,
-            "status": "1",
-            "tenant_id": "b48110a0286411ef994a3043d7ee537e",
-            "token_num": 0,
-            "update_date": "Mon, 17 Jun 2024 16:00:05 GMT",
-            "update_time": 1718611205876,
-            "vector_similarity_weight": 0.3
-        }
-    ],
-    "message": "List datasets successfully!"
-}
-```
-
-## Delete dataset
-
-This method deletes a dataset for a specific user.
-
-### Request
-
-#### Request URI
-
-| Method | Request URI             |
-|--------|-------------------------|
-| DELETE | `/dataset/{dataset_id}` |
-
-#### Request parameter
-
-| Name         |  Type  | Required | Description                                                                                                                                                      |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `dataset_id` | string | Yes      | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.                                                                                |
-
-### Response 
-
-```json
-{
-  "code": 0,
-  "message": "Remove dataset: 9cefaefc2e2611ef916b3043d7ee537e successfully"
-}
-```  
-
-### Get the details of the specific dataset
-
-This method gets the details of the specific dataset. 
-
-### Request
-
-#### Request URI
-
-| Method   | Request URI             |
-|----------|-------------------------|
-| GET      | `/dataset/{dataset_id}` |
-
-#### Request parameter
-
-| Name         |  Type  | Required | Description                                                                                                                                                      |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `dataset_id` | string | Yes      | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.                                                                                |
-
-### Response 
-
-```json
-{
-    "code": 0,
-    "data": {
-        "avatar": null,
-        "chunk_num": 0, 
-        "description": null,
-        "doc_num": 0,
-        "embd_id": "BAAI/bge-large-zh-v1.5",
-        "id": "060323022e3511efa8263043d7ee537e", 
-        "language": "Chinese", 
-        "name": "test(1)", 
-        "parser_config": 
-        {
-            "pages": [[1, 1000000]]
-        }, 
-        "parser_id": "naive", 
-        "permission": "me", 
-        "token_num": 0
-  }, 
-    "message": "success"
-}
-```
-
-### Update the details of the specific dataset
-
-This method updates the details of the specific dataset. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI             |
-|--------|-------------------------|
-| PUT    | `/dataset/{dataset_id}` |
-
-#### Request parameter
-
-You are required to input at least one parameter.
-
-| Name                 | Type   | Required | Description                                                           |
-|----------------------|--------|----------|-----------------------------------------------------------------------|
-| `name`               | string | No       | The name of the knowledge base, from which you get the document list. |
-| `description`        | string | No       | The description of the knowledge base.                                |
-| `permission`         | string | No       | The permission for the knowledge base, default:me.                    |
-| `language`           | string | No       | The language of the knowledge base.                                   |
-| `chunk_method`       | string | No       | The chunk method of the knowledge base.                               |
-| `embedding_model_id` | string | No       | The embedding model id of the knowledge base.                         |
-| `photo`              | string | No       | The photo of the knowledge base.                                      |
-| `layout_recognize`   | bool   | No       | The layout recognize of the knowledge base.                           |
-| `token_num`          | int    | No       | The token number of the knowledge base.                               |
-| `id`                 | string | No       | The id of the knowledge base.                                         |
-
-### Response 
-
-### Successful response
-
-```json
-{
-    "code": 0,
-    "data": {
-        "avatar": null,
-        "chunk_num": 0,
-        "create_date": "Wed, 19 Jun 2024 20:33:34 GMT",
-        "create_time": 1718800414518, 
-        "created_by": "b48110a0286411ef994a3043d7ee537e", 
-        "description": "new_description1", 
-        "doc_num": 0, 
-        "embd_id": "BAAI/bge-large-zh-v1.5", 
-        "id": "24f9f17a2e3811ef820e3043d7ee537e", 
-        "language": "English", 
-        "name": "new_name", 
-        "parser_config": 
-        {
-            "pages": [[1, 1000000]]
-        },
-        "parser_id": "naive", 
-        "permission": "me", 
-        "similarity_threshold": 0.2, 
-        "status": "1", 
-        "tenant_id": "b48110a0286411ef994a3043d7ee537e", 
-        "token_num": 0, 
-        "update_date": "Wed, 19 Jun 2024 20:33:34 GMT", 
-        "update_time": 1718800414529, 
-        "vector_similarity_weight": 0.3
-  }, 
-    "message": "success"
-}
-```
-
-### Response for the operating error
-
-```json
-{
-    "code": 103, 
-    "message": "Only the owner of knowledgebase is authorized for this operation!"
-}
-```
-
-### Response for no parameter
-```json
-{ 
-    "code": 102, 
-    "message": "Please input at least one parameter that you want to update!"
-}
-```
-
-------------------------------------------------------------------------------------------------------------------------------
-
-## Upload documents
-
-This method uploads documents for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                       |
-|--------|-----------------------------------|
-| POST   | `/dataset/{dataset_id}/documents` |
-
-
-#### Request parameter
-
-| Name         |  Type  | Required |        Description                                         |
-|--------------|--------|----------|------------------------------------------------------------|
-| `dataset_id` | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID. |
-
-### Response 
-
-### Successful response
-
-```json
-{
-      "code": 0,
-      "data": [
-        {
-          "created_by": "b48110a0286411ef994a3043d7ee537e",
-          "id": "859584a0379211efb1a23043d7ee537e",
-          "kb_id": "8591349a379211ef92213043d7ee537e",
-          "location": "test.txt",
-          "name": "test.txt",
-          "parser_config": {
-            "pages": [
-              [1, 1000000]
-            ]
-          },
-          "parser_id": "naive",
-          "size": 0,
-          "thumbnail": null,
-          "type": "doc"
-        },
-        {
-          "created_by": "b48110a0286411ef994a3043d7ee537e",
-          "id": "8596f18c379211efb1a23043d7ee537e",
-          "kb_id": "8591349a379211ef92213043d7ee537e",
-          "location": "test1.txt",
-          "name": "test1.txt",
-          "parser_config": {
-            "pages": [
-              [1, 1000000]
-            ]
-          },
-          "parser_id": "naive",
-          "size": 0,
-          "thumbnail": null,
-          "type": "doc"
-        }
-      ],
-      "message": "success"
-}
-```
-
-### Response for nonexistent files
-
-```json
-{
-      "code": "RetCode.DATA_ERROR",
-      "message": "The file test_data/imagination.txt does not exist"
-}
-```
-
-### Response for nonexistent dataset
-
-```json
-{
-      "code": 102,
-      "message": "Can't find this dataset"
-}
-```
-
-### Response for the number of files exceeding the limit
-
-```json
-{
-      "code": 102,
-      "message": "You try to upload 512 files, which exceeds the maximum number of uploading files: 256"
-}
-```
-### Response for uploading without files.
-
-```json
-{
-    "code": 101,
-    "message": "None is not string."
-}
-```
-
-## Delete documents
-
-This method deletes documents for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                       |
-|--------|-----------------------------------|
-| DELETE | `/dataset/{dataset_id}/documents/{document_id}` |
-
-
-#### Request parameter
-
-| Name          |  Type  | Required | Description                                                                         |
-|---------------|--------|----------|-------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.   |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID. |
-
-### Response 
-
-### Successful response
-
-```json
-{
-      "code": 0,
-      "data": true,
-      "message": "success"
-}
-```
-
-### Response for deleting a document that does not exist
-
-```json
-{
-      "code": 102,
-      "message": "Document 111 not found!"
-}
-```
-### Response for deleting documents from a non-existent dataset
-
-```json
-{
-      "code": 101,
-      "message": "The document f7aba1ec379b11ef8e853043d7ee537e is not in the dataset: 000, but in the dataset: f7a7ccf2379b11ef83223043d7ee537e."
-}
-```
-
-## List documents
-
-This method lists documents for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                       |
-|--------|-----------------------------------|
-| GET    | `/dataset/{dataset_id}/documents` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------|
-| `dataset_id` | string | Yes      | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.                          |
-| `offset`     | int    | No       | The start of the listed documents. Default: 0                                                              |
-| `count`      | int    | No       | The total count of the listed documents. Default: -1, meaning all the later part of documents from the start. |
-| `order_by`   | string | No       | Default: `create_time`                                                                                     |
-| `descend`    | bool   | No       | The order of listing documents. Default: True                                                              |
-| `keywords`   | string | No       | The searching keywords of listing documents. Default: ""                                                   |
-
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": 0,
-      "data": {
-        "docs": [
-          {
-            "chunk_num": 0,
-            "create_date": "Mon, 01 Jul 2024 19:24:10 GMT",
-            "create_time": 1719833050046,
-            "created_by": "b48110a0286411ef994a3043d7ee537e",
-            "id": "6fb6f588379c11ef87023043d7ee537e",
-            "kb_id": "6fb1c9e6379c11efa3523043d7ee537e",
-            "location": "empty.txt",
-            "name": "empty.txt",
-            "parser_config": {
-              "pages": [
-                [1, 1000000]
-              ]
-            },
-            "parser_id": "naive",
-            "process_begin_at": null,
-            "process_duation": 0.0,
-            "progress": 0.0,
-            "progress_msg": "",
-            "run": "0",
-            "size": 0,
-            "source_type": "local",
-            "status": "1",
-            "thumbnail": null,
-            "token_num": 0,
-            "type": "doc",
-            "update_date": "Mon, 01 Jul 2024 19:24:10 GMT",
-            "update_time": 1719833050046
-          },
-          {
-            "chunk_num": 0,
-            "create_date": "Mon, 01 Jul 2024 19:24:10 GMT",
-            "create_time": 1719833050037,
-            "created_by": "b48110a0286411ef994a3043d7ee537e",
-            "id": "6fb59c60379c11ef87023043d7ee537e",
-            "kb_id": "6fb1c9e6379c11efa3523043d7ee537e",
-            "location": "test.txt",
-            "name": "test.txt",
-            "parser_config": {
-              "pages": [
-                [1, 1000000]
-              ]
-            },
-            "parser_id": "naive",
-            "process_begin_at": null,
-            "process_duation": 0.0,
-            "progress": 0.0,
-            "progress_msg": "",
-            "run": "0",
-            "size": 0,
-            "source_type": "local",
-            "status": "1",
-            "thumbnail": null,
-            "token_num": 0,
-            "type": "doc",
-            "update_date": "Mon, 01 Jul 2024 19:24:10 GMT",
-            "update_time": 1719833050037
-          }
-        ],
-        "total": 2
-      },
-      "message": "success"
-}
-```
-
-### Response for listing documents with IndexError
-
-```json
-{
-      "code": 100,
-      "message": "IndexError('Offset is out of the valid range.')"
-}
-```
-## Update the details of the document
-
-This method updates the details, including the name, enable and template type of a specific document for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                                     |
-|--------|-------------------------------------------------|
-| PUT    | `/dataset/{dataset_id}/documents/{document_id}` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.   |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID. |
-
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": 0,
-      "data": {
-        "chunk_num": 0,
-        "create_date": "Mon, 15 Jul 2024 16:55:03 GMT",
-        "create_time": 1721033703914,
-        "created_by": "b48110a0286411ef994a3043d7ee537e",
-        "id": "ed30167a428711efab193043d7ee537e",
-        "kb_id": "ed2d8770428711efaf583043d7ee537e",
-        "location": "test.txt",
-        "name": "new_name.txt",
-        "parser_config": {
-          "pages": [
-            [1, 1000000]
-          ]
-        },
-        "parser_id": "naive",
-        "process_begin_at": null,
-        "process_duration": 0.0,
-        "progress": 0.0,
-        "progress_msg": "",
-        "run": "0",
-        "size": 14,
-        "source_type": "local",
-        "status": "1",
-        "thumbnail": null,
-        "token_num": 0,
-        "type": "doc",
-        "update_date": "Mon, 15 Jul 2024 16:55:03 GMT",
-        "update_time": 1721033703934
-      },
-      "message": "Success"
-}
-```
-
-### Response for updating a document which does not exist.
-
-```json
-{
-      "code": 101,
-      "message": "This document weird_doc_id cannot be found!"
-}
-```
-
-### Response for updating a document without giving parameters.
-```json
-{
-      "code": 102,
-      "message": "Please input at least one parameter that you want to update!"
-}
-```
-
-### Response for updating a document in the nonexistent dataset.
-```json
-{
-      "code": 102,
-      "message": "This dataset fake_dataset_id cannot be found!"
-}
-```
-
-### Response for updating a document with an extension name that differs from its original.
-```json
-{
-      "code": 101,
-      "data": false,
-      "message": "The extension of file cannot be changed"
-}
-```
-
-### Response for updating a document with a duplicate name.
-```json
-{
-      "code": 101,
-      "message": "Duplicated document name in the same dataset."
-}
-```
-
-### Response for updating a document's illegal parameter.
-```json
-{
-      "code": 101,
-      "message": "illegal_parameter is an illegal parameter."
-}
-```
-
-### Response for updating a document's name without its name value.
-```json
-{
-      "code": 102,
-      "message": "There is no new name."
-}
-```
-
-### Response for updating a document's with giving illegal enable's value.
-```json
-{
-      "code": 102,
-      "message": "Illegal value '?' for 'enable' field."
-}
-```
-
-## Download the document
-
-This method downloads a specific document for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                                     |
-|--------|-------------------------------------------------|
-| GET    | `/dataset/{dataset_id}/documents/{document_id}` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.   |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID. |
-
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": "0",
-      "data": "b'test\\ntest\\ntest'"
-}
-```
-
-### Response for downloading a document which does not exist.
-
-```json
-{
-      "code": 101,
-      "message": "This document 'imagination.txt' cannot be found!"
-}
-```
-
-### Response for downloading a document in the nonexistent dataset.
-```json
-{
-      "code": 102,
-      "message": "This dataset 'imagination' cannot be found!"
-}
-```
-
-### Response for downloading an empty document.
-```json
-{
-      "code": 102,
-      "message": "This file is empty."
-}
-```
-
-## Start parsing a document
-
-This method enables a specific document to start parsing for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                                            |
-|--------|--------------------------------------------------------|
-| POST   | `/dataset/{dataset_id}/documents/{document_id}/status` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                |
-|--------------|--------|----------|------------------------------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.   |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID. |
-
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": 0,
-      "message": ""
-}
-```
-
-### Response for parsing a document which does not exist.
-
-```json
-{
-      "code": 101,
-      "message": "This document 'imagination.txt' cannot be found!"
-}
-```
-
-### Response for parsing a document in the nonexistent dataset.
-```json
-{
-      "code": 102,
-      "message": "This dataset 'imagination' cannot be found!"
-}
-```
-
-### Response for parsing an empty document.
-```json
-{
-      "code": 0,
-      "message": "Empty data in the document: empty.txt;"
-}
-```
-
-## Start parsing multiple documents
-
-This method enables multiple documents, including all documents in the specific dataset or specified documents, to start parsing for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                                           |
-|--------|-------------------------------------------------------|
-| POST   | `/dataset/{dataset_id}/documents/status` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                                       |
-|--------------|--------|----------|-----------------------------------------------------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.                                                 |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID.                                               |
-| `doc_ids` | list | No | The document IDs of the documents that the user would like to parse. Default: None, means all documents in the specified dataset. |
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": 0,
-      "data": true,
-      "message": ""
-}
-```
-
-### Response for parsing documents which does not exist.
-
-```json
-{
-      "code": 101,
-      "message": "This document 'imagination.txt' cannot be found!"
-}
-```
-
-### Response for parsing documents in the nonexistent dataset.
-```json
-{
-      "code": 102,
-      "message": "This dataset 'imagination' cannot be found!"
-}
-```
-
-### Response for parsing documents, one of which is empty.
-```json
-{
-      "code": 0,
-      "data": true,
-      "message": "Empty data in the document: empty.txt; "
-}
-```
-
-## Show the parsing status of the document
-
-This method shows the parsing status of the document for a specific user. 
-
-### Request
-
-#### Request URI
-
-| Method | Request URI                                           |
-|--------|-------------------------------------------------------|
-| GET    | `/dataset/{dataset_id}/documents/status` |
-
-
-#### Request parameter
-
-| Name         | Type   | Required | Description                                                                                                                       |
-|--------------|--------|----------|-----------------------------------------------------------------------------------------------------------------------------------|
-| `dataset_id`  | string |   Yes    | The ID of the dataset. Call ['GET' /dataset](#create-dataset) to retrieve the ID.                                                 |
-| `document_id` | string |   Yes    | The ID of the document. Call ['GET' /document](#list-documents) to retrieve the ID.                                               |
-
-### Response 
-
-### Successful Response 
-
-```json
-{
-      "code": 0,
-      "data": {
-            "progress": 0.0,
-            "status": "RUNNING"
-      },
-      "message": "success"
-}
-```
-
-### Response for showing the parsing status of a document which does not exist.
-
-```json
-{
-      "code": 102,
-      "message": "This document: 'imagination.txt' is not a valid document."
-}
-```
-
-### Response for showing the parsing status of a document in the nonexistent dataset.
-```json
-{
-      "code": 102,
-      "message": "This dataset 'imagination' cannot be found!"
-}
-```

download_deps.py

@@ -1,7 +1,13 @@
 #!/usr/bin/env python3
 
 from huggingface_hub import snapshot_download
+import nltk
 import os
+import urllib.request
+
+urls = [
+    "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
+]
 
 repos = [
     "InfiniFlow/text_concat_xgb_v1.0",
@@ -12,13 +18,24 @@ repos = [
     "maidalun1020/bce-reranker-base_v1",
 ]
 
-
 def download_model(repo_id):
-    local_dir = os.path.join("huggingface.co", repo_id)
+    local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
     os.makedirs(local_dir, exist_ok=True)
     snapshot_download(repo_id=repo_id, local_dir=local_dir)
 
 
 if __name__ == "__main__":
+    for url in urls:
+        filename = url.split("/")[-1]
+        print(f"Downloading {url}...")
+        if not os.path.exists(filename):
+            urllib.request.urlretrieve(url, filename)
+
+    local_dir = os.path.abspath('nltk_data')
+    for data in ['wordnet', 'punkt', 'punkt_tab']:
+        print(f"Downloading nltk {data}...")
+        nltk.download(data, download_dir=local_dir)
+
     for repo_id in repos:
+        print(f"Downloading huggingface repo {repo_id}...")
         download_model(repo_id)

graphrag/entity_resolution.py

@@ -13,7 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
+import itertools
 import logging
 import re
 import traceback
@@ -93,16 +93,8 @@ class EntityResolution:
             node_clusters[graph.nodes[node]['entity_type']].append(node)
 
         candidate_resolution = {entity_type: [] for entity_type in entity_types}
-        for node_cluster in node_clusters.items():
-            candidate_resolution_tmp = []
-            for a in node_cluster[1]:
-                for b in node_cluster[1]:
-                    if a == b:
-                        continue
-                    if self.is_similarity(a, b) and (b, a) not in candidate_resolution_tmp:
-                        candidate_resolution_tmp.append((a, b))
-            if candidate_resolution_tmp:
-                candidate_resolution[node_cluster[0]] = candidate_resolution_tmp
+        for k, v in node_clusters.items():
+            candidate_resolution[k] = [(a, b) for a, b in itertools.combinations(v, 2) if self.is_similarity(a, b)]
 
         gen_conf = {"temperature": 0.5}
         resolution_result = set()

graphrag/graph_extractor.py

@@ -164,6 +164,7 @@ class GraphExtractor:
         text = perform_variable_replacements(self._extraction_prompt, variables=variables)
         gen_conf = {"temperature": 0.3}
         response = self._llm.chat(text, [{"role": "user", "content": "Output:"}], gen_conf)
+        if response.find("**ERROR**") >= 0: raise Exception(response)
         token_count = num_tokens_from_string(text + response)
 
         results = response or ""

graphrag/index.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import os
 from concurrent.futures import ThreadPoolExecutor
 import json
 from functools import reduce
@@ -64,7 +65,8 @@ def build_knowledge_graph_chunks(tenant_id: str, chunks: List[str], callback, en
     texts, graphs = [], []
     cnt = 0
     threads = []
-    exe = ThreadPoolExecutor(max_workers=50)
+    max_workers = int(os.environ.get('GRAPH_EXTRACTOR_MAX_WORKERS', 50))
+    exe = ThreadPoolExecutor(max_workers=max_workers)
     for i in range(len(chunks)):
         tkn_cnt = num_tokens_from_string(chunks[i])
         if cnt+tkn_cnt >= left_token_count and texts:

graphrag/mind_map_extractor.py

@@ -16,6 +16,7 @@
 
 import collections
 import logging
+import os
 import re
 import logging
 import traceback
@@ -65,17 +66,20 @@ class MindMapExtractor:
         if isinstance(obj, str):
             obj = [obj]
         if isinstance(obj, list):
-            for i in obj: keyset.add(i)
-            return [{"id": re.sub(r"\*+", "", i), "children": []} for i in obj if re.sub(r"\*+", "", i)]
+            keyset.update(obj)
+            obj = [re.sub(r"\*+", "", i) for i in obj]
+            return [{"id": i, "children": []} for i in obj if i]
         arr = []
         for k, v in obj.items():
             k = self._key(k)
-            if not k or k in keyset: continue
-            keyset.add(k)
-            arr.append({
-                "id": k,
-                "children": self._be_children(v, keyset)
-            })
+            if k and k not in keyset:
+                keyset.add(k)
+                arr.append(
+                    {
+                        "id": k,
+                        "children": self._be_children(v, keyset)
+                    }
+                )
         return arr
 
     def __call__(
@@ -86,7 +90,8 @@ class MindMapExtractor:
             prompt_variables = {}
 
         try:
-            exe = ThreadPoolExecutor(max_workers=12)
+            max_workers = int(os.environ.get('MINDMAP_EXTRACTOR_MAX_WORKERS', 12))
+            exe = ThreadPoolExecutor(max_workers=max_workers)
             threads = []
             token_count = max(self._llm.max_length * 0.8, self._llm.max_length-512)
             texts = []
@@ -110,15 +115,22 @@ class MindMapExtractor:
                 return MindMapResult(output={"id": "root", "children": []})
 
             merge_json = reduce(self._merge, res)
-            if len(merge_json.keys()) > 1:
-                keyset = set(
-                    [re.sub(r"\*+", "", k) for k, v in merge_json.items() if isinstance(v, dict) and re.sub(r"\*+", "", k)])
-                merge_json = {"id": "root",
-                          "children": [{"id": self._key(k), "children": self._be_children(v, keyset)} for k, v in
-                                       merge_json.items() if isinstance(v, dict) and self._key(k)]}
+            if len(merge_json) > 1:
+                keys = [re.sub(r"\*+", "", k) for k, v in merge_json.items() if isinstance(v, dict)]
+                keyset = set(i for i in keys if i)
+                merge_json = {
+                    "id": "root",
+                    "children": [
+                        {
+                            "id": self._key(k),
+                            "children": self._be_children(v, keyset)
+                        }
+                        for k, v in merge_json.items() if isinstance(v, dict) and self._key(k)
+                    ]
+                }
             else:
                 k = self._key(list(merge_json.keys())[0])
-                merge_json = {"id": k, "children": self._be_children(list(merge_json.items())[0][1], set([k]))}
+                merge_json = {"id": k, "children": self._be_children(list(merge_json.items())[0][1], {k})}
 
         except Exception as e:
             logging.exception("error mind graph")

intergrations/chatgpt-on-wechat/plugins/README.md

@@ -0,0 +1,57 @@
+RAGFlow Chat Plugin for ChatGPT-on-WeChat
+=========================================
+
+This folder contains the source code for the `ragflow_chat` plugin, which extends the core functionality of the RAGFlow API to support conversational interactions using Retrieval-Augmented Generation (RAG). This plugin integrates seamlessly with the [ChatGPT-on-WeChat](https://github.com/zhayujie/chatgpt-on-wechat) project, enabling WeChat and other platforms to leverage the knowledge retrieval capabilities provided by RAGFlow in chat interactions.
+
+### Features
+* **Conversational Interactions**: Combine WeChat's conversational interface with powerful RAG (Retrieval-Augmented Generation) capabilities.
+* **Knowledge-Based Responses**: Enrich conversations by retrieving relevant data from external knowledge sources and incorporating them into chat responses.
+* **Multi-Platform Support**: Works across WeChat, WeCom, and various other platforms supported by the ChatGPT-on-WeChat framework.
+
+### Plugin vs. ChatGPT-on-WeChat Configurations
+**Note**: There are two distinct configuration files used in this setup—one for the ChatGPT-on-WeChat core project and another specific to the `ragflow_chat` plugin. It is important to configure both correctly to ensure smooth integration.
+
+#### ChatGPT-on-WeChat Root Configuration (`config.json`)
+This file is located in the root directory of the [ChatGPT-on-WeChat](https://github.com/zhayujie/chatgpt-on-wechat) project and is responsible for defining the communication channels and overall behavior. For example, it handles the configuration for WeChat, WeCom, and other services like Feishu and DingTalk.
+
+Example `config.json` (for WeChat channel):
+```json
+{
+  "channel_type": "wechatmp",
+  "wechatmp_app_id": "YOUR_APP_ID",
+  "wechatmp_app_secret": "YOUR_APP_SECRET",
+  "wechatmp_token": "YOUR_TOKEN",
+  "wechatmp_port": 80,
+  ...
+}
+```
+
+This file can also be modified to support other communication platforms, such as:
+- **Personal WeChat** (`channel_type: wx`)
+- **WeChat Public Account** (`wechatmp` or `wechatmp_service`)
+- **WeChat Work (WeCom)** (`wechatcom_app`)
+- **Feishu** (`feishu`)
+- **DingTalk** (`dingtalk`)
+
+For detailed configuration options, see the official [LinkAI documentation](https://docs.link-ai.tech/cow/multi-platform/wechat-mp).
+
+#### RAGFlow Chat Plugin Configuration (`plugins/ragflow_chat/config.json`)
+This configuration is specific to the `ragflow_chat` plugin and is used to set up communication with the RAGFlow server. Ensure that your RAGFlow server is running, and update the plugin's `config.json` file with your server details:
+
+Example `config.json` (for `ragflow_chat`):
+```json
+{
+  "ragflow_api_key": "YOUR_API_KEY",
+  "ragflow_host": "127.0.0.1:80"
+}
+```
+
+This file must be configured to point to your RAGFlow instance, with the `ragflow_api_key` and `ragflow_host` fields set appropriately. The `ragflow_host` is typically your server's address and port number, and the `ragflow_api_key` is obtained from your RAGFlow API setup.
+
+### Requirements
+Before you can use this plugin, ensure the following are in place:
+
+1. You have installed and configured [ChatGPT-on-WeChat](https://github.com/zhayujie/chatgpt-on-wechat).
+2. You have deployed and are running the [RAGFlow](https://github.com/infiniflow/ragflow) server.
+   
+Make sure both `config.json` files (ChatGPT-on-WeChat and RAGFlow Chat Plugin) are correctly set up as per the examples above.

intergrations/chatgpt-on-wechat/plugins/__init__.py

@@ -0,0 +1 @@
+from .ragflow_chat import *

intergrations/chatgpt-on-wechat/plugins/config.json

@@ -0,0 +1,4 @@
+{
+    "api_key": "ragflow-***",
+    "host_address": "127.0.0.1:80"
+}

intergrations/chatgpt-on-wechat/plugins/ragflow_chat.py

@@ -0,0 +1,114 @@
+import requests
+import json
+from bridge.context import Context, ContextType  # Import Context, ContextType
+from bridge.reply import Reply, ReplyType  # Import Reply, ReplyType
+from bridge import *
+from common.log import logger
+from config import conf
+from plugins import Plugin, register  # Import Plugin and register
+from plugins.event import Event, EventContext, EventAction  # Import event-related classes
+
+@register(name="RAGFlowChat", desc="Use RAGFlow API to chat", version="1.0", author="Your Name")
+class RAGFlowChat(Plugin):
+    def __init__(self):
+        super().__init__()
+        # Load plugin configuration
+        self.cfg = self.load_config()
+        # Bind event handling function
+        self.handlers[Event.ON_HANDLE_CONTEXT] = self.on_handle_context
+        # Store conversation_id for each user
+        self.conversations = {}
+        logger.info("[RAGFlowChat] Plugin initialized")
+
+    def on_handle_context(self, e_context: EventContext):
+        context = e_context['context']
+        if context.type != ContextType.TEXT:
+            return  # Only process text messages
+
+        user_input = context.content.strip()
+        session_id = context['session_id']
+
+        # Call RAGFlow API to get a reply
+        reply_text = self.get_ragflow_reply(user_input, session_id)
+        if reply_text:
+            reply = Reply()
+            reply.type = ReplyType.TEXT
+            reply.content = reply_text
+            e_context['reply'] = reply
+            e_context.action = EventAction.BREAK_PASS  # Skip the default processing logic
+        else:
+            # If no reply is received, pass to the next plugin or default logic
+            e_context.action = EventAction.CONTINUE
+
+    def get_ragflow_reply(self, user_input, session_id):
+        # Get API_KEY and host address from the configuration
+        api_key = self.cfg.get("api_key")
+        host_address = self.cfg.get("host_address")
+        user_id = session_id  # Use session_id as user_id
+
+        if not api_key or not host_address:
+            logger.error("[RAGFlowChat] Missing configuration")
+            return "The plugin configuration is incomplete. Please check the configuration."
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json"
+        }
+
+        # Step 1: Get or create conversation_id
+        conversation_id = self.conversations.get(user_id)
+        if not conversation_id:
+            # Create a new conversation
+            url_new_conversation = f"http://{host_address}/v1/api/new_conversation"
+            params_new_conversation = {
+                "user_id": user_id
+            }
+            try:
+                response = requests.get(url_new_conversation, headers=headers, params=params_new_conversation)
+                logger.debug(f"[RAGFlowChat] New conversation response: {response.text}")
+                if response.status_code == 200:
+                    data = response.json()
+                    if data.get("retcode") == 0:
+                        conversation_id = data["data"]["id"]
+                        self.conversations[user_id] = conversation_id
+                    else:
+                        logger.error(f"[RAGFlowChat] Failed to create conversation: {data.get('retmsg')}")
+                        return f"Sorry, unable to create a conversation: {data.get('retmsg')}"
+                else:
+                    logger.error(f"[RAGFlowChat] HTTP error when creating conversation: {response.status_code}")
+                    return f"Sorry, unable to connect to RAGFlow API (create conversation). HTTP status code: {response.status_code}"
+            except Exception as e:
+                logger.exception(f"[RAGFlowChat] Exception when creating conversation: {e}")
+                return f"Sorry, an internal error occurred: {str(e)}"
+
+        # Step 2: Send the message and get a reply
+        url_completion = f"http://{host_address}/v1/api/completion"
+        payload_completion = {
+            "conversation_id": conversation_id,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": user_input
+                }
+            ],
+            "quote": False,
+            "stream": False
+        }
+
+        try:
+            response = requests.post(url_completion, headers=headers, json=payload_completion)
+            logger.debug(f"[RAGFlowChat] Completion response: {response.text}")
+            if response.status_code == 200:
+                data = response.json()
+                if data.get("retcode") == 0:
+                    answer = data["data"]["answer"]
+                    return answer
+                else:
+                    logger.error(f"[RAGFlowChat] Failed to get answer: {data.get('retmsg')}")
+                    return f"Sorry, unable to get a reply: {data.get('retmsg')}"
+            else:
+                logger.error(f"[RAGFlowChat] HTTP error when getting answer: {response.status_code}")
+                return f"Sorry, unable to connect to RAGFlow API (get reply). HTTP status code: {response.status_code}"
+        except Exception as e:
+            logger.exception(f"[RAGFlowChat] Exception when getting answer: {e}")
+            return f"Sorry, an internal error occurred: {str(e)}"

intergrations/chatgpt-on-wechat/plugins/requirements.txt

@@ -0,0 +1 @@
+requests

pyproject.toml

@@ -24,7 +24,8 @@ cachetools = "5.3.3"
 chardet = "5.2.0"
 cn2an = "0.5.22"
 cohere = "5.6.2"
-dashscope = "1.14.1"
+Crawl4AI = "0.3.8"
+dashscope = "1.20.11"
 deepl = "1.18.0"
 demjson3 = "3.0.6"
 discord-py = "2.3.2"
@@ -55,7 +56,7 @@ nltk = "3.9.1"
 numpy = "1.26.4"
 ollama = "0.2.1"
 onnxruntime = "1.19.2"
-openai = "1.12.0"
+openai = "1.45.0"
 opencv-python = "4.10.0.84"
 opencv-python-headless = "4.10.0.84"
 openpyxl = "3.1.2"
@@ -63,7 +64,7 @@ ormsgpack = "1.5.0"
 pandas = "2.2.2"
 pdfplumber = "0.10.4"
 peewee = "3.17.1"
-pillow = "10.3.0"
+pillow = "10.4.0"
 protobuf = "5.27.2"
 psycopg2-binary = "2.9.9"
 pyclipper = "1.3.0.post5"
@@ -92,13 +93,13 @@ strenum = "0.4.15"
 tabulate = "0.9.0"
 tencentcloud-sdk-python = "3.0.1215"
 tika = "2.6.0"
-tiktoken = "0.6.0"
+tiktoken = "0.7.0"
 umap_learn = "0.5.6"
 vertexai = "1.64.0"
 volcengine = "1.0.146"
 voyageai = "0.2.3"
 webdriver-manager = "4.0.1"
-werkzeug = "3.0.3"
+werkzeug = "3.0.6"
 wikipedia = "1.4.0"
 word2number = "1.1"
 xgboost = "1.5.0"

rag/app/audio.py

@@ -10,9 +10,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import io
 import re
-import numpy as np
 
 from api.db import LLMType
 from rag.nlp import rag_tokenizer

rag/app/book.py

@@ -15,9 +15,9 @@ import re
 from io import BytesIO
 
 from deepdoc.parser.utils import get_text
-from rag.nlp import bullets_category, is_english, tokenize, remove_contents_table, \
-    hierarchical_merge, make_colon_as_title, naive_merge, random_choices, tokenize_table, add_positions, \
-    tokenize_chunks, find_codec
+from rag.nlp import bullets_category, is_english,remove_contents_table, \
+    hierarchical_merge, make_colon_as_title, naive_merge, random_choices, tokenize_table, \
+    tokenize_chunks
 from rag.nlp import rag_tokenizer
 from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
 

rag/app/laws.py

@@ -10,7 +10,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import copy
 from tika import parser
 import re
 from io import BytesIO
@@ -18,8 +17,8 @@ from docx import Document
 
 from api.db import ParserType
 from deepdoc.parser.utils import get_text
-from rag.nlp import bullets_category, is_english, tokenize, remove_contents_table, hierarchical_merge, \
-    make_colon_as_title, add_positions, tokenize_chunks, find_codec, docx_question_level
+from rag.nlp import bullets_category, remove_contents_table, hierarchical_merge, \
+    make_colon_as_title, tokenize_chunks, docx_question_level
 from rag.nlp import rag_tokenizer
 from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
 from rag.settings import cron_logger

rag/app/manual.py

@@ -19,13 +19,13 @@ import re
 
 from api.db import ParserType
 from io import BytesIO
-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, docx_question_level
-from deepdoc.parser import PdfParser, PlainParser
+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level
 from rag.utils import num_tokens_from_string
-from deepdoc.parser import PdfParser, ExcelParser, DocxParser
+from deepdoc.parser import PdfParser, PlainParser, DocxParser
 from docx import Document
 from PIL import Image
 
+
 class Pdf(PdfParser):
     def __init__(self):
         self.model_speciess = ParserType.MANUAL.value
@@ -67,9 +67,11 @@ class Pdf(PdfParser):
         return [(b["text"], b.get("layout_no", ""), self.get_position(b, zoomin))
                 for i, b in enumerate(self.boxes)], tbls
 
+
 class Docx(DocxParser):
     def __init__(self):
         pass
+
     def get_picture(self, document, paragraph):
         img = paragraph._element.xpath('.//pic:pic')
         if not img:
@@ -80,6 +82,7 @@ class Docx(DocxParser):
         image = related_part.image
         image = Image.open(BytesIO(image.blob))
         return image
+
     def concat_img(self, img1, img2):
         if img1 and not img2:
             return img1
@@ -160,6 +163,7 @@ class Docx(DocxParser):
             tbls.append(((None, html), ""))
         return ti_list, tbls
 
+
 def chunk(filename, binary=None, from_page=0, to_page=100000,
           lang="Chinese", callback=None, **kwargs):
     """
@@ -244,6 +248,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         res = tokenize_table(tbls, doc, eng)
         res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
         return res
+
     if re.search(r"\.docx$", filename, re.IGNORECASE):
         docx_parser = Docx()
         ti_list, tbls = docx_parser(filename, binary,
@@ -259,8 +264,6 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         raise NotImplementedError("file type not supported yet(pdf and docx supported)")
     
 
-
-
 if __name__ == "__main__":
     import sys
 

rag/app/naive.py

@@ -16,14 +16,16 @@ from docx import Document
 from timeit import default_timer as timer
 import re
 from deepdoc.parser.pdf_parser import PlainParser
-from rag.nlp import rag_tokenizer, naive_merge, tokenize_table, tokenize_chunks, find_codec, concat_img, naive_merge_docx, tokenize_chunks_docx
+from rag.nlp import rag_tokenizer, naive_merge, tokenize_table, tokenize_chunks, find_codec, concat_img, \
+    naive_merge_docx, tokenize_chunks_docx
 from deepdoc.parser import PdfParser, ExcelParser, DocxParser, HtmlParser, JsonParser, MarkdownParser, TxtParser
 from rag.settings import cron_logger
 from rag.utils import num_tokens_from_string
 from PIL import Image
 from functools import reduce
 from markdown import markdown
-from docx.image.exceptions import UnrecognizedImageError
+from docx.image.exceptions import UnrecognizedImageError, UnexpectedEndOfFileError, InvalidImageStreamError
+
 
 class Docx(DocxParser):
     def __init__(self):
@@ -41,6 +43,12 @@ class Docx(DocxParser):
         except UnrecognizedImageError:
             print("Unrecognized image format. Skipping image.")
             return None
+        except UnexpectedEndOfFileError:
+            print("EOF was unexpectedly encountered while reading an image stream. Skipping image.")
+            return None
+        except InvalidImageStreamError:
+            print("The recognized image stream appears to be corrupted. Skipping image.")
+            return None
         try:
             image = Image.open(BytesIO(image_blob)).convert('RGB')
             return image
@@ -93,14 +101,14 @@ class Docx(DocxParser):
 
         tbls = []
         for tb in self.doc.tables:
-            html= "<table>"
+            html = "<table>"
             for r in tb.rows:
                 html += "<tr>"
                 i = 0
                 while i < len(r.cells):
                     span = 1
                     c = r.cells[i]
-                    for j in range(i+1, len(r.cells)):
+                    for j in range(i + 1, len(r.cells)):
                         if c.text == r.cells[j].text:
                             span += 1
                             i = j
@@ -135,9 +143,9 @@ class Pdf(PdfParser):
         self._text_merge()
         callback(0.67, "Text merging finished")
         tbls = self._extract_table_figure(True, zoomin, True, True)
-        #self._naive_vertical_merge()
+        # self._naive_vertical_merge()
         self._concat_downward()
-        #self._filter_forpages()
+        # self._filter_forpages()
 
         cron_logger.info("layouts: {}".format(timer() - start))
         return [(b["text"], self._line_tag(b, zoomin))
@@ -146,8 +154,6 @@ class Pdf(PdfParser):
 
 class Markdown(MarkdownParser):
     def __call__(self, filename, binary=None):
-        txt = ""
-        tbls = []
         if binary:
             encoding = find_codec(binary)
             txt = binary.decode(encoding, errors="ignore")
@@ -159,11 +165,15 @@ class Markdown(MarkdownParser):
         tbls = []
         for sec in remainder.split("\n"):
             if num_tokens_from_string(sec) > 10 * self.chunk_token_num:
-                sections.append((sec[:int(len(sec)/2)], ""))
-                sections.append((sec[int(len(sec)/2):], ""))
+                sections.append((sec[:int(len(sec) / 2)], ""))
+                sections.append((sec[int(len(sec) / 2):], ""))
             else:
-                sections.append((sec, ""))
-        print(tables)
+                if sections and sections[-1][0].strip().find("#") == 0:
+                    sec_, _ = sections.pop(-1)
+                    sections.append((sec_+"\n"+sec, ""))
+                else:
+                    sections.append((sec, ""))
+
         for table in tables:
             tbls.append(((None, markdown(table, extensions=['markdown.extensions.tables'])), ""))
         return sections, tbls
@@ -192,7 +202,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
     if re.search(r"\.docx$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         sections, tbls = Docx()(filename, binary)
-        res = tokenize_table(tbls, doc, eng)    # just for table
+        res = tokenize_table(tbls, doc, eng)  # just for table
 
         callback(0.8, "Finish parsing.")
         st = timer()
@@ -230,7 +240,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
                                parser_config.get("chunk_token_num", 128),
                                parser_config.get("delimiter", "\n!?;。；！？"))
         callback(0.8, "Finish parsing.")
-    
+
     elif re.search(r"\.(md|markdown)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         sections, tbls = Markdown(int(parser_config.get("chunk_token_num", 128)))(filename, binary)
@@ -277,7 +287,9 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
 if __name__ == "__main__":
     import sys
 
+
     def dummy(prog=None, msg=""):
         pass
 
+
     chunk(sys.argv[1], from_page=0, to_page=10, callback=dummy)

rag/app/paper.py

@@ -12,13 +12,11 @@
 #
 import copy
 import re
-from collections import Counter
 
 from api.db import ParserType
 from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks
 from deepdoc.parser import PdfParser, PlainParser
 import numpy as np
-from rag.utils import num_tokens_from_string
 
 
 class Pdf(PdfParser):
@@ -135,7 +133,6 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         Only pdf is supported.
         The abstract of the paper will be sliced as an entire chunk, and will not be sliced partly.
     """
-    pdf_parser = None
     if re.search(r"\.pdf$", filename, re.IGNORECASE):
         if not kwargs.get("parser_config", {}).get("layout_recognize", True):
             pdf_parser = PlainParser()

rag/app/qa.py

@@ -14,7 +14,6 @@ import re
 from copy import deepcopy
 from io import BytesIO
 from timeit import default_timer as timer
-from nltk import word_tokenize
 from openpyxl import load_workbook
 
 from deepdoc.parser.utils import get_text
@@ -69,6 +68,7 @@ class Excel(ExcelParser):
             [rmPrefix(q) for q, _ in random_choices(res, k=30) if len(q) > 1])
         return res
 
+
 class Pdf(PdfParser):
     def __call__(self, filename, binary=None, from_page=0,
                  to_page=100000, zoomin=3, callback=None):
@@ -156,6 +156,7 @@ class Pdf(PdfParser):
         if last_q:
             qai_list.append((last_q, last_a, *self.crop(last_tag, need_position=True)))
         return qai_list, tbls
+
     def get_tbls_info(self, tbls, tbl_index):
         if tbl_index >= len(tbls):
             return 1, 0, 0, 0, 0, '@@0\t0\t0\t0\t0##', ''
@@ -167,10 +168,13 @@ class Pdf(PdfParser):
         tbl_tag = "@@{}\t{:.1f}\t{:.1f}\t{:.1f}\t{:.1f}##" \
             .format(tbl_pn, tbl_left, tbl_right, tbl_top, tbl_bottom)
         tbl_text = ''.join(tbls[tbl_index][0][1])
-        return tbl_pn, tbl_left, tbl_right, tbl_top, tbl_bottom, tbl_tag, tbl_text
+        return tbl_pn, tbl_left, tbl_right, tbl_top, tbl_bottom, tbl_tag,
+
+
 class Docx(DocxParser):
     def __init__(self):
         pass
+
     def get_picture(self, document, paragraph):
         img = paragraph._element.xpath('.//pic:pic')
         if not img:
@@ -243,6 +247,7 @@ class Docx(DocxParser):
             tbls.append(((None, html), ""))
         return qai_list, tbls
 
+
 def rmPrefix(txt):
     return re.sub(
         r"^(问题|答案|回答|user|assistant|Q|A|Question|Answer|问|答)[\t:： ]+", "", txt.strip(), flags=re.IGNORECASE)
@@ -259,6 +264,7 @@ def beAdocPdf(d, q, a, eng, image, poss):
     add_positions(d, poss)
     return d
 
+
 def beAdocDocx(d, q, a, eng, image):
     qprefix = "Question: " if eng else "问题："
     aprefix = "Answer: " if eng else "回答："
@@ -269,6 +275,7 @@ def beAdocDocx(d, q, a, eng, image):
     d["image"] = image
     return d
 
+
 def beAdoc(d, q, a, eng):
     qprefix = "Question: " if eng else "问题："
     aprefix = "Answer: " if eng else "回答："
@@ -283,6 +290,7 @@ def mdQuestionLevel(s):
     match = re.match(r'#*', s)
     return (len(match.group(0)), s.lstrip('#').lstrip()) if match else (0, s)
 
+
 def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
     """
         Excel and csv(txt) format files are supported.
@@ -307,6 +315,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
         for q, a in excel_parser(filename, binary, callback):
             res.append(beAdoc(deepcopy(doc), q, a, eng))
         return res
+
     elif re.search(r"\.(txt|csv)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         txt = get_text(filename, binary)
@@ -340,16 +349,16 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
             f"{len(fails)} failure, line: %s..." % (",".join(fails[:3])) if fails else "")))
 
         return res
+
     elif re.search(r"\.pdf$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         pdf_parser = Pdf()
         qai_list, tbls = pdf_parser(filename if not binary else binary,
                                     from_page=0, to_page=10000, callback=callback)
-        
-
         for q, a, image, poss in qai_list:
             res.append(beAdocPdf(deepcopy(doc), q, a, eng, image, poss))
         return res
+
     elif re.search(r"\.(md|markdown)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         txt = get_text(filename, binary)
@@ -385,6 +394,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
             if sum_question:
                 res.append(beAdoc(deepcopy(doc), sum_question, markdown(last_answer, extensions=['markdown.extensions.tables']), eng))
         return res
+
     elif re.search(r"\.docx$", filename, re.IGNORECASE):
         docx_parser = Docx()
         qai_list, tbls = docx_parser(filename, binary,