Compare commits

..

30 Commits

Author SHA1 Message Date
cfdccebb17 Feat: Fixed an issue where modifying fields in the agent operator caused the loss of structured data. #10427 (#11388)
### What problem does this PR solve?

Feat: Fixed an issue where modifying fields in the agent operator caused
the loss of structured data. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 20:11:53 +08:00
980a883033 Docs: minor (#11385)
### What problem does this PR solve?

### Type of change

- [x] Documentation Update
2025-11-19 19:41:21 +08:00
02d429f0ca Doc: Optimize read me (#11386)
### What problem does this PR solve?

Users currently can’t view `git checkout v0.22.1` directly. They need to
scroll the code block all the way to the right to see it.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 19:40:55 +08:00
9c24d5d44a Fix some multilingual issues (#11382)
### What problem does this PR solve?

Fix some multilingual issues

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 19:14:43 +08:00
0cc5d7a8a6 Feat: If a query variable in a data manipulation operator is deleted, a warning message should be displayed to the user. #10427 #11255 (#11384)
### What problem does this PR solve?

Feat: If a query variable in a data manipulation operator is deleted, a
warning message should be displayed to the user. #10427 #11255

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 19:10:57 +08:00
c43bf1dcf5 Fix: refine error msg. (#11380)
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 19:10:45 +08:00
f76b8279dd Doc: Added v0.22.1 release notes (#11383)
### What problem does this PR solve?


### Type of change


- [x] Documentation Update
2025-11-19 18:40:06 +08:00
db5ec89dc5 Feat: The key for the begin operator can only contain alphanumeric characters and underscores. #10427 (#11377)
### What problem does this PR solve?

Feat: The key for the begin operator can only contain alphanumeric
characters and underscores. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 16:16:57 +08:00
1c201c4d54 Fix: circle imports issue. (#11374)
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 16:13:21 +08:00
ba78d0f0c2 Feat: Structured data will still be stored in outputs for compatibility with older versions. #10427 (#11368)
### What problem does this PR solve?

Feat: Structured data will still be stored in outputs for compatibility
with older versions. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 15:15:51 +08:00
add8c63458 Add release notes (#11372)
### What problem does this PR solve?

As title.

### Type of change

- [x] Documentation Update

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2025-11-19 14:48:41 +08:00
83661efdaf Update README for supporting Gemini 3 Pro (#11369)
### What problem does this PR solve?

As title

### Type of change

- [x] Documentation Update

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2025-11-19 14:16:03 +08:00
971197d595 Feat: Set the outputs type of list operation. #10427 (#11366)
### What problem does this PR solve?

Feat: Set the outputs type of list operation. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 13:59:43 +08:00
0884e9a4d9 Fix: bbox not included in mineru output (#11365)
### What problem does this PR solve?

Fix: bbox not included in mineru output #11315

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 13:59:32 +08:00
2de42f00b8 Fix: component list operation issue. (#11364)
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 13:19:44 +08:00
e8fe580d7a Feat: add Gemini 3 Pro preview (#11361)
### What problem does this PR solve?

Add Gemini 3 Pro preview.

Change `GenerativeModel` to `genai`.

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 13:17:22 +08:00
62505164d5 chore(template): introducing variable aggregator to customer service template (#11352)
### What problem does this PR solve?
Update customer service template

### Type of change
- [x] Other (please describe):
2025-11-19 12:28:06 +08:00
d1dcf3b43c Refactor /stats API (#11363)
### What problem does this PR solve?

One loop to get better performance

### Type of change

- [x] Refactoring

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2025-11-19 12:27:45 +08:00
f84662d2ee Fix: Fixed an issue where variable aggregation operators could not be connected to other operators. #10427 (#11358)
### What problem does this PR solve?

Fix: Fixed an issue where variable aggregation operators could not be
connected to other operators. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-19 10:29:26 +08:00
1cb6b7f5dd Update version info to v0.22.1 (#11346)
### What problem does this PR solve?

As title

### Type of change

- [x] Other (please describe): Update version info

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2025-11-19 09:50:23 +08:00
023f509501 Fix: variable assigner issue. (#11351)
### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-19 09:49:40 +08:00
50bc53a1f5 Fix: Modify the personal center style #10703 (#11347)
### What problem does this PR solve?

Fix: Modify the personal center style

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-18 20:07:17 +08:00
8cd4882596 Feat: Display variables in the variable assignment node. #10427 (#11349)
### What problem does this PR solve?

Feat: Display variables in the variable assignment node. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-18 20:07:04 +08:00
35e5fade93 Feat: new component variable assigner (#11050)
### What problem does this PR solve?
issue:
https://github.com/infiniflow/ragflow/issues/10427
change:
new component variable assigner
### Type of change

- [x] New Feature (non-breaking change which adds functionality)
2025-11-18 19:14:38 +08:00
4942a23290 Feat: Add a switch to control the display of structured output to the agent form. #10427 (#11344)
### What problem does this PR solve?

Feat: Add a switch to control the display of structured output to the
agent form. #10427

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
2025-11-18 18:58:36 +08:00
d1716d865a Feat: Alter flask to Quart for async API serving. (#11275)
### What problem does this PR solve?

#11277

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
2025-11-18 17:05:16 +08:00
c2b7c305fa Fix: crop index may out of range (#11341)
### What problem does this PR solve?

Crop index may out of range. #11323


### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-18 17:01:54 +08:00
341e5904c8 Fix: No results can be found through the API /api/v1/dify/retrieval (#11338)
### What problem does this PR solve?

No results can be found through the API /api/v1/dify/retrieval. #11307 

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-18 15:42:31 +08:00
ded9bf80c5 Fix:limit random sampling range in check_embedding (#11337)
### What problem does this PR solve?
issue:
[#11319](https://github.com/infiniflow/ragflow/issues/11319)
change:
limit random sampling range in check_embedding

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-18 15:24:27 +08:00
fea157ba08 Fix: manual parser with mineru (#11336)
### What problem does this PR solve?

Fix: manual parser with mineru #11320
Fix: missing parameter in mineru #11334
Fix: add outlines parameter for pdf parsers

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2025-11-18 15:22:52 +08:00
135 changed files with 6309 additions and 5071 deletions

View File

@ -22,7 +22,7 @@
<img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@ -85,6 +85,7 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
## 🔥 Latest Updates ## 🔥 Latest Updates
- 2025-11-19 Supports Gemini 3 Pro.
- 2025-11-12 Supports data synchronization from Confluence, AWS S3, Discord, Google Drive. - 2025-11-12 Supports data synchronization from Confluence, AWS S3, Discord, Google Drive.
- 2025-10-23 Supports MinerU & Docling as document parsing methods. - 2025-10-23 Supports MinerU & Docling as document parsing methods.
- 2025-10-15 Supports orchestrable ingestion pipeline. - 2025-10-15 Supports orchestrable ingestion pipeline.
@ -93,8 +94,6 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io).
- 2025-05-23 Adds a Python/JavaScript code executor component to Agent. - 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
- 2025-05-05 Supports cross-language query. - 2025-05-05 Supports cross-language query.
- 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files. - 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
- 2024-12-18 Upgrades Document Layout Analysis model in DeepDoc.
- 2024-08-22 Support text to SQL statements through RAG.
## 🎉 Stay Tuned ## 🎉 Stay Tuned
@ -188,12 +187,13 @@ releases! 🌟
> All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64. > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
> If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system. > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
> The command below downloads the `v0.22.0` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.22.0`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server. > The command below downloads the `v0.22.1` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.22.1`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases), e.g.: git checkout v0.22.0 # git checkout v0.22.1
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
# This steps ensures the **entrypoint.sh** file in the code matches the Docker image version. # This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@ -85,6 +85,7 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
## 🔥 Pembaruan Terbaru ## 🔥 Pembaruan Terbaru
- 2025-11-19 Mendukung Gemini 3 Pro.
- 2025-11-12 Mendukung sinkronisasi data dari Confluence, AWS S3, Discord, Google Drive. - 2025-11-12 Mendukung sinkronisasi data dari Confluence, AWS S3, Discord, Google Drive.
- 2025-10-23 Mendukung MinerU & Docling sebagai metode penguraian dokumen. - 2025-10-23 Mendukung MinerU & Docling sebagai metode penguraian dokumen.
- 2025-10-15 Dukungan untuk jalur data yang terorkestrasi. - 2025-10-15 Dukungan untuk jalur data yang terorkestrasi.
@ -186,12 +187,13 @@ Coba demo kami di [https://demo.ragflow.io](https://demo.ragflow.io).
> Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64. > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
> Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image). > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
> Perintah di bawah ini mengunduh edisi v0.22.0 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.22.0, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server. > Perintah di bawah ini mengunduh edisi v0.22.1 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.22.1, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases), contoh: git checkout v0.22.0 # git checkout v0.22.1
# Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases)
# This steps ensures the **entrypoint.sh** file in the code matches the Docker image version. # This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@ -66,6 +66,7 @@
## 🔥 最新情報 ## 🔥 最新情報
- 2025-11-19 Gemini 3 Proをサポートしています
- 2025-11-12 Confluence、AWS S3、Discord、Google Drive からのデータ同期をサポートします。 - 2025-11-12 Confluence、AWS S3、Discord、Google Drive からのデータ同期をサポートします。
- 2025-10-23 ドキュメント解析方法として MinerU と Docling をサポートします。 - 2025-10-23 ドキュメント解析方法として MinerU と Docling をサポートします。
- 2025-10-15 オーケストレーションされたデータパイプラインのサポート。 - 2025-10-15 オーケストレーションされたデータパイプラインのサポート。
@ -166,12 +167,13 @@
> 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
> ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
> 以下のコマンドは、RAGFlow Docker イメージの v0.22.0 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.22.0 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。 > 以下のコマンドは、RAGFlow Docker イメージの v0.22.1 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.22.1 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases) 例: git checkout v0.22.0 # git checkout v0.22.1
# 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases)
# この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。 # この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@ -67,6 +67,7 @@
## 🔥 업데이트 ## 🔥 업데이트
- 2025-11-19 Gemini 3 Pro를 지원합니다.
- 2025-11-12 Confluence, AWS S3, Discord, Google Drive에서 데이터 동기화를 지원합니다. - 2025-11-12 Confluence, AWS S3, Discord, Google Drive에서 데이터 동기화를 지원합니다.
- 2025-10-23 문서 파싱 방법으로 MinerU 및 Docling을 지원합니다. - 2025-10-23 문서 파싱 방법으로 MinerU 및 Docling을 지원합니다.
- 2025-10-15 조정된 데이터 파이프라인 지원. - 2025-10-15 조정된 데이터 파이프라인 지원.
@ -168,12 +169,13 @@
> 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다. > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
> ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image). > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
> 아래 명령어는 RAGFlow Docker 이미지의 v0.22.0 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.22.0과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오. > 아래 명령어는 RAGFlow Docker 이미지의 v0.22.1 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.22.1과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases), e.g.: git checkout v0.22.0 # git checkout v0.22.1
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
# 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다. # 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다.
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@ -86,6 +86,7 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
## 🔥 Últimas Atualizações ## 🔥 Últimas Atualizações
- 19-11-2025 Suporta Gemini 3 Pro.
- 12-11-2025 Suporta a sincronização de dados do Confluence, AWS S3, Discord e Google Drive. - 12-11-2025 Suporta a sincronização de dados do Confluence, AWS S3, Discord e Google Drive.
- 23-10-2025 Suporta MinerU e Docling como métodos de análise de documentos. - 23-10-2025 Suporta MinerU e Docling como métodos de análise de documentos.
- 15-10-2025 Suporte para pipelines de dados orquestrados. - 15-10-2025 Suporte para pipelines de dados orquestrados.
@ -186,12 +187,13 @@ Experimente nossa demo em [https://demo.ragflow.io](https://demo.ragflow.io).
> Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64. > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
> Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema. > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
> O comando abaixo baixa a edição`v0.22.0` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.22.0`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor. > O comando abaixo baixa a edição`v0.22.1` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.22.1`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases), ex.: git checkout v0.22.0 # git checkout v0.22.1
# Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases)
# Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker. # Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker.
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@ -85,6 +85,7 @@
## 🔥 近期更新 ## 🔥 近期更新
- 2025-11-19 支援 Gemini 3 Pro.
- 2025-11-12 支援從 Confluence、AWS S3、Discord、Google Drive 進行資料同步。 - 2025-11-12 支援從 Confluence、AWS S3、Discord、Google Drive 進行資料同步。
- 2025-10-23 支援 MinerU 和 Docling 作為文件解析方法。 - 2025-10-23 支援 MinerU 和 Docling 作為文件解析方法。
- 2025-10-15 支援可編排的資料管道。 - 2025-10-15 支援可編排的資料管道。
@ -185,12 +186,13 @@
> 所有 Docker 映像檔都是為 x86 平台建置的。目前,我們不提供 ARM64 平台的 Docker 映像檔。 > 所有 Docker 映像檔都是為 x86 平台建置的。目前,我們不提供 ARM64 平台的 Docker 映像檔。
> 如果您使用的是 ARM64 平台,請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。 > 如果您使用的是 ARM64 平台,請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.22.0`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.22.0` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。 > 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.22.1`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.22.1` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# 可選使用穩定版標籤查看發佈https://github.com/infiniflow/ragflow/releasesgit checkout v0.22.0 # git checkout v0.22.1
# 可選使用穩定版標籤查看發佈https://github.com/infiniflow/ragflow/releases
# 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。 # 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -22,7 +22,7 @@
<img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99"> <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
</a> </a>
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank"> <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.0"> <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.22.1">
</a> </a>
<a href="https://github.com/infiniflow/ragflow/releases/latest"> <a href="https://github.com/infiniflow/ragflow/releases/latest">
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release"> <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@ -85,6 +85,7 @@
## 🔥 近期更新 ## 🔥 近期更新
- 2025-11-19 支持 Gemini 3 Pro.
- 2025-11-12 支持从 Confluence、AWS S3、Discord、Google Drive 进行数据同步。 - 2025-11-12 支持从 Confluence、AWS S3、Discord、Google Drive 进行数据同步。
- 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。 - 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。
- 2025-10-15 支持可编排的数据管道。 - 2025-10-15 支持可编排的数据管道。
@ -186,12 +187,13 @@
> 请注意,目前官方提供的所有 Docker 镜像均基于 x86 架构构建,并不提供基于 ARM64 的 Docker 镜像。 > 请注意,目前官方提供的所有 Docker 镜像均基于 x86 架构构建,并不提供基于 ARM64 的 Docker 镜像。
> 如果你的操作系统是 ARM64 架构,请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。 > 如果你的操作系统是 ARM64 架构,请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
> 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.22.0`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.22.0` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。 > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.22.1`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.22.1` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
```bash ```bash
$ cd ragflow/docker $ cd ragflow/docker
# 可选使用稳定版本标签查看发布https://github.com/infiniflow/ragflow/releases例如git checkout v0.22.0 # git checkout v0.22.1
# 可选使用稳定版本标签查看发布https://github.com/infiniflow/ragflow/releases
# 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。 # 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。
# Use CPU for DeepDoc tasks: # Use CPU for DeepDoc tasks:

View File

@ -48,7 +48,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
1. Ensure the Admin Service is running. 1. Ensure the Admin Service is running.
2. Install ragflow-cli. 2. Install ragflow-cli.
```bash ```bash
pip install ragflow-cli==0.22.0 pip install ragflow-cli==0.22.1
``` ```
3. Launch the CLI client: 3. Launch the CLI client:
```bash ```bash

View File

@ -20,8 +20,10 @@ import logging
import time import time
import threading import threading
import traceback import traceback
from werkzeug.serving import run_simple
from flask import Flask from flask import Flask
from flask_login import LoginManager
from werkzeug.serving import run_simple
from routes import admin_bp from routes import admin_bp
from common.log_utils import init_root_logger from common.log_utils import init_root_logger
from common.constants import SERVICE_CONF from common.constants import SERVICE_CONF
@ -30,7 +32,6 @@ from common import settings
from config import load_configurations, SERVICE_CONFIGS from config import load_configurations, SERVICE_CONFIGS
from auth import init_default_admin, setup_auth from auth import init_default_admin, setup_auth
from flask_session import Session from flask_session import Session
from flask_login import LoginManager
from common.versions import get_ragflow_version from common.versions import get_ragflow_version
stop_event = threading.Event() stop_event = threading.Event()

View File

@ -19,7 +19,8 @@ import logging
import uuid import uuid
from functools import wraps from functools import wraps
from datetime import datetime from datetime import datetime
from flask import request, jsonify
from flask import jsonify, request
from flask_login import current_user, login_user from flask_login import current_user, login_user
from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
@ -30,7 +31,7 @@ from common.constants import ActiveEnum, StatusEnum
from api.utils.crypt import decrypt from api.utils.crypt import decrypt
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.time_utils import current_timestamp, datetime_format, get_format_time from common.time_utils import current_timestamp, datetime_format, get_format_time
from common.connection_utils import construct_response from common.connection_utils import sync_construct_response
from common import settings from common import settings
@ -129,7 +130,7 @@ def login_admin(email: str, password: str):
user.last_login_time = get_format_time() user.last_login_time = get_format_time()
user.save() user.save()
msg = "Welcome back!" msg = "Welcome back!"
return construct_response(data=resp, auth=user.get_id(), message=msg) return sync_construct_response(data=resp, auth=user.get_id(), message=msg)
def check_admin(username: str, password: str): def check_admin(username: str, password: str):

View File

@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
from flask import jsonify from flask import jsonify

View File

@ -17,7 +17,7 @@
import secrets import secrets
from flask import Blueprint, request from flask import Blueprint, request
from flask_login import current_user, logout_user, login_required from flask_login import current_user, login_required, logout_user
from auth import login_verify, login_admin, check_admin_auth from auth import login_verify, login_admin, check_admin_auth
from responses import success_response, error_response from responses import success_response, error_response

View File

@ -25,7 +25,6 @@ from typing import Any, Union, Tuple
from agent.component import component_class from agent.component import component_class
from agent.component.base import ComponentBase from agent.component.base import ComponentBase
from api.db.services.file_service import FileService
from api.db.services.task_service import has_canceled from api.db.services.task_service import has_canceled
from common.misc_utils import get_uuid, hash_str2int from common.misc_utils import get_uuid, hash_str2int
from common.exceptions import TaskCanceledException from common.exceptions import TaskCanceledException
@ -217,6 +216,38 @@ class Graph:
else: else:
cur = getattr(cur, key, None) cur = getattr(cur, key, None)
return cur return cur
def set_variable_value(self, exp: str,value):
exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
if exp.find("@") < 0:
self.globals[exp] = value
return
cpn_id, var_nm = exp.split("@")
cpn = self.get_component(cpn_id)
if not cpn:
raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
parts = var_nm.split(".", 1)
root_key = parts[0]
rest = parts[1] if len(parts) > 1 else ""
if not rest:
cpn["obj"].set_output(root_key, value)
return
root_val = cpn["obj"].output(root_key)
if not root_val:
root_val = {}
cpn["obj"].set_output(root_key, self.set_variable_param_value(root_val,rest,value))
def set_variable_param_value(self, obj: Any, path: str, value) -> Any:
cur = obj
keys = path.split('.')
if not path:
return value
for key in keys:
if key not in cur or not isinstance(cur[key], dict):
cur[key] = {}
cur = cur[key]
cur[keys[-1]] = value
return obj
def is_canceled(self) -> bool: def is_canceled(self) -> bool:
return has_canceled(self.task_id) return has_canceled(self.task_id)
@ -270,7 +301,7 @@ class Canvas(Graph):
self.retrieval = [] self.retrieval = []
self.memory = [] self.memory = []
for k in self.globals.keys(): for k in self.globals.keys():
if k.startswith("sys."): if k.startswith("sys.") or k.startswith("env."):
if isinstance(self.globals[k], str): if isinstance(self.globals[k], str):
self.globals[k] = "" self.globals[k] = ""
elif isinstance(self.globals[k], int): elif isinstance(self.globals[k], int):
@ -284,7 +315,7 @@ class Canvas(Graph):
else: else:
self.globals[k] = None self.globals[k] = None
def run(self, **kwargs): async def run(self, **kwargs):
st = time.perf_counter() st = time.perf_counter()
self.message_id = get_uuid() self.message_id = get_uuid()
created_at = int(time.time()) created_at = int(time.time())
@ -549,6 +580,7 @@ class Canvas(Graph):
return self.components[cpnnm]["obj"].get_input_elements() return self.components[cpnnm]["obj"].get_input_elements()
def get_files(self, files: Union[None, list[dict]]) -> list[str]: def get_files(self, files: Union[None, list[dict]]) -> list[str]:
from api.db.services.file_service import FileService
if not files: if not files:
return [] return []
def image_to_base64(file): def image_to_base64(file):

View File

@ -163,12 +163,7 @@ class Agent(LLM, ToolBase):
downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else [] downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
ex = self.exception_handler() ex = self.exception_handler()
output_structure=None if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not (ex and ex["goto"]):
try:
output_structure=self._param.outputs['structured']
except Exception:
pass
if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]):
self.set_output("content", partial(self.stream_output_with_tools, prompt, msg, user_defined_prompt)) self.set_output("content", partial(self.stream_output_with_tools, prompt, msg, user_defined_prompt))
return return

View File

@ -1,3 +1,18 @@
#
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from abc import ABC from abc import ABC
import ast import ast
import os import os

View File

@ -32,6 +32,7 @@ class IterationParam(ComponentParamBase):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.items_ref = "" self.items_ref = ""
self.veriable={}
def get_input_form(self) -> dict[str, dict]: def get_input_form(self) -> dict[str, dict]:
return { return {

View File

@ -47,7 +47,9 @@ class ListOperations(ComponentBase,ABC):
def _invoke(self, **kwargs): def _invoke(self, **kwargs):
self.input_objects=[] self.input_objects=[]
inputs = getattr(self._param, "query", None) inputs = getattr(self._param, "query", None)
self.inputs=self._canvas.get_variable_value(inputs) self.inputs = self._canvas.get_variable_value(inputs)
if not isinstance(self.inputs, list):
raise TypeError("The input of List Operations should be an array.")
self.set_input_value(inputs, self.inputs) self.set_input_value(inputs, self.inputs)
if self._param.operations == "topN": if self._param.operations == "topN":
self._topN() self._topN()

View File

@ -249,7 +249,7 @@ class LLM(ComponentBase):
downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else [] downstreams = self._canvas.get_component(self._id)["downstream"] if self._canvas.get_component(self._id) else []
ex = self.exception_handler() ex = self.exception_handler()
if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not output_structure and not (ex and ex["goto"]): if any([self._canvas.get_component_obj(cid).component_name.lower()=="message" for cid in downstreams]) and not (ex and ex["goto"]):
self.set_output("content", partial(self._stream_output, prompt, msg)) self.set_output("content", partial(self._stream_output, prompt, msg))
return return

View File

@ -17,7 +17,6 @@ import json
import os import os
import random import random
import re import re
import pypandoc
import logging import logging
import tempfile import tempfile
from functools import partial from functools import partial
@ -30,6 +29,7 @@ from common.connection_utils import timeout
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common import settings from common import settings
class MessageParam(ComponentParamBase): class MessageParam(ComponentParamBase):
""" """
Define the Message component parameters. Define the Message component parameters.
@ -176,6 +176,10 @@ class Message(ComponentBase):
return "" return ""
def _convert_content(self, content): def _convert_content(self, content):
if not self._param.output_format:
return
import pypandoc
doc_id = get_uuid() doc_id = get_uuid()
if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx"}: if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx"}:

View File

@ -0,0 +1,192 @@
#
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from abc import ABC
import os
import numbers
from agent.component.base import ComponentBase, ComponentParamBase
from api.utils.api_utils import timeout
class VariableAssignerParam(ComponentParamBase):
"""
Define the Variable Assigner component parameters.
"""
def __init__(self):
super().__init__()
self.variables=[]
def check(self):
return True
def get_input_form(self) -> dict[str, dict]:
return {
"items": {
"type": "json",
"name": "Items"
}
}
class VariableAssigner(ComponentBase,ABC):
component_name = "VariableAssigner"
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
def _invoke(self, **kwargs):
if not isinstance(self._param.variables,list):
return
else:
for item in self._param.variables:
if any([not item.get("variable"), not item.get("operator"), not item.get("parameter")]):
assert "Variable is not complete."
variable=item["variable"]
operator=item["operator"]
parameter=item["parameter"]
variable_value=self._canvas.get_variable_value(variable)
new_variable=self._operate(variable_value,operator,parameter)
self._canvas.set_variable_value(variable, new_variable)
def _operate(self,variable,operator,parameter):
if operator == "overwrite":
return self._overwrite(parameter)
elif operator == "clear":
return self._clear(variable)
elif operator == "set":
return self._set(variable,parameter)
elif operator == "append":
return self._append(variable,parameter)
elif operator == "extend":
return self._extend(variable,parameter)
elif operator == "remove_first":
return self._remove_first(variable)
elif operator == "remove_last":
return self._remove_last(variable)
elif operator == "+=":
return self._add(variable,parameter)
elif operator == "-=":
return self._subtract(variable,parameter)
elif operator == "*=":
return self._multiply(variable,parameter)
elif operator == "/=":
return self._divide(variable,parameter)
else:
return
def _overwrite(self,parameter):
return self._canvas.get_variable_value(parameter)
def _clear(self,variable):
if isinstance(variable,list):
return []
elif isinstance(variable,str):
return ""
elif isinstance(variable,dict):
return {}
elif isinstance(variable,int):
return 0
elif isinstance(variable,float):
return 0.0
elif isinstance(variable,bool):
return False
else:
return None
def _set(self,variable,parameter):
if variable is None:
return self._canvas.get_value_with_variable(parameter)
elif isinstance(variable,str):
return self._canvas.get_value_with_variable(parameter)
elif isinstance(variable,bool):
return parameter
elif isinstance(variable,int):
return parameter
elif isinstance(variable,float):
return parameter
else:
return parameter
def _append(self,variable,parameter):
parameter=self._canvas.get_variable_value(parameter)
if variable is None:
variable=[]
if not isinstance(variable,list):
return "ERROR:VARIABLE_NOT_LIST"
elif len(variable)!=0 and not isinstance(parameter,type(variable[0])):
return "ERROR:PARAMETER_NOT_LIST_ELEMENT_TYPE"
else:
variable.append(parameter)
return variable
def _extend(self,variable,parameter):
parameter=self._canvas.get_variable_value(parameter)
if variable is None:
variable=[]
if not isinstance(variable,list):
return "ERROR:VARIABLE_NOT_LIST"
elif not isinstance(parameter,list):
return "ERROR:PARAMETER_NOT_LIST"
elif len(variable)!=0 and len(parameter)!=0 and not isinstance(parameter[0],type(variable[0])):
return "ERROR:PARAMETER_NOT_LIST_ELEMENT_TYPE"
else:
return variable + parameter
def _remove_first(self,variable):
if len(variable)==0:
return variable
if not isinstance(variable,list):
return "ERROR:VARIABLE_NOT_LIST"
else:
return variable[1:]
def _remove_last(self,variable):
if len(variable)==0:
return variable
if not isinstance(variable,list):
return "ERROR:VARIABLE_NOT_LIST"
else:
return variable[:-1]
def is_number(self, value):
if isinstance(value, bool):
return False
return isinstance(value, numbers.Number)
def _add(self,variable,parameter):
if self.is_number(variable) and self.is_number(parameter):
return variable + parameter
else:
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
def _subtract(self,variable,parameter):
if self.is_number(variable) and self.is_number(parameter):
return variable - parameter
else:
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
def _multiply(self,variable,parameter):
if self.is_number(variable) and self.is_number(parameter):
return variable * parameter
else:
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
def _divide(self,variable,parameter):
if self.is_number(variable) and self.is_number(parameter):
if parameter==0:
return "ERROR:DIVIDE_BY_ZERO"
else:
return variable/parameter
else:
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
def thoughts(self) -> str:
return "Assign variables from canvas."

File diff suppressed because one or more lines are too long

View File

@ -18,12 +18,11 @@ import sys
import logging import logging
from importlib.util import module_from_spec, spec_from_file_location from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path from pathlib import Path
from flask import Blueprint, Flask from quart import Blueprint, Quart, request, g, current_app, session
from werkzeug.wrappers.request import Request from werkzeug.wrappers.request import Request
from flask_cors import CORS
from flasgger import Swagger from flasgger import Swagger
from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
from quart_cors import cors
from common.constants import StatusEnum from common.constants import StatusEnum
from api.db.db_models import close_connection from api.db.db_models import close_connection
from api.db.services import UserService from api.db.services import UserService
@ -31,17 +30,20 @@ from api.utils.json_encode import CustomJSONEncoder
from api.utils import commands from api.utils import commands
from flask_mail import Mail from flask_mail import Mail
from flask_session import Session from quart_auth import Unauthorized
from flask_login import LoginManager
from common import settings from common import settings
from api.utils.api_utils import server_error_response from api.utils.api_utils import server_error_response
from api.constants import API_VERSION from api.constants import API_VERSION
from common.misc_utils import get_uuid
settings.init_settings()
__all__ = ["app"] __all__ = ["app"]
Request.json = property(lambda self: self.get_json(force=True, silent=True)) Request.json = property(lambda self: self.get_json(force=True, silent=True))
app = Flask(__name__) app = Quart(__name__)
app = cors(app, allow_origin="*")
smtp_mail_server = Mail() smtp_mail_server = Mail()
# Add this at the beginning of your file to configure Swagger UI # Add this at the beginning of your file to configure Swagger UI
@ -76,7 +78,6 @@ swagger = Swagger(
}, },
) )
CORS(app, supports_credentials=True, max_age=2592000)
app.url_map.strict_slashes = False app.url_map.strict_slashes = False
app.json_encoder = CustomJSONEncoder app.json_encoder = CustomJSONEncoder
app.errorhandler(Exception)(server_error_response) app.errorhandler(Exception)(server_error_response)
@ -84,17 +85,143 @@ app.errorhandler(Exception)(server_error_response)
## convince for dev and debug ## convince for dev and debug
# app.config["LOGIN_DISABLED"] = True # app.config["LOGIN_DISABLED"] = True
app.config["SESSION_PERMANENT"] = False app.config["SESSION_PERMANENT"] = False
app.config["SESSION_TYPE"] = "filesystem" app.config["SESSION_TYPE"] = "redis"
app.config["SESSION_REDIS"] = settings.decrypt_database_config(name="redis")
app.config["MAX_CONTENT_LENGTH"] = int( app.config["MAX_CONTENT_LENGTH"] = int(
os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024) os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
) )
app.config['SECRET_KEY'] = settings.SECRET_KEY
Session(app) app.secret_key = settings.SECRET_KEY
login_manager = LoginManager()
login_manager.init_app(app)
commands.register_commands(app) commands.register_commands(app)
from functools import wraps
from typing import ParamSpec, TypeVar
from collections.abc import Awaitable, Callable
from werkzeug.local import LocalProxy
T = TypeVar("T")
P = ParamSpec("P")
def _load_user():
jwt = Serializer(secret_key=settings.SECRET_KEY)
authorization = request.headers.get("Authorization")
g.user = None
if not authorization:
return
try:
access_token = str(jwt.loads(authorization))
if not access_token or not access_token.strip():
logging.warning("Authentication attempt with empty access token")
return None
# Access tokens should be UUIDs (32 hex characters)
if len(access_token.strip()) < 32:
logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
return None
user = UserService.query(
access_token=access_token, status=StatusEnum.VALID.value
)
if user:
if not user[0].access_token or not user[0].access_token.strip():
logging.warning(f"User {user[0].email} has empty access_token in database")
return None
g.user = user[0]
return user[0]
except Exception as e:
logging.warning(f"load_user got exception {e}")
current_user = LocalProxy(_load_user)
def login_required(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
"""A decorator to restrict route access to authenticated users.
This should be used to wrap a route handler (or view function) to
enforce that only authenticated requests can access it. Note that
it is important that this decorator be wrapped by the route
decorator and not vice, versa, as below.
.. code-block:: python
@app.route('/')
@login_required
async def index():
...
If the request is not authenticated a
`quart.exceptions.Unauthorized` exception will be raised.
"""
@wraps(func)
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
if not current_user:# or not session.get("_user_id"):
raise Unauthorized()
else:
return await current_app.ensure_async(func)(*args, **kwargs)
return wrapper
def login_user(user, remember=False, duration=None, force=False, fresh=True):
"""
Logs a user in. You should pass the actual user object to this. If the
user's `is_active` property is ``False``, they will not be logged in
unless `force` is ``True``.
This will return ``True`` if the log in attempt succeeds, and ``False`` if
it fails (i.e. because the user is inactive).
:param user: The user object to log in.
:type user: object
:param remember: Whether to remember the user after their session expires.
Defaults to ``False``.
:type remember: bool
:param duration: The amount of time before the remember cookie expires. If
``None`` the value set in the settings is used. Defaults to ``None``.
:type duration: :class:`datetime.timedelta`
:param force: If the user is inactive, setting this to ``True`` will log
them in regardless. Defaults to ``False``.
:type force: bool
:param fresh: setting this to ``False`` will log in the user with a session
marked as not "fresh". Defaults to ``True``.
:type fresh: bool
"""
if not force and not user.is_active:
return False
session["_user_id"] = user.id
session["_fresh"] = fresh
session["_id"] = get_uuid()
return True
def logout_user():
"""
Logs a user out. (You do not need to pass the actual user.) This will
also clean up the remember me cookie if it exists.
"""
if "_user_id" in session:
session.pop("_user_id")
if "_fresh" in session:
session.pop("_fresh")
if "_id" in session:
session.pop("_id")
COOKIE_NAME = "remember_token"
cookie_name = current_app.config.get("REMEMBER_COOKIE_NAME", COOKIE_NAME)
if cookie_name in request.cookies:
session["_remember"] = "clear"
if "_remember_seconds" in session:
session.pop("_remember_seconds")
return True
def search_pages_path(page_path): def search_pages_path(page_path):
app_path_list = [ app_path_list = [
@ -142,40 +269,6 @@ client_urls_prefix = [
] ]
@login_manager.request_loader
def load_user(web_request):
jwt = Serializer(secret_key=settings.SECRET_KEY)
authorization = web_request.headers.get("Authorization")
if authorization:
try:
access_token = str(jwt.loads(authorization))
if not access_token or not access_token.strip():
logging.warning("Authentication attempt with empty access token")
return None
# Access tokens should be UUIDs (32 hex characters)
if len(access_token.strip()) < 32:
logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
return None
user = UserService.query(
access_token=access_token, status=StatusEnum.VALID.value
)
if user:
if not user[0].access_token or not user[0].access_token.strip():
logging.warning(f"User {user[0].email} has empty access_token in database")
return None
return user[0]
else:
return None
except Exception as e:
logging.warning(f"load_user got exception {e}")
return None
else:
return None
@app.teardown_request @app.teardown_request
def _db_close(exception): def _db_close(exception):
if exception: if exception:

View File

@ -14,20 +14,20 @@
# limitations under the License. # limitations under the License.
# #
from datetime import datetime, timedelta from datetime import datetime, timedelta
from flask import request from quart import request
from flask_login import login_required, current_user
from api.db.db_models import APIToken from api.db.db_models import APIToken
from api.db.services.api_service import APITokenService, API4ConversationService from api.db.services.api_service import APITokenService, API4ConversationService
from api.db.services.user_service import UserTenantService from api.db.services.user_service import UserTenantService
from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \ from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
generate_confirmation_token generate_confirmation_token
from common.time_utils import current_timestamp, datetime_format from common.time_utils import current_timestamp, datetime_format
from api.apps import login_required, current_user
@manager.route('/new_token', methods=['POST']) # noqa: F821 @manager.route('/new_token', methods=['POST']) # noqa: F821
@login_required @login_required
def new_token(): async def new_token():
req = request.json req = await request.json
try: try:
tenants = UserTenantService.query(user_id=current_user.id) tenants = UserTenantService.query(user_id=current_user.id)
if not tenants: if not tenants:
@ -72,8 +72,8 @@ def token_list():
@manager.route('/rm', methods=['POST']) # noqa: F821 @manager.route('/rm', methods=['POST']) # noqa: F821
@validate_request("tokens", "tenant_id") @validate_request("tokens", "tenant_id")
@login_required @login_required
def rm(): async def rm():
req = request.json req = await request.json
try: try:
for token in req["tokens"]: for token in req["tokens"]:
APITokenService.filter_delete( APITokenService.filter_delete(
@ -101,14 +101,18 @@ def stats():
"to_date", "to_date",
datetime.now().strftime("%Y-%m-%d %H:%M:%S")), datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
"agent" if "canvas_id" in request.args else None) "agent" if "canvas_id" in request.args else None)
res = {
"pv": [(o["dt"], o["pv"]) for o in objs], res = {"pv": [], "uv": [], "speed": [], "tokens": [], "round": [], "thumb_up": []}
"uv": [(o["dt"], o["uv"]) for o in objs],
"speed": [(o["dt"], float(o["tokens"]) / (float(o["duration"] + 0.1))) for o in objs], for obj in objs:
"tokens": [(o["dt"], float(o["tokens"]) / 1000.) for o in objs], dt = obj["dt"]
"round": [(o["dt"], o["round"]) for o in objs], res["pv"].append((dt, obj["pv"]))
"thumb_up": [(o["dt"], o["thumb_up"]) for o in objs] res["uv"].append((dt, obj["uv"]))
} res["speed"].append((dt, float(obj["tokens"]) / (float(obj["duration"]) + 0.1))) # +0.1 to avoid division by zero
res["tokens"].append((dt, float(obj["tokens"]) / 1000.0)) # convert to thousands
res["round"].append((dt, obj["round"]))
res["thumb_up"].append((dt, obj["thumb_up"]))
return get_json_result(data=res) return get_json_result(data=res)
except Exception as e: except Exception as e:
return server_error_response(e) return server_error_response(e)

View File

@ -18,12 +18,8 @@ import logging
import re import re
import sys import sys
from functools import partial from functools import partial
import flask
import trio import trio
from flask import request, Response from quart import request, Response, make_response
from flask_login import login_required, current_user
from agent.component import LLM from agent.component import LLM
from api.db import CanvasCategory, FileType from api.db import CanvasCategory, FileType
from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService, API4ConversationService from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService, API4ConversationService
@ -35,7 +31,8 @@ from api.db.services.user_service import TenantService
from api.db.services.user_canvas_version import UserCanvasVersionService from api.db.services.user_canvas_version import UserCanvasVersionService
from common.constants import RetCode from common.constants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result, \
request_json
from agent.canvas import Canvas from agent.canvas import Canvas
from peewee import MySQLDatabase, PostgresqlDatabase from peewee import MySQLDatabase, PostgresqlDatabase
from api.db.db_models import APIToken, Task from api.db.db_models import APIToken, Task
@ -46,6 +43,7 @@ from rag.flow.pipeline import Pipeline
from rag.nlp import search from rag.nlp import search
from rag.utils.redis_conn import REDIS_CONN from rag.utils.redis_conn import REDIS_CONN
from common import settings from common import settings
from api.apps import login_required, current_user
@manager.route('/templates', methods=['GET']) # noqa: F821 @manager.route('/templates', methods=['GET']) # noqa: F821
@ -57,8 +55,9 @@ def templates():
@manager.route('/rm', methods=['POST']) # noqa: F821 @manager.route('/rm', methods=['POST']) # noqa: F821
@validate_request("canvas_ids") @validate_request("canvas_ids")
@login_required @login_required
def rm(): async def rm():
for i in request.json["canvas_ids"]: req = await request_json()
for i in req["canvas_ids"]:
if not UserCanvasService.accessible(i, current_user.id): if not UserCanvasService.accessible(i, current_user.id):
return get_json_result( return get_json_result(
data=False, message='Only owner of canvas authorized for this operation.', data=False, message='Only owner of canvas authorized for this operation.',
@ -70,8 +69,8 @@ def rm():
@manager.route('/set', methods=['POST']) # noqa: F821 @manager.route('/set', methods=['POST']) # noqa: F821
@validate_request("dsl", "title") @validate_request("dsl", "title")
@login_required @login_required
def save(): async def save():
req = request.json req = await request_json()
if not isinstance(req["dsl"], str): if not isinstance(req["dsl"], str):
req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False) req["dsl"] = json.dumps(req["dsl"], ensure_ascii=False)
req["dsl"] = json.loads(req["dsl"]) req["dsl"] = json.loads(req["dsl"])
@ -129,8 +128,8 @@ def getsse(canvas_id):
@manager.route('/completion', methods=['POST']) # noqa: F821 @manager.route('/completion', methods=['POST']) # noqa: F821
@validate_request("id") @validate_request("id")
@login_required @login_required
def run(): async def run():
req = request.json req = await request_json()
query = req.get("query", "") query = req.get("query", "")
files = req.get("files", []) files = req.get("files", [])
inputs = req.get("inputs", {}) inputs = req.get("inputs", {})
@ -160,10 +159,10 @@ def run():
except Exception as e: except Exception as e:
return server_error_response(e) return server_error_response(e)
def sse(): async def sse():
nonlocal canvas, user_id nonlocal canvas, user_id
try: try:
for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs): async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n" yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
cvs.dsl = json.loads(str(canvas)) cvs.dsl = json.loads(str(canvas))
@ -179,15 +178,15 @@ def run():
resp.headers.add_header("Connection", "keep-alive") resp.headers.add_header("Connection", "keep-alive")
resp.headers.add_header("X-Accel-Buffering", "no") resp.headers.add_header("X-Accel-Buffering", "no")
resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8") resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
resp.call_on_close(lambda: canvas.cancel_task()) #resp.call_on_close(lambda: canvas.cancel_task())
return resp return resp
@manager.route('/rerun', methods=['POST']) # noqa: F821 @manager.route('/rerun', methods=['POST']) # noqa: F821
@validate_request("id", "dsl", "component_id") @validate_request("id", "dsl", "component_id")
@login_required @login_required
def rerun(): async def rerun():
req = request.json req = await request_json()
doc = PipelineOperationLogService.get_documents_info(req["id"]) doc = PipelineOperationLogService.get_documents_info(req["id"])
if not doc: if not doc:
return get_data_error_result(message="Document not found.") return get_data_error_result(message="Document not found.")
@ -224,8 +223,8 @@ def cancel(task_id):
@manager.route('/reset', methods=['POST']) # noqa: F821 @manager.route('/reset', methods=['POST']) # noqa: F821
@validate_request("id") @validate_request("id")
@login_required @login_required
def reset(): async def reset():
req = request.json req = await request_json()
if not UserCanvasService.accessible(req["id"], current_user.id): if not UserCanvasService.accessible(req["id"], current_user.id):
return get_json_result( return get_json_result(
data=False, message='Only owner of canvas authorized for this operation.', data=False, message='Only owner of canvas authorized for this operation.',
@ -245,7 +244,7 @@ def reset():
@manager.route("/upload/<canvas_id>", methods=["POST"]) # noqa: F821 @manager.route("/upload/<canvas_id>", methods=["POST"]) # noqa: F821
def upload(canvas_id): async def upload(canvas_id):
e, cvs = UserCanvasService.get_by_canvas_id(canvas_id) e, cvs = UserCanvasService.get_by_canvas_id(canvas_id)
if not e: if not e:
return get_data_error_result(message="canvas not found.") return get_data_error_result(message="canvas not found.")
@ -311,7 +310,8 @@ def upload(canvas_id):
except Exception as e: except Exception as e:
return server_error_response(e) return server_error_response(e)
file = request.files['file'] files = await request.files
file = files['file']
try: try:
DocumentService.check_doc_health(user_id, file.filename) DocumentService.check_doc_health(user_id, file.filename)
return get_json_result(data=structured(file.filename, filename_type(file.filename), file.read(), file.content_type)) return get_json_result(data=structured(file.filename, filename_type(file.filename), file.read(), file.content_type))
@ -342,8 +342,8 @@ def input_form():
@manager.route('/debug', methods=['POST']) # noqa: F821 @manager.route('/debug', methods=['POST']) # noqa: F821
@validate_request("id", "component_id", "params") @validate_request("id", "component_id", "params")
@login_required @login_required
def debug(): async def debug():
req = request.json req = await request_json()
if not UserCanvasService.accessible(req["id"], current_user.id): if not UserCanvasService.accessible(req["id"], current_user.id):
return get_json_result( return get_json_result(
data=False, message='Only owner of canvas authorized for this operation.', data=False, message='Only owner of canvas authorized for this operation.',
@ -374,8 +374,8 @@ def debug():
@manager.route('/test_db_connect', methods=['POST']) # noqa: F821 @manager.route('/test_db_connect', methods=['POST']) # noqa: F821
@validate_request("db_type", "database", "username", "host", "port", "password") @validate_request("db_type", "database", "username", "host", "port", "password")
@login_required @login_required
def test_db_connect(): async def test_db_connect():
req = request.json req = await request_json()
try: try:
if req["db_type"] in ["mysql", "mariadb"]: if req["db_type"] in ["mysql", "mariadb"]:
db = MySQLDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"], db = MySQLDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
@ -519,8 +519,8 @@ def list_canvas():
@manager.route('/setting', methods=['POST']) # noqa: F821 @manager.route('/setting', methods=['POST']) # noqa: F821
@validate_request("id", "title", "permission") @validate_request("id", "title", "permission")
@login_required @login_required
def setting(): async def setting():
req = request.json req = await request_json()
req["user_id"] = current_user.id req["user_id"] = current_user.id
if not UserCanvasService.accessible(req["id"], current_user.id): if not UserCanvasService.accessible(req["id"], current_user.id):
@ -601,8 +601,8 @@ def prompts():
@manager.route('/download', methods=['GET']) # noqa: F821 @manager.route('/download', methods=['GET']) # noqa: F821
def download(): async def download():
id = request.args.get("id") id = request.args.get("id")
created_by = request.args.get("created_by") created_by = request.args.get("created_by")
blob = FileService.get_blob(created_by, id) blob = FileService.get_blob(created_by, id)
return flask.make_response(blob) return await make_response(blob)

View File

@ -18,8 +18,7 @@ import json
import re import re
import xxhash import xxhash
from flask import request from quart import request
from flask_login import current_user, login_required
from api.db.services.dialog_service import meta_filter from api.db.services.dialog_service import meta_filter
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
@ -27,7 +26,8 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import LLMBundle from api.db.services.llm_service import LLMBundle
from api.db.services.search_service import SearchService from api.db.services.search_service import SearchService
from api.db.services.user_service import UserTenantService from api.db.services.user_service import UserTenantService
from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \
request_json
from rag.app.qa import beAdoc, rmPrefix from rag.app.qa import beAdoc, rmPrefix
from rag.app.tag import label_question from rag.app.tag import label_question
from rag.nlp import rag_tokenizer, search from rag.nlp import rag_tokenizer, search
@ -35,13 +35,14 @@ from rag.prompts.generator import gen_meta_filter, cross_languages, keyword_extr
from common.string_utils import remove_redundant_spaces from common.string_utils import remove_redundant_spaces
from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD
from common import settings from common import settings
from api.apps import login_required, current_user
@manager.route('/list', methods=['POST']) # noqa: F821 @manager.route('/list', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("doc_id") @validate_request("doc_id")
def list_chunk(): async def list_chunk():
req = request.json req = await request_json()
doc_id = req["doc_id"] doc_id = req["doc_id"]
page = int(req.get("page", 1)) page = int(req.get("page", 1))
size = int(req.get("size", 30)) size = int(req.get("size", 30))
@ -121,8 +122,8 @@ def get():
@manager.route('/set', methods=['POST']) # noqa: F821 @manager.route('/set', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("doc_id", "chunk_id", "content_with_weight") @validate_request("doc_id", "chunk_id", "content_with_weight")
def set(): async def set():
req = request.json req = await request_json()
d = { d = {
"id": req["chunk_id"], "id": req["chunk_id"],
"content_with_weight": req["content_with_weight"]} "content_with_weight": req["content_with_weight"]}
@ -178,8 +179,8 @@ def set():
@manager.route('/switch', methods=['POST']) # noqa: F821 @manager.route('/switch', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("chunk_ids", "available_int", "doc_id") @validate_request("chunk_ids", "available_int", "doc_id")
def switch(): async def switch():
req = request.json req = await request_json()
try: try:
e, doc = DocumentService.get_by_id(req["doc_id"]) e, doc = DocumentService.get_by_id(req["doc_id"])
if not e: if not e:
@ -198,8 +199,8 @@ def switch():
@manager.route('/rm', methods=['POST']) # noqa: F821 @manager.route('/rm', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("chunk_ids", "doc_id") @validate_request("chunk_ids", "doc_id")
def rm(): async def rm():
req = request.json req = await request_json()
try: try:
e, doc = DocumentService.get_by_id(req["doc_id"]) e, doc = DocumentService.get_by_id(req["doc_id"])
if not e: if not e:
@ -222,8 +223,8 @@ def rm():
@manager.route('/create', methods=['POST']) # noqa: F821 @manager.route('/create', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("doc_id", "content_with_weight") @validate_request("doc_id", "content_with_weight")
def create(): async def create():
req = request.json req = await request_json()
chunck_id = xxhash.xxh64((req["content_with_weight"] + req["doc_id"]).encode("utf-8")).hexdigest() chunck_id = xxhash.xxh64((req["content_with_weight"] + req["doc_id"]).encode("utf-8")).hexdigest()
d = {"id": chunck_id, "content_ltks": rag_tokenizer.tokenize(req["content_with_weight"]), d = {"id": chunck_id, "content_ltks": rag_tokenizer.tokenize(req["content_with_weight"]),
"content_with_weight": req["content_with_weight"]} "content_with_weight": req["content_with_weight"]}
@ -280,8 +281,8 @@ def create():
@manager.route('/retrieval_test', methods=['POST']) # noqa: F821 @manager.route('/retrieval_test', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("kb_id", "question") @validate_request("kb_id", "question")
def retrieval_test(): async def retrieval_test():
req = request.json req = await request_json()
page = int(req.get("page", 1)) page = int(req.get("page", 1))
size = int(req.get("size", 30)) size = int(req.get("size", 30))
question = req["question"] question = req["question"]

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
import asyncio
import json import json
import logging import logging
import time import time
@ -20,8 +21,7 @@ import uuid
from html import escape from html import escape
from typing import Any from typing import Any
from flask import make_response, request from quart import request, make_response
from flask_login import current_user, login_required
from google_auth_oauthlib.flow import Flow from google_auth_oauthlib.flow import Flow
from api.db import InputType from api.db import InputType
@ -32,12 +32,13 @@ from common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, Docum
from common.data_source.google_util.constant import GOOGLE_DRIVE_WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES from common.data_source.google_util.constant import GOOGLE_DRIVE_WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from rag.utils.redis_conn import REDIS_CONN from rag.utils.redis_conn import REDIS_CONN
from api.apps import login_required, current_user
@manager.route("/set", methods=["POST"]) # noqa: F821 @manager.route("/set", methods=["POST"]) # noqa: F821
@login_required @login_required
def set_connector(): async def set_connector():
req = request.json req = await request.json
if req.get("id"): if req.get("id"):
conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req} conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
ConnectorService.update_by_id(req["id"], conn) ConnectorService.update_by_id(req["id"], conn)
@ -57,7 +58,7 @@ def set_connector():
} }
ConnectorService.save(**conn) ConnectorService.save(**conn)
time.sleep(1) await asyncio.sleep(1)
e, conn = ConnectorService.get_by_id(req["id"]) e, conn = ConnectorService.get_by_id(req["id"])
return get_json_result(data=conn.to_dict()) return get_json_result(data=conn.to_dict())
@ -88,8 +89,8 @@ def list_logs(connector_id):
@manager.route("/<connector_id>/resume", methods=["PUT"]) # noqa: F821 @manager.route("/<connector_id>/resume", methods=["PUT"]) # noqa: F821
@login_required @login_required
def resume(connector_id): async def resume(connector_id):
req = request.json req = await request.json
if req.get("resume"): if req.get("resume"):
ConnectorService.resume(connector_id, TaskStatus.SCHEDULE) ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
else: else:
@ -100,8 +101,8 @@ def resume(connector_id):
@manager.route("/<connector_id>/rebuild", methods=["PUT"]) # noqa: F821 @manager.route("/<connector_id>/rebuild", methods=["PUT"]) # noqa: F821
@login_required @login_required
@validate_request("kb_id") @validate_request("kb_id")
def rebuild(connector_id): async def rebuild(connector_id):
req = request.json req = await request.json
err = ConnectorService.rebuild(req["kb_id"], connector_id, current_user.id) err = ConnectorService.rebuild(req["kb_id"], connector_id, current_user.id)
if err: if err:
return get_json_result(data=False, message=err, code=RetCode.SERVER_ERROR) return get_json_result(data=False, message=err, code=RetCode.SERVER_ERROR)
@ -145,7 +146,7 @@ def _get_web_client_config(credentials: dict[str, Any]) -> dict[str, Any]:
return {"web": web_section} return {"web": web_section}
def _render_web_oauth_popup(flow_id: str, success: bool, message: str): async def _render_web_oauth_popup(flow_id: str, success: bool, message: str):
status = "success" if success else "error" status = "success" if success else "error"
auto_close = "window.close();" if success else "" auto_close = "window.close();" if success else ""
escaped_message = escape(message) escaped_message = escape(message)
@ -163,7 +164,7 @@ def _render_web_oauth_popup(flow_id: str, success: bool, message: str):
payload_json=payload_json, payload_json=payload_json,
auto_close=auto_close, auto_close=auto_close,
) )
response = make_response(html, 200) response = await make_response(html, 200)
response.headers["Content-Type"] = "text/html; charset=utf-8" response.headers["Content-Type"] = "text/html; charset=utf-8"
return response return response
@ -171,14 +172,14 @@ def _render_web_oauth_popup(flow_id: str, success: bool, message: str):
@manager.route("/google-drive/oauth/web/start", methods=["POST"]) # noqa: F821 @manager.route("/google-drive/oauth/web/start", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("credentials") @validate_request("credentials")
def start_google_drive_web_oauth(): async def start_google_drive_web_oauth():
if not GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI: if not GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI:
return get_json_result( return get_json_result(
code=RetCode.SERVER_ERROR, code=RetCode.SERVER_ERROR,
message="Google Drive OAuth redirect URI is not configured on the server.", message="Google Drive OAuth redirect URI is not configured on the server.",
) )
req = request.json or {} req = await request.json or {}
raw_credentials = req.get("credentials", "") raw_credentials = req.get("credentials", "")
try: try:
credentials = _load_credentials(raw_credentials) credentials = _load_credentials(raw_credentials)
@ -230,31 +231,31 @@ def start_google_drive_web_oauth():
@manager.route("/google-drive/oauth/web/callback", methods=["GET"]) # noqa: F821 @manager.route("/google-drive/oauth/web/callback", methods=["GET"]) # noqa: F821
def google_drive_web_oauth_callback(): async def google_drive_web_oauth_callback():
state_id = request.args.get("state") state_id = request.args.get("state")
error = request.args.get("error") error = request.args.get("error")
error_description = request.args.get("error_description") or error error_description = request.args.get("error_description") or error
if not state_id: if not state_id:
return _render_web_oauth_popup("", False, "Missing OAuth state parameter.") return await _render_web_oauth_popup("", False, "Missing OAuth state parameter.")
state_cache = REDIS_CONN.get(_web_state_cache_key(state_id)) state_cache = REDIS_CONN.get(_web_state_cache_key(state_id))
if not state_cache: if not state_cache:
return _render_web_oauth_popup(state_id, False, "Authorization session expired. Please restart from the main window.") return await _render_web_oauth_popup(state_id, False, "Authorization session expired. Please restart from the main window.")
state_obj = json.loads(state_cache) state_obj = json.loads(state_cache)
client_config = state_obj.get("client_config") client_config = state_obj.get("client_config")
if not client_config: if not client_config:
REDIS_CONN.delete(_web_state_cache_key(state_id)) REDIS_CONN.delete(_web_state_cache_key(state_id))
return _render_web_oauth_popup(state_id, False, "Authorization session was invalid. Please retry.") return await _render_web_oauth_popup(state_id, False, "Authorization session was invalid. Please retry.")
if error: if error:
REDIS_CONN.delete(_web_state_cache_key(state_id)) REDIS_CONN.delete(_web_state_cache_key(state_id))
return _render_web_oauth_popup(state_id, False, error_description or "Authorization was cancelled.") return await _render_web_oauth_popup(state_id, False, error_description or "Authorization was cancelled.")
code = request.args.get("code") code = request.args.get("code")
if not code: if not code:
return _render_web_oauth_popup(state_id, False, "Missing authorization code from Google.") return await _render_web_oauth_popup(state_id, False, "Missing authorization code from Google.")
try: try:
flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE]) flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE])
@ -263,7 +264,7 @@ def google_drive_web_oauth_callback():
except Exception as exc: # pragma: no cover - defensive except Exception as exc: # pragma: no cover - defensive
logging.exception("Failed to exchange Google OAuth code: %s", exc) logging.exception("Failed to exchange Google OAuth code: %s", exc)
REDIS_CONN.delete(_web_state_cache_key(state_id)) REDIS_CONN.delete(_web_state_cache_key(state_id))
return _render_web_oauth_popup(state_id, False, "Failed to exchange tokens with Google. Please retry.") return await _render_web_oauth_popup(state_id, False, "Failed to exchange tokens with Google. Please retry.")
creds_json = flow.credentials.to_json() creds_json = flow.credentials.to_json()
result_payload = { result_payload = {
@ -273,14 +274,14 @@ def google_drive_web_oauth_callback():
REDIS_CONN.set_obj(_web_result_cache_key(state_id), result_payload, WEB_FLOW_TTL_SECS) REDIS_CONN.set_obj(_web_result_cache_key(state_id), result_payload, WEB_FLOW_TTL_SECS)
REDIS_CONN.delete(_web_state_cache_key(state_id)) REDIS_CONN.delete(_web_state_cache_key(state_id))
return _render_web_oauth_popup(state_id, True, "Authorization completed successfully.") return await _render_web_oauth_popup(state_id, True, "Authorization completed successfully.")
@manager.route("/google-drive/oauth/web/result", methods=["POST"]) # noqa: F821 @manager.route("/google-drive/oauth/web/result", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("flow_id") @validate_request("flow_id")
def poll_google_drive_web_result(): async def poll_google_drive_web_result():
req = request.json or {} req = await request.json or {}
flow_id = req.get("flow_id") flow_id = req.get("flow_id")
cache_raw = REDIS_CONN.get(_web_result_cache_key(flow_id)) cache_raw = REDIS_CONN.get(_web_result_cache_key(flow_id))
if not cache_raw: if not cache_raw:

View File

@ -17,8 +17,8 @@ import json
import re import re
import logging import logging
from copy import deepcopy from copy import deepcopy
from flask import Response, request from quart import Response, request
from flask_login import current_user, login_required from api.apps import current_user, login_required
from api.db.db_models import APIToken from api.db.db_models import APIToken
from api.db.services.conversation_service import ConversationService, structure_answer from api.db.services.conversation_service import ConversationService, structure_answer
from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap from api.db.services.dialog_service import DialogService, ask, chat, gen_mindmap
@ -34,8 +34,8 @@ from common.constants import RetCode, LLMType
@manager.route("/set", methods=["POST"]) # noqa: F821 @manager.route("/set", methods=["POST"]) # noqa: F821
@login_required @login_required
def set_conversation(): async def set_conversation():
req = request.json req = await request.json
conv_id = req.get("conversation_id") conv_id = req.get("conversation_id")
is_new = req.get("is_new") is_new = req.get("is_new")
name = req.get("name", "New conversation") name = req.get("name", "New conversation")
@ -128,8 +128,9 @@ def getsse(dialog_id):
@manager.route("/rm", methods=["POST"]) # noqa: F821 @manager.route("/rm", methods=["POST"]) # noqa: F821
@login_required @login_required
def rm(): async def rm():
conv_ids = request.json["conversation_ids"] req = await request.json
conv_ids = req["conversation_ids"]
try: try:
for cid in conv_ids: for cid in conv_ids:
exist, conv = ConversationService.get_by_id(cid) exist, conv = ConversationService.get_by_id(cid)
@ -165,8 +166,8 @@ def list_conversation():
@manager.route("/completion", methods=["POST"]) # noqa: F821 @manager.route("/completion", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("conversation_id", "messages") @validate_request("conversation_id", "messages")
def completion(): async def completion():
req = request.json req = await request.json
msg = [] msg = []
for m in req["messages"]: for m in req["messages"]:
if m["role"] == "system": if m["role"] == "system":
@ -250,8 +251,8 @@ def completion():
@manager.route("/tts", methods=["POST"]) # noqa: F821 @manager.route("/tts", methods=["POST"]) # noqa: F821
@login_required @login_required
def tts(): async def tts():
req = request.json req = await request.json
text = req["text"] text = req["text"]
tenants = TenantService.get_info_by(current_user.id) tenants = TenantService.get_info_by(current_user.id)
@ -283,8 +284,8 @@ def tts():
@manager.route("/delete_msg", methods=["POST"]) # noqa: F821 @manager.route("/delete_msg", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("conversation_id", "message_id") @validate_request("conversation_id", "message_id")
def delete_msg(): async def delete_msg():
req = request.json req = await request.json
e, conv = ConversationService.get_by_id(req["conversation_id"]) e, conv = ConversationService.get_by_id(req["conversation_id"])
if not e: if not e:
return get_data_error_result(message="Conversation not found!") return get_data_error_result(message="Conversation not found!")
@ -306,8 +307,8 @@ def delete_msg():
@manager.route("/thumbup", methods=["POST"]) # noqa: F821 @manager.route("/thumbup", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("conversation_id", "message_id") @validate_request("conversation_id", "message_id")
def thumbup(): async def thumbup():
req = request.json req = await request.json
e, conv = ConversationService.get_by_id(req["conversation_id"]) e, conv = ConversationService.get_by_id(req["conversation_id"])
if not e: if not e:
return get_data_error_result(message="Conversation not found!") return get_data_error_result(message="Conversation not found!")
@ -333,8 +334,8 @@ def thumbup():
@manager.route("/ask", methods=["POST"]) # noqa: F821 @manager.route("/ask", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("question", "kb_ids") @validate_request("question", "kb_ids")
def ask_about(): async def ask_about():
req = request.json req = await request.json
uid = current_user.id uid = current_user.id
search_id = req.get("search_id", "") search_id = req.get("search_id", "")
@ -365,8 +366,8 @@ def ask_about():
@manager.route("/mindmap", methods=["POST"]) # noqa: F821 @manager.route("/mindmap", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("question", "kb_ids") @validate_request("question", "kb_ids")
def mindmap(): async def mindmap():
req = request.json req = await request.json
search_id = req.get("search_id", "") search_id = req.get("search_id", "")
search_app = SearchService.get_detail(search_id) if search_id else {} search_app = SearchService.get_detail(search_id) if search_id else {}
search_config = search_app.get("search_config", {}) if search_app else {} search_config = search_app.get("search_config", {}) if search_app else {}
@ -383,8 +384,8 @@ def mindmap():
@manager.route("/related_questions", methods=["POST"]) # noqa: F821 @manager.route("/related_questions", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("question") @validate_request("question")
def related_questions(): async def related_questions():
req = request.json req = await request.json
search_id = req.get("search_id", "") search_id = req.get("search_id", "")
search_config = {} search_config = {}

View File

@ -14,8 +14,7 @@
# limitations under the License. # limitations under the License.
# #
from flask import request from quart import request
from flask_login import login_required, current_user
from api.db.services import duplicate_name from api.db.services import duplicate_name
from api.db.services.dialog_service import DialogService from api.db.services.dialog_service import DialogService
from common.constants import StatusEnum from common.constants import StatusEnum
@ -26,13 +25,14 @@ from api.utils.api_utils import server_error_response, get_data_error_result, va
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.constants import RetCode from common.constants import RetCode
from api.utils.api_utils import get_json_result from api.utils.api_utils import get_json_result
from api.apps import login_required, current_user
@manager.route('/set', methods=['POST']) # noqa: F821 @manager.route('/set', methods=['POST']) # noqa: F821
@validate_request("prompt_config") @validate_request("prompt_config")
@login_required @login_required
def set_dialog(): async def set_dialog():
req = request.json req = await request.json
dialog_id = req.get("dialog_id", "") dialog_id = req.get("dialog_id", "")
is_create = not dialog_id is_create = not dialog_id
name = req.get("name", "New Dialog") name = req.get("name", "New Dialog")
@ -169,18 +169,19 @@ def list_dialogs():
@manager.route('/next', methods=['POST']) # noqa: F821 @manager.route('/next', methods=['POST']) # noqa: F821
@login_required @login_required
def list_dialogs_next(): async def list_dialogs_next():
keywords = request.args.get("keywords", "") args = request.args
page_number = int(request.args.get("page", 0)) keywords = args.get("keywords", "")
items_per_page = int(request.args.get("page_size", 0)) page_number = int(args.get("page", 0))
parser_id = request.args.get("parser_id") items_per_page = int(args.get("page_size", 0))
orderby = request.args.get("orderby", "create_time") parser_id = args.get("parser_id")
if request.args.get("desc", "true").lower() == "false": orderby = args.get("orderby", "create_time")
if args.get("desc", "true").lower() == "false":
desc = False desc = False
else: else:
desc = True desc = True
req = request.get_json() req = await request.get_json()
owner_ids = req.get("owner_ids", []) owner_ids = req.get("owner_ids", [])
try: try:
if not owner_ids: if not owner_ids:
@ -207,8 +208,8 @@ def list_dialogs_next():
@manager.route('/rm', methods=['POST']) # noqa: F821 @manager.route('/rm', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("dialog_ids") @validate_request("dialog_ids")
def rm(): async def rm():
req = request.json req = await request.json
dialog_list=[] dialog_list=[]
tenants = UserTenantService.query(user_id=current_user.id) tenants = UserTenantService.query(user_id=current_user.id)
try: try:

View File

@ -18,11 +18,8 @@ import os.path
import pathlib import pathlib
import re import re
from pathlib import Path from pathlib import Path
from quart import request, make_response
import flask from api.apps import current_user, login_required
from flask import request
from flask_login import current_user, login_required
from api.common.check_team_permission import check_kb_team_permission from api.common.check_team_permission import check_kb_team_permission
from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
from api.db import VALID_FILE_TYPES, FileType from api.db import VALID_FILE_TYPES, FileType
@ -39,7 +36,7 @@ from api.utils.api_utils import (
get_data_error_result, get_data_error_result,
get_json_result, get_json_result,
server_error_response, server_error_response,
validate_request, validate_request, request_json,
) )
from api.utils.file_utils import filename_type, thumbnail from api.utils.file_utils import filename_type, thumbnail
from common.file_utils import get_project_base_directory from common.file_utils import get_project_base_directory
@ -53,14 +50,16 @@ from common import settings
@manager.route("/upload", methods=["POST"]) # noqa: F821 @manager.route("/upload", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("kb_id") @validate_request("kb_id")
def upload(): async def upload():
kb_id = request.form.get("kb_id") form = await request.form
kb_id = form.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
if "file" not in request.files: files = await request.files
if "file" not in files:
return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
@ -87,12 +86,13 @@ def upload():
@manager.route("/web_crawl", methods=["POST"]) # noqa: F821 @manager.route("/web_crawl", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("kb_id", "name", "url") @validate_request("kb_id", "name", "url")
def web_crawl(): async def web_crawl():
kb_id = request.form.get("kb_id") form = await request.form
kb_id = form.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
name = request.form.get("name") name = form.get("name")
url = request.form.get("url") url = form.get("url")
if not is_valid_url(url): if not is_valid_url(url):
return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
e, kb = KnowledgebaseService.get_by_id(kb_id) e, kb = KnowledgebaseService.get_by_id(kb_id)
@ -152,8 +152,8 @@ def web_crawl():
@manager.route("/create", methods=["POST"]) # noqa: F821 @manager.route("/create", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("name", "kb_id") @validate_request("name", "kb_id")
def create(): async def create():
req = request.json req = await request_json()
kb_id = req["kb_id"] kb_id = req["kb_id"]
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
@ -208,7 +208,7 @@ def create():
@manager.route("/list", methods=["POST"]) # noqa: F821 @manager.route("/list", methods=["POST"]) # noqa: F821
@login_required @login_required
def list_docs(): async def list_docs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
@ -230,7 +230,7 @@ def list_docs():
create_time_from = int(request.args.get("create_time_from", 0)) create_time_from = int(request.args.get("create_time_from", 0))
create_time_to = int(request.args.get("create_time_to", 0)) create_time_to = int(request.args.get("create_time_to", 0))
req = request.get_json() req = await request.get_json()
run_status = req.get("run_status", []) run_status = req.get("run_status", [])
if run_status: if run_status:
@ -270,8 +270,8 @@ def list_docs():
@manager.route("/filter", methods=["POST"]) # noqa: F821 @manager.route("/filter", methods=["POST"]) # noqa: F821
@login_required @login_required
def get_filter(): async def get_filter():
req = request.get_json() req = await request.get_json()
kb_id = req.get("kb_id") kb_id = req.get("kb_id")
if not kb_id: if not kb_id:
@ -308,8 +308,8 @@ def get_filter():
@manager.route("/infos", methods=["POST"]) # noqa: F821 @manager.route("/infos", methods=["POST"]) # noqa: F821
@login_required @login_required
def doc_infos(): async def doc_infos():
req = request.json req = await request_json()
doc_ids = req["doc_ids"] doc_ids = req["doc_ids"]
for doc_id in doc_ids: for doc_id in doc_ids:
if not DocumentService.accessible(doc_id, current_user.id): if not DocumentService.accessible(doc_id, current_user.id):
@ -340,8 +340,8 @@ def thumbnails():
@manager.route("/change_status", methods=["POST"]) # noqa: F821 @manager.route("/change_status", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_ids", "status") @validate_request("doc_ids", "status")
def change_status(): async def change_status():
req = request.get_json() req = await request.get_json()
doc_ids = req.get("doc_ids", []) doc_ids = req.get("doc_ids", [])
status = str(req.get("status", "")) status = str(req.get("status", ""))
@ -380,8 +380,8 @@ def change_status():
@manager.route("/rm", methods=["POST"]) # noqa: F821 @manager.route("/rm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_id") @validate_request("doc_id")
def rm(): async def rm():
req = request.json req = await request_json()
doc_ids = req["doc_id"] doc_ids = req["doc_id"]
if isinstance(doc_ids, str): if isinstance(doc_ids, str):
doc_ids = [doc_ids] doc_ids = [doc_ids]
@ -401,8 +401,8 @@ def rm():
@manager.route("/run", methods=["POST"]) # noqa: F821 @manager.route("/run", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_ids", "run") @validate_request("doc_ids", "run")
def run(): async def run():
req = request.json req = await request_json()
for doc_id in req["doc_ids"]: for doc_id in req["doc_ids"]:
if not DocumentService.accessible(doc_id, current_user.id): if not DocumentService.accessible(doc_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
@ -448,8 +448,8 @@ def run():
@manager.route("/rename", methods=["POST"]) # noqa: F821 @manager.route("/rename", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_id", "name") @validate_request("doc_id", "name")
def rename(): async def rename():
req = request.json req = await request_json()
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:
@ -495,14 +495,14 @@ def rename():
@manager.route("/get/<doc_id>", methods=["GET"]) # noqa: F821 @manager.route("/get/<doc_id>", methods=["GET"]) # noqa: F821
# @login_required # @login_required
def get(doc_id): async def get(doc_id):
try: try:
e, doc = DocumentService.get_by_id(doc_id) e, doc = DocumentService.get_by_id(doc_id)
if not e: if not e:
return get_data_error_result(message="Document not found!") return get_data_error_result(message="Document not found!")
b, n = File2DocumentService.get_storage_address(doc_id=doc_id) b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
response = flask.make_response(settings.STORAGE_IMPL.get(b, n)) response = await make_response(settings.STORAGE_IMPL.get(b, n))
ext = re.search(r"\.([^.]+)$", doc.name.lower()) ext = re.search(r"\.([^.]+)$", doc.name.lower())
ext = ext.group(1) if ext else None ext = ext.group(1) if ext else None
@ -520,12 +520,12 @@ def get(doc_id):
@manager.route("/download/<attachment_id>", methods=["GET"]) # noqa: F821 @manager.route("/download/<attachment_id>", methods=["GET"]) # noqa: F821
@login_required @login_required
def download_attachment(attachment_id): async def download_attachment(attachment_id):
try: try:
ext = request.args.get("ext", "markdown") ext = request.args.get("ext", "markdown")
data = settings.STORAGE_IMPL.get(current_user.id, attachment_id) data = settings.STORAGE_IMPL.get(current_user.id, attachment_id)
# data = settings.STORAGE_IMPL.get("eb500d50bb0411f0907561d2782adda5", attachment_id) # data = settings.STORAGE_IMPL.get("eb500d50bb0411f0907561d2782adda5", attachment_id)
response = flask.make_response(data) response = await make_response(data)
response.headers.set("Content-Type", CONTENT_TYPE_MAP.get(ext, f"application/{ext}")) response.headers.set("Content-Type", CONTENT_TYPE_MAP.get(ext, f"application/{ext}"))
return response return response
@ -537,9 +537,9 @@ def download_attachment(attachment_id):
@manager.route("/change_parser", methods=["POST"]) # noqa: F821 @manager.route("/change_parser", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_id") @validate_request("doc_id")
def change_parser(): async def change_parser():
req = request.json req = await request_json()
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
@ -590,13 +590,13 @@ def change_parser():
@manager.route("/image/<image_id>", methods=["GET"]) # noqa: F821 @manager.route("/image/<image_id>", methods=["GET"]) # noqa: F821
# @login_required # @login_required
def get_image(image_id): async def get_image(image_id):
try: try:
arr = image_id.split("-") arr = image_id.split("-")
if len(arr) != 2: if len(arr) != 2:
return get_data_error_result(message="Image not found.") return get_data_error_result(message="Image not found.")
bkt, nm = image_id.split("-") bkt, nm = image_id.split("-")
response = flask.make_response(settings.STORAGE_IMPL.get(bkt, nm)) response = await make_response(settings.STORAGE_IMPL.get(bkt, nm))
response.headers.set("Content-Type", "image/JPEG") response.headers.set("Content-Type", "image/JPEG")
return response return response
except Exception as e: except Exception as e:
@ -606,24 +606,25 @@ def get_image(image_id):
@manager.route("/upload_and_parse", methods=["POST"]) # noqa: F821 @manager.route("/upload_and_parse", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("conversation_id") @validate_request("conversation_id")
def upload_and_parse(): async def upload_and_parse():
if "file" not in request.files: files = await request.file
if "file" not in files:
return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
doc_ids = doc_upload_and_parse(request.form.get("conversation_id"), file_objs, current_user.id) form = await request.form
doc_ids = doc_upload_and_parse(form.get("conversation_id"), file_objs, current_user.id)
return get_json_result(data=doc_ids) return get_json_result(data=doc_ids)
@manager.route("/parse", methods=["POST"]) # noqa: F821 @manager.route("/parse", methods=["POST"]) # noqa: F821
@login_required @login_required
def parse(): async def parse():
url = request.json.get("url") if request.json else "" url = await request.json.get("url") if await request.json else ""
if url: if url:
if not is_valid_url(url): if not is_valid_url(url):
return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
@ -664,10 +665,11 @@ def parse():
txt = FileService.parse_docs([f], current_user.id) txt = FileService.parse_docs([f], current_user.id)
return get_json_result(data=txt) return get_json_result(data=txt)
if "file" not in request.files: files = await request.files
if "file" not in files:
return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = files.getlist("file")
txt = FileService.parse_docs(file_objs, current_user.id) txt = FileService.parse_docs(file_objs, current_user.id)
return get_json_result(data=txt) return get_json_result(data=txt)
@ -676,8 +678,8 @@ def parse():
@manager.route("/set_meta", methods=["POST"]) # noqa: F821 @manager.route("/set_meta", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("doc_id", "meta") @validate_request("doc_id", "meta")
def set_meta(): async def set_meta():
req = request.json req = await request_json()
if not DocumentService.accessible(req["doc_id"], current_user.id): if not DocumentService.accessible(req["doc_id"], current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
try: try:

View File

@ -19,8 +19,8 @@ from pathlib import Path
from api.db.services.file2document_service import File2DocumentService from api.db.services.file2document_service import File2DocumentService
from api.db.services.file_service import FileService from api.db.services.file_service import FileService
from flask import request from quart import request
from flask_login import login_required, current_user from api.apps import login_required, current_user
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
@ -33,8 +33,8 @@ from api.utils.api_utils import get_json_result
@manager.route('/convert', methods=['POST']) # noqa: F821 @manager.route('/convert', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("file_ids", "kb_ids") @validate_request("file_ids", "kb_ids")
def convert(): async def convert():
req = request.json req = await request.json
kb_ids = req["kb_ids"] kb_ids = req["kb_ids"]
file_ids = req["file_ids"] file_ids = req["file_ids"]
file2documents = [] file2documents = []
@ -103,8 +103,8 @@ def convert():
@manager.route('/rm', methods=['POST']) # noqa: F821 @manager.route('/rm', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("file_ids") @validate_request("file_ids")
def rm(): async def rm():
req = request.json req = await request.json
file_ids = req["file_ids"] file_ids = req["file_ids"]
if not file_ids: if not file_ids:
return get_json_result( return get_json_result(

View File

@ -17,10 +17,8 @@ import logging
import os import os
import pathlib import pathlib
import re import re
from quart import request, make_response
import flask from api.apps import login_required, current_user
from flask import request
from flask_login import login_required, current_user
from api.common.check_team_permission import check_file_team_permission from api.common.check_team_permission import check_file_team_permission
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
@ -40,17 +38,19 @@ from common import settings
@manager.route('/upload', methods=['POST']) # noqa: F821 @manager.route('/upload', methods=['POST']) # noqa: F821
@login_required @login_required
# @validate_request("parent_id") # @validate_request("parent_id")
def upload(): async def upload():
pf_id = request.form.get("parent_id") form = await request.form
pf_id = form.get("parent_id")
if not pf_id: if not pf_id:
root_folder = FileService.get_root_folder(current_user.id) root_folder = FileService.get_root_folder(current_user.id)
pf_id = root_folder["id"] pf_id = root_folder["id"]
if 'file' not in request.files: files = await request.files
if 'file' not in files:
return get_json_result( return get_json_result(
data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR) data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist('file') file_objs = files.getlist('file')
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == '': if file_obj.filename == '':
@ -123,10 +123,10 @@ def upload():
@manager.route('/create', methods=['POST']) # noqa: F821 @manager.route('/create', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("name") @validate_request("name")
def create(): async def create():
req = request.json req = await request.json
pf_id = request.json.get("parent_id") pf_id = await request.json.get("parent_id")
input_file_type = request.json.get("type") input_file_type = await request.json.get("type")
if not pf_id: if not pf_id:
root_folder = FileService.get_root_folder(current_user.id) root_folder = FileService.get_root_folder(current_user.id)
pf_id = root_folder["id"] pf_id = root_folder["id"]
@ -238,8 +238,8 @@ def get_all_parent_folders():
@manager.route("/rm", methods=["POST"]) # noqa: F821 @manager.route("/rm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("file_ids") @validate_request("file_ids")
def rm(): async def rm():
req = request.json req = await request.json
file_ids = req["file_ids"] file_ids = req["file_ids"]
def _delete_single_file(file): def _delete_single_file(file):
@ -299,8 +299,8 @@ def rm():
@manager.route('/rename', methods=['POST']) # noqa: F821 @manager.route('/rename', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("file_id", "name") @validate_request("file_id", "name")
def rename(): async def rename():
req = request.json req = await request.json
try: try:
e, file = FileService.get_by_id(req["file_id"]) e, file = FileService.get_by_id(req["file_id"])
if not e: if not e:
@ -338,7 +338,7 @@ def rename():
@manager.route('/get/<file_id>', methods=['GET']) # noqa: F821 @manager.route('/get/<file_id>', methods=['GET']) # noqa: F821
@login_required @login_required
def get(file_id): async def get(file_id):
try: try:
e, file = FileService.get_by_id(file_id) e, file = FileService.get_by_id(file_id)
if not e: if not e:
@ -351,7 +351,7 @@ def get(file_id):
b, n = File2DocumentService.get_storage_address(file_id=file_id) b, n = File2DocumentService.get_storage_address(file_id=file_id)
blob = settings.STORAGE_IMPL.get(b, n) blob = settings.STORAGE_IMPL.get(b, n)
response = flask.make_response(blob) response = await make_response(blob)
ext = re.search(r"\.([^.]+)$", file.name.lower()) ext = re.search(r"\.([^.]+)$", file.name.lower())
ext = ext.group(1) if ext else None ext = ext.group(1) if ext else None
if ext: if ext:
@ -368,8 +368,8 @@ def get(file_id):
@manager.route("/mv", methods=["POST"]) # noqa: F821 @manager.route("/mv", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("src_file_ids", "dest_file_id") @validate_request("src_file_ids", "dest_file_id")
def move(): async def move():
req = request.json req = await request.json
try: try:
file_ids = req["src_file_ids"] file_ids = req["src_file_ids"]
dest_parent_id = req["dest_file_id"] dest_parent_id = req["dest_file_id"]

View File

@ -18,11 +18,9 @@ import logging
import random import random
import re import re
from flask import request from quart import request
from flask_login import login_required, current_user
import numpy as np import numpy as np
from api.db.services.connector_service import Connector2KbService from api.db.services.connector_service import Connector2KbService
from api.db.services.llm_service import LLMBundle from api.db.services.llm_service import LLMBundle
from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks
@ -31,7 +29,8 @@ from api.db.services.file_service import FileService
from api.db.services.pipeline_operation_log_service import PipelineOperationLogService from api.db.services.pipeline_operation_log_service import PipelineOperationLogService
from api.db.services.task_service import TaskService, GRAPH_RAPTOR_FAKE_DOC_ID from api.db.services.task_service import TaskService, GRAPH_RAPTOR_FAKE_DOC_ID
from api.db.services.user_service import TenantService, UserTenantService from api.db.services.user_service import TenantService, UserTenantService
from api.utils.api_utils import get_error_data_result, server_error_response, get_data_error_result, validate_request, not_allowed_parameters from api.utils.api_utils import get_error_data_result, server_error_response, get_data_error_result, validate_request, not_allowed_parameters, \
request_json
from api.db import VALID_FILE_TYPES from api.db import VALID_FILE_TYPES
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.db_models import File from api.db.db_models import File
@ -42,27 +41,28 @@ from rag.utils.redis_conn import REDIS_CONN
from rag.utils.doc_store_conn import OrderByExpr from rag.utils.doc_store_conn import OrderByExpr
from common.constants import RetCode, PipelineTaskType, StatusEnum, VALID_TASK_STATUS, FileSource, LLMType, PAGERANK_FLD from common.constants import RetCode, PipelineTaskType, StatusEnum, VALID_TASK_STATUS, FileSource, LLMType, PAGERANK_FLD
from common import settings from common import settings
from api.apps import login_required, current_user
@manager.route('/create', methods=['post']) # noqa: F821 @manager.route('/create', methods=['post']) # noqa: F821
@login_required @login_required
@validate_request("name") @validate_request("name")
def create(): async def create():
req = request.json req = await request_json()
req = KnowledgebaseService.create_with_name( e, res = KnowledgebaseService.create_with_name(
name = req.pop("name", None), name = req.pop("name", None),
tenant_id = current_user.id, tenant_id = current_user.id,
parser_id = req.pop("parser_id", None), parser_id = req.pop("parser_id", None),
**req **req
) )
code = req.get("code") if not e:
if code: return res
return get_data_error_result(code=code, message=req.get("message"))
try: try:
if not KnowledgebaseService.save(**req): if not KnowledgebaseService.save(**res):
return get_data_error_result() return get_data_error_result()
return get_json_result(data={"kb_id":req["id"]}) return get_json_result(data={"kb_id":res["id"]})
except Exception as e: except Exception as e:
return server_error_response(e) return server_error_response(e)
@ -71,8 +71,8 @@ def create():
@login_required @login_required
@validate_request("kb_id", "name", "description", "parser_id") @validate_request("kb_id", "name", "description", "parser_id")
@not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by") @not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
def update(): async def update():
req = request.json req = await request_json()
if not isinstance(req["name"], str): if not isinstance(req["name"], str):
return get_data_error_result(message="Dataset name must be string.") return get_data_error_result(message="Dataset name must be string.")
if req["name"].strip() == "": if req["name"].strip() == "":
@ -170,18 +170,19 @@ def detail():
@manager.route('/list', methods=['POST']) # noqa: F821 @manager.route('/list', methods=['POST']) # noqa: F821
@login_required @login_required
def list_kbs(): async def list_kbs():
keywords = request.args.get("keywords", "") args = request.args
page_number = int(request.args.get("page", 0)) keywords = args.get("keywords", "")
items_per_page = int(request.args.get("page_size", 0)) page_number = int(args.get("page", 0))
parser_id = request.args.get("parser_id") items_per_page = int(args.get("page_size", 0))
orderby = request.args.get("orderby", "create_time") parser_id = args.get("parser_id")
if request.args.get("desc", "true").lower() == "false": orderby = args.get("orderby", "create_time")
if args.get("desc", "true").lower() == "false":
desc = False desc = False
else: else:
desc = True desc = True
req = request.get_json() req = await request_json()
owner_ids = req.get("owner_ids", []) owner_ids = req.get("owner_ids", [])
try: try:
if not owner_ids: if not owner_ids:
@ -203,11 +204,12 @@ def list_kbs():
except Exception as e: except Exception as e:
return server_error_response(e) return server_error_response(e)
@manager.route('/rm', methods=['post']) # noqa: F821 @manager.route('/rm', methods=['post']) # noqa: F821
@login_required @login_required
@validate_request("kb_id") @validate_request("kb_id")
def rm(): async def rm():
req = request.json req = await request_json()
if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id): if not KnowledgebaseService.accessible4deletion(req["kb_id"], current_user.id):
return get_json_result( return get_json_result(
data=False, data=False,
@ -283,8 +285,8 @@ def list_tags_from_kbs():
@manager.route('/<kb_id>/rm_tags', methods=['POST']) # noqa: F821 @manager.route('/<kb_id>/rm_tags', methods=['POST']) # noqa: F821
@login_required @login_required
def rm_tags(kb_id): async def rm_tags(kb_id):
req = request.json req = await request_json()
if not KnowledgebaseService.accessible(kb_id, current_user.id): if not KnowledgebaseService.accessible(kb_id, current_user.id):
return get_json_result( return get_json_result(
data=False, data=False,
@ -303,8 +305,8 @@ def rm_tags(kb_id):
@manager.route('/<kb_id>/rename_tag', methods=['POST']) # noqa: F821 @manager.route('/<kb_id>/rename_tag', methods=['POST']) # noqa: F821
@login_required @login_required
def rename_tags(kb_id): async def rename_tags(kb_id):
req = request.json req = await request_json()
if not KnowledgebaseService.accessible(kb_id, current_user.id): if not KnowledgebaseService.accessible(kb_id, current_user.id):
return get_json_result( return get_json_result(
data=False, data=False,
@ -407,7 +409,7 @@ def get_basic_info():
@manager.route("/list_pipeline_logs", methods=["POST"]) # noqa: F821 @manager.route("/list_pipeline_logs", methods=["POST"]) # noqa: F821
@login_required @login_required
def list_pipeline_logs(): async def list_pipeline_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
@ -426,7 +428,7 @@ def list_pipeline_logs():
if create_date_to > create_date_from: if create_date_to > create_date_from:
return get_data_error_result(message="Create data filter is abnormal.") return get_data_error_result(message="Create data filter is abnormal.")
req = request.get_json() req = await request_json()
operation_status = req.get("operation_status", []) operation_status = req.get("operation_status", [])
if operation_status: if operation_status:
@ -451,7 +453,7 @@ def list_pipeline_logs():
@manager.route("/list_pipeline_dataset_logs", methods=["POST"]) # noqa: F821 @manager.route("/list_pipeline_dataset_logs", methods=["POST"]) # noqa: F821
@login_required @login_required
def list_pipeline_dataset_logs(): async def list_pipeline_dataset_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
@ -468,7 +470,7 @@ def list_pipeline_dataset_logs():
if create_date_to > create_date_from: if create_date_to > create_date_from:
return get_data_error_result(message="Create data filter is abnormal.") return get_data_error_result(message="Create data filter is abnormal.")
req = request.get_json() req = await request_json()
operation_status = req.get("operation_status", []) operation_status = req.get("operation_status", [])
if operation_status: if operation_status:
@ -485,12 +487,12 @@ def list_pipeline_dataset_logs():
@manager.route("/delete_pipeline_logs", methods=["POST"]) # noqa: F821 @manager.route("/delete_pipeline_logs", methods=["POST"]) # noqa: F821
@login_required @login_required
def delete_pipeline_logs(): async def delete_pipeline_logs():
kb_id = request.args.get("kb_id") kb_id = request.args.get("kb_id")
if not kb_id: if not kb_id:
return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR) return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
req = request.get_json() req = await request_json()
log_ids = req.get("log_ids", []) log_ids = req.get("log_ids", [])
PipelineOperationLogService.delete_by_ids(log_ids) PipelineOperationLogService.delete_by_ids(log_ids)
@ -514,8 +516,8 @@ def pipeline_log_detail():
@manager.route("/run_graphrag", methods=["POST"]) # noqa: F821 @manager.route("/run_graphrag", methods=["POST"]) # noqa: F821
@login_required @login_required
def run_graphrag(): async def run_graphrag():
req = request.json req = await request_json()
kb_id = req.get("kb_id", "") kb_id = req.get("kb_id", "")
if not kb_id: if not kb_id:
@ -583,8 +585,8 @@ def trace_graphrag():
@manager.route("/run_raptor", methods=["POST"]) # noqa: F821 @manager.route("/run_raptor", methods=["POST"]) # noqa: F821
@login_required @login_required
def run_raptor(): async def run_raptor():
req = request.json req = await request_json()
kb_id = req.get("kb_id", "") kb_id = req.get("kb_id", "")
if not kb_id: if not kb_id:
@ -652,8 +654,8 @@ def trace_raptor():
@manager.route("/run_mindmap", methods=["POST"]) # noqa: F821 @manager.route("/run_mindmap", methods=["POST"]) # noqa: F821
@login_required @login_required
def run_mindmap(): async def run_mindmap():
req = request.json req = await request_json()
kb_id = req.get("kb_id", "") kb_id = req.get("kb_id", "")
if not kb_id: if not kb_id:
@ -768,7 +770,7 @@ def delete_kb_task():
@manager.route("/check_embedding", methods=["post"]) # noqa: F821 @manager.route("/check_embedding", methods=["post"]) # noqa: F821
@login_required @login_required
def check_embedding(): async def check_embedding():
def _guess_vec_field(src: dict) -> str | None: def _guess_vec_field(src: dict) -> str | None:
for k in src or {}: for k in src or {}:
@ -819,7 +821,7 @@ def check_embedding():
return [] return []
n = min(n, total) n = min(n, total)
offsets = sorted(random.sample(range(total), n)) offsets = sorted(random.sample(range(min(total,1000)), n))
out = [] out = []
for off in offsets: for off in offsets:
@ -859,7 +861,7 @@ def check_embedding():
def _clean(s: str) -> str: def _clean(s: str) -> str:
s = re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", s or "") s = re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", s or "")
return s if s else "None" return s if s else "None"
req = request.json req = await request_json()
kb_id = req.get("kb_id", "") kb_id = req.get("kb_id", "")
embd_id = req.get("embd_id", "") embd_id = req.get("embd_id", "")
n = int(req.get("check_num", 5)) n = int(req.get("check_num", 5))
@ -884,6 +886,7 @@ def check_embedding():
try: try:
v, _ = emb_mdl.encode([title, txt_in]) v, _ = emb_mdl.encode([title, txt_in])
assert len(v[1]) == len(ck["vector"]), f"The dimension ({len(v[1])}) of given embedding model is different from the original ({len(ck['vector'])})"
sim_content = _cos_sim(v[1], ck["vector"]) sim_content = _cos_sim(v[1], ck["vector"])
title_w = 0.1 title_w = 0.1
qv_mix = title_w * v[0] + (1 - title_w) * v[1] qv_mix = title_w * v[0] + (1 - title_w) * v[1]
@ -893,8 +896,8 @@ def check_embedding():
if sim_mix > sim: if sim_mix > sim:
sim = sim_mix sim = sim_mix
mode = "title+content" mode = "title+content"
except Exception: except Exception as e:
return get_error_data_result(message="embedding failure") return get_error_data_result(message=f"Embedding failure. {e}")
eff_sims.append(sim) eff_sims.append(sim)
results.append({ results.append({

View File

@ -15,8 +15,8 @@
# #
from flask import request from quart import request
from flask_login import current_user, login_required from api.apps import current_user, login_required
from langfuse import Langfuse from langfuse import Langfuse
from api.db.db_models import DB from api.db.db_models import DB
@ -27,8 +27,8 @@ from api.utils.api_utils import get_error_data_result, get_json_result, server_e
@manager.route("/api_key", methods=["POST", "PUT"]) # noqa: F821 @manager.route("/api_key", methods=["POST", "PUT"]) # noqa: F821
@login_required @login_required
@validate_request("secret_key", "public_key", "host") @validate_request("secret_key", "public_key", "host")
def set_api_key(): async def set_api_key():
req = request.get_json() req = await request.get_json()
secret_key = req.get("secret_key", "") secret_key = req.get("secret_key", "")
public_key = req.get("public_key", "") public_key = req.get("public_key", "")
host = req.get("host", "") host = req.get("host", "")

View File

@ -16,8 +16,9 @@
import logging import logging
import json import json
import os import os
from flask import request from quart import request
from flask_login import login_required, current_user
from api.apps import login_required, current_user
from api.db.services.tenant_llm_service import LLMFactoriesService, TenantLLMService from api.db.services.tenant_llm_service import LLMFactoriesService, TenantLLMService
from api.db.services.llm_service import LLMService from api.db.services.llm_service import LLMService
from api.utils.api_utils import server_error_response, get_data_error_result, validate_request from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
@ -52,8 +53,8 @@ def factories():
@manager.route("/set_api_key", methods=["POST"]) # noqa: F821 @manager.route("/set_api_key", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("llm_factory", "api_key") @validate_request("llm_factory", "api_key")
def set_api_key(): async def set_api_key():
req = request.json req = await request.json
# test if api key works # test if api key works
chat_passed, embd_passed, rerank_passed = False, False, False chat_passed, embd_passed, rerank_passed = False, False, False
factory = req["llm_factory"] factory = req["llm_factory"]
@ -122,8 +123,8 @@ def set_api_key():
@manager.route("/add_llm", methods=["POST"]) # noqa: F821 @manager.route("/add_llm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("llm_factory") @validate_request("llm_factory")
def add_llm(): async def add_llm():
req = request.json req = await request.json
factory = req["llm_factory"] factory = req["llm_factory"]
api_key = req.get("api_key", "x") api_key = req.get("api_key", "x")
llm_name = req.get("llm_name") llm_name = req.get("llm_name")
@ -142,11 +143,11 @@ def add_llm():
elif factory == "Tencent Hunyuan": elif factory == "Tencent Hunyuan":
req["api_key"] = apikey_json(["hunyuan_sid", "hunyuan_sk"]) req["api_key"] = apikey_json(["hunyuan_sid", "hunyuan_sk"])
return set_api_key() return await set_api_key()
elif factory == "Tencent Cloud": elif factory == "Tencent Cloud":
req["api_key"] = apikey_json(["tencent_cloud_sid", "tencent_cloud_sk"]) req["api_key"] = apikey_json(["tencent_cloud_sid", "tencent_cloud_sk"])
return set_api_key() return await set_api_key()
elif factory == "Bedrock": elif factory == "Bedrock":
# For Bedrock, due to its special authentication method # For Bedrock, due to its special authentication method
@ -267,8 +268,8 @@ def add_llm():
@manager.route("/delete_llm", methods=["POST"]) # noqa: F821 @manager.route("/delete_llm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("llm_factory", "llm_name") @validate_request("llm_factory", "llm_name")
def delete_llm(): async def delete_llm():
req = request.json req = await request.json
TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]]) TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]])
return get_json_result(data=True) return get_json_result(data=True)
@ -276,8 +277,8 @@ def delete_llm():
@manager.route("/enable_llm", methods=["POST"]) # noqa: F821 @manager.route("/enable_llm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("llm_factory", "llm_name") @validate_request("llm_factory", "llm_name")
def enable_llm(): async def enable_llm():
req = request.json req = await request.json
TenantLLMService.filter_update( TenantLLMService.filter_update(
[TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]], {"status": str(req.get("status", "1"))} [TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"], TenantLLM.llm_name == req["llm_name"]], {"status": str(req.get("status", "1"))}
) )
@ -287,8 +288,8 @@ def enable_llm():
@manager.route("/delete_factory", methods=["POST"]) # noqa: F821 @manager.route("/delete_factory", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("llm_factory") @validate_request("llm_factory")
def delete_factory(): async def delete_factory():
req = request.json req = await request.json
TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"]]) TenantLLMService.filter_delete([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == req["llm_factory"]])
return get_json_result(data=True) return get_json_result(data=True)

View File

@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
from flask import Response, request from quart import Response, request
from flask_login import current_user, login_required from api.apps import current_user, login_required
from api.db.db_models import MCPServer from api.db.db_models import MCPServer
from api.db.services.mcp_server_service import MCPServerService from api.db.services.mcp_server_service import MCPServerService
@ -30,7 +30,7 @@ from common.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_too
@manager.route("/list", methods=["POST"]) # noqa: F821 @manager.route("/list", methods=["POST"]) # noqa: F821
@login_required @login_required
def list_mcp() -> Response: async def list_mcp() -> Response:
keywords = request.args.get("keywords", "") keywords = request.args.get("keywords", "")
page_number = int(request.args.get("page", 0)) page_number = int(request.args.get("page", 0))
items_per_page = int(request.args.get("page_size", 0)) items_per_page = int(request.args.get("page_size", 0))
@ -40,7 +40,7 @@ def list_mcp() -> Response:
else: else:
desc = True desc = True
req = request.get_json() req = await request.get_json()
mcp_ids = req.get("mcp_ids", []) mcp_ids = req.get("mcp_ids", [])
try: try:
servers = MCPServerService.get_servers(current_user.id, mcp_ids, 0, 0, orderby, desc, keywords) or [] servers = MCPServerService.get_servers(current_user.id, mcp_ids, 0, 0, orderby, desc, keywords) or []
@ -72,8 +72,8 @@ def detail() -> Response:
@manager.route("/create", methods=["POST"]) # noqa: F821 @manager.route("/create", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("name", "url", "server_type") @validate_request("name", "url", "server_type")
def create() -> Response: async def create() -> Response:
req = request.get_json() req = await request.get_json()
server_type = req.get("server_type", "") server_type = req.get("server_type", "")
if server_type not in VALID_MCP_SERVER_TYPES: if server_type not in VALID_MCP_SERVER_TYPES:
@ -127,8 +127,8 @@ def create() -> Response:
@manager.route("/update", methods=["POST"]) # noqa: F821 @manager.route("/update", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_id") @validate_request("mcp_id")
def update() -> Response: async def update() -> Response:
req = request.get_json() req = await request.get_json()
mcp_id = req.get("mcp_id", "") mcp_id = req.get("mcp_id", "")
e, mcp_server = MCPServerService.get_by_id(mcp_id) e, mcp_server = MCPServerService.get_by_id(mcp_id)
@ -183,8 +183,8 @@ def update() -> Response:
@manager.route("/rm", methods=["POST"]) # noqa: F821 @manager.route("/rm", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_ids") @validate_request("mcp_ids")
def rm() -> Response: async def rm() -> Response:
req = request.get_json() req = await request.get_json()
mcp_ids = req.get("mcp_ids", []) mcp_ids = req.get("mcp_ids", [])
try: try:
@ -201,8 +201,8 @@ def rm() -> Response:
@manager.route("/import", methods=["POST"]) # noqa: F821 @manager.route("/import", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcpServers") @validate_request("mcpServers")
def import_multiple() -> Response: async def import_multiple() -> Response:
req = request.get_json() req = await request.get_json()
servers = req.get("mcpServers", {}) servers = req.get("mcpServers", {})
if not servers: if not servers:
return get_data_error_result(message="No MCP servers provided.") return get_data_error_result(message="No MCP servers provided.")
@ -268,8 +268,8 @@ def import_multiple() -> Response:
@manager.route("/export", methods=["POST"]) # noqa: F821 @manager.route("/export", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_ids") @validate_request("mcp_ids")
def export_multiple() -> Response: async def export_multiple() -> Response:
req = request.get_json() req = await request.get_json()
mcp_ids = req.get("mcp_ids", []) mcp_ids = req.get("mcp_ids", [])
if not mcp_ids: if not mcp_ids:
@ -300,8 +300,8 @@ def export_multiple() -> Response:
@manager.route("/list_tools", methods=["POST"]) # noqa: F821 @manager.route("/list_tools", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_ids") @validate_request("mcp_ids")
def list_tools() -> Response: async def list_tools() -> Response:
req = request.get_json() req = await request.get_json()
mcp_ids = req.get("mcp_ids", []) mcp_ids = req.get("mcp_ids", [])
if not mcp_ids: if not mcp_ids:
return get_data_error_result(message="No MCP server IDs provided.") return get_data_error_result(message="No MCP server IDs provided.")
@ -347,8 +347,8 @@ def list_tools() -> Response:
@manager.route("/test_tool", methods=["POST"]) # noqa: F821 @manager.route("/test_tool", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_id", "tool_name", "arguments") @validate_request("mcp_id", "tool_name", "arguments")
def test_tool() -> Response: async def test_tool() -> Response:
req = request.get_json() req = await request.get_json()
mcp_id = req.get("mcp_id", "") mcp_id = req.get("mcp_id", "")
if not mcp_id: if not mcp_id:
return get_data_error_result(message="No MCP server ID provided.") return get_data_error_result(message="No MCP server ID provided.")
@ -380,8 +380,8 @@ def test_tool() -> Response:
@manager.route("/cache_tools", methods=["POST"]) # noqa: F821 @manager.route("/cache_tools", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("mcp_id", "tools") @validate_request("mcp_id", "tools")
def cache_tool() -> Response: async def cache_tool() -> Response:
req = request.get_json() req = await request.get_json()
mcp_id = req.get("mcp_id", "") mcp_id = req.get("mcp_id", "")
if not mcp_id: if not mcp_id:
return get_data_error_result(message="No MCP server ID provided.") return get_data_error_result(message="No MCP server ID provided.")
@ -403,8 +403,8 @@ def cache_tool() -> Response:
@manager.route("/test_mcp", methods=["POST"]) # noqa: F821 @manager.route("/test_mcp", methods=["POST"]) # noqa: F821
@validate_request("url", "server_type") @validate_request("url", "server_type")
def test_mcp() -> Response: async def test_mcp() -> Response:
req = request.get_json() req = await request.get_json()
url = req.get("url", "") url = req.get("url", "")
if not url: if not url:

View File

@ -15,8 +15,8 @@
# #
from flask import Response from quart import Response
from flask_login import login_required from api.apps import login_required
from api.utils.api_utils import get_json_result from api.utils.api_utils import get_json_result
from plugin import GlobalPluginManager from plugin import GlobalPluginManager

View File

@ -27,7 +27,7 @@ from common.constants import RetCode
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
from api.utils.api_utils import get_result from api.utils.api_utils import get_result
from flask import request, Response from quart import request, Response
@manager.route('/agents', methods=['GET']) # noqa: F821 @manager.route('/agents', methods=['GET']) # noqa: F821
@ -52,8 +52,8 @@ def list_agents(tenant_id):
@manager.route("/agents", methods=["POST"]) # noqa: F821 @manager.route("/agents", methods=["POST"]) # noqa: F821
@token_required @token_required
def create_agent(tenant_id: str): async def create_agent(tenant_id: str):
req: dict[str, Any] = cast(dict[str, Any], request.json) req: dict[str, Any] = cast(dict[str, Any], await request.json)
req["user_id"] = tenant_id req["user_id"] = tenant_id
if req.get("dsl") is not None: if req.get("dsl") is not None:
@ -89,8 +89,8 @@ def create_agent(tenant_id: str):
@manager.route("/agents/<agent_id>", methods=["PUT"]) # noqa: F821 @manager.route("/agents/<agent_id>", methods=["PUT"]) # noqa: F821
@token_required @token_required
def update_agent(tenant_id: str, agent_id: str): async def update_agent(tenant_id: str, agent_id: str):
req: dict[str, Any] = {k: v for k, v in cast(dict[str, Any], request.json).items() if v is not None} req: dict[str, Any] = {k: v for k, v in cast(dict[str, Any], (await request.json)).items() if v is not None}
req["user_id"] = tenant_id req["user_id"] = tenant_id
if req.get("dsl") is not None: if req.get("dsl") is not None:
@ -135,8 +135,8 @@ def delete_agent(tenant_id: str, agent_id: str):
@manager.route('/webhook/<agent_id>', methods=['POST']) # noqa: F821 @manager.route('/webhook/<agent_id>', methods=['POST']) # noqa: F821
@token_required @token_required
def webhook(tenant_id: str, agent_id: str): async def webhook(tenant_id: str, agent_id: str):
req = request.json req = await request.json
if not UserCanvasService.accessible(req["id"], tenant_id): if not UserCanvasService.accessible(req["id"], tenant_id):
return get_json_result( return get_json_result(
data=False, message='Only owner of canvas authorized for this operation.', data=False, message='Only owner of canvas authorized for this operation.',

View File

@ -14,22 +14,20 @@
# limitations under the License. # limitations under the License.
# #
import logging import logging
from quart import request
from flask import request
from api.db.services.dialog_service import DialogService from api.db.services.dialog_service import DialogService
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.tenant_llm_service import TenantLLMService from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.constants import RetCode, StatusEnum from common.constants import RetCode, StatusEnum
from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required, request_json
@manager.route("/chats", methods=["POST"]) # noqa: F821 @manager.route("/chats", methods=["POST"]) # noqa: F821
@token_required @token_required
def create(tenant_id): async def create(tenant_id):
req = request.json req = await request_json()
ids = [i for i in req.get("dataset_ids", []) if i] ids = [i for i in req.get("dataset_ids", []) if i]
for kb_id in ids: for kb_id in ids:
kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id) kbs = KnowledgebaseService.accessible(kb_id=kb_id, user_id=tenant_id)
@ -145,10 +143,10 @@ def create(tenant_id):
@manager.route("/chats/<chat_id>", methods=["PUT"]) # noqa: F821 @manager.route("/chats/<chat_id>", methods=["PUT"]) # noqa: F821
@token_required @token_required
def update(tenant_id, chat_id): async def update(tenant_id, chat_id):
if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value): if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
return get_error_data_result(message="You do not own the chat") return get_error_data_result(message="You do not own the chat")
req = request.json req = await request_json()
ids = req.get("dataset_ids", []) ids = req.get("dataset_ids", [])
if "show_quotation" in req: if "show_quotation" in req:
req["do_refer"] = req.pop("show_quotation") req["do_refer"] = req.pop("show_quotation")
@ -228,10 +226,10 @@ def update(tenant_id, chat_id):
@manager.route("/chats", methods=["DELETE"]) # noqa: F821 @manager.route("/chats", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def delete(tenant_id): async def delete_chats(tenant_id):
errors = [] errors = []
success_count = 0 success_count = 0
req = request.json req = await request_json()
if not req: if not req:
ids = None ids = None
else: else:
@ -251,8 +249,8 @@ def delete(tenant_id):
errors.append(f"Assistant({id}) not found.") errors.append(f"Assistant({id}) not found.")
continue continue
temp_dict = {"status": StatusEnum.INVALID.value} temp_dict = {"status": StatusEnum.INVALID.value}
DialogService.update_by_id(id, temp_dict) success_count += DialogService.update_by_id(id, temp_dict)
success_count += 1 print(success_count, "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$", flush=True)
if errors: if errors:
if success_count > 0: if success_count > 0:

View File

@ -18,7 +18,7 @@
import logging import logging
import os import os
import json import json
from flask import request from quart import request
from peewee import OperationalError from peewee import OperationalError
from api.db.db_models import File from api.db.db_models import File
from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks
@ -54,7 +54,7 @@ from common import settings
@manager.route("/datasets", methods=["POST"]) # noqa: F821 @manager.route("/datasets", methods=["POST"]) # noqa: F821
@token_required @token_required
def create(tenant_id): async def create(tenant_id):
""" """
Create a new dataset. Create a new dataset.
--- ---
@ -116,16 +116,19 @@ def create(tenant_id):
# | embedding_model| embd_id | # | embedding_model| embd_id |
# | chunk_method | parser_id | # | chunk_method | parser_id |
req, err = validate_and_parse_json_request(request, CreateDatasetReq) req, err = await validate_and_parse_json_request(request, CreateDatasetReq)
if err is not None: if err is not None:
return get_error_argument_result(err) return get_error_argument_result(err)
req = KnowledgebaseService.create_with_name( e, req = KnowledgebaseService.create_with_name(
name = req.pop("name", None), name = req.pop("name", None),
tenant_id = tenant_id, tenant_id = tenant_id,
parser_id = req.pop("parser_id", None), parser_id = req.pop("parser_id", None),
**req **req
) )
if not e:
return req
# Insert embedding model(embd id) # Insert embedding model(embd id)
ok, t = TenantService.get_by_id(tenant_id) ok, t = TenantService.get_by_id(tenant_id)
if not ok: if not ok:
@ -152,7 +155,7 @@ def create(tenant_id):
@manager.route("/datasets", methods=["DELETE"]) # noqa: F821 @manager.route("/datasets", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def delete(tenant_id): async def delete(tenant_id):
""" """
Delete datasets. Delete datasets.
--- ---
@ -190,7 +193,7 @@ def delete(tenant_id):
schema: schema:
type: object type: object
""" """
req, err = validate_and_parse_json_request(request, DeleteDatasetReq) req, err = await validate_and_parse_json_request(request, DeleteDatasetReq)
if err is not None: if err is not None:
return get_error_argument_result(err) return get_error_argument_result(err)
@ -250,7 +253,7 @@ def delete(tenant_id):
@manager.route("/datasets/<dataset_id>", methods=["PUT"]) # noqa: F821 @manager.route("/datasets/<dataset_id>", methods=["PUT"]) # noqa: F821
@token_required @token_required
def update(tenant_id, dataset_id): async def update(tenant_id, dataset_id):
""" """
Update a dataset. Update a dataset.
--- ---
@ -316,7 +319,7 @@ def update(tenant_id, dataset_id):
# | embedding_model| embd_id | # | embedding_model| embd_id |
# | chunk_method | parser_id | # | chunk_method | parser_id |
extras = {"dataset_id": dataset_id} extras = {"dataset_id": dataset_id}
req, err = validate_and_parse_json_request(request, UpdateDatasetReq, extras=extras, exclude_unset=True) req, err = await validate_and_parse_json_request(request, UpdateDatasetReq, extras=extras, exclude_unset=True)
if err is not None: if err is not None:
return get_error_argument_result(err) return get_error_argument_result(err)

View File

@ -15,7 +15,7 @@
# #
import logging import logging
from flask import request, jsonify from quart import request, jsonify
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.knowledgebase_service import KnowledgebaseService
@ -29,7 +29,7 @@ from common import settings
@manager.route('/dify/retrieval', methods=['POST']) # noqa: F821 @manager.route('/dify/retrieval', methods=['POST']) # noqa: F821
@apikey_required @apikey_required
@validate_request("knowledge_id", "query") @validate_request("knowledge_id", "query")
def retrieval(tenant_id): async def retrieval(tenant_id):
""" """
Dify-compatible retrieval API Dify-compatible retrieval API
--- ---
@ -113,7 +113,7 @@ def retrieval(tenant_id):
404: 404:
description: Knowledge base or document not found description: Knowledge base or document not found
""" """
req = request.json req = await request.json
question = req["query"] question = req["query"]
kb_id = req["knowledge_id"] kb_id = req["knowledge_id"]
use_kg = req.get("use_kg", False) use_kg = req.get("use_kg", False)
@ -131,12 +131,10 @@ def retrieval(tenant_id):
return build_error_result(message="Knowledgebase not found!", code=RetCode.NOT_FOUND) return build_error_result(message="Knowledgebase not found!", code=RetCode.NOT_FOUND)
embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id) embd_mdl = LLMBundle(kb.tenant_id, LLMType.EMBEDDING.value, llm_name=kb.embd_id)
print(metadata_condition) if metadata_condition:
# print("after", convert_conditions(metadata_condition)) doc_ids.extend(meta_filter(metas, convert_conditions(metadata_condition)))
doc_ids.extend(meta_filter(metas, convert_conditions(metadata_condition))) if not doc_ids and metadata_condition:
# print("doc_ids", doc_ids) doc_ids = ["-999"]
if not doc_ids and metadata_condition is not None:
doc_ids = ['-999']
ranks = settings.retriever.retrieval( ranks = settings.retriever.retrieval(
question, question,
embd_mdl, embd_mdl,

View File

@ -20,7 +20,7 @@ import re
from io import BytesIO from io import BytesIO
import xxhash import xxhash
from flask import request, send_file from quart import request, send_file
from peewee import OperationalError from peewee import OperationalError
from pydantic import BaseModel, Field, validator from pydantic import BaseModel, Field, validator
@ -35,7 +35,8 @@ from api.db.services.llm_service import LLMBundle
from api.db.services.tenant_llm_service import TenantLLMService from api.db.services.tenant_llm_service import TenantLLMService
from api.db.services.task_service import TaskService, queue_tasks from api.db.services.task_service import TaskService, queue_tasks
from api.db.services.dialog_service import meta_filter, convert_conditions from api.db.services.dialog_service import meta_filter, convert_conditions
from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required from api.utils.api_utils import check_duplicate_ids, construct_json_result, get_error_data_result, get_parser_config, get_result, server_error_response, token_required, \
request_json
from rag.app.qa import beAdoc, rmPrefix from rag.app.qa import beAdoc, rmPrefix
from rag.app.tag import label_question from rag.app.tag import label_question
from rag.nlp import rag_tokenizer, search from rag.nlp import rag_tokenizer, search
@ -69,7 +70,7 @@ class Chunk(BaseModel):
@manager.route("/datasets/<dataset_id>/documents", methods=["POST"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/documents", methods=["POST"]) # noqa: F821
@token_required @token_required
def upload(dataset_id, tenant_id): async def upload(dataset_id, tenant_id):
""" """
Upload documents to a dataset. Upload documents to a dataset.
--- ---
@ -130,9 +131,11 @@ def upload(dataset_id, tenant_id):
type: string type: string
description: Processing status. description: Processing status.
""" """
if "file" not in request.files: form = await request.form
files = await request.files
if "file" not in files:
return get_error_data_result(message="No file part!", code=RetCode.ARGUMENT_ERROR) return get_error_data_result(message="No file part!", code=RetCode.ARGUMENT_ERROR)
file_objs = request.files.getlist("file") file_objs = files.getlist("file")
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == "": if file_obj.filename == "":
return get_result(message="No file selected!", code=RetCode.ARGUMENT_ERROR) return get_result(message="No file selected!", code=RetCode.ARGUMENT_ERROR)
@ -155,7 +158,7 @@ def upload(dataset_id, tenant_id):
e, kb = KnowledgebaseService.get_by_id(dataset_id) e, kb = KnowledgebaseService.get_by_id(dataset_id)
if not e: if not e:
raise LookupError(f"Can't find the dataset with ID {dataset_id}!") raise LookupError(f"Can't find the dataset with ID {dataset_id}!")
err, files = FileService.upload_document(kb, file_objs, tenant_id, parent_path=request.form.get("parent_path")) err, files = FileService.upload_document(kb, file_objs, tenant_id, parent_path=form.get("parent_path"))
if err: if err:
return get_result(message="\n".join(err), code=RetCode.SERVER_ERROR) return get_result(message="\n".join(err), code=RetCode.SERVER_ERROR)
# rename key's name # rename key's name
@ -179,7 +182,7 @@ def upload(dataset_id, tenant_id):
@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["PUT"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["PUT"]) # noqa: F821
@token_required @token_required
def update_doc(tenant_id, dataset_id, document_id): async def update_doc(tenant_id, dataset_id, document_id):
""" """
Update a document within a dataset. Update a document within a dataset.
--- ---
@ -228,7 +231,7 @@ def update_doc(tenant_id, dataset_id, document_id):
schema: schema:
type: object type: object
""" """
req = request.json req = await request_json()
if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id): if not KnowledgebaseService.query(id=dataset_id, tenant_id=tenant_id):
return get_error_data_result(message="You don't own the dataset.") return get_error_data_result(message="You don't own the dataset.")
e, kb = KnowledgebaseService.get_by_id(dataset_id) e, kb = KnowledgebaseService.get_by_id(dataset_id)
@ -359,7 +362,7 @@ def update_doc(tenant_id, dataset_id, document_id):
@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["GET"]) # noqa: F821
@token_required @token_required
def download(tenant_id, dataset_id, document_id): async def download(tenant_id, dataset_id, document_id):
""" """
Download a document from a dataset. Download a document from a dataset.
--- ---
@ -409,10 +412,10 @@ def download(tenant_id, dataset_id, document_id):
return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR) return construct_json_result(message="This file is empty.", code=RetCode.DATA_ERROR)
file = BytesIO(file_stream) file = BytesIO(file_stream)
# Use send_file with a proper filename and MIME type # Use send_file with a proper filename and MIME type
return send_file( return await send_file(
file, file,
as_attachment=True, as_attachment=True,
download_name=doc[0].name, attachment_filename=doc[0].name,
mimetype="application/octet-stream", # Set a default MIME type mimetype="application/octet-stream", # Set a default MIME type
) )
@ -589,7 +592,7 @@ def list_docs(dataset_id, tenant_id):
@manager.route("/datasets/<dataset_id>/documents", methods=["DELETE"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/documents", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def delete(tenant_id, dataset_id): async def delete(tenant_id, dataset_id):
""" """
Delete documents from a dataset. Delete documents from a dataset.
--- ---
@ -628,7 +631,7 @@ def delete(tenant_id, dataset_id):
""" """
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id): if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ") return get_error_data_result(message=f"You don't own the dataset {dataset_id}. ")
req = request.json req = await request_json()
if not req: if not req:
doc_ids = None doc_ids = None
else: else:
@ -699,7 +702,7 @@ def delete(tenant_id, dataset_id):
@manager.route("/datasets/<dataset_id>/chunks", methods=["POST"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/chunks", methods=["POST"]) # noqa: F821
@token_required @token_required
def parse(tenant_id, dataset_id): async def parse(tenant_id, dataset_id):
""" """
Start parsing documents into chunks. Start parsing documents into chunks.
--- ---
@ -738,7 +741,7 @@ def parse(tenant_id, dataset_id):
""" """
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id): if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.") return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
req = request.json req = await request_json()
if not req.get("document_ids"): if not req.get("document_ids"):
return get_error_data_result("`document_ids` is required") return get_error_data_result("`document_ids` is required")
doc_list = req.get("document_ids") doc_list = req.get("document_ids")
@ -782,7 +785,7 @@ def parse(tenant_id, dataset_id):
@manager.route("/datasets/<dataset_id>/chunks", methods=["DELETE"]) # noqa: F821 @manager.route("/datasets/<dataset_id>/chunks", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def stop_parsing(tenant_id, dataset_id): async def stop_parsing(tenant_id, dataset_id):
""" """
Stop parsing documents into chunks. Stop parsing documents into chunks.
--- ---
@ -821,7 +824,7 @@ def stop_parsing(tenant_id, dataset_id):
""" """
if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id): if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
return get_error_data_result(message=f"You don't own the dataset {dataset_id}.") return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
req = request.json req = await request_json()
if not req.get("document_ids"): if not req.get("document_ids"):
return get_error_data_result("`document_ids` is required") return get_error_data_result("`document_ids` is required")
@ -1023,7 +1026,7 @@ def list_chunks(tenant_id, dataset_id, document_id):
"/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["POST"] "/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["POST"]
) )
@token_required @token_required
def add_chunk(tenant_id, dataset_id, document_id): async def add_chunk(tenant_id, dataset_id, document_id):
""" """
Add a chunk to a document. Add a chunk to a document.
--- ---
@ -1093,7 +1096,7 @@ def add_chunk(tenant_id, dataset_id, document_id):
if not doc: if not doc:
return get_error_data_result(message=f"You don't own the document {document_id}.") return get_error_data_result(message=f"You don't own the document {document_id}.")
doc = doc[0] doc = doc[0]
req = request.json req = await request_json()
if not str(req.get("content", "")).strip(): if not str(req.get("content", "")).strip():
return get_error_data_result(message="`content` is required") return get_error_data_result(message="`content` is required")
if "important_keywords" in req: if "important_keywords" in req:
@ -1152,7 +1155,7 @@ def add_chunk(tenant_id, dataset_id, document_id):
"datasets/<dataset_id>/documents/<document_id>/chunks", methods=["DELETE"] "datasets/<dataset_id>/documents/<document_id>/chunks", methods=["DELETE"]
) )
@token_required @token_required
def rm_chunk(tenant_id, dataset_id, document_id): async def rm_chunk(tenant_id, dataset_id, document_id):
""" """
Remove chunks from a document. Remove chunks from a document.
--- ---
@ -1199,7 +1202,7 @@ def rm_chunk(tenant_id, dataset_id, document_id):
docs = DocumentService.get_by_ids([document_id]) docs = DocumentService.get_by_ids([document_id])
if not docs: if not docs:
raise LookupError(f"Can't find the document with ID {document_id}!") raise LookupError(f"Can't find the document with ID {document_id}!")
req = request.json req = await request_json()
condition = {"doc_id": document_id} condition = {"doc_id": document_id}
if "chunk_ids" in req: if "chunk_ids" in req:
unique_chunk_ids, duplicate_messages = check_duplicate_ids(req["chunk_ids"], "chunk") unique_chunk_ids, duplicate_messages = check_duplicate_ids(req["chunk_ids"], "chunk")
@ -1223,7 +1226,7 @@ def rm_chunk(tenant_id, dataset_id, document_id):
"/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["PUT"] "/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["PUT"]
) )
@token_required @token_required
def update_chunk(tenant_id, dataset_id, document_id, chunk_id): async def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
""" """
Update a chunk within a document. Update a chunk within a document.
--- ---
@ -1285,7 +1288,7 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
if not doc: if not doc:
return get_error_data_result(message=f"You don't own the document {document_id}.") return get_error_data_result(message=f"You don't own the document {document_id}.")
doc = doc[0] doc = doc[0]
req = request.json req = await request_json()
if "content" in req: if "content" in req:
content = req["content"] content = req["content"]
else: else:
@ -1327,7 +1330,7 @@ def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
@manager.route("/retrieval", methods=["POST"]) # noqa: F821 @manager.route("/retrieval", methods=["POST"]) # noqa: F821
@token_required @token_required
def retrieval_test(tenant_id): async def retrieval_test(tenant_id):
""" """
Retrieve chunks based on a query. Retrieve chunks based on a query.
--- ---
@ -1408,7 +1411,7 @@ def retrieval_test(tenant_id):
format: float format: float
description: Similarity score. description: Similarity score.
""" """
req = request.json req = await request_json()
if not req.get("dataset_ids"): if not req.get("dataset_ids"):
return get_error_data_result("`dataset_ids` is required.") return get_error_data_result("`dataset_ids` is required.")
kb_ids = req["dataset_ids"] kb_ids = req["dataset_ids"]

View File

@ -17,9 +17,7 @@
import pathlib import pathlib
import re import re
from quart import request, make_response
import flask
from flask import request
from pathlib import Path from pathlib import Path
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
@ -37,7 +35,7 @@ from common import settings
@manager.route('/file/upload', methods=['POST']) # noqa: F821 @manager.route('/file/upload', methods=['POST']) # noqa: F821
@token_required @token_required
def upload(tenant_id): async def upload(tenant_id):
""" """
Upload a file to the system. Upload a file to the system.
--- ---
@ -79,15 +77,17 @@ def upload(tenant_id):
type: string type: string
description: File type (e.g., document, folder) description: File type (e.g., document, folder)
""" """
pf_id = request.form.get("parent_id") form = await request.form
files = await request.files
pf_id = form.get("parent_id")
if not pf_id: if not pf_id:
root_folder = FileService.get_root_folder(tenant_id) root_folder = FileService.get_root_folder(tenant_id)
pf_id = root_folder["id"] pf_id = root_folder["id"]
if 'file' not in request.files: if 'file' not in files:
return get_json_result(data=False, message='No file part!', code=400) return get_json_result(data=False, message='No file part!', code=400)
file_objs = request.files.getlist('file') file_objs = files.getlist('file')
for file_obj in file_objs: for file_obj in file_objs:
if file_obj.filename == '': if file_obj.filename == '':
@ -151,7 +151,7 @@ def upload(tenant_id):
@manager.route('/file/create', methods=['POST']) # noqa: F821 @manager.route('/file/create', methods=['POST']) # noqa: F821
@token_required @token_required
def create(tenant_id): async def create(tenant_id):
""" """
Create a new file or folder. Create a new file or folder.
--- ---
@ -193,9 +193,9 @@ def create(tenant_id):
type: type:
type: string type: string
""" """
req = request.json req = await request.json
pf_id = request.json.get("parent_id") pf_id = await request.json.get("parent_id")
input_file_type = request.json.get("type") input_file_type = await request.json.get("type")
if not pf_id: if not pf_id:
root_folder = FileService.get_root_folder(tenant_id) root_folder = FileService.get_root_folder(tenant_id)
pf_id = root_folder["id"] pf_id = root_folder["id"]
@ -450,7 +450,7 @@ def get_all_parent_folders(tenant_id):
@manager.route('/file/rm', methods=['POST']) # noqa: F821 @manager.route('/file/rm', methods=['POST']) # noqa: F821
@token_required @token_required
def rm(tenant_id): async def rm(tenant_id):
""" """
Delete one or multiple files/folders. Delete one or multiple files/folders.
--- ---
@ -481,7 +481,7 @@ def rm(tenant_id):
type: boolean type: boolean
example: true example: true
""" """
req = request.json req = await request.json
file_ids = req["file_ids"] file_ids = req["file_ids"]
try: try:
for file_id in file_ids: for file_id in file_ids:
@ -524,7 +524,7 @@ def rm(tenant_id):
@manager.route('/file/rename', methods=['POST']) # noqa: F821 @manager.route('/file/rename', methods=['POST']) # noqa: F821
@token_required @token_required
def rename(tenant_id): async def rename(tenant_id):
""" """
Rename a file. Rename a file.
--- ---
@ -556,7 +556,7 @@ def rename(tenant_id):
type: boolean type: boolean
example: true example: true
""" """
req = request.json req = await request.json
try: try:
e, file = FileService.get_by_id(req["file_id"]) e, file = FileService.get_by_id(req["file_id"])
if not e: if not e:
@ -585,7 +585,7 @@ def rename(tenant_id):
@manager.route('/file/get/<file_id>', methods=['GET']) # noqa: F821 @manager.route('/file/get/<file_id>', methods=['GET']) # noqa: F821
@token_required @token_required
def get(tenant_id, file_id): async def get(tenant_id, file_id):
""" """
Download a file. Download a file.
--- ---
@ -619,7 +619,7 @@ def get(tenant_id, file_id):
b, n = File2DocumentService.get_storage_address(file_id=file_id) b, n = File2DocumentService.get_storage_address(file_id=file_id)
blob = settings.STORAGE_IMPL.get(b, n) blob = settings.STORAGE_IMPL.get(b, n)
response = flask.make_response(blob) response = await make_response(blob)
ext = re.search(r"\.([^.]+)$", file.name) ext = re.search(r"\.([^.]+)$", file.name)
if ext: if ext:
if file.type == FileType.VISUAL.value: if file.type == FileType.VISUAL.value:
@ -633,7 +633,7 @@ def get(tenant_id, file_id):
@manager.route('/file/mv', methods=['POST']) # noqa: F821 @manager.route('/file/mv', methods=['POST']) # noqa: F821
@token_required @token_required
def move(tenant_id): async def move(tenant_id):
""" """
Move one or multiple files to another folder. Move one or multiple files to another folder.
--- ---
@ -667,7 +667,7 @@ def move(tenant_id):
type: boolean type: boolean
example: true example: true
""" """
req = request.json req = await request.json
try: try:
file_ids = req["src_file_ids"] file_ids = req["src_file_ids"]
parent_id = req["dest_file_id"] parent_id = req["dest_file_id"]
@ -693,8 +693,8 @@ def move(tenant_id):
@manager.route('/file/convert', methods=['POST']) # noqa: F821 @manager.route('/file/convert', methods=['POST']) # noqa: F821
@token_required @token_required
def convert(tenant_id): async def convert(tenant_id):
req = request.json req = await request.json
kb_ids = req["kb_ids"] kb_ids = req["kb_ids"]
file_ids = req["file_ids"] file_ids = req["file_ids"]
file2documents = [] file2documents = []

View File

@ -18,7 +18,7 @@ import re
import time import time
import tiktoken import tiktoken
from flask import Response, jsonify, request from quart import Response, jsonify, request
from agent.canvas import Canvas from agent.canvas import Canvas
from api.db.db_models import APIToken from api.db.db_models import APIToken
@ -44,8 +44,8 @@ from common import settings
@manager.route("/chats/<chat_id>/sessions", methods=["POST"]) # noqa: F821 @manager.route("/chats/<chat_id>/sessions", methods=["POST"]) # noqa: F821
@token_required @token_required
def create(tenant_id, chat_id): async def create(tenant_id, chat_id):
req = request.json req = await request.json
req["dialog_id"] = chat_id req["dialog_id"] = chat_id
dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value) dia = DialogService.query(tenant_id=tenant_id, id=req["dialog_id"], status=StatusEnum.VALID.value)
if not dia: if not dia:
@ -97,8 +97,8 @@ def create_agent_session(tenant_id, agent_id):
@manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PUT"]) # noqa: F821 @manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PUT"]) # noqa: F821
@token_required @token_required
def update(tenant_id, chat_id, session_id): async def update(tenant_id, chat_id, session_id):
req = request.json req = await request.json
req["dialog_id"] = chat_id req["dialog_id"] = chat_id
conv_id = session_id conv_id = session_id
conv = ConversationService.query(id=conv_id, dialog_id=chat_id) conv = ConversationService.query(id=conv_id, dialog_id=chat_id)
@ -119,8 +119,8 @@ def update(tenant_id, chat_id, session_id):
@manager.route("/chats/<chat_id>/completions", methods=["POST"]) # noqa: F821 @manager.route("/chats/<chat_id>/completions", methods=["POST"]) # noqa: F821
@token_required @token_required
def chat_completion(tenant_id, chat_id): async def chat_completion(tenant_id, chat_id):
req = request.json req = await request.json
if not req: if not req:
req = {"question": ""} req = {"question": ""}
if not req.get("session_id"): if not req.get("session_id"):
@ -149,7 +149,7 @@ def chat_completion(tenant_id, chat_id):
@manager.route("/chats_openai/<chat_id>/chat/completions", methods=["POST"]) # noqa: F821 @manager.route("/chats_openai/<chat_id>/chat/completions", methods=["POST"]) # noqa: F821
@validate_request("model", "messages") # noqa: F821 @validate_request("model", "messages") # noqa: F821
@token_required @token_required
def chat_completion_openai_like(tenant_id, chat_id): async def chat_completion_openai_like(tenant_id, chat_id):
""" """
OpenAI-like chat completion API that simulates the behavior of OpenAI's completions endpoint. OpenAI-like chat completion API that simulates the behavior of OpenAI's completions endpoint.
@ -206,7 +206,7 @@ def chat_completion_openai_like(tenant_id, chat_id):
if reference: if reference:
print(completion.choices[0].message.reference) print(completion.choices[0].message.reference)
""" """
req = request.get_json() req = await request.get_json()
need_reference = bool(req.get("reference", False)) need_reference = bool(req.get("reference", False))
@ -383,8 +383,8 @@ def chat_completion_openai_like(tenant_id, chat_id):
@manager.route("/agents_openai/<agent_id>/chat/completions", methods=["POST"]) # noqa: F821 @manager.route("/agents_openai/<agent_id>/chat/completions", methods=["POST"]) # noqa: F821
@validate_request("model", "messages") # noqa: F821 @validate_request("model", "messages") # noqa: F821
@token_required @token_required
def agents_completion_openai_compatibility(tenant_id, agent_id): async def agents_completion_openai_compatibility(tenant_id, agent_id):
req = request.json req = await request.json
tiktokenenc = tiktoken.get_encoding("cl100k_base") tiktokenenc = tiktoken.get_encoding("cl100k_base")
messages = req.get("messages", []) messages = req.get("messages", [])
if not messages: if not messages:
@ -443,8 +443,8 @@ def agents_completion_openai_compatibility(tenant_id, agent_id):
@manager.route("/agents/<agent_id>/completions", methods=["POST"]) # noqa: F821 @manager.route("/agents/<agent_id>/completions", methods=["POST"]) # noqa: F821
@token_required @token_required
def agent_completions(tenant_id, agent_id): async def agent_completions(tenant_id, agent_id):
req = request.json req = await request.json
if req.get("stream", True): if req.get("stream", True):
@ -610,13 +610,13 @@ def list_agent_session(tenant_id, agent_id):
@manager.route("/chats/<chat_id>/sessions", methods=["DELETE"]) # noqa: F821 @manager.route("/chats/<chat_id>/sessions", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def delete(tenant_id, chat_id): async def delete(tenant_id, chat_id):
if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value): if not DialogService.query(id=chat_id, tenant_id=tenant_id, status=StatusEnum.VALID.value):
return get_error_data_result(message="You don't own the chat") return get_error_data_result(message="You don't own the chat")
errors = [] errors = []
success_count = 0 success_count = 0
req = request.json req = await request.json
convs = ConversationService.query(dialog_id=chat_id) convs = ConversationService.query(dialog_id=chat_id)
if not req: if not req:
ids = None ids = None
@ -661,10 +661,10 @@ def delete(tenant_id, chat_id):
@manager.route("/agents/<agent_id>/sessions", methods=["DELETE"]) # noqa: F821 @manager.route("/agents/<agent_id>/sessions", methods=["DELETE"]) # noqa: F821
@token_required @token_required
def delete_agent_session(tenant_id, agent_id): async def delete_agent_session(tenant_id, agent_id):
errors = [] errors = []
success_count = 0 success_count = 0
req = request.json req = await request.json
cvs = UserCanvasService.query(user_id=tenant_id, id=agent_id) cvs = UserCanvasService.query(user_id=tenant_id, id=agent_id)
if not cvs: if not cvs:
return get_error_data_result(f"You don't own the agent {agent_id}") return get_error_data_result(f"You don't own the agent {agent_id}")
@ -716,8 +716,8 @@ def delete_agent_session(tenant_id, agent_id):
@manager.route("/sessions/ask", methods=["POST"]) # noqa: F821 @manager.route("/sessions/ask", methods=["POST"]) # noqa: F821
@token_required @token_required
def ask_about(tenant_id): async def ask_about(tenant_id):
req = request.json req = await request.json
if not req.get("question"): if not req.get("question"):
return get_error_data_result("`question` is required.") return get_error_data_result("`question` is required.")
if not req.get("dataset_ids"): if not req.get("dataset_ids"):
@ -755,8 +755,8 @@ def ask_about(tenant_id):
@manager.route("/sessions/related_questions", methods=["POST"]) # noqa: F821 @manager.route("/sessions/related_questions", methods=["POST"]) # noqa: F821
@token_required @token_required
def related_questions(tenant_id): async def related_questions(tenant_id):
req = request.json req = await request.json
if not req.get("question"): if not req.get("question"):
return get_error_data_result("`question` is required.") return get_error_data_result("`question` is required.")
question = req["question"] question = req["question"]
@ -806,8 +806,8 @@ Related search terms:
@manager.route("/chatbots/<dialog_id>/completions", methods=["POST"]) # noqa: F821 @manager.route("/chatbots/<dialog_id>/completions", methods=["POST"]) # noqa: F821
def chatbot_completions(dialog_id): async def chatbot_completions(dialog_id):
req = request.json req = await request.json
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
@ -856,8 +856,8 @@ def chatbots_inputs(dialog_id):
@manager.route("/agentbots/<agent_id>/completions", methods=["POST"]) # noqa: F821 @manager.route("/agentbots/<agent_id>/completions", methods=["POST"]) # noqa: F821
def agent_bot_completions(agent_id): async def agent_bot_completions(agent_id):
req = request.json req = await request.json
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
@ -901,7 +901,7 @@ def begin_inputs(agent_id):
@manager.route("/searchbots/ask", methods=["POST"]) # noqa: F821 @manager.route("/searchbots/ask", methods=["POST"]) # noqa: F821
@validate_request("question", "kb_ids") @validate_request("question", "kb_ids")
def ask_about_embedded(): async def ask_about_embedded():
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
return get_error_data_result(message='Authorization is not valid!"') return get_error_data_result(message='Authorization is not valid!"')
@ -910,7 +910,7 @@ def ask_about_embedded():
if not objs: if not objs:
return get_error_data_result(message='Authentication error: API key is invalid!"') return get_error_data_result(message='Authentication error: API key is invalid!"')
req = request.json req = await request.json
uid = objs[0].tenant_id uid = objs[0].tenant_id
search_id = req.get("search_id", "") search_id = req.get("search_id", "")
@ -940,7 +940,7 @@ def ask_about_embedded():
@manager.route("/searchbots/retrieval_test", methods=["POST"]) # noqa: F821 @manager.route("/searchbots/retrieval_test", methods=["POST"]) # noqa: F821
@validate_request("kb_id", "question") @validate_request("kb_id", "question")
def retrieval_test_embedded(): async def retrieval_test_embedded():
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
return get_error_data_result(message='Authorization is not valid!"') return get_error_data_result(message='Authorization is not valid!"')
@ -949,7 +949,7 @@ def retrieval_test_embedded():
if not objs: if not objs:
return get_error_data_result(message='Authentication error: API key is invalid!"') return get_error_data_result(message='Authentication error: API key is invalid!"')
req = request.json req = await request.json
page = int(req.get("page", 1)) page = int(req.get("page", 1))
size = int(req.get("size", 30)) size = int(req.get("size", 30))
question = req["question"] question = req["question"]
@ -1039,7 +1039,7 @@ def retrieval_test_embedded():
@manager.route("/searchbots/related_questions", methods=["POST"]) # noqa: F821 @manager.route("/searchbots/related_questions", methods=["POST"]) # noqa: F821
@validate_request("question") @validate_request("question")
def related_questions_embedded(): async def related_questions_embedded():
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
return get_error_data_result(message='Authorization is not valid!"') return get_error_data_result(message='Authorization is not valid!"')
@ -1048,7 +1048,7 @@ def related_questions_embedded():
if not objs: if not objs:
return get_error_data_result(message='Authentication error: API key is invalid!"') return get_error_data_result(message='Authentication error: API key is invalid!"')
req = request.json req = await request.json
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
if not tenant_id: if not tenant_id:
return get_error_data_result(message="permission denined.") return get_error_data_result(message="permission denined.")
@ -1115,7 +1115,7 @@ def detail_share_embedded():
@manager.route("/searchbots/mindmap", methods=["POST"]) # noqa: F821 @manager.route("/searchbots/mindmap", methods=["POST"]) # noqa: F821
@validate_request("question", "kb_ids") @validate_request("question", "kb_ids")
def mindmap(): async def mindmap():
token = request.headers.get("Authorization").split() token = request.headers.get("Authorization").split()
if len(token) != 2: if len(token) != 2:
return get_error_data_result(message='Authorization is not valid!"') return get_error_data_result(message='Authorization is not valid!"')
@ -1125,7 +1125,7 @@ def mindmap():
return get_error_data_result(message='Authentication error: API key is invalid!"') return get_error_data_result(message='Authentication error: API key is invalid!"')
tenant_id = objs[0].tenant_id tenant_id = objs[0].tenant_id
req = request.json req = await request.json
search_id = req.get("search_id", "") search_id = req.get("search_id", "")
search_app = SearchService.get_detail(search_id) if search_id else {} search_app = SearchService.get_detail(search_id) if search_id else {}

View File

@ -14,8 +14,8 @@
# limitations under the License. # limitations under the License.
# #
from flask import request from quart import request
from flask_login import current_user, login_required from api.apps import current_user, login_required
from api.constants import DATASET_NAME_LIMIT from api.constants import DATASET_NAME_LIMIT
from api.db.db_models import DB from api.db.db_models import DB
@ -30,8 +30,8 @@ from api.utils.api_utils import get_data_error_result, get_json_result, not_allo
@manager.route("/create", methods=["post"]) # noqa: F821 @manager.route("/create", methods=["post"]) # noqa: F821
@login_required @login_required
@validate_request("name") @validate_request("name")
def create(): async def create():
req = request.get_json() req = await request.get_json()
search_name = req["name"] search_name = req["name"]
description = req.get("description", "") description = req.get("description", "")
if not isinstance(search_name, str): if not isinstance(search_name, str):
@ -65,8 +65,8 @@ def create():
@login_required @login_required
@validate_request("search_id", "name", "search_config", "tenant_id") @validate_request("search_id", "name", "search_config", "tenant_id")
@not_allowed_parameters("id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by") @not_allowed_parameters("id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by")
def update(): async def update():
req = request.get_json() req = await request.get_json()
if not isinstance(req["name"], str): if not isinstance(req["name"], str):
return get_data_error_result(message="Search name must be string.") return get_data_error_result(message="Search name must be string.")
if req["name"].strip() == "": if req["name"].strip() == "":
@ -140,7 +140,7 @@ def detail():
@manager.route("/list", methods=["POST"]) # noqa: F821 @manager.route("/list", methods=["POST"]) # noqa: F821
@login_required @login_required
def list_search_app(): async def list_search_app():
keywords = request.args.get("keywords", "") keywords = request.args.get("keywords", "")
page_number = int(request.args.get("page", 0)) page_number = int(request.args.get("page", 0))
items_per_page = int(request.args.get("page_size", 0)) items_per_page = int(request.args.get("page_size", 0))
@ -150,7 +150,7 @@ def list_search_app():
else: else:
desc = True desc = True
req = request.get_json() req = await request.get_json()
owner_ids = req.get("owner_ids", []) owner_ids = req.get("owner_ids", [])
try: try:
if not owner_ids: if not owner_ids:
@ -173,8 +173,8 @@ def list_search_app():
@manager.route("/rm", methods=["post"]) # noqa: F821 @manager.route("/rm", methods=["post"]) # noqa: F821
@login_required @login_required
@validate_request("search_id") @validate_request("search_id")
def rm(): async def rm():
req = request.get_json() req = await request.get_json()
search_id = req["search_id"] search_id = req["search_id"]
if not SearchService.accessible4deletion(search_id, current_user.id): if not SearchService.accessible4deletion(search_id, current_user.id):
return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)

View File

@ -17,7 +17,7 @@ import logging
from datetime import datetime from datetime import datetime
import json import json
from flask_login import login_required, current_user from api.apps import login_required, current_user
from api.db.db_models import APIToken from api.db.db_models import APIToken
from api.db.services.api_service import APITokenService from api.db.services.api_service import APITokenService
@ -34,7 +34,7 @@ from common.time_utils import current_timestamp, datetime_format
from timeit import default_timer as timer from timeit import default_timer as timer
from rag.utils.redis_conn import REDIS_CONN from rag.utils.redis_conn import REDIS_CONN
from flask import jsonify from quart import jsonify
from api.utils.health_utils import run_health_checks from api.utils.health_utils import run_health_checks
from common import settings from common import settings

View File

@ -14,10 +14,7 @@
# limitations under the License. # limitations under the License.
# #
from flask import request from quart import request
from flask_login import login_required, current_user
from api.apps import smtp_mail_server
from api.db import UserTenantRole from api.db import UserTenantRole
from api.db.db_models import UserTenant from api.db.db_models import UserTenant
from api.db.services.user_service import UserTenantService, UserService from api.db.services.user_service import UserTenantService, UserService
@ -28,6 +25,7 @@ from common.time_utils import delta_seconds
from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
from api.utils.web_utils import send_invite_email from api.utils.web_utils import send_invite_email
from common import settings from common import settings
from api.apps import smtp_mail_server, login_required, current_user
@manager.route("/<tenant_id>/user/list", methods=["GET"]) # noqa: F821 @manager.route("/<tenant_id>/user/list", methods=["GET"]) # noqa: F821
@ -51,14 +49,14 @@ def user_list(tenant_id):
@manager.route('/<tenant_id>/user', methods=['POST']) # noqa: F821 @manager.route('/<tenant_id>/user', methods=['POST']) # noqa: F821
@login_required @login_required
@validate_request("email") @validate_request("email")
def create(tenant_id): async def create(tenant_id):
if current_user.id != tenant_id: if current_user.id != tenant_id:
return get_json_result( return get_json_result(
data=False, data=False,
message='No authorization.', message='No authorization.',
code=RetCode.AUTHENTICATION_ERROR) code=RetCode.AUTHENTICATION_ERROR)
req = request.json req = await request.json
invite_user_email = req["email"] invite_user_email = req["email"]
invite_users = UserService.query(email=invite_user_email) invite_users = UserService.query(email=invite_user_email)
if not invite_users: if not invite_users:

View File

@ -22,8 +22,7 @@ import secrets
import time import time
from datetime import datetime from datetime import datetime
from flask import redirect, request, session, make_response from quart import redirect, request, session, make_response
from flask_login import current_user, login_required, login_user, logout_user
from werkzeug.security import check_password_hash, generate_password_hash from werkzeug.security import check_password_hash, generate_password_hash
from api.apps.auth import get_auth_client from api.apps.auth import get_auth_client
@ -45,7 +44,7 @@ from api.utils.api_utils import (
) )
from api.utils.crypt import decrypt from api.utils.crypt import decrypt
from rag.utils.redis_conn import REDIS_CONN from rag.utils.redis_conn import REDIS_CONN
from api.apps import smtp_mail_server from api.apps import smtp_mail_server, login_required, current_user, login_user, logout_user
from api.utils.web_utils import ( from api.utils.web_utils import (
send_email_html, send_email_html,
OTP_LENGTH, OTP_LENGTH,
@ -61,7 +60,7 @@ from common import settings
@manager.route("/login", methods=["POST", "GET"]) # noqa: F821 @manager.route("/login", methods=["POST", "GET"]) # noqa: F821
def login(): async def login():
""" """
User login endpoint. User login endpoint.
--- ---
@ -91,10 +90,11 @@ def login():
schema: schema:
type: object type: object
""" """
if not request.json: json_body = await request.json
if not json_body:
return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Unauthorized!") return get_json_result(data=False, code=RetCode.AUTHENTICATION_ERROR, message="Unauthorized!")
email = request.json.get("email", "") email = json_body.get("email", "")
users = UserService.query(email=email) users = UserService.query(email=email)
if not users: if not users:
return get_json_result( return get_json_result(
@ -103,7 +103,7 @@ def login():
message=f"Email: {email} is not registered!", message=f"Email: {email} is not registered!",
) )
password = request.json.get("password") password = json_body.get("password")
try: try:
password = decrypt(password) password = decrypt(password)
except BaseException: except BaseException:
@ -125,7 +125,8 @@ def login():
user.update_date = (datetime_format(datetime.now()),) user.update_date = (datetime_format(datetime.now()),)
user.save() user.save()
msg = "Welcome back!" msg = "Welcome back!"
return construct_response(data=response_data, auth=user.get_id(), message=msg)
return await construct_response(data=response_data, auth=user.get_id(), message=msg)
else: else:
return get_json_result( return get_json_result(
data=False, data=False,
@ -501,7 +502,7 @@ def log_out():
@manager.route("/setting", methods=["POST"]) # noqa: F821 @manager.route("/setting", methods=["POST"]) # noqa: F821
@login_required @login_required
def setting_user(): async def setting_user():
""" """
Update user settings. Update user settings.
--- ---
@ -530,7 +531,7 @@ def setting_user():
type: object type: object
""" """
update_dict = {} update_dict = {}
request_data = request.json request_data = await request.json
if request_data.get("password"): if request_data.get("password"):
new_password = request_data.get("new_password") new_password = request_data.get("new_password")
if not check_password_hash(current_user.password, decrypt(request_data["password"])): if not check_password_hash(current_user.password, decrypt(request_data["password"])):
@ -660,7 +661,7 @@ def user_register(user_id, user):
@manager.route("/register", methods=["POST"]) # noqa: F821 @manager.route("/register", methods=["POST"]) # noqa: F821
@validate_request("nickname", "email", "password") @validate_request("nickname", "email", "password")
def user_add(): async def user_add():
""" """
Register a new user. Register a new user.
--- ---
@ -697,7 +698,7 @@ def user_add():
code=RetCode.OPERATING_ERROR, code=RetCode.OPERATING_ERROR,
) )
req = request.json req = await request.json
email_address = req["email"] email_address = req["email"]
# Validate the email address # Validate the email address
@ -737,7 +738,7 @@ def user_add():
raise Exception(f"Same email: {email_address} exists!") raise Exception(f"Same email: {email_address} exists!")
user = users[0] user = users[0]
login_user(user) login_user(user)
return construct_response( return await construct_response(
data=user.to_json(), data=user.to_json(),
auth=user.get_id(), auth=user.get_id(),
message=f"{nickname}, welcome aboard!", message=f"{nickname}, welcome aboard!",
@ -793,7 +794,7 @@ def tenant_info():
@manager.route("/set_tenant_info", methods=["POST"]) # noqa: F821 @manager.route("/set_tenant_info", methods=["POST"]) # noqa: F821
@login_required @login_required
@validate_request("tenant_id", "asr_id", "embd_id", "img2txt_id", "llm_id") @validate_request("tenant_id", "asr_id", "embd_id", "img2txt_id", "llm_id")
def set_tenant_info(): async def set_tenant_info():
""" """
Update tenant information. Update tenant information.
--- ---
@ -830,7 +831,7 @@ def set_tenant_info():
schema: schema:
type: object type: object
""" """
req = request.json req = await request.json
try: try:
tid = req.pop("tenant_id") tid = req.pop("tenant_id")
TenantService.update_by_id(tid, req) TenantService.update_by_id(tid, req)
@ -840,7 +841,7 @@ def set_tenant_info():
@manager.route("/forget/captcha", methods=["GET"]) # noqa: F821 @manager.route("/forget/captcha", methods=["GET"]) # noqa: F821
def forget_get_captcha(): async def forget_get_captcha():
""" """
GET /forget/captcha?email=<email> GET /forget/captcha?email=<email>
- Generate an image captcha and cache it in Redis under key captcha:{email} with TTL = OTP_TTL_SECONDS. - Generate an image captcha and cache it in Redis under key captcha:{email} with TTL = OTP_TTL_SECONDS.
@ -862,19 +863,19 @@ def forget_get_captcha():
from captcha.image import ImageCaptcha from captcha.image import ImageCaptcha
image = ImageCaptcha(width=300, height=120, font_sizes=[50, 60, 70]) image = ImageCaptcha(width=300, height=120, font_sizes=[50, 60, 70])
img_bytes = image.generate(captcha_text).read() img_bytes = image.generate(captcha_text).read()
response = make_response(img_bytes) response = await make_response(img_bytes)
response.headers.set("Content-Type", "image/JPEG") response.headers.set("Content-Type", "image/JPEG")
return response return response
@manager.route("/forget/otp", methods=["POST"]) # noqa: F821 @manager.route("/forget/otp", methods=["POST"]) # noqa: F821
def forget_send_otp(): async def forget_send_otp():
""" """
POST /forget/otp POST /forget/otp
- Verify the image captcha stored at captcha:{email} (case-insensitive). - Verify the image captcha stored at captcha:{email} (case-insensitive).
- On success, generate an email OTP (AZ with length = OTP_LENGTH), store hash + salt (and timestamp) in Redis with TTL, reset attempts and cooldown, and send the OTP via email. - On success, generate an email OTP (AZ with length = OTP_LENGTH), store hash + salt (and timestamp) in Redis with TTL, reset attempts and cooldown, and send the OTP via email.
""" """
req = request.get_json() req = await request.get_json()
email = req.get("email") or "" email = req.get("email") or ""
captcha = (req.get("captcha") or "").strip() captcha = (req.get("captcha") or "").strip()
@ -935,12 +936,12 @@ def forget_send_otp():
@manager.route("/forget", methods=["POST"]) # noqa: F821 @manager.route("/forget", methods=["POST"]) # noqa: F821
def forget(): async def forget():
""" """
POST: Verify email + OTP and reset password, then log the user in. POST: Verify email + OTP and reset password, then log the user in.
Request JSON: { email, otp, new_password, confirm_new_password } Request JSON: { email, otp, new_password, confirm_new_password }
""" """
req = request.get_json() req = await request.get_json()
email = req.get("email") or "" email = req.get("email") or ""
otp = (req.get("otp") or "").strip() otp = (req.get("otp") or "").strip()
new_pwd = req.get("new_password") new_pwd = req.get("new_password")

View File

@ -25,7 +25,7 @@ from datetime import datetime, timezone
from enum import Enum from enum import Enum
from functools import wraps from functools import wraps
from flask_login import UserMixin from quart_auth import AuthUser
from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer from itsdangerous.url_safe import URLSafeTimedSerializer as Serializer
from peewee import InterfaceError, OperationalError, BigIntegerField, BooleanField, CharField, CompositeKey, DateTimeField, Field, FloatField, IntegerField, Metadata, Model, TextField from peewee import InterfaceError, OperationalError, BigIntegerField, BooleanField, CharField, CompositeKey, DateTimeField, Field, FloatField, IntegerField, Metadata, Model, TextField
from playhouse.migrate import MySQLMigrator, PostgresqlMigrator, migrate from playhouse.migrate import MySQLMigrator, PostgresqlMigrator, migrate
@ -595,7 +595,7 @@ def fill_db_model_object(model_object, human_model_dict):
return model_object return model_object
class User(DataBaseModel, UserMixin): class User(DataBaseModel, AuthUser):
id = CharField(max_length=32, primary_key=True) id = CharField(max_length=32, primary_key=True)
access_token = CharField(max_length=255, null=True, index=True) access_token = CharField(max_length=255, null=True, index=True)
nickname = CharField(max_length=100, null=False, help_text="nicky name", index=True) nickname = CharField(max_length=100, null=False, help_text="nicky name", index=True)

View File

@ -24,7 +24,6 @@ from api.db import InputType
from api.db.db_models import Connector, SyncLogs, Connector2Kb, Knowledgebase from api.db.db_models import Connector, SyncLogs, Connector2Kb, Knowledgebase
from api.db.services.common_service import CommonService from api.db.services.common_service import CommonService
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
from api.db.services.file_service import FileService
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.constants import TaskStatus from common.constants import TaskStatus
from common.time_utils import current_timestamp, timestamp_to_date from common.time_utils import current_timestamp, timestamp_to_date
@ -68,6 +67,7 @@ class ConnectorService(CommonService):
@classmethod @classmethod
def rebuild(cls, kb_id:str, connector_id: str, tenant_id:str): def rebuild(cls, kb_id:str, connector_id: str, tenant_id:str):
from api.db.services.file_service import FileService
e, conn = cls.get_by_id(connector_id) e, conn = cls.get_by_id(connector_id)
if not e: if not e:
return None return None
@ -191,6 +191,7 @@ class SyncLogsService(CommonService):
@classmethod @classmethod
def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True): def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True):
from api.db.services.file_service import FileService
if not docs: if not docs:
return None return None

View File

@ -41,6 +41,7 @@ from rag.utils.redis_conn import REDIS_CONN
from rag.utils.doc_store_conn import OrderByExpr from rag.utils.doc_store_conn import OrderByExpr
from common import settings from common import settings
class DocumentService(CommonService): class DocumentService(CommonService):
model = Document model = Document

View File

@ -18,7 +18,6 @@ import re
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from pathlib import Path from pathlib import Path
from flask_login import current_user
from peewee import fn from peewee import fn
from api.db import KNOWLEDGEBASE_FOLDER_NAME, FileType from api.db import KNOWLEDGEBASE_FOLDER_NAME, FileType
@ -184,6 +183,7 @@ class FileService(CommonService):
@classmethod @classmethod
@DB.connection_context() @DB.connection_context()
def create_folder(cls, file, parent_id, name, count): def create_folder(cls, file, parent_id, name, count):
from api.apps import current_user
# Recursively create folder structure # Recursively create folder structure
# Args: # Args:
# file: Current file object # file: Current file object
@ -508,6 +508,7 @@ class FileService(CommonService):
@staticmethod @staticmethod
def parse(filename, blob, img_base64=True, tenant_id=None): def parse(filename, blob, img_base64=True, tenant_id=None):
from rag.app import audio, email, naive, picture, presentation from rag.app import audio, email, naive, picture, presentation
from api.apps import current_user
def dummy(prog=None, msg=""): def dummy(prog=None, msg=""):
pass pass

View File

@ -24,9 +24,10 @@ from common.time_utils import current_timestamp, datetime_format
from api.db.services import duplicate_name from api.db.services import duplicate_name
from api.db.services.user_service import TenantService from api.db.services.user_service import TenantService
from common.misc_utils import get_uuid from common.misc_utils import get_uuid
from common.constants import StatusEnum, RetCode from common.constants import StatusEnum
from api.constants import DATASET_NAME_LIMIT from api.constants import DATASET_NAME_LIMIT
from api.utils.api_utils import get_parser_config from api.utils.api_utils import get_parser_config, get_data_error_result
class KnowledgebaseService(CommonService): class KnowledgebaseService(CommonService):
"""Service class for managing knowledge base operations. """Service class for managing knowledge base operations.
@ -391,12 +392,12 @@ class KnowledgebaseService(CommonService):
""" """
# Validate name # Validate name
if not isinstance(name, str): if not isinstance(name, str):
return {"code": RetCode.DATA_ERROR, "message": "Dataset name must be string."} return False, get_data_error_result(message="Dataset name must be string.")
dataset_name = name.strip() dataset_name = name.strip()
if len(dataset_name) == 0: if dataset_name == "":
return {"code": RetCode.DATA_ERROR, "message": "Dataset name can't be empty."} return False, get_data_error_result(message="Dataset name can't be empty.")
if len(dataset_name.encode("utf-8")) > DATASET_NAME_LIMIT: if len(dataset_name.encode("utf-8")) > DATASET_NAME_LIMIT:
return {"code": RetCode.DATA_ERROR, "message": f"Dataset name length is {len(dataset_name)} which is larger than {DATASET_NAME_LIMIT}"} return False, get_data_error_result(message=f"Dataset name length is {len(dataset_name)} which is larger than {DATASET_NAME_LIMIT}")
# Deduplicate name within tenant # Deduplicate name within tenant
dataset_name = duplicate_name( dataset_name = duplicate_name(
@ -409,7 +410,7 @@ class KnowledgebaseService(CommonService):
# Verify tenant exists # Verify tenant exists
ok, _t = TenantService.get_by_id(tenant_id) ok, _t = TenantService.get_by_id(tenant_id)
if not ok: if not ok:
return {"code": RetCode.DATA_ERROR, "message": "Tenant does not exist."} return False, get_data_error_result(message="Tenant not found.")
# Build payload # Build payload
kb_id = get_uuid() kb_id = get_uuid()
@ -425,7 +426,7 @@ class KnowledgebaseService(CommonService):
# Update parser_config (always override with validated default/merged config) # Update parser_config (always override with validated default/merged config)
payload["parser_config"] = get_parser_config(parser_id, kwargs.get("parser_config")) payload["parser_config"] = get_parser_config(parser_id, kwargs.get("parser_config"))
return payload return True, payload
@classmethod @classmethod

View File

@ -31,7 +31,6 @@ import traceback
import threading import threading
import uuid import uuid
from werkzeug.serving import run_simple
from api.apps import app, smtp_mail_server from api.apps import app, smtp_mail_server
from api.db.runtime_config import RuntimeConfig from api.db.runtime_config import RuntimeConfig
from api.db.services.document_service import DocumentService from api.db.services.document_service import DocumentService
@ -153,14 +152,7 @@ if __name__ == '__main__':
# start http server # start http server
try: try:
logging.info("RAGFlow HTTP server start...") logging.info("RAGFlow HTTP server start...")
run_simple( app.run(host=settings.HOST_IP, port=settings.HOST_PORT)
hostname=settings.HOST_IP,
port=settings.HOST_PORT,
application=app,
threaded=True,
use_reloader=RuntimeConfig.DEBUG,
use_debugger=RuntimeConfig.DEBUG,
)
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
stop_event.set() stop_event.set()

View File

@ -15,6 +15,7 @@
# #
import functools import functools
import inspect
import json import json
import logging import logging
import os import os
@ -24,14 +25,12 @@ from functools import wraps
import requests import requests
import trio import trio
from flask import ( from quart import (
Response, Response,
jsonify, jsonify,
request
) )
from flask_login import current_user
from flask import (
request as flask_request,
)
from peewee import OperationalError from peewee import OperationalError
from common.constants import ActiveEnum from common.constants import ActiveEnum
@ -46,6 +45,12 @@ from common import settings
requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder) requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder)
async def request_json():
try:
return await request.json
except Exception:
return {}
def serialize_for_json(obj): def serialize_for_json(obj):
""" """
Recursively serialize objects to make them JSON serializable. Recursively serialize objects to make them JSON serializable.
@ -105,31 +110,37 @@ def server_error_response(e):
def validate_request(*args, **kwargs): def validate_request(*args, **kwargs):
def process_args(input_arguments):
no_arguments = []
error_arguments = []
for arg in args:
if arg not in input_arguments:
no_arguments.append(arg)
for k, v in kwargs.items():
config_value = input_arguments.get(k, None)
if config_value is None:
no_arguments.append(k)
elif isinstance(v, (tuple, list)):
if config_value not in v:
error_arguments.append((k, set(v)))
elif config_value != v:
error_arguments.append((k, v))
if no_arguments or error_arguments:
error_string = ""
if no_arguments:
error_string += "required argument are missing: {}; ".format(",".join(no_arguments))
if error_arguments:
error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
return error_string
def wrapper(func): def wrapper(func):
@wraps(func) @wraps(func)
def decorated_function(*_args, **_kwargs): async def decorated_function(*_args, **_kwargs):
input_arguments = flask_request.json or flask_request.form.to_dict() errs = process_args(await request.json or (await request.form).to_dict())
no_arguments = [] if errs:
error_arguments = [] return get_json_result(code=RetCode.ARGUMENT_ERROR, message=errs)
for arg in args: if inspect.iscoroutinefunction(func):
if arg not in input_arguments: return await func(*_args, **_kwargs)
no_arguments.append(arg)
for k, v in kwargs.items():
config_value = input_arguments.get(k, None)
if config_value is None:
no_arguments.append(k)
elif isinstance(v, (tuple, list)):
if config_value not in v:
error_arguments.append((k, set(v)))
elif config_value != v:
error_arguments.append((k, v))
if no_arguments or error_arguments:
error_string = ""
if no_arguments:
error_string += "required argument are missing: {}; ".format(",".join(no_arguments))
if error_arguments:
error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments]))
return get_json_result(code=RetCode.ARGUMENT_ERROR, message=error_string)
return func(*_args, **_kwargs) return func(*_args, **_kwargs)
return decorated_function return decorated_function
@ -138,30 +149,34 @@ def validate_request(*args, **kwargs):
def not_allowed_parameters(*params): def not_allowed_parameters(*params):
def decorator(f): def decorator(func):
def wrapper(*args, **kwargs): async def wrapper(*args, **kwargs):
input_arguments = flask_request.json or flask_request.form.to_dict() input_arguments = await request.json or (await request.form).to_dict()
for param in params: for param in params:
if param in input_arguments: if param in input_arguments:
return get_json_result(code=RetCode.ARGUMENT_ERROR, message=f"Parameter {param} isn't allowed") return get_json_result(code=RetCode.ARGUMENT_ERROR, message=f"Parameter {param} isn't allowed")
return f(*args, **kwargs) if inspect.iscoroutinefunction(func):
return await func(*args, **kwargs)
return func(*args, **kwargs)
return wrapper return wrapper
return decorator return decorator
def active_required(f): def active_required(func):
@wraps(f) @wraps(func)
def wrapper(*args, **kwargs): async def wrapper(*args, **kwargs):
from api.db.services import UserService from api.db.services import UserService
from api.apps import current_user
user_id = current_user.id user_id = current_user.id
usr = UserService.filter_by_id(user_id) usr = UserService.filter_by_id(user_id)
# check is_active # check is_active
if not usr or not usr.is_active == ActiveEnum.ACTIVE.value: if not usr or not usr.is_active == ActiveEnum.ACTIVE.value:
return get_json_result(code=RetCode.FORBIDDEN, message="User isn't active, please activate first.") return get_json_result(code=RetCode.FORBIDDEN, message="User isn't active, please activate first.")
return f(*args, **kwargs) if inspect.iscoroutinefunction(func):
return await func(*args, **kwargs)
return func(*args, **kwargs)
return wrapper return wrapper
@ -173,12 +188,15 @@ def get_json_result(code: RetCode = RetCode.SUCCESS, message="success", data=Non
def apikey_required(func): def apikey_required(func):
@wraps(func) @wraps(func)
def decorated_function(*args, **kwargs): async def decorated_function(*args, **kwargs):
token = flask_request.headers.get("Authorization").split()[1] token = request.headers.get("Authorization").split()[1]
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return build_error_result(message="API-KEY is invalid!", code=RetCode.FORBIDDEN) return build_error_result(message="API-KEY is invalid!", code=RetCode.FORBIDDEN)
kwargs["tenant_id"] = objs[0].tenant_id kwargs["tenant_id"] = objs[0].tenant_id
if inspect.iscoroutinefunction(func):
return await func(*args, **kwargs)
return func(*args, **kwargs) return func(*args, **kwargs)
return decorated_function return decorated_function
@ -199,23 +217,38 @@ def construct_json_result(code: RetCode = RetCode.SUCCESS, message="success", da
def token_required(func): def token_required(func):
@wraps(func) def get_tenant_id(**kwargs):
def decorated_function(*args, **kwargs):
if os.environ.get("DISABLE_SDK"): if os.environ.get("DISABLE_SDK"):
return get_json_result(data=False, message="`Authorization` can't be empty") return False, get_json_result(data=False, message="`Authorization` can't be empty")
authorization_str = flask_request.headers.get("Authorization") authorization_str = request.headers.get("Authorization")
if not authorization_str: if not authorization_str:
return get_json_result(data=False, message="`Authorization` can't be empty") return False, get_json_result(data=False, message="`Authorization` can't be empty")
authorization_list = authorization_str.split() authorization_list = authorization_str.split()
if len(authorization_list) < 2: if len(authorization_list) < 2:
return get_json_result(data=False, message="Please check your authorization format.") return False, get_json_result(data=False, message="Please check your authorization format.")
token = authorization_list[1] token = authorization_list[1]
objs = APIToken.query(token=token) objs = APIToken.query(token=token)
if not objs: if not objs:
return get_json_result(data=False, message="Authentication error: API key is invalid!", code=RetCode.AUTHENTICATION_ERROR) return False, get_json_result(data=False, message="Authentication error: API key is invalid!", code=RetCode.AUTHENTICATION_ERROR)
kwargs["tenant_id"] = objs[0].tenant_id kwargs["tenant_id"] = objs[0].tenant_id
return True, kwargs
@wraps(func)
def decorated_function(*args, **kwargs):
e, kwargs = get_tenant_id(**kwargs)
if not e:
return kwargs
return func(*args, **kwargs) return func(*args, **kwargs)
@wraps(func)
async def adecorated_function(*args, **kwargs):
e, kwargs = get_tenant_id(**kwargs)
if not e:
return kwargs
return await func(*args, **kwargs)
if inspect.iscoroutinefunction(func):
return adecorated_function
return decorated_function return decorated_function

View File

@ -18,7 +18,7 @@ import base64
import click import click
import re import re
from flask import Flask from quart import Quart
from werkzeug.security import generate_password_hash from werkzeug.security import generate_password_hash
from api.db.services import UserService from api.db.services import UserService
@ -73,6 +73,7 @@ def reset_email(email, new_email, email_confirm):
UserService.update_user(user[0].id,user_dict) UserService.update_user(user[0].id,user_dict)
click.echo(click.style('Congratulations!, email has been reset.', fg='green')) click.echo(click.style('Congratulations!, email has been reset.', fg='green'))
def register_commands(app: Flask):
def register_commands(app: Quart):
app.cli.add_command(reset_password) app.cli.add_command(reset_password)
app.cli.add_command(reset_email) app.cli.add_command(reset_email)

View File

@ -17,7 +17,7 @@ from collections import Counter
from typing import Annotated, Any, Literal from typing import Annotated, Any, Literal
from uuid import UUID from uuid import UUID
from flask import Request from quart import Request
from pydantic import ( from pydantic import (
BaseModel, BaseModel,
ConfigDict, ConfigDict,
@ -32,7 +32,7 @@ from werkzeug.exceptions import BadRequest, UnsupportedMediaType
from api.constants import DATASET_NAME_LIMIT from api.constants import DATASET_NAME_LIMIT
def validate_and_parse_json_request(request: Request, validator: type[BaseModel], *, extras: dict[str, Any] | None = None, exclude_unset: bool = False) -> tuple[dict[str, Any] | None, str | None]: async def validate_and_parse_json_request(request: Request, validator: type[BaseModel], *, extras: dict[str, Any] | None = None, exclude_unset: bool = False) -> tuple[dict[str, Any] | None, str | None]:
""" """
Validates and parses JSON requests through a multi-stage validation pipeline. Validates and parses JSON requests through a multi-stage validation pipeline.
@ -81,7 +81,7 @@ def validate_and_parse_json_request(request: Request, validator: type[BaseModel]
from the final output after validation from the final output after validation
""" """
try: try:
payload = request.get_json() or {} payload = await request.get_json() or {}
except UnsupportedMediaType: except UnsupportedMediaType:
return None, f"Unsupported content type: Expected application/json, got {request.content_type}" return None, f"Unsupported content type: Expected application/json, got {request.content_type}"
except BadRequest: except BadRequest:

View File

@ -23,7 +23,7 @@ from urllib.parse import urlparse
from api.apps import smtp_mail_server from api.apps import smtp_mail_server
from flask_mail import Message from flask_mail import Message
from flask import render_template_string from quart import render_template_string
from api.utils.email_templates import EMAIL_TEMPLATES from api.utils.email_templates import EMAIL_TEMPLATES
from selenium import webdriver from selenium import webdriver
from selenium.common.exceptions import TimeoutException from selenium.common.exceptions import TimeoutException

View File

@ -21,7 +21,7 @@ from typing import Any, Callable, Coroutine, Optional, Type, Union
import asyncio import asyncio
import trio import trio
from functools import wraps from functools import wraps
from flask import make_response, jsonify from quart import make_response, jsonify
from common.constants import RetCode from common.constants import RetCode
TimeoutException = Union[Type[BaseException], BaseException] TimeoutException = Union[Type[BaseException], BaseException]
@ -103,7 +103,7 @@ def timeout(seconds: float | int | str = None, attempts: int = 2, *, exception:
return decorator return decorator
def construct_response(code=RetCode.SUCCESS, message="success", data=None, auth=None): async def construct_response(code=RetCode.SUCCESS, message="success", data=None, auth=None):
result_dict = {"code": code, "message": message, "data": data} result_dict = {"code": code, "message": message, "data": data}
response_dict = {} response_dict = {}
for key, value in result_dict.items(): for key, value in result_dict.items():
@ -111,7 +111,27 @@ def construct_response(code=RetCode.SUCCESS, message="success", data=None, auth=
continue continue
else: else:
response_dict[key] = value response_dict[key] = value
response = make_response(jsonify(response_dict)) response = await make_response(jsonify(response_dict))
if auth:
response.headers["Authorization"] = auth
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Method"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Allow-Headers"] = "*"
response.headers["Access-Control-Expose-Headers"] = "Authorization"
return response
def sync_construct_response(code=RetCode.SUCCESS, message="success", data=None, auth=None):
import flask
result_dict = {"code": code, "message": message, "data": data}
response_dict = {}
for key, value in result_dict.items():
if value is None and key != "code":
continue
else:
response_dict[key] = value
response = flask.make_response(flask.jsonify(response_dict))
if auth: if auth:
response.headers["Authorization"] = auth response.headers["Authorization"] = auth
response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Origin"] = "*"

View File

@ -1429,6 +1429,13 @@
"status": "1", "status": "1",
"rank": "980", "rank": "980",
"llm": [ "llm": [
{
"llm_name": "gemini-3-pro-preview",
"tags": "LLM,CHAT,1M,IMAGE2TEXT",
"max_tokens": 1048576,
"model_type": "image2text",
"is_tools": true
},
{ {
"llm_name": "gemini-2.5-flash", "llm_name": "gemini-2.5-flash",
"tags": "LLM,CHAT,1024K,IMAGE2TEXT", "tags": "LLM,CHAT,1024K,IMAGE2TEXT",
@ -4841,7 +4848,7 @@
] ]
}, },
{ {
"name": "JieKou.AI", "name": "Jiekou.AI",
"logo": "", "logo": "",
"tags": "LLM,TEXT EMBEDDING,TEXT RE-RANK", "tags": "LLM,TEXT EMBEDDING,TEXT RE-RANK",
"status": "1", "status": "1",
@ -5474,4 +5481,4 @@
] ]
} }
] ]
} }

View File

@ -61,7 +61,9 @@ class DoclingParser(RAGFlowPdfParser):
self.page_images: list[Image.Image] = [] self.page_images: list[Image.Image] = []
self.page_from = 0 self.page_from = 0
self.page_to = 10_000 self.page_to = 10_000
self.outlines = []
def check_installation(self) -> bool: def check_installation(self) -> bool:
if DocumentConverter is None: if DocumentConverter is None:
self.logger.warning("[Docling] 'docling' is not importable, please: pip install docling") self.logger.warning("[Docling] 'docling' is not importable, please: pip install docling")

View File

@ -59,6 +59,7 @@ class MinerUParser(RAGFlowPdfParser):
self.mineru_api = mineru_api.rstrip("/") self.mineru_api = mineru_api.rstrip("/")
self.mineru_server_url = mineru_server_url.rstrip("/") self.mineru_server_url = mineru_server_url.rstrip("/")
self.using_api = False self.using_api = False
self.outlines = []
self.logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
def _extract_zip_no_root(self, zip_path, extract_to, root_dir): def _extract_zip_no_root(self, zip_path, extract_to, root_dir):
@ -317,7 +318,7 @@ class MinerUParser(RAGFlowPdfParser):
def _line_tag(self, bx): def _line_tag(self, bx):
pn = [bx["page_idx"] + 1] pn = [bx["page_idx"] + 1]
positions = bx["bbox"] positions = bx.get("bbox", (0, 0, 0, 0))
x0, top, x1, bott = positions x0, top, x1, bott = positions
if hasattr(self, "page_images") and self.page_images and len(self.page_images) > bx["page_idx"]: if hasattr(self, "page_images") and self.page_images and len(self.page_images) > bx["page_idx"]:
@ -337,12 +338,54 @@ class MinerUParser(RAGFlowPdfParser):
return None, None return None, None
return return
if not getattr(self, "page_images", None):
self.logger.warning("[MinerU] crop called without page images; skipping image generation.")
if need_position:
return None, None
return
page_count = len(self.page_images)
filtered_poss = []
for pns, left, right, top, bottom in poss:
if not pns:
self.logger.warning("[MinerU] Empty page index list in crop; skipping this position.")
continue
valid_pns = [p for p in pns if 0 <= p < page_count]
if not valid_pns:
self.logger.warning(f"[MinerU] All page indices {pns} out of range for {page_count} pages; skipping.")
continue
filtered_poss.append((valid_pns, left, right, top, bottom))
poss = filtered_poss
if not poss:
self.logger.warning("[MinerU] No valid positions after filtering; skip cropping.")
if need_position:
return None, None
return
max_width = max(np.max([right - left for (_, left, right, _, _) in poss]), 6) max_width = max(np.max([right - left for (_, left, right, _, _) in poss]), 6)
GAP = 6 GAP = 6
pos = poss[0] pos = poss[0]
poss.insert(0, ([pos[0][0]], pos[1], pos[2], max(0, pos[3] - 120), max(pos[3] - GAP, 0))) first_page_idx = pos[0][0]
poss.insert(0, ([first_page_idx], pos[1], pos[2], max(0, pos[3] - 120), max(pos[3] - GAP, 0)))
pos = poss[-1] pos = poss[-1]
poss.append(([pos[0][-1]], pos[1], pos[2], min(self.page_images[pos[0][-1]].size[1], pos[4] + GAP), min(self.page_images[pos[0][-1]].size[1], pos[4] + 120))) last_page_idx = pos[0][-1]
if not (0 <= last_page_idx < page_count):
self.logger.warning(f"[MinerU] Last page index {last_page_idx} out of range for {page_count} pages; skipping crop.")
if need_position:
return None, None
return
last_page_height = self.page_images[last_page_idx].size[1]
poss.append(
(
[last_page_idx],
pos[1],
pos[2],
min(last_page_height, pos[4] + GAP),
min(last_page_height, pos[4] + 120),
)
)
positions = [] positions = []
for ii, (pns, left, right, top, bottom) in enumerate(poss): for ii, (pns, left, right, top, bottom) in enumerate(poss):
@ -352,7 +395,14 @@ class MinerUParser(RAGFlowPdfParser):
bottom = top + 2 bottom = top + 2
for pn in pns[1:]: for pn in pns[1:]:
bottom += self.page_images[pn - 1].size[1] if 0 <= pn - 1 < page_count:
bottom += self.page_images[pn - 1].size[1]
else:
self.logger.warning(f"[MinerU] Page index {pn}-1 out of range for {page_count} pages during crop; skipping height accumulation.")
if not (0 <= pns[0] < page_count):
self.logger.warning(f"[MinerU] Base page index {pns[0]} out of range for {page_count} pages during crop; skipping this segment.")
continue
img0 = self.page_images[pns[0]] img0 = self.page_images[pns[0]]
x0, y0, x1, y1 = int(left), int(top), int(right), int(min(bottom, img0.size[1])) x0, y0, x1, y1 = int(left), int(top), int(right), int(min(bottom, img0.size[1]))
@ -363,6 +413,9 @@ class MinerUParser(RAGFlowPdfParser):
bottom -= img0.size[1] bottom -= img0.size[1]
for pn in pns[1:]: for pn in pns[1:]:
if not (0 <= pn < page_count):
self.logger.warning(f"[MinerU] Page index {pn} out of range for {page_count} pages during crop; skipping this page.")
continue
page = self.page_images[pn] page = self.page_images[pn]
x0, y0, x1, y1 = int(left), 0, int(right), int(min(bottom, page.size[1])) x0, y0, x1, y1 = int(left), 0, int(right), int(min(bottom, page.size[1]))
cimgp = page.crop((x0, y0, x1, y1)) cimgp = page.crop((x0, y0, x1, y1))

View File

@ -1252,24 +1252,77 @@ class RAGFlowPdfParser:
return None, None return None, None
return return
if not getattr(self, "page_images", None):
logging.warning("crop called without page images; skipping image generation.")
if need_position:
return None, None
return
page_count = len(self.page_images)
filtered_poss = []
for pns, left, right, top, bottom in poss:
if not pns:
logging.warning("Empty page index list in crop; skipping this position.")
continue
valid_pns = [p for p in pns if 0 <= p < page_count]
if not valid_pns:
logging.warning(f"All page indices {pns} out of range for {page_count} pages; skipping.")
continue
filtered_poss.append((valid_pns, left, right, top, bottom))
poss = filtered_poss
if not poss:
logging.warning("No valid positions after filtering; skip cropping.")
if need_position:
return None, None
return
max_width = max(np.max([right - left for (_, left, right, _, _) in poss]), 6) max_width = max(np.max([right - left for (_, left, right, _, _) in poss]), 6)
GAP = 6 GAP = 6
pos = poss[0] pos = poss[0]
poss.insert(0, ([pos[0][0]], pos[1], pos[2], max(0, pos[3] - 120), max(pos[3] - GAP, 0))) first_page_idx = pos[0][0]
poss.insert(0, ([first_page_idx], pos[1], pos[2], max(0, pos[3] - 120), max(pos[3] - GAP, 0)))
pos = poss[-1] pos = poss[-1]
poss.append(([pos[0][-1]], pos[1], pos[2], min(self.page_images[pos[0][-1]].size[1] / ZM, pos[4] + GAP), min(self.page_images[pos[0][-1]].size[1] / ZM, pos[4] + 120))) last_page_idx = pos[0][-1]
if not (0 <= last_page_idx < page_count):
logging.warning(f"Last page index {last_page_idx} out of range for {page_count} pages; skipping crop.")
if need_position:
return None, None
return
last_page_height = self.page_images[last_page_idx].size[1] / ZM
poss.append(
(
[last_page_idx],
pos[1],
pos[2],
min(last_page_height, pos[4] + GAP),
min(last_page_height, pos[4] + 120),
)
)
positions = [] positions = []
for ii, (pns, left, right, top, bottom) in enumerate(poss): for ii, (pns, left, right, top, bottom) in enumerate(poss):
right = left + max_width right = left + max_width
bottom *= ZM bottom *= ZM
for pn in pns[1:]: for pn in pns[1:]:
bottom += self.page_images[pn - 1].size[1] if 0 <= pn - 1 < page_count:
bottom += self.page_images[pn - 1].size[1]
else:
logging.warning(f"Page index {pn}-1 out of range for {page_count} pages during crop; skipping height accumulation.")
if not (0 <= pns[0] < page_count):
logging.warning(f"Base page index {pns[0]} out of range for {page_count} pages during crop; skipping this segment.")
continue
imgs.append(self.page_images[pns[0]].crop((left * ZM, top * ZM, right * ZM, min(bottom, self.page_images[pns[0]].size[1])))) imgs.append(self.page_images[pns[0]].crop((left * ZM, top * ZM, right * ZM, min(bottom, self.page_images[pns[0]].size[1]))))
if 0 < ii < len(poss) - 1: if 0 < ii < len(poss) - 1:
positions.append((pns[0] + self.page_from, left, right, top, min(bottom, self.page_images[pns[0]].size[1]) / ZM)) positions.append((pns[0] + self.page_from, left, right, top, min(bottom, self.page_images[pns[0]].size[1]) / ZM))
bottom -= self.page_images[pns[0]].size[1] bottom -= self.page_images[pns[0]].size[1]
for pn in pns[1:]: for pn in pns[1:]:
if not (0 <= pn < page_count):
logging.warning(f"Page index {pn} out of range for {page_count} pages during crop; skipping this page.")
continue
imgs.append(self.page_images[pn].crop((left * ZM, 0, right * ZM, min(bottom, self.page_images[pn].size[1])))) imgs.append(self.page_images[pn].crop((left * ZM, 0, right * ZM, min(bottom, self.page_images[pn].size[1]))))
if 0 < ii < len(poss) - 1: if 0 < ii < len(poss) - 1:
positions.append((pn + self.page_from, left, right, 0, min(bottom, self.page_images[pn].size[1]) / ZM)) positions.append((pn + self.page_from, left, right, 0, min(bottom, self.page_images[pn].size[1]) / ZM))

View File

@ -47,6 +47,7 @@ class TencentCloudAPIClient:
self.secret_id = secret_id self.secret_id = secret_id
self.secret_key = secret_key self.secret_key = secret_key
self.region = region self.region = region
self.outlines = []
# Create credentials # Create credentials
self.cred = credential.Credential(secret_id, secret_key) self.cred = credential.Credential(secret_id, secret_key)

View File

@ -106,11 +106,11 @@ ADMIN_SVR_HTTP_PORT=9381
SVR_MCP_PORT=9382 SVR_MCP_PORT=9382
# The RAGFlow Docker image to download. v0.22+ doesn't include embedding models. # The RAGFlow Docker image to download. v0.22+ doesn't include embedding models.
RAGFLOW_IMAGE=infiniflow/ragflow:v0.22.0 RAGFLOW_IMAGE=infiniflow/ragflow:v0.22.1
# If you cannot download the RAGFlow Docker image: # If you cannot download the RAGFlow Docker image:
# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.22.0 # RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:v0.22.1
# RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.22.0 # RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:v0.22.1
# #
# - For the `nightly` edition, uncomment either of the following: # - For the `nightly` edition, uncomment either of the following:
# RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly # RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly

View File

@ -77,7 +77,7 @@ The [.env](./.env) file contains important environment variables for Docker.
- `SVR_HTTP_PORT` - `SVR_HTTP_PORT`
The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`. The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`.
- `RAGFLOW-IMAGE` - `RAGFLOW-IMAGE`
The Docker image edition. Defaults to `infiniflow/ragflow:v0.22.0`. The RAGFlow Docker image does not include embedding models. The Docker image edition. Defaults to `infiniflow/ragflow:v0.22.1`. The RAGFlow Docker image does not include embedding models.
> [!TIP] > [!TIP]

View File

@ -97,7 +97,7 @@ RAGFlow utilizes MinIO as its object storage solution, leveraging its scalabilit
- `SVR_HTTP_PORT` - `SVR_HTTP_PORT`
The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`. The port used to expose RAGFlow's HTTP API service to the host machine, allowing **external** access to the service running inside the Docker container. Defaults to `9380`.
- `RAGFLOW-IMAGE` - `RAGFLOW-IMAGE`
The Docker image edition. Defaults to `infiniflow/ragflow:v0.22.0` (the RAGFlow Docker image without embedding models). The Docker image edition. Defaults to `infiniflow/ragflow:v0.22.1` (the RAGFlow Docker image without embedding models).
:::tip NOTE :::tip NOTE
If you cannot download the RAGFlow Docker image, try the following mirrors. If you cannot download the RAGFlow Docker image, try the following mirrors.

View File

@ -47,7 +47,7 @@ After building the infiniflow/ragflow:nightly image, you are ready to launch a f
1. Edit Docker Compose Configuration 1. Edit Docker Compose Configuration
Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.22.0` to `infiniflow/ragflow:nightly` to use the pre-built image. Open the `docker/.env` file. Find the `RAGFLOW_IMAGE` setting and change the image reference from `infiniflow/ragflow:v0.22.1` to `infiniflow/ragflow:nightly` to use the pre-built image.
2. Launch the Service 2. Launch the Service

View File

@ -133,7 +133,7 @@ See [Run retrieval test](./run_retrieval_test.md) for details.
## Search for dataset ## Search for dataset
As of RAGFlow v0.22.0, the search feature is still in a rudimentary form, supporting only dataset search by name. As of RAGFlow v0.22.1, the search feature is still in a rudimentary form, supporting only dataset search by name.
![search dataset](https://raw.githubusercontent.com/infiniflow/ragflow-docs/main/images/search_datasets.jpg) ![search dataset](https://raw.githubusercontent.com/infiniflow/ragflow-docs/main/images/search_datasets.jpg)

View File

@ -87,4 +87,4 @@ RAGFlow's file management allows you to download an uploaded file:
![download_file](https://github.com/infiniflow/ragflow/assets/93570324/cf3b297f-7d9b-4522-bf5f-4f45743e4ed5) ![download_file](https://github.com/infiniflow/ragflow/assets/93570324/cf3b297f-7d9b-4522-bf5f-4f45743e4ed5)
> As of RAGFlow v0.22.0, bulk download is not supported, nor can you download an entire folder. > As of RAGFlow v0.22.1, bulk download is not supported, nor can you download an entire folder.

View File

@ -46,7 +46,7 @@ The Admin CLI and Admin Service form a client-server architectural suite for RAG
2. Install ragflow-cli. 2. Install ragflow-cli.
```bash ```bash
pip install ragflow-cli==0.22.0 pip install ragflow-cli==0.22.1
``` ```
3. Launch the CLI client: 3. Launch the CLI client:

View File

@ -48,16 +48,16 @@ To upgrade RAGFlow, you must upgrade **both** your code **and** your Docker imag
git clone https://github.com/infiniflow/ragflow.git git clone https://github.com/infiniflow/ragflow.git
``` ```
2. Switch to the latest, officially published release, e.g., `v0.22.0`: 2. Switch to the latest, officially published release, e.g., `v0.22.1`:
```bash ```bash
git checkout -f v0.22.0 git checkout -f v0.22.1
``` ```
3. Update **ragflow/docker/.env**: 3. Update **ragflow/docker/.env**:
```bash ```bash
RAGFLOW_IMAGE=infiniflow/ragflow:v0.22.0 RAGFLOW_IMAGE=infiniflow/ragflow:v0.22.1
``` ```
4. Update the RAGFlow image and restart RAGFlow: 4. Update the RAGFlow image and restart RAGFlow:
@ -78,10 +78,10 @@ No, you do not need to. Upgrading RAGFlow in itself will *not* remove your uploa
1. From an environment with Internet access, pull the required Docker image. 1. From an environment with Internet access, pull the required Docker image.
2. Save the Docker image to a **.tar** file. 2. Save the Docker image to a **.tar** file.
```bash ```bash
docker save -o ragflow.v0.22.0.tar infiniflow/ragflow:v0.22.0 docker save -o ragflow.v0.22.1.tar infiniflow/ragflow:v0.22.1
``` ```
3. Copy the **.tar** file to the target server. 3. Copy the **.tar** file to the target server.
4. Load the **.tar** file into Docker: 4. Load the **.tar** file into Docker:
```bash ```bash
docker load -i ragflow.v0.22.0.tar docker load -i ragflow.v0.22.1.tar
``` ```

View File

@ -44,7 +44,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
`vm.max_map_count`. This value sets the maximum number of memory map areas a process may have. Its default value is 65530. While most applications require fewer than a thousand maps, reducing this value can result in abnormal behaviors, and the system will throw out-of-memory errors when a process reaches the limitation. `vm.max_map_count`. This value sets the maximum number of memory map areas a process may have. Its default value is 65530. While most applications require fewer than a thousand maps, reducing this value can result in abnormal behaviors, and the system will throw out-of-memory errors when a process reaches the limitation.
RAGFlow v0.22.0 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component. RAGFlow v0.22.1 uses Elasticsearch or [Infinity](https://github.com/infiniflow/infinity) for multiple recall. Setting the value of `vm.max_map_count` correctly is crucial to the proper functioning of the Elasticsearch component.
<Tabs <Tabs
defaultValue="linux" defaultValue="linux"
@ -184,7 +184,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
```bash ```bash
$ git clone https://github.com/infiniflow/ragflow.git $ git clone https://github.com/infiniflow/ragflow.git
$ cd ragflow/docker $ cd ragflow/docker
$ git checkout -f v0.22.0 $ git checkout -f v0.22.1
``` ```
3. Use the pre-built Docker images and start up the server: 3. Use the pre-built Docker images and start up the server:
@ -200,7 +200,7 @@ This section provides instructions on setting up the RAGFlow server on Linux. If
| RAGFlow image tag | Image size (GB) | Stable? | | RAGFlow image tag | Image size (GB) | Stable? |
| ------------------- | --------------- | ------------------------ | | ------------------- | --------------- | ------------------------ |
| v0.22.0 | &approx;2 | Stable release | | v0.22.1 | &approx;2 | Stable release |
| nightly | &approx;2 | _Unstable_ nightly build | | nightly | &approx;2 | _Unstable_ nightly build |
```mdx-code-block ```mdx-code-block

View File

@ -7,6 +7,32 @@ slug: /release_notes
Key features, improvements and bug fixes in the latest releases. Key features, improvements and bug fixes in the latest releases.
## v0.22.1
Released on November 19, 2025.
### Improvements
- Agent:
- Supports exporting Agent outputs in Word or Markdown formats.
- Adds a **List operations** component.
- Adds a **Variable aggregator** component.
- Data sources:
- Supports S3-compatible data sources, e.g., MinIO.
- Adds data synchronization with JIRA.
- Continues the redesign of the **Profile** page layouts.
- Upgrades the Flask web framework from synchronous to asynchronous, increasing concurrency and preventing blocking issues caused when requesting upstream LLM services.
### Fixed issues
- A v0.22.0 issue: Users failed to parse uploaded files or switch embedding model in a dataset containing parsed files using a built-in model from a `-full` RAGFlow edition.
- Image concatenated in Word documents. [#11310](https://github.com/infiniflow/ragflow/pull/11310)
- Mixed images and text were not correctly displayed in the chat history.
### Newly supported models
- Gemini 3 Pro Preview
## v0.22.0 ## v0.22.0
Released on November 12, 2025. Released on November 12, 2025.
@ -77,7 +103,7 @@ Released on October 15, 2025.
- Redesigns RAGFlow's Login and Registration pages. - Redesigns RAGFlow's Login and Registration pages.
- Upgrades RAGFlow's document engine Infinity to v0.6.0. - Upgrades RAGFlow's document engine Infinity to v0.6.0.
### Added models ### Newly supported models
- Tongyi Qwen 3 series - Tongyi Qwen 3 series
- Claude Sonnet 4.5 - Claude Sonnet 4.5
@ -100,7 +126,7 @@ Released on September 10, 2025.
- **Execute SQL** component enhanced: Replaces the original variable reference component with a text input field, allowing users to write free-form SQL queries and reference variables. See [here](./guides/agent/agent_component_reference/execute_sql.md). - **Execute SQL** component enhanced: Replaces the original variable reference component with a text input field, allowing users to write free-form SQL queries and reference variables. See [here](./guides/agent/agent_component_reference/execute_sql.md).
- Chat: Re-enables **Reasoning** and **Cross-language search**. - Chat: Re-enables **Reasoning** and **Cross-language search**.
### Added models ### Newly supported models
- Meituan LongCat - Meituan LongCat
- Kimi: kimi-k2-turbo-preview and kimi-k2-0905-preview - Kimi: kimi-k2-turbo-preview and kimi-k2-0905-preview
@ -139,7 +165,7 @@ Released on August 27, 2025.
- Improves Markdown file parsing, with AST support to avoid unintended chunking. - Improves Markdown file parsing, with AST support to avoid unintended chunking.
- Enhances HTML parsing, supporting bs4-based HTML tag traversal. - Enhances HTML parsing, supporting bs4-based HTML tag traversal.
### Added models ### Newly supported models
ZHIPU GLM-4.5 ZHIPU GLM-4.5
@ -200,7 +226,7 @@ Released on August 8, 2025.
- The **Retrieval** component now supports the dynamic specification of dataset names using variables. - The **Retrieval** component now supports the dynamic specification of dataset names using variables.
- The user interface now includes a French language option. - The user interface now includes a French language option.
### Added Models ### Newly supported models
- GPT-5 - GPT-5
- Claude 4.1 - Claude 4.1
@ -264,7 +290,7 @@ Released on June 23, 2025.
- Added support for models installed via Ollama or VLLM when creating a dataset through the API. [#8069](https://github.com/infiniflow/ragflow/pull/8069) - Added support for models installed via Ollama or VLLM when creating a dataset through the API. [#8069](https://github.com/infiniflow/ragflow/pull/8069)
- Enabled role-based authentication for S3 bucket access. [#8149](https://github.com/infiniflow/ragflow/pull/8149) - Enabled role-based authentication for S3 bucket access. [#8149](https://github.com/infiniflow/ragflow/pull/8149)
### Added models ### Newly supported models
- Qwen 3 Embedding. [#8184](https://github.com/infiniflow/ragflow/pull/8184) - Qwen 3 Embedding. [#8184](https://github.com/infiniflow/ragflow/pull/8184)
- Voyage Multimodal 3. [#7987](https://github.com/infiniflow/ragflow/pull/7987) - Voyage Multimodal 3. [#7987](https://github.com/infiniflow/ragflow/pull/7987)

View File

@ -56,7 +56,7 @@ env:
ragflow: ragflow:
image: image:
repository: infiniflow/ragflow repository: infiniflow/ragflow
tag: v0.22.0 tag: v0.22.1
pullPolicy: IfNotPresent pullPolicy: IfNotPresent
pullSecrets: [] pullSecrets: []
# Optional service configuration overrides # Optional service configuration overrides

View File

@ -1,6 +1,6 @@
[project] [project]
name = "ragflow" name = "ragflow"
version = "0.22.0" version = "0.22.1"
description = "[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data." description = "[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It offers a streamlined RAG workflow for businesses of any scale, combining LLM (Large Language Models) to provide truthful question-answering capabilities, backed by well-founded citations from various complex formatted data."
authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }] authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }]
license-files = ["LICENSE"] license-files = ["LICENSE"]
@ -86,6 +86,9 @@ dependencies = [
"python-pptx>=1.0.2,<2.0.0", "python-pptx>=1.0.2,<2.0.0",
"pywencai==0.12.2", "pywencai==0.12.2",
"qianfan==0.4.6", "qianfan==0.4.6",
"quart-auth==0.11.0",
"quart-cors==0.8.0",
"Quart==0.20.0",
"ranx==0.3.20", "ranx==0.3.20",
"readability-lxml==0.8.1", "readability-lxml==0.8.1",
"valkey==6.0.2", "valkey==6.0.2",

View File

@ -216,6 +216,30 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
**kwargs **kwargs
) )
def _normalize_section(section):
# pad section to length 3: (txt, sec_id, poss)
if len(section) == 1:
section = (section[0], "", [])
elif len(section) == 2:
section = (section[0], "", section[1])
elif len(section) != 3:
raise ValueError(f"Unexpected section length: {len(section)} (value={section!r})")
txt, sec_id, poss = section
if isinstance(poss, str):
poss = pdf_parser.extract_positions(poss)
first = poss[0] # tuple: ([pn], x1, x2, y1, y2)
pn = first[0]
if isinstance(pn, list):
pn = pn[0] # [pn] -> pn
poss[0] = (pn, *first[1:])
return (txt, sec_id, poss)
sections = [_normalize_section(sec) for sec in sections]
if not sections and not tbls: if not sections and not tbls:
return [] return []

View File

@ -70,6 +70,7 @@ def by_mineru(filename, binary=None, from_page=0, to_page=100000, lang="Chinese"
callback=callback, callback=callback,
output_dir=os.environ.get("MINERU_OUTPUT_DIR", ""), output_dir=os.environ.get("MINERU_OUTPUT_DIR", ""),
backend=os.environ.get("MINERU_BACKEND", "pipeline"), backend=os.environ.get("MINERU_BACKEND", "pipeline"),
server_url=os.environ.get("MINERU_SERVER_URL", ""),
delete_output=bool(int(os.environ.get("MINERU_DELETE_OUTPUT", 1))), delete_output=bool(int(os.environ.get("MINERU_DELETE_OUTPUT", 1))),
) )
return sections, tables, pdf_parser return sections, tables, pdf_parser

View File

@ -14,24 +14,27 @@
# limitations under the License. # limitations under the License.
# #
import re
import base64 import base64
import json import json
import os
import tempfile
import logging import logging
import os
import re
import tempfile
from abc import ABC from abc import ABC
from copy import deepcopy from copy import deepcopy
from io import BytesIO from io import BytesIO
from pathlib import Path from pathlib import Path
from urllib.parse import urljoin from urllib.parse import urljoin
import requests import requests
from openai import OpenAI from openai import OpenAI
from openai.lib.azure import AzureOpenAI from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI from zhipuai import ZhipuAI
from common.token_utils import num_tokens_from_string, total_token_count_from_response
from rag.nlp import is_english from rag.nlp import is_english
from rag.prompts.generator import vision_llm_describe_prompt from rag.prompts.generator import vision_llm_describe_prompt
from common.token_utils import num_tokens_from_string, total_token_count_from_response
class Base(ABC): class Base(ABC):
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -70,12 +73,7 @@ class Base(ABC):
pmpt = [{"type": "text", "text": text}] pmpt = [{"type": "text", "text": text}]
for img in images: for img in images:
pmpt.append({ pmpt.append({"type": "image_url", "image_url": {"url": img if isinstance(img, str) and img.startswith("data:") else f"data:image/png;base64,{img}"}})
"type": "image_url",
"image_url": {
"url": img if isinstance(img, str) and img.startswith("data:") else f"data:image/png;base64,{img}"
}
})
return pmpt return pmpt
def chat(self, system, history, gen_conf, images=None, **kwargs): def chat(self, system, history, gen_conf, images=None, **kwargs):
@ -128,7 +126,7 @@ class Base(ABC):
try: try:
image.save(buffered, format="JPEG") image.save(buffered, format="JPEG")
except Exception: except Exception:
# reset buffer before saving PNG # reset buffer before saving PNG
buffered.seek(0) buffered.seek(0)
buffered.truncate() buffered.truncate()
image.save(buffered, format="PNG") image.save(buffered, format="PNG")
@ -158,7 +156,7 @@ class Base(ABC):
try: try:
image.save(buffered, format="JPEG") image.save(buffered, format="JPEG")
except Exception: except Exception:
# reset buffer before saving PNG # reset buffer before saving PNG
buffered.seek(0) buffered.seek(0)
buffered.truncate() buffered.truncate()
image.save(buffered, format="PNG") image.save(buffered, format="PNG")
@ -176,18 +174,13 @@ class Base(ABC):
"请用中文详细描述一下图中的内容,比如时间,地点,人物,事情,人物心情等,如果有数据请提取出数据。" "请用中文详细描述一下图中的内容,比如时间,地点,人物,事情,人物心情等,如果有数据请提取出数据。"
if self.lang.lower() == "chinese" if self.lang.lower() == "chinese"
else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out.", else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out.",
b64 b64,
) ),
} }
] ]
def vision_llm_prompt(self, b64, prompt=None): def vision_llm_prompt(self, b64, prompt=None):
return [ return [{"role": "user", "content": self._image_prompt(prompt if prompt else vision_llm_describe_prompt(), b64)}]
{
"role": "user",
"content": self._image_prompt(prompt if prompt else vision_llm_describe_prompt(), b64)
}
]
class GptV4(Base): class GptV4(Base):
@ -208,7 +201,7 @@ class GptV4(Base):
model=self.model_name, model=self.model_name,
messages=self.prompt(b64), messages=self.prompt(b64),
extra_body=self.extra_body, extra_body=self.extra_body,
unused = None, unused=None,
) )
return res.choices[0].message.content.strip(), total_token_count_from_response(res) return res.choices[0].message.content.strip(), total_token_count_from_response(res)
@ -219,7 +212,7 @@ class GptV4(Base):
messages=self.vision_llm_prompt(b64, prompt), messages=self.vision_llm_prompt(b64, prompt),
extra_body=self.extra_body, extra_body=self.extra_body,
) )
return res.choices[0].message.content.strip(),total_token_count_from_response(res) return res.choices[0].message.content.strip(), total_token_count_from_response(res)
class AzureGptV4(GptV4): class AzureGptV4(GptV4):
@ -324,14 +317,12 @@ class Zhipu4V(GptV4):
self.lang = lang self.lang = lang
Base.__init__(self, **kwargs) Base.__init__(self, **kwargs)
def _clean_conf(self, gen_conf): def _clean_conf(self, gen_conf):
if "max_tokens" in gen_conf: if "max_tokens" in gen_conf:
del gen_conf["max_tokens"] del gen_conf["max_tokens"]
gen_conf = self._clean_conf_plealty(gen_conf) gen_conf = self._clean_conf_plealty(gen_conf)
return gen_conf return gen_conf
def _clean_conf_plealty(self, gen_conf): def _clean_conf_plealty(self, gen_conf):
if "presence_penalty" in gen_conf: if "presence_penalty" in gen_conf:
del gen_conf["presence_penalty"] del gen_conf["presence_penalty"]
@ -339,24 +330,17 @@ class Zhipu4V(GptV4):
del gen_conf["frequency_penalty"] del gen_conf["frequency_penalty"]
return gen_conf return gen_conf
def _request(self, msg, stream, gen_conf={}): def _request(self, msg, stream, gen_conf={}):
response = requests.post( response = requests.post(
self.base_url, self.base_url,
json={ json={"model": self.model_name, "messages": msg, "stream": stream, **gen_conf},
"model": self.model_name, headers={
"messages": msg, "Authorization": f"Bearer {self.api_key}",
"stream": stream, "Content-Type": "application/json",
**gen_conf
}, },
headers= {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
) )
return response.json() return response.json()
def chat(self, system, history, gen_conf, images=None, stream=False, **kwargs): def chat(self, system, history, gen_conf, images=None, stream=False, **kwargs):
if system and history and history[0].get("role") != "system": if system and history and history[0].get("role") != "system":
history.insert(0, {"role": "system", "content": system}) history.insert(0, {"role": "system", "content": system})
@ -369,10 +353,9 @@ class Zhipu4V(GptV4):
cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip() cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip()
return cleaned, total_token_count_from_response(response) return cleaned, total_token_count_from_response(response)
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs): def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN
from rag.nlp import is_chinese from rag.nlp import is_chinese
if system and history and history[0].get("role") != "system": if system and history and history[0].get("role") != "system":
@ -402,44 +385,24 @@ class Zhipu4V(GptV4):
yield tk_count yield tk_count
def describe(self, image): def describe(self, image):
return self.describe_with_prompt(image) return self.describe_with_prompt(image)
def describe_with_prompt(self, image, prompt=None): def describe_with_prompt(self, image, prompt=None):
b64 = self.image2base64(image) b64 = self.image2base64(image)
if prompt is None: if prompt is None:
prompt = "Describe this image." prompt = "Describe this image."
# Chat messages # Chat messages
messages = [ messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": b64}}, {"type": "text", "text": prompt}]}]
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": { "url": b64 }
},
{
"type": "text",
"text": prompt
}
]
}
]
resp = self.client.chat.completions.create( resp = self.client.chat.completions.create(model=self.model_name, messages=messages, stream=False)
model=self.model_name,
messages=messages,
stream=False
)
content = resp.choices[0].message.content.strip() content = resp.choices[0].message.content.strip()
cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip() cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip()
return cleaned, num_tokens_from_string(cleaned) return cleaned, num_tokens_from_string(cleaned)
class StepFunCV(GptV4): class StepFunCV(GptV4):
_FACTORY_NAME = "StepFun" _FACTORY_NAME = "StepFun"
@ -452,6 +415,7 @@ class StepFunCV(GptV4):
self.lang = lang self.lang = lang
Base.__init__(self, **kwargs) Base.__init__(self, **kwargs)
class VolcEngineCV(GptV4): class VolcEngineCV(GptV4):
_FACTORY_NAME = "VolcEngine" _FACTORY_NAME = "VolcEngine"
@ -464,6 +428,7 @@ class VolcEngineCV(GptV4):
self.lang = lang self.lang = lang
Base.__init__(self, **kwargs) Base.__init__(self, **kwargs)
class LmStudioCV(GptV4): class LmStudioCV(GptV4):
_FACTORY_NAME = "LM-Studio" _FACTORY_NAME = "LM-Studio"
@ -502,13 +467,7 @@ class TogetherAICV(GptV4):
class YiCV(GptV4): class YiCV(GptV4):
_FACTORY_NAME = "01.AI" _FACTORY_NAME = "01.AI"
def __init__( def __init__(self, key, model_name, lang="Chinese", base_url="https://api.lingyiwanwu.com/v1", **kwargs):
self,
key,
model_name,
lang="Chinese",
base_url="https://api.lingyiwanwu.com/v1", **kwargs
):
if not base_url: if not base_url:
base_url = "https://api.lingyiwanwu.com/v1" base_url = "https://api.lingyiwanwu.com/v1"
super().__init__(key, model_name, lang, base_url, **kwargs) super().__init__(key, model_name, lang, base_url, **kwargs)
@ -517,13 +476,7 @@ class YiCV(GptV4):
class SILICONFLOWCV(GptV4): class SILICONFLOWCV(GptV4):
_FACTORY_NAME = "SILICONFLOW" _FACTORY_NAME = "SILICONFLOW"
def __init__( def __init__(self, key, model_name, lang="Chinese", base_url="https://api.siliconflow.cn/v1", **kwargs):
self,
key,
model_name,
lang="Chinese",
base_url="https://api.siliconflow.cn/v1", **kwargs
):
if not base_url: if not base_url:
base_url = "https://api.siliconflow.cn/v1" base_url = "https://api.siliconflow.cn/v1"
super().__init__(key, model_name, lang, base_url, **kwargs) super().__init__(key, model_name, lang, base_url, **kwargs)
@ -532,13 +485,7 @@ class SILICONFLOWCV(GptV4):
class OpenRouterCV(GptV4): class OpenRouterCV(GptV4):
_FACTORY_NAME = "OpenRouter" _FACTORY_NAME = "OpenRouter"
def __init__( def __init__(self, key, model_name, lang="Chinese", base_url="https://openrouter.ai/api/v1", **kwargs):
self,
key,
model_name,
lang="Chinese",
base_url="https://openrouter.ai/api/v1", **kwargs
):
if not base_url: if not base_url:
base_url = "https://openrouter.ai/api/v1" base_url = "https://openrouter.ai/api/v1"
api_key = json.loads(key).get("api_key", "") api_key = json.loads(key).get("api_key", "")
@ -549,6 +496,7 @@ class OpenRouterCV(GptV4):
provider_order = json.loads(key).get("provider_order", "") provider_order = json.loads(key).get("provider_order", "")
self.extra_body = {} self.extra_body = {}
if provider_order: if provider_order:
def _to_order_list(x): def _to_order_list(x):
if x is None: if x is None:
return [] return []
@ -557,6 +505,7 @@ class OpenRouterCV(GptV4):
if isinstance(x, (list, tuple)): if isinstance(x, (list, tuple)):
return [str(s).strip() for s in x if str(s).strip()] return [str(s).strip() for s in x if str(s).strip()]
return [] return []
provider_cfg = {} provider_cfg = {}
provider_order = _to_order_list(provider_order) provider_order = _to_order_list(provider_order)
provider_cfg["order"] = provider_order provider_cfg["order"] = provider_order
@ -616,18 +565,18 @@ class OllamaCV(Base):
def __init__(self, key, model_name, lang="Chinese", **kwargs): def __init__(self, key, model_name, lang="Chinese", **kwargs):
from ollama import Client from ollama import Client
self.client = Client(host=kwargs["base_url"]) self.client = Client(host=kwargs["base_url"])
self.model_name = model_name self.model_name = model_name
self.lang = lang self.lang = lang
self.keep_alive = kwargs.get("ollama_keep_alive", int(os.environ.get("OLLAMA_KEEP_ALIVE", -1))) self.keep_alive = kwargs.get("ollama_keep_alive", int(os.environ.get("OLLAMA_KEEP_ALIVE", -1)))
Base.__init__(self, **kwargs) Base.__init__(self, **kwargs)
def _clean_img(self, img): def _clean_img(self, img):
if not isinstance(img, str): if not isinstance(img, str):
return img return img
#remove the header like "data/*;base64," # remove the header like "data/*;base64,"
if img.startswith("data:") and ";base64," in img: if img.startswith("data:") and ";base64," in img:
img = img.split(";base64,")[1] img = img.split(";base64,")[1]
return img return img
@ -687,12 +636,7 @@ class OllamaCV(Base):
def chat(self, system, history, gen_conf, images=None, **kwargs): def chat(self, system, history, gen_conf, images=None, **kwargs):
try: try:
response = self.client.chat( response = self.client.chat(model=self.model_name, messages=self._form_history(system, history, images), options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
model=self.model_name,
messages=self._form_history(system, history, images),
options=self._clean_conf(gen_conf),
keep_alive=self.keep_alive
)
ans = response["message"]["content"].strip() ans = response["message"]["content"].strip()
return ans, response["eval_count"] + response.get("prompt_eval_count", 0) return ans, response["eval_count"] + response.get("prompt_eval_count", 0)
@ -702,13 +646,7 @@ class OllamaCV(Base):
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs): def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
ans = "" ans = ""
try: try:
response = self.client.chat( response = self.client.chat(model=self.model_name, messages=self._form_history(system, history, images), stream=True, options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
model=self.model_name,
messages=self._form_history(system, history, images),
stream=True,
options=self._clean_conf(gen_conf),
keep_alive=self.keep_alive
)
for resp in response: for resp in response:
if resp["done"]: if resp["done"]:
yield resp.get("prompt_eval_count", 0) + resp.get("eval_count", 0) yield resp.get("prompt_eval_count", 0) + resp.get("eval_count", 0)
@ -723,29 +661,80 @@ class GeminiCV(Base):
_FACTORY_NAME = "Gemini" _FACTORY_NAME = "Gemini"
def __init__(self, key, model_name="gemini-1.0-pro-vision-latest", lang="Chinese", **kwargs): def __init__(self, key, model_name="gemini-1.0-pro-vision-latest", lang="Chinese", **kwargs):
from google.generativeai import GenerativeModel, client from google import genai
client.configure(api_key=key) self.api_key = key
_client = client.get_default_generative_client()
self.api_key=key
self.model_name = model_name self.model_name = model_name
self.model = GenerativeModel(model_name=self.model_name) self.client = genai.Client(api_key=key)
self.model._client = _client
self.lang = lang self.lang = lang
Base.__init__(self, **kwargs) Base.__init__(self, **kwargs)
logging.info(f"[GeminiCV] Initialized with model={self.model_name} lang={self.lang}")
def _image_to_part(self, image):
from google.genai import types
if isinstance(image, str) and image.startswith("data:") and ";base64," in image:
header, b64data = image.split(",", 1)
mime = header.split(":", 1)[1].split(";", 1)[0]
data = base64.b64decode(b64data)
else:
data_url = self.image2base64(image)
header, b64data = data_url.split(",", 1)
mime = header.split(":", 1)[1].split(";", 1)[0]
data = base64.b64decode(b64data)
return types.Part(
inline_data=types.Blob(
mime_type=mime,
data=data,
)
)
def _form_history(self, system, history, images=None): def _form_history(self, system, history, images=None):
hist = [] from google.genai import types
if system:
hist.append({"role": "user", "parts": [system, history[0]["content"]]}) contents = []
images = images or []
system_len = len(system) if isinstance(system, str) else 0
history_len = len(history) if history else 0
images_len = len(images)
logging.info(f"[GeminiCV] _form_history called: system_len={system_len} history_len={history_len} images_len={images_len}")
image_parts = []
for img in images: for img in images:
hist[0]["parts"].append(("data:image/jpeg;base64," + img) if img[:4]!="data" else img) try:
for h in history[1:]: image_parts.append(self._image_to_part(img))
hist.append({"role": "user" if h["role"]=="user" else "model", "parts": [h["content"]]}) except Exception:
return hist continue
remaining_history = history or []
if system or remaining_history:
parts = []
if system:
parts.append(types.Part(text=system))
if remaining_history:
first = remaining_history[0]
parts.append(types.Part(text=first.get("content", "")))
remaining_history = remaining_history[1:]
parts.extend(image_parts)
contents.append(types.Content(role="user", parts=parts))
elif image_parts:
contents.append(types.Content(role="user", parts=image_parts))
role_map = {"user": "user", "assistant": "model", "system": "user"}
for h in remaining_history:
role = role_map.get(h.get("role"), "user")
contents.append(
types.Content(
role=role,
parts=[types.Part(text=h.get("content", ""))],
)
)
return contents
def describe(self, image): def describe(self, image):
from PIL.Image import open from google.genai import types
prompt = ( prompt = (
"请用中文详细描述一下图中的内容,比如时间,地点,人物,事情,人物心情等,如果有数据请提取出数据。" "请用中文详细描述一下图中的内容,比如时间,地点,人物,事情,人物心情等,如果有数据请提取出数据。"
@ -753,74 +742,104 @@ class GeminiCV(Base):
else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out." else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out."
) )
if image is bytes: contents = [
with BytesIO(image) as bio: types.Content(
with open(bio) as img: role="user",
input = [prompt, img] parts=[
res = self.model.generate_content(input) types.Part(text=prompt),
return res.text, total_token_count_from_response(res) self._image_to_part(image),
else: ],
b64 = self.image2base64_rawvalue(image) )
with BytesIO(base64.b64decode(b64)) as bio: ]
with open(bio) as img:
input = [prompt, img] res = self.client.models.generate_content(
res = self.model.generate_content(input) model=self.model_name,
return res.text, total_token_count_from_response(res) contents=contents,
)
return res.text, total_token_count_from_response(res)
def describe_with_prompt(self, image, prompt=None): def describe_with_prompt(self, image, prompt=None):
from PIL.Image import open from google.genai import types
vision_prompt = prompt if prompt else vision_llm_describe_prompt() vision_prompt = prompt if prompt else vision_llm_describe_prompt()
if image is bytes: contents = [
with BytesIO(image) as bio: types.Content(
with open(bio) as img: role="user",
input = [vision_prompt, img] parts=[
res = self.model.generate_content(input) types.Part(text=vision_prompt),
return res.text, total_token_count_from_response(res) self._image_to_part(image),
else: ],
b64 = self.image2base64_rawvalue(image) )
with BytesIO(base64.b64decode(b64)) as bio: ]
with open(bio) as img:
input = [vision_prompt, img]
res = self.model.generate_content(input)
return res.text, total_token_count_from_response(res)
res = self.client.models.generate_content(
model=self.model_name,
contents=contents,
)
return res.text, total_token_count_from_response(res)
def chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs): def chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs):
if video_bytes: if video_bytes:
try: try:
size = len(video_bytes) if video_bytes else 0
logging.info(f"[GeminiCV] chat called with video: filename={filename} size={size}")
summary, summary_num_tokens = self._process_video(video_bytes, filename) summary, summary_num_tokens = self._process_video(video_bytes, filename)
return summary, summary_num_tokens return summary, summary_num_tokens
except Exception as e: except Exception as e:
logging.info(f"[GeminiCV] chat video error: {e}")
return "**ERROR**: " + str(e), 0 return "**ERROR**: " + str(e), 0
generation_config = dict(temperature=gen_conf.get("temperature", 0.3), top_p=gen_conf.get("top_p", 0.7)) from google.genai import types
history_len = len(history) if history else 0
images_len = len(images) if images else 0
logging.info(f"[GeminiCV] chat called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
generation_config = types.GenerateContentConfig(
temperature=gen_conf.get("temperature", 0.3),
top_p=gen_conf.get("top_p", 0.7),
)
try: try:
response = self.model.generate_content( response = self.client.models.generate_content(
self._form_history(system, history, images), model=self.model_name,
generation_config=generation_config) contents=self._form_history(system, history, images),
config=generation_config,
)
ans = response.text ans = response.text
return ans, total_token_count_from_response(ans) logging.info("[GeminiCV] chat completed")
return ans, total_token_count_from_response(response)
except Exception as e: except Exception as e:
logging.warning(f"[GeminiCV] chat error: {e}")
return "**ERROR**: " + str(e), 0 return "**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs): def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
ans = "" ans = ""
response = None response = None
try: try:
generation_config = dict(temperature=gen_conf.get("temperature", 0.3), top_p=gen_conf.get("top_p", 0.7)) from google.genai import types
response = self.model.generate_content(
self._form_history(system, history, images), generation_config = types.GenerateContentConfig(
generation_config=generation_config, temperature=gen_conf.get("temperature", 0.3),
stream=True, top_p=gen_conf.get("top_p", 0.7),
)
history_len = len(history) if history else 0
images_len = len(images) if images else 0
logging.info(f"[GeminiCV] chat_streamly called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
response_stream = self.client.models.generate_content_stream(
model=self.model_name,
contents=self._form_history(system, history, images),
config=generation_config,
) )
for resp in response: for chunk in response_stream:
if not resp.text: if chunk.text:
continue ans += chunk.text
ans = resp.text yield chunk.text
yield ans logging.info("[GeminiCV] chat_streamly completed")
except Exception as e: except Exception as e:
logging.warning(f"[GeminiCV] chat_streamly error: {e}")
yield ans + "\n**ERROR**: " + str(e) yield ans + "\n**ERROR**: " + str(e)
yield total_token_count_from_response(response) yield total_token_count_from_response(response)
@ -830,17 +849,15 @@ class GeminiCV(Base):
from google.genai import types from google.genai import types
video_size_mb = len(video_bytes) / (1024 * 1024) video_size_mb = len(video_bytes) / (1024 * 1024)
client = genai.Client(api_key=self.api_key) client = self.client if hasattr(self, "client") else genai.Client(api_key=self.api_key)
logging.info(f"[GeminiCV] _process_video called: filename={filename} size_mb={video_size_mb:.2f}")
tmp_path = None tmp_path = None
try: try:
if video_size_mb <= 20: if video_size_mb <= 20:
response = client.models.generate_content( response = client.models.generate_content(
model="models/gemini-2.5-flash", model="models/gemini-2.5-flash",
contents=types.Content(parts=[ contents=types.Content(parts=[types.Part(inline_data=types.Blob(data=video_bytes, mime_type="video/mp4")), types.Part(text="Please summarize the video in proper sentences.")]),
types.Part(inline_data=types.Blob(data=video_bytes, mime_type="video/mp4")),
types.Part(text="Please summarize the video in proper sentences.")
])
) )
else: else:
logging.info(f"Video size {video_size_mb:.2f}MB exceeds 20MB. Using Files API...") logging.info(f"Video size {video_size_mb:.2f}MB exceeds 20MB. Using Files API...")
@ -850,16 +867,13 @@ class GeminiCV(Base):
tmp_path = Path(tmp.name) tmp_path = Path(tmp.name)
uploaded_file = client.files.upload(file=tmp_path) uploaded_file = client.files.upload(file=tmp_path)
response = client.models.generate_content( response = client.models.generate_content(model="gemini-2.5-flash", contents=[uploaded_file, "Please summarize this video in proper sentences."])
model="gemini-2.5-flash",
contents=[uploaded_file, "Please summarize this video in proper sentences."]
)
summary = response.text or "" summary = response.text or ""
logging.info(f"Video summarized: {summary[:32]}...") logging.info(f"[GeminiCV] Video summarized: {summary[:32]}...")
return summary, num_tokens_from_string(summary) return summary, num_tokens_from_string(summary)
except Exception as e: except Exception as e:
logging.error(f"Video processing failed: {e}") logging.warning(f"[GeminiCV] Video processing failed: {e}")
raise raise
finally: finally:
if tmp_path and tmp_path.exists(): if tmp_path and tmp_path.exists():
@ -869,13 +883,7 @@ class GeminiCV(Base):
class NvidiaCV(Base): class NvidiaCV(Base):
_FACTORY_NAME = "NVIDIA" _FACTORY_NAME = "NVIDIA"
def __init__( def __init__(self, key, model_name, lang="Chinese", base_url="https://ai.api.nvidia.com/v1/vlm", **kwargs):
self,
key,
model_name,
lang="Chinese",
base_url="https://ai.api.nvidia.com/v1/vlm", **kwargs
):
if not base_url: if not base_url:
base_url = ("https://ai.api.nvidia.com/v1/vlm",) base_url = ("https://ai.api.nvidia.com/v1/vlm",)
self.lang = lang self.lang = lang
@ -920,9 +928,7 @@ class NvidiaCV(Base):
"content-type": "application/json", "content-type": "application/json",
"Authorization": f"Bearer {self.key}", "Authorization": f"Bearer {self.key}",
}, },
json={ json={"messages": msg, **gen_conf},
"messages": msg, **gen_conf
},
) )
return response.json() return response.json()
@ -930,18 +936,12 @@ class NvidiaCV(Base):
b64 = self.image2base64(image) b64 = self.image2base64(image)
vision_prompt = self.vision_llm_prompt(b64, prompt) if prompt else self.vision_llm_prompt(b64) vision_prompt = self.vision_llm_prompt(b64, prompt) if prompt else self.vision_llm_prompt(b64)
response = self._request(vision_prompt) response = self._request(vision_prompt)
return ( return (response["choices"][0]["message"]["content"].strip(), total_token_count_from_response(response))
response["choices"][0]["message"]["content"].strip(),
total_token_count_from_response(response)
)
def chat(self, system, history, gen_conf, images=None, **kwargs): def chat(self, system, history, gen_conf, images=None, **kwargs):
try: try:
response = self._request(self._form_history(system, history, images), gen_conf) response = self._request(self._form_history(system, history, images), gen_conf)
return ( return (response["choices"][0]["message"]["content"].strip(), total_token_count_from_response(response))
response["choices"][0]["message"]["content"].strip(),
total_token_count_from_response(response)
)
except Exception as e: except Exception as e:
return "**ERROR**: " + str(e), 0 return "**ERROR**: " + str(e), 0
@ -950,7 +950,7 @@ class NvidiaCV(Base):
try: try:
response = self._request(self._form_history(system, history, images), gen_conf) response = self._request(self._form_history(system, history, images), gen_conf)
cnt = response["choices"][0]["message"]["content"] cnt = response["choices"][0]["message"]["content"]
total_tokens += total_token_count_from_response(response) total_tokens += total_token_count_from_response(response)
for resp in cnt: for resp in cnt:
yield resp yield resp
except Exception as e: except Exception as e:
@ -978,14 +978,15 @@ class AnthropicCV(Base):
return text return text
pmpt = [{"type": "text", "text": text}] pmpt = [{"type": "text", "text": text}]
for img in images: for img in images:
pmpt.append({ pmpt.append(
"type": "image", {
"source": { "type": "image",
"type": "base64", "source": {
"media_type": (img.split(":")[1].split(";")[0] if isinstance(img, str) and img[:4] == "data" else "image/png"), "type": "base64",
"data": (img.split(",")[1] if isinstance(img, str) and img[:4] == "data" else img) "media_type": (img.split(":")[1].split(";")[0] if isinstance(img, str) and img[:4] == "data" else "image/png"),
}, "data": (img.split(",")[1] if isinstance(img, str) and img[:4] == "data" else img),
} },
}
) )
return pmpt return pmpt

View File

@ -52,7 +52,7 @@ def chunks_format(reference):
"similarity": chunk.get("similarity"), "similarity": chunk.get("similarity"),
"vector_similarity": chunk.get("vector_similarity"), "vector_similarity": chunk.get("vector_similarity"),
"term_similarity": chunk.get("term_similarity"), "term_similarity": chunk.get("term_similarity"),
"doc_type": chunk.get("doc_type_kwd"), "doc_type": get_value(chunk, "doc_type_kwd", "doc_type"),
} }
for chunk in reference.get("chunks", []) for chunk in reference.get("chunks", [])
] ]

View File

@ -1,6 +1,6 @@
[project] [project]
name = "ragflow-sdk" name = "ragflow-sdk"
version = "0.22.0" version = "0.22.1"
description = "Python client sdk of [RAGFlow](https://github.com/infiniflow/ragflow). RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding." description = "Python client sdk of [RAGFlow](https://github.com/infiniflow/ragflow). RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding."
authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }] authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }]
license = { text = "Apache License, Version 2.0" } license = { text = "Apache License, Version 2.0" }

View File

@ -101,10 +101,10 @@ def test_invalid_name_dataset(get_auth):
# create dataset # create dataset
# with pytest.raises(Exception) as e: # with pytest.raises(Exception) as e:
res = create_dataset(get_auth, 0) res = create_dataset(get_auth, 0)
assert res['code'] == 100 assert res['code'] != 0
res = create_dataset(get_auth, "") res = create_dataset(get_auth, "")
assert res['code'] == 102 assert res['code'] != 0
long_string = "" long_string = ""
@ -112,7 +112,7 @@ def test_invalid_name_dataset(get_auth):
long_string += random.choice(string.ascii_letters + string.digits) long_string += random.choice(string.ascii_letters + string.digits)
res = create_dataset(get_auth, long_string) res = create_dataset(get_auth, long_string)
assert res['code'] == 102 assert res['code'] != 0
print(res) print(res)

2
sdk/python/uv.lock generated
View File

@ -353,7 +353,7 @@ wheels = [
[[package]] [[package]]
name = "ragflow-sdk" name = "ragflow-sdk"
version = "0.22.0" version = "0.22.1"
source = { virtual = "." } source = { virtual = "." }
dependencies = [ dependencies = [
{ name = "beartype" }, { name = "beartype" },

6737
uv.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -140,6 +140,16 @@
<path d="M0 0h1024v1024H0z" opacity=".01"></path> <path d="M0 0h1024v1024H0z" opacity=".01"></path>
<path d="M867.072 141.184H156.032a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64z m0.832 226.368H403.2a32 32 0 0 0 0 64h464.704a32 32 0 0 0 0-64zM403.2 573.888h464.704a32 32 0 0 1 0 64H403.2a32 32 0 0 1 0-64z m464.704 226.368H156.864a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64zM137.472 367.552v270.336l174.528-122.24-174.528-148.096z" ></path> <path d="M867.072 141.184H156.032a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64z m0.832 226.368H403.2a32 32 0 0 0 0 64h464.704a32 32 0 0 0 0-64zM403.2 573.888h464.704a32 32 0 0 1 0 64H403.2a32 32 0 0 1 0-64z m464.704 226.368H156.864a32 32 0 0 0 0 64h711.04a32 32 0 0 0 0-64zM137.472 367.552v270.336l174.528-122.24-174.528-148.096z" ></path>
</symbol>` + </symbol>` +
` <symbol id="icon-a-listoperations" viewBox="0 0 1024 1024">
<path d="M341.376 96a32 32 0 0 1 0 64h-128a10.688 10.688 0 0 0-10.688 10.688v682.624a10.752 10.752 0 0 0 10.688 10.688h128a32 32 0 0 1 0 64h-128a74.688 74.688 0 0 1-74.688-74.688V170.688A74.688 74.688 0 0 1 213.376 96h128z m469.312 0a74.688 74.688 0 0 1 74.688 74.688v682.624a74.752 74.752 0 0 1-74.688 74.688h-128a32 32 0 1 1 0-64h128a10.752 10.752 0 0 0 10.688-10.688V170.688a10.752 10.752 0 0 0-10.688-10.688h-128a32 32 0 1 1 0-64h128zM357.248 464.256a48 48 0 0 1 0 95.488l-4.928 0.256H352a48 48 0 0 1 0-96h0.32l4.928 0.256z m155.072-0.256a48 48 0 1 1 0 96H512a48 48 0 0 1 0-96h0.32z m160 0a48 48 0 0 1 0 96H672a48 48 0 0 1 0-96h0.32z" ></path>
</symbol>` +
`<symbol id="icon-aggregator" viewBox="0 0 1024 1024">
<path d="M949.312 533.312a32 32 0 0 1-9.344 22.592l-170.688 170.688a32 32 0 0 1-45.248-45.248l116.032-116.032H478.208l-10.176-0.128a202.688 202.688 0 0 1-135.36-59.264L41.344 214.592a32 32 0 1 1 45.312-45.248l291.264 291.328 10.24 9.344a138.688 138.688 0 0 0 89.344 31.296h362.56L724.032 385.28a32 32 0 0 1 45.248-45.248l170.688 170.624a32 32 0 0 1 9.344 22.656zM299.968 638.656a32 32 0 0 1 0 45.248L86.656 897.28a32 32 0 0 1-45.312-45.248L254.72 638.72a32 32 0 0 1 45.312 0z" ></path>
</symbol>` +
`<symbol id="icon-a-ariableassigner" viewBox="0 0 1024 1024">
<path d="M509.056 64c123.136 0 235.072 48.512 317.12 130.56l-41.024 37.312C714.24 161.024 617.216 119.936 509.056 119.936a391.808 391.808 0 1 0 0 783.552 392.448 392.448 0 0 0 294.784-134.272l41.024 37.312c-82.048 93.248-201.472 149.248-335.808 149.248-246.272 3.712-447.744-197.76-447.744-444.032S262.784 64 509.056 64z m-63.424 186.56a29.184 29.184 0 0 1 14.912 14.912l160.448 444.032c3.712 14.912-3.712 26.112-18.56 33.536-14.976 3.776-26.24-3.648-33.664-14.848l-48.512-149.248H341.12l-59.712 149.248a27.648 27.648 0 0 1-33.6 14.848c-14.912-3.712-18.56-18.624-14.848-33.536l179.008-444.032c3.776-11.136 22.4-18.624 33.6-14.912zM889.6 530.432c14.976 0 26.176 11.2 26.176 26.112a25.472 25.472 0 0 1-26.176 26.112h-212.608a25.472 25.472 0 0 1-26.112-26.112c0-14.912 11.2-26.112 26.112-26.112H889.6z m-529.792 0h141.824L434.432 351.36l-74.624 179.2zM889.6 411.008c14.912 0 26.176 11.2 26.176 26.112a25.536 25.536 0 0 1-26.176 26.112h-212.608a25.536 25.536 0 0 1-26.112-26.112c0-14.912 11.2-26.112 26.112-26.112H889.6z" ></path>
</symbol>
` +
'</svg>'), '</svg>'),
((h) => { ((h) => {
var a = (l = (l = document.getElementsByTagName('script'))[ var a = (l = (l = document.getElementsByTagName('script'))[

View File

@ -496,7 +496,7 @@ const DynamicForm = {
<Textarea <Textarea
{...finalFieldProps} {...finalFieldProps}
placeholder={field.placeholder} placeholder={field.placeholder}
className="resize-none" // className="resize-none"
/> />
); );
}} }}
@ -544,7 +544,7 @@ const DynamicForm = {
render={({ field: formField }) => ( render={({ field: formField }) => (
<FormItem <FormItem
className={cn('flex items-center w-full', { className={cn('flex items-center w-full', {
'flex-row items-start space-x-3 space-y-0': 'flex-row items-center space-x-3 space-y-0':
!field.horizontal, !field.horizontal,
})} })}
> >

View File

@ -1,15 +1,16 @@
import { FileIconMap } from '@/constants/file'; import { FileIconMap } from '@/constants/file';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { getExtension } from '@/utils/document-util'; import { getExtension } from '@/utils/document-util';
import { CSSProperties } from 'react';
type IconFontType = { type IconFontType = {
name: string; name: string;
className?: string; className?: string;
style?: CSSProperties;
}; };
export const IconFont = ({ name, className }: IconFontType) => ( export const IconFont = ({ name, className, style }: IconFontType) => (
<svg className={cn('size-4', className)}> <svg className={cn('size-4', className)} style={style}>
<use xlinkHref={`#icon-${name}`} /> <use xlinkHref={`#icon-${name}`} />
</svg> </svg>
); );

View File

@ -48,7 +48,7 @@ const JsonSchemaVisualizer: FC<JsonSchemaVisualizerProps> = ({
try { try {
const parsedJson = JSON.parse(value); const parsedJson = JSON.parse(value);
if (onChange) { if (onChange && typeof parsedJson !== 'number') {
onChange(parsedJson); onChange(parsedJson);
} }
} catch (_error) { } catch (_error) {

View File

@ -8,7 +8,10 @@ type KeyInputProps = {
} & Omit<InputProps, 'onChange'>; } & Omit<InputProps, 'onChange'>;
export const KeyInput = forwardRef<HTMLInputElement, KeyInputProps>( export const KeyInput = forwardRef<HTMLInputElement, KeyInputProps>(
function KeyInput({ value, onChange, searchValue = /[^a-zA-Z0-9_]/g }, ref) { function KeyInput(
{ value, onChange, searchValue = /[^a-zA-Z0-9_]/g, ...props },
ref,
) {
const handleChange = useCallback( const handleChange = useCallback(
(e: ChangeEvent<HTMLInputElement>) => { (e: ChangeEvent<HTMLInputElement>) => {
const value = e.target.value ?? ''; const value = e.target.value ?? '';
@ -18,6 +21,6 @@ export const KeyInput = forwardRef<HTMLInputElement, KeyInputProps>(
[onChange, searchValue], [onChange, searchValue],
); );
return <Input value={value} onChange={handleChange} ref={ref} />; return <Input {...props} value={value} onChange={handleChange} ref={ref} />;
}, },
); );

View File

@ -6,6 +6,7 @@ interface NumberInputProps {
value?: number; value?: number;
onChange?: (value: number) => void; onChange?: (value: number) => void;
height?: number | string; height?: number | string;
min?: number;
} }
const NumberInput: React.FC<NumberInputProps> = ({ const NumberInput: React.FC<NumberInputProps> = ({
@ -13,6 +14,7 @@ const NumberInput: React.FC<NumberInputProps> = ({
value: initialValue, value: initialValue,
onChange, onChange,
height, height,
min = 0,
}) => { }) => {
const [value, setValue] = useState<number>(() => { const [value, setValue] = useState<number>(() => {
return initialValue ?? 0; return initialValue ?? 0;
@ -76,6 +78,7 @@ const NumberInput: React.FC<NumberInputProps> = ({
onChange={handleChange} onChange={handleChange}
className="w-full flex-1 text-center bg-transparent focus:outline-none" className="w-full flex-1 text-center bg-transparent focus:outline-none"
style={style} style={style}
min={min}
/> />
<button <button
type="button" type="button"

View File

@ -2,7 +2,6 @@
'use client'; 'use client';
import { EyeIcon, EyeOffIcon } from 'lucide-react';
import React, { useId, useState } from 'react'; import React, { useId, useState } from 'react';
import { Input, InputProps } from '../ui/input'; import { Input, InputProps } from '../ui/input';
@ -33,11 +32,11 @@ export default React.forwardRef<HTMLInputElement, InputProps>(
aria-pressed={isVisible} aria-pressed={isVisible}
aria-controls="password" aria-controls="password"
> >
{isVisible ? ( {/* {isVisible ? (
<EyeOffIcon size={16} aria-hidden="true" /> <EyeOffIcon size={16} aria-hidden="true" />
) : ( ) : (
<EyeIcon size={16} aria-hidden="true" /> <EyeIcon size={16} aria-hidden="true" />
)} )} */}
</button> </button>
</div> </div>
</div> </div>

View File

@ -140,7 +140,7 @@ export const SelectWithSearch = forwardRef<
ref={ref} ref={ref}
disabled={disabled} disabled={disabled}
className={cn( className={cn(
'!bg-bg-input hover:bg-background border-input w-full justify-between px-3 font-normal outline-offset-0 outline-none focus-visible:outline-[3px] [&_svg]:pointer-events-auto', '!bg-bg-input hover:bg-background border-border-button w-full justify-between px-3 font-normal outline-offset-0 outline-none focus-visible:outline-[3px] [&_svg]:pointer-events-auto',
triggerClassName, triggerClassName,
)} )}
> >
@ -166,7 +166,7 @@ export const SelectWithSearch = forwardRef<
)} )}
<ChevronDownIcon <ChevronDownIcon
size={16} size={16}
className="text-muted-foreground/80 shrink-0 ml-2" className="text-text-disabled shrink-0 ml-2"
aria-hidden="true" aria-hidden="true"
/> />
</div> </div>

View File

@ -41,7 +41,7 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
type={isPasswordInput && showPassword ? 'text' : type} type={isPasswordInput && showPassword ? 'text' : type}
className={cn( className={cn(
'peer/input', 'peer/input',
'flex h-8 w-full rounded-md border-0.5 border-input bg-bg-input px-3 py-2 outline-none text-sm text-text-primary', 'flex h-8 w-full rounded-md border-0.5 border-border-button bg-bg-input px-3 py-2 outline-none text-sm text-text-primary',
'file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled', 'file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled',
'focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-accent-primary', 'focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-accent-primary',
'disabled:cursor-not-allowed disabled:opacity-50 transition-colors', 'disabled:cursor-not-allowed disabled:opacity-50 transition-colors',

View File

@ -54,7 +54,7 @@ const Textarea = forwardRef<HTMLTextAreaElement, TextareaProps>(
return ( return (
<textarea <textarea
className={cn( className={cn(
'flex min-h-[80px] w-full bg-bg-input rounded-md border border-input px-3 py-2 text-base ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 md:text-sm overflow-hidden', 'flex min-h-[80px] w-full bg-bg-input rounded-md border border-input px-3 py-2 text-base ring-offset-background placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-accent-primary focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 md:text-sm overflow-hidden',
className, className,
)} )}
rows={autoSize?.minRows ?? props.rows ?? undefined} rows={autoSize?.minRows ?? props.rows ?? undefined}

View File

@ -4,7 +4,7 @@ import * as TooltipPrimitive from '@radix-ui/react-tooltip';
import * as React from 'react'; import * as React from 'react';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { Info } from 'lucide-react'; import { CircleQuestionMark } from 'lucide-react';
const TooltipProvider = TooltipPrimitive.Provider; const TooltipProvider = TooltipPrimitive.Provider;
@ -39,7 +39,7 @@ export const FormTooltip = ({ tooltip }: { tooltip: React.ReactNode }) => {
e.preventDefault(); // Prevent clicking the tooltip from triggering form save e.preventDefault(); // Prevent clicking the tooltip from triggering form save
}} }}
> >
<Info className="size-3 ml-2" /> <CircleQuestionMark className="size-3 ml-2" />
</TooltipTrigger> </TooltipTrigger>
<TooltipContent>{tooltip}</TooltipContent> <TooltipContent>{tooltip}</TooltipContent>
</Tooltip> </Tooltip>

View File

@ -61,9 +61,10 @@ function buildLlmOptionsWithIcon(x: IThirdOAIModel) {
<div className="flex items-center justify-center gap-6"> <div className="flex items-center justify-center gap-6">
<LlmIcon <LlmIcon
name={getLLMIconName(x.fid, x.llm_name)} name={getLLMIconName(x.fid, x.llm_name)}
width={26} width={24}
height={26} height={24}
size={'small'} size={'small'}
imgClass="size-6"
/> />
<span>{getRealModelName(x.llm_name)}</span> <span>{getRealModelName(x.llm_name)}</span>
</div> </div>

View File

@ -12,9 +12,12 @@ export const LogicalOperatorIcon = function OperatorIcon({
return ( return (
<IconFont <IconFont
name={icon} name={icon}
className={cn('size-4', { className={cn('size-4')}
'rotate-180': value === ComparisonOperator.GreatThan, style={
})} value === ComparisonOperator.GreatThan
? { transform: 'rotate(180deg)' }
: undefined
}
></IconFont> ></IconFont>
); );
} }

View File

@ -1,6 +1,7 @@
export default { export default {
translation: { translation: {
common: { common: {
confirm: 'Confirm',
back: 'Back', back: 'Back',
noResults: 'No results.', noResults: 'No results.',
selectPlaceholder: 'select value', selectPlaceholder: 'select value',
@ -694,6 +695,7 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
tocEnhanceTip: ` During the parsing of the document, table of contents information was generated (see the 'Enable Table of Contents Extraction' option in the General method). This allows the large model to return table of contents items relevant to the user's query, thereby using these items to retrieve related chunks and apply weighting to these chunks during the sorting process. This approach is derived from mimicking the behavioral logic of how humans search for knowledge in books.`, tocEnhanceTip: ` During the parsing of the document, table of contents information was generated (see the 'Enable Table of Contents Extraction' option in the General method). This allows the large model to return table of contents items relevant to the user's query, thereby using these items to retrieve related chunks and apply weighting to these chunks during the sorting process. This approach is derived from mimicking the behavioral logic of how humans search for knowledge in books.`,
}, },
setting: { setting: {
edit: 'Edit',
cropTip: cropTip:
'Drag the selection area to choose the cropping position of the image, and scroll to zoom in/out', 'Drag the selection area to choose the cropping position of the image, and scroll to zoom in/out',
cropImage: 'Crop image', cropImage: 'Crop image',
@ -816,8 +818,7 @@ Example: https://fsn1.your-objectstorage.com`,
modelsToBeAdded: 'Models to be added', modelsToBeAdded: 'Models to be added',
addTheModel: 'Add', addTheModel: 'Add',
apiKey: 'API-Key', apiKey: 'API-Key',
apiKeyMessage: apiKeyMessage: 'Please enter the API key',
'Please enter the API key (for locally deployed model,ignore this).',
apiKeyTip: apiKeyTip:
'The API key can be obtained by registering the corresponding LLM supplier.', 'The API key can be obtained by registering the corresponding LLM supplier.',
showMoreModels: 'View models', showMoreModels: 'View models',
@ -1060,7 +1061,7 @@ Example: https://fsn1.your-objectstorage.com`,
apiKeyPlaceholder: apiKeyPlaceholder:
'YOUR_API_KEY (obtained from https://serpapi.com/manage-api-key)', 'YOUR_API_KEY (obtained from https://serpapi.com/manage-api-key)',
flowStart: 'Start', flowStart: 'Start',
flowNum: 'Num', flowNum: 'N',
test: 'Test', test: 'Test',
extractDepth: 'Extract Depth', extractDepth: 'Extract Depth',
format: 'Format', format: 'Format',
@ -1644,6 +1645,7 @@ The variable aggregation node (originally the variable assignment node) is a cru
beginInputTip: beginInputTip:
'By defining input parameters, this content can be accessed by other components in subsequent processes.', 'By defining input parameters, this content can be accessed by other components in subsequent processes.',
query: 'Query variables', query: 'Query variables',
queryRequired: 'Query is required',
queryTip: 'Select the variable you want to use', queryTip: 'Select the variable you want to use',
agent: 'Agent', agent: 'Agent',
addAgent: 'Add Agent', addAgent: 'Add Agent',
@ -1852,6 +1854,19 @@ Important structured information may include: names, dates, locations, events, k
asc: 'Ascending', asc: 'Ascending',
desc: 'Descending', desc: 'Descending',
}, },
variableAssignerLogicalOperatorOptions: {
overwrite: 'Overwritten By',
clear: 'Clear',
set: 'Set',
'+=': 'Add',
'-=': 'Subtract',
'*=': 'Multiply',
'/=': 'Divide',
append: 'Append',
extend: 'Extend',
removeFirst: 'Remove first',
removeLast: 'Remove last',
},
}, },
llmTools: { llmTools: {
bad_calculator: { bad_calculator: {

View File

@ -1,6 +1,7 @@
export default { export default {
translation: { translation: {
common: { common: {
confirm: '确定',
back: '返回', back: '返回',
noResults: '无结果。', noResults: '无结果。',
selectPlaceholder: '请选择', selectPlaceholder: '请选择',
@ -684,6 +685,7 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
tocEnhanceTip: `解析文档时生成了目录信息见General方法的启用目录抽取让大模型返回和用户问题相关的目录项从而利用目录项拿到相关chunk对这些chunk在排序中进行加权。这种方法来源于模仿人类查询书本中知识的行为逻辑`, tocEnhanceTip: `解析文档时生成了目录信息见General方法的启用目录抽取让大模型返回和用户问题相关的目录项从而利用目录项拿到相关chunk对这些chunk在排序中进行加权。这种方法来源于模仿人类查询书本中知识的行为逻辑`,
}, },
setting: { setting: {
edit: '编辑',
cropTip: '拖动选区可以选择要图片的裁剪位置,滚动可以放大/缩小选区', cropTip: '拖动选区可以选择要图片的裁剪位置,滚动可以放大/缩小选区',
cropImage: '剪裁图片', cropImage: '剪裁图片',
selectModelPlaceholder: '请选择模型', selectModelPlaceholder: '请选择模型',
@ -786,7 +788,7 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
modelsToBeAdded: '待添加的模型', modelsToBeAdded: '待添加的模型',
addTheModel: '添加', addTheModel: '添加',
apiKey: 'API-Key', apiKey: 'API-Key',
apiKeyMessage: '请输入api key(如果是本地部署的模型,请忽略它)', apiKeyMessage: '请输入api key',
apiKeyTip: 'API key可以通过注册相应的LLM供应商来获取。', apiKeyTip: 'API key可以通过注册相应的LLM供应商来获取。',
showMoreModels: '展示更多模型', showMoreModels: '展示更多模型',
hideModels: '隐藏模型', hideModels: '隐藏模型',
@ -1547,6 +1549,7 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
task: '任务', task: '任务',
beginInputTip: '通过定义输入参数,此内容可以被后续流程中的其他组件访问。', beginInputTip: '通过定义输入参数,此内容可以被后续流程中的其他组件访问。',
query: '查询变量', query: '查询变量',
queryRequired: '查询变量是必填项',
queryTip: '选择您想要使用的变量', queryTip: '选择您想要使用的变量',
agent: '智能体', agent: '智能体',
addAgent: '添加智能体', addAgent: '添加智能体',
@ -1713,6 +1716,19 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`,
asc: '升序', asc: '升序',
desc: '降序', desc: '降序',
}, },
variableAssignerLogicalOperatorOptions: {
overwrite: '覆盖',
clear: '清除',
set: '设置',
add: '加',
subtract: '减',
multiply: '乘',
divide: '除',
append: '追加',
extend: '扩展',
removeFirst: '移除第一个',
removeLast: '移除最后一个',
},
}, },
footer: { footer: {
profile: 'All rights reserved @ React', profile: 'All rights reserved @ React',

View File

@ -11,11 +11,12 @@ export function DataOperationsNode({
}: NodeProps<BaseNode<DataOperationsFormSchemaType>>) { }: NodeProps<BaseNode<DataOperationsFormSchemaType>>) {
const { data } = props; const { data } = props;
const { t } = useTranslation(); const { t } = useTranslation();
const operations = data.form?.operations;
return ( return (
<RagNode {...props}> <RagNode {...props}>
<LabelCard> <LabelCard>
{t(`flow.operationsOptions.${camelCase(data.form?.operations)}`)} {operations && t(`flow.operationsOptions.${camelCase(operations)}`)}
</LabelCard> </LabelCard>
</RagNode> </RagNode>
); );

View File

@ -81,7 +81,6 @@ export function AccordionOperators({
Operator.DataOperations, Operator.DataOperations,
Operator.VariableAssigner, Operator.VariableAssigner,
Operator.ListOperations, Operator.ListOperations,
Operator.VariableAssigner,
Operator.VariableAggregator, Operator.VariableAggregator,
]} ]}
isCustomDropdown={isCustomDropdown} isCustomDropdown={isCustomDropdown}

Some files were not shown because too many files have changed in this diff Show More