mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-02-05 18:15:06 +08:00
6f31c5fed2
### What problem does this PR solve? This PR adds MySQL and PostgreSQL as data source connectors, allowing users to import data directly from relational databases into RAGFlow for RAG workflows. Many users store their knowledge in databases (product catalogs, documentation, FAQs, etc.) and currently have no way to sync this data into RAGFlow without exporting to files first. This feature lets them connect directly to their databases, run SQL queries, and automatically create documents from the results. Closes #763 Closes #11560 ### Type of change - [ ] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [ ] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): ### What this PR does **New capabilities:** - Connect to MySQL and PostgreSQL databases - Run custom SQL queries to extract data - Map database columns to document content (vectorized) and metadata (searchable) - Support incremental sync using a timestamp column - Full frontend UI with connection form and tooltips **Files changed:** Backend: - `common/constants.py` - Added MYSQL/POSTGRESQL to FileSource enum - `common/data_source/config.py` - Added to DocumentSource enum - `common/data_source/rdbms_connector.py` - New connector (368 lines) - `common/data_source/__init__.py` - Exported the connector - `rag/svr/sync_data_source.py` - Added MySQL and PostgreSQL sync classes - `pyproject.toml` - Added mysql-connector-python dependency Frontend: - `web/src/pages/user-setting/data-source/constant/index.tsx` - Form fields - `web/src/locales/en.ts` - English translations - `web/src/assets/svg/data-source/mysql.svg` - MySQL icon - `web/src/assets/svg/data-source/postgresql.svg` - PostgreSQL icon ### Testing done Tested with MySQL 8.0 and PostgreSQL 16: - Connection validation works correctly - Full sync imports all query results as documents - Incremental sync only fetches rows updated since last sync - Custom SQL queries filter data as expected - Invalid credentials show clear error messages - Lint checks pass (`ruff check` returns no errors) --------- Co-authored-by: mkdev11 <YOUR_GITHUB_ID+MkDev11@users.noreply.github.com>
85 lines
3.0 KiB
Python
85 lines
3.0 KiB
Python
|
|
"""
|
|
Thanks to https://github.com/onyx-dot-app/onyx
|
|
|
|
Content of this directory is under the "MIT Expat" license as defined below.
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in all
|
|
copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
SOFTWARE.
|
|
"""
|
|
|
|
from .blob_connector import BlobStorageConnector
|
|
from .slack_connector import SlackConnector
|
|
from .gmail_connector import GmailConnector
|
|
from .notion_connector import NotionConnector
|
|
from .confluence_connector import ConfluenceConnector
|
|
from .discord_connector import DiscordConnector
|
|
from .dropbox_connector import DropboxConnector
|
|
from .google_drive.connector import GoogleDriveConnector
|
|
from .jira.connector import JiraConnector
|
|
from .sharepoint_connector import SharePointConnector
|
|
from .teams_connector import TeamsConnector
|
|
from .moodle_connector import MoodleConnector
|
|
from .airtable_connector import AirtableConnector
|
|
from .asana_connector import AsanaConnector
|
|
from .imap_connector import ImapConnector
|
|
from .zendesk_connector import ZendeskConnector
|
|
from .seafile_connector import SeaFileConnector
|
|
from .rdbms_connector import RDBMSConnector
|
|
from .config import BlobType, DocumentSource
|
|
from .models import Document, TextSection, ImageSection, BasicExpertInfo
|
|
from .exceptions import (
|
|
ConnectorMissingCredentialError,
|
|
ConnectorValidationError,
|
|
CredentialExpiredError,
|
|
InsufficientPermissionsError,
|
|
UnexpectedValidationError
|
|
)
|
|
|
|
__all__ = [
|
|
"BlobStorageConnector",
|
|
"SlackConnector",
|
|
"GmailConnector",
|
|
"NotionConnector",
|
|
"ConfluenceConnector",
|
|
"DiscordConnector",
|
|
"DropboxConnector",
|
|
"GoogleDriveConnector",
|
|
"JiraConnector",
|
|
"SharePointConnector",
|
|
"TeamsConnector",
|
|
"MoodleConnector",
|
|
"BlobType",
|
|
"DocumentSource",
|
|
"Document",
|
|
"TextSection",
|
|
"ImageSection",
|
|
"BasicExpertInfo",
|
|
"ConnectorMissingCredentialError",
|
|
"ConnectorValidationError",
|
|
"CredentialExpiredError",
|
|
"InsufficientPermissionsError",
|
|
"UnexpectedValidationError",
|
|
"AirtableConnector",
|
|
"AsanaConnector",
|
|
"ImapConnector",
|
|
"ZendeskConnector",
|
|
"SeaFileConnector",
|
|
"RDBMSConnector",
|
|
]
|