Feat: add gmail connector (#11549)

### What problem does this PR solve?

_Briefly describe what this PR aims to solve. Include background context
that will help reviewers understand the purpose of the PR._

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Billy Bao
2025-11-28 13:09:40 +08:00
committed by GitHub
parent 982ed233a2
commit cf7fdd274b
20 changed files with 856 additions and 108 deletions

View File

@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="52 42 88 66">
<path fill="#4285f4" d="M58 108h14V74L52 59v43c0 3.32 2.69 6 6 6"/>
<path fill="#34a853" d="M120 108h14c3.32 0 6-2.69 6-6V59l-20 15"/>
<path fill="#fbbc04" d="M120 48v26l20-15v-8c0-7.42-8.47-11.65-14.4-7.2"/>
<path fill="#ea4335" d="M72 74V48l24 18 24-18v26L96 92"/>
<path fill="#c5221f" d="M52 51v8l20 15V48l-5.6-4.2c-5.94-4.45-14.4-.22-14.4 7.2"/>
</svg>

After

Width:  |  Height:  |  Size: 419 B

View File

@ -739,6 +739,7 @@ Example: Virtual Hosted Style`,
'Sync pages and databases from Notion for knowledge retrieval.',
google_driveDescription:
'Connect your Google Drive via OAuth and sync specific folders or drives.',
gmailDescription: 'Connect your Gmail via OAuth to sync emails.',
webdavDescription: 'Connect to WebDAV servers to sync files.',
webdavRemotePathTip:
'Optional: Specify a folder path on the WebDAV server (e.g., /Documents). Leave empty to sync from root.',
@ -750,6 +751,10 @@ Example: Virtual Hosted Style`,
'Comma-separated emails whose "My Drive" contents should be indexed (include the primary admin).',
google_driveSharedFoldersTip:
'Comma-separated Google Drive folder links to crawl.',
gmailPrimaryAdminTip:
'Primary admin email with Gmail / Workspace access, used to enumerate domain users and as the default sync account.',
gmailTokenTip:
'Upload the OAuth JSON generated from Google Console. If it only contains client credentials, run the browser-based verification once to mint long-lived refresh tokens.',
dropboxDescription:
'Connect your Dropbox to sync files and folders from a chosen account.',
dropboxAccessTokenTip:

View File

@ -736,6 +736,8 @@ export default {
'Синхронизируйте страницы и базы данных из Notion для извлечения знаний.',
google_driveDescription:
'Подключите ваш Google Drive через OAuth и синхронизируйте определенные папки или диски.',
gmailDescription:
'Подключите ваш Gmail / Google Workspace аккаунт для синхронизации писем и их метаданных, чтобы построить корпоративную почтовую базу знаний и поиск с учетом прав доступа.',
google_driveTokenTip:
'Загрузите JSON токена OAuth, сгенерированный из помощника OAuth или Google Cloud Console. Вы также можете загрузить client_secret JSON из "установленного" или "веб" приложения. Если это ваша первая синхронизация, откроется окно браузера для завершения согласия OAuth. Если JSON уже содержит токен обновления, он будет автоматически повторно использован.',
google_drivePrimaryAdminTip:
@ -744,6 +746,10 @@ export default {
'Электронные почты через запятую, чье содержимое "Мой диск" должно индексироваться (включите основного администратора).',
google_driveSharedFoldersTip:
'Ссылки на папки Google Drive через запятую для обхода.',
gmailPrimaryAdminTip:
'Основной административный email с доступом к Gmail / Workspace, используется для перечисления пользователей домена и как аккаунт синхронизации по умолчанию.',
gmailTokenTip:
'Загрузите OAuth JSON, сгенерированный в Google Console. Если он содержит только учетные данные клиента, выполните одноразовое подтверждение в браузере, чтобы получить долгоживущие токены обновления.',
jiraDescription:
'Подключите ваше рабочее пространство Jira для синхронизации задач, комментариев и вложений.',
jiraBaseUrlTip:

View File

@ -718,6 +718,7 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
notionDescription: ' 同步 Notion 页面与数据库,用于知识检索。',
google_driveDescription:
'通过 OAuth 连接 Google Drive并同步指定的文件夹或云端硬盘。',
gmailDescription: '通过 OAuth 连接 Gmail用于同步邮件。',
google_driveTokenTip:
'请上传由 OAuth helper 或 Google Cloud Console 导出的 OAuth token JSON。也支持上传 “installed” 或 “web” 类型的 client_secret JSON。若为首次同步将自动弹出浏览器完成 OAuth 授权流程;如果该 JSON 已包含 refresh token将会被自动复用。',
google_drivePrimaryAdminTip: '拥有相应 Drive 访问权限的管理员邮箱。',
@ -725,6 +726,10 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
'需要索引其 “我的云端硬盘” 的邮箱,多个邮箱用逗号分隔(建议包含管理员)。',
google_driveSharedFoldersTip:
'需要同步的 Google Drive 文件夹链接,多个链接用逗号分隔。',
gmailPrimaryAdminTip:
'拥有 Gmail / Workspace 访问权限的主要管理员邮箱,用于列出域内用户并作为默认同步账号。',
gmailTokenTip:
'请上传由 Google Console 生成的 OAuth JSON。如果仅包含 client credentials请通过浏览器授权一次以获取长期有效的刷新 Token。',
dropboxDescription: '连接 Dropbox同步指定账号下的文件与文件夹。',
dropboxAccessTokenTip:
'请在 Dropbox App Console 生成 Access Token并勾选 files.metadata.read、files.content.read、sharing.read 等必要权限。',

View File

@ -47,6 +47,7 @@ const AddDataSourceModal = ({
}
open={visible || false}
onOpenChange={(open) => !open && hideModal?.()}
maskClosable={false}
// onOk={() => handleOk()}
okText={t('common.confirm')}
cancelText={t('common.cancel')}

View File

@ -0,0 +1,391 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { FileUploader } from '@/components/file-uploader';
import { Button } from '@/components/ui/button';
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog';
import message from '@/components/ui/message';
import { FileMimeType } from '@/constants/common';
import {
pollGmailWebAuthResult,
startGmailWebAuth,
} from '@/services/data-source-service';
import { Loader2 } from 'lucide-react';
export type GmailTokenFieldProps = {
value?: string;
onChange: (value: any) => void;
placeholder?: string;
};
const credentialHasRefreshToken = (content: string) => {
try {
const parsed = JSON.parse(content);
return Boolean(parsed?.refresh_token);
} catch {
return false;
}
};
const describeCredentials = (content?: string) => {
if (!content) return '';
try {
const parsed = JSON.parse(content);
if (parsed?.refresh_token) {
return 'Uploaded OAuth tokens with a refresh token.';
}
if (parsed?.installed || parsed?.web) {
return 'Client credentials detected. Complete verification to mint long-lived tokens.';
}
return 'Stored Google credential JSON.';
} catch {
return '';
}
};
const GmailTokenField = ({
value,
onChange,
placeholder,
}: GmailTokenFieldProps) => {
const [files, setFiles] = useState<File[]>([]);
const [pendingCredentials, setPendingCredentials] = useState<string>('');
const [dialogOpen, setDialogOpen] = useState(false);
const [webAuthLoading, setWebAuthLoading] = useState(false);
const [webFlowId, setWebFlowId] = useState<string | null>(null);
const [webStatus, setWebStatus] = useState<
'idle' | 'waiting' | 'success' | 'error'
>('idle');
const [webStatusMessage, setWebStatusMessage] = useState('');
const webFlowIdRef = useRef<string | null>(null);
const webPollTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const clearWebState = useCallback(() => {
if (webPollTimerRef.current) {
clearTimeout(webPollTimerRef.current);
webPollTimerRef.current = null;
}
webFlowIdRef.current = null;
setWebFlowId(null);
setWebStatus('idle');
setWebStatusMessage('');
}, []);
useEffect(() => {
return () => {
if (webPollTimerRef.current) {
clearTimeout(webPollTimerRef.current);
}
};
}, []);
useEffect(() => {
webFlowIdRef.current = webFlowId;
}, [webFlowId]);
const credentialSummary = useMemo(() => describeCredentials(value), [value]);
const hasVerifiedTokens = useMemo(
() => Boolean(value && credentialHasRefreshToken(value)),
[value],
);
const hasUploadedButUnverified = useMemo(
() => Boolean(value && !hasVerifiedTokens),
[hasVerifiedTokens, value],
);
const resetDialog = useCallback(
(shouldResetState: boolean) => {
setDialogOpen(false);
clearWebState();
if (shouldResetState) {
setPendingCredentials('');
setFiles([]);
}
},
[clearWebState],
);
const fetchWebResult = useCallback(
async (flowId: string) => {
try {
const { data } = await pollGmailWebAuthResult({
flow_id: flowId,
});
if (data.code === 0 && data.data?.credentials) {
onChange(data.data.credentials);
setPendingCredentials('');
message.success('Gmail credentials verified.');
resetDialog(false);
return;
}
if (data.code === 106) {
setWebStatus('waiting');
setWebStatusMessage('Authorization confirmed. Finalizing tokens...');
if (webPollTimerRef.current) {
clearTimeout(webPollTimerRef.current);
}
webPollTimerRef.current = setTimeout(
() => fetchWebResult(flowId),
1500,
);
return;
}
message.error(data.message || 'Authorization failed.');
clearWebState();
} catch (err) {
message.error('Unable to retrieve authorization result.');
clearWebState();
}
},
[clearWebState, onChange, resetDialog],
);
useEffect(() => {
const handler = (event: MessageEvent) => {
const payload = event.data;
if (!payload || payload.type !== 'ragflow-gmail-oauth') {
return;
}
if (!payload.flowId) {
return;
}
if (webFlowIdRef.current && webFlowIdRef.current !== payload.flowId) {
return;
}
if (payload.status === 'success') {
setWebStatus('waiting');
setWebStatusMessage('Authorization confirmed. Finalizing tokens...');
fetchWebResult(payload.flowId);
} else {
message.error(
payload.message || 'Authorization window reported an error.',
);
clearWebState();
}
};
window.addEventListener('message', handler);
return () => window.removeEventListener('message', handler);
}, [clearWebState, fetchWebResult]);
const handleValueChange = useCallback(
(nextFiles: File[]) => {
if (!nextFiles.length) {
setFiles([]);
onChange('');
setPendingCredentials('');
clearWebState();
return;
}
const file = nextFiles[nextFiles.length - 1];
file
.text()
.then((text) => {
try {
JSON.parse(text);
} catch {
message.error('Invalid JSON file.');
setFiles([]);
clearWebState();
return;
}
setFiles([file]);
clearWebState();
if (credentialHasRefreshToken(text)) {
onChange(text);
setPendingCredentials('');
message.success('Gmail OAuth credentials uploaded.');
return;
}
setPendingCredentials(text);
setDialogOpen(true);
message.info(
'Client configuration uploaded. Verification is required to finish setup.',
);
})
.catch(() => {
message.error('Unable to read the uploaded file.');
setFiles([]);
});
},
[clearWebState, onChange],
);
const handleStartWebAuthorization = useCallback(async () => {
if (!pendingCredentials) {
message.error('No Google credential file detected.');
return;
}
setWebAuthLoading(true);
clearWebState();
try {
const { data } = await startGmailWebAuth({
credentials: pendingCredentials,
});
if (data.code === 0 && data.data?.authorization_url) {
const flowId = data.data.flow_id;
const popup = window.open(
data.data.authorization_url,
'ragflow-gmail-oauth',
'width=600,height=720',
);
if (!popup) {
message.error(
'Popup was blocked. Please allow popups for this site.',
);
return;
}
popup.focus();
webFlowIdRef.current = flowId;
setWebFlowId(flowId);
setWebStatus('waiting');
setWebStatusMessage('Complete the Google consent in the popup window.');
} else {
message.error(data.message || 'Failed to start browser authorization.');
}
} catch (err) {
message.error('Failed to start browser authorization.');
} finally {
setWebAuthLoading(false);
}
}, [clearWebState, pendingCredentials]);
const handleManualWebCheck = useCallback(() => {
if (!webFlowId) {
message.info('Start browser authorization first.');
return;
}
setWebStatus('waiting');
setWebStatusMessage('Checking authorization status...');
fetchWebResult(webFlowId);
}, [fetchWebResult, webFlowId]);
const handleCancel = useCallback(() => {
message.warning(
'Verification canceled. Upload the credential again to restart.',
);
resetDialog(true);
}, [resetDialog]);
return (
<div className="flex flex-col gap-3">
{(credentialSummary ||
hasVerifiedTokens ||
hasUploadedButUnverified ||
pendingCredentials) && (
<div className="flex flex-wrap items-center gap-3 rounded-md border border-dashed border-muted-foreground/40 bg-muted/20 px-3 py-2 text-xs text-muted-foreground">
<div className="flex flex-wrap items-center gap-2">
{hasVerifiedTokens ? (
<span className="rounded-full bg-emerald-100 px-2 py-0.5 text-[11px] font-semibold uppercase tracking-wide text-emerald-700">
Verified
</span>
) : null}
{hasUploadedButUnverified ? (
<span className="rounded-full bg-amber-100 px-2 py-0.5 text-[11px] font-semibold uppercase tracking-wide text-amber-700">
Needs authorization
</span>
) : null}
{pendingCredentials && !hasVerifiedTokens ? (
<span className="rounded-full bg-blue-100 px-2 py-0.5 text-[11px] font-semibold uppercase tracking-wide text-blue-700">
Uploaded (pending)
</span>
) : null}
</div>
{credentialSummary ? (
<p className="m-0">{credentialSummary}</p>
) : null}
</div>
)}
<FileUploader
className="py-4 border-[0.5px] bg-bg-card text-text-secondary"
value={files}
onValueChange={handleValueChange}
accept={{ '*.json': [FileMimeType.Json] }}
maxFileCount={1}
description={'Upload your Gmail OAuth JSON file.'}
/>
<Dialog
open={dialogOpen}
onOpenChange={(open) => {
if (!open && dialogOpen) {
handleCancel();
}
}}
>
<DialogContent
onPointerDownOutside={(e) => e.preventDefault()}
onInteractOutside={(e) => e.preventDefault()}
onEscapeKeyDown={(e) => e.preventDefault()}
>
<DialogHeader>
<DialogTitle>Complete Gmail verification</DialogTitle>
<DialogDescription>
The uploaded client credentials do not contain a refresh token.
Run the verification flow once to mint reusable tokens.
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="rounded-md border border-dashed border-muted-foreground/40 bg-muted/10 px-4 py-4 text-sm text-muted-foreground">
<div className="text-sm font-semibold text-foreground">
Authorize in browser
</div>
<p className="mt-2">
We will open Google&apos;s consent page in a new window. Sign in
with the admin account, grant access, and return here. Your
credentials will update automatically.
</p>
{webStatus !== 'idle' && (
<p
className={`mt-2 text-xs ${
webStatus === 'error'
? 'text-destructive'
: 'text-muted-foreground'
}`}
>
{webStatusMessage}
</p>
)}
<div className="mt-3 flex flex-wrap gap-2">
<Button
onClick={handleStartWebAuthorization}
disabled={webAuthLoading}
>
{webAuthLoading && (
<Loader2 className="mr-2 size-4 animate-spin" />
)}
Authorize with Google
</Button>
{webFlowId ? (
<Button
variant="outline"
onClick={handleManualWebCheck}
disabled={webStatus === 'success'}
>
Refresh status
</Button>
) : null}
</div>
</div>
</div>
<DialogFooter className="pt-2">
<Button variant="ghost" onClick={handleCancel}>
Cancel
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
);
};
export default GmailTokenField;

View File

@ -1,5 +1,3 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { FileUploader } from '@/components/file-uploader';
import { Button } from '@/components/ui/button';
import {
@ -17,6 +15,7 @@ import {
startGoogleDriveWebAuth,
} from '@/services/data-source-service';
import { Loader2 } from 'lucide-react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
type GoogleDriveTokenFieldProps = {
value?: string;
@ -313,12 +312,16 @@ const GoogleDriveTokenField = ({
<Dialog
open={dialogOpen}
onOpenChange={(open) => {
if (!open) {
if (!open && dialogOpen) {
handleCancel();
}
}}
>
<DialogContent>
<DialogContent
onPointerDownOutside={(e) => e.preventDefault()}
onInteractOutside={(e) => e.preventDefault()}
onEscapeKeyDown={(e) => e.preventDefault()}
>
<DialogHeader>
<DialogTitle>Complete Google verification</DialogTitle>
<DialogDescription>
@ -326,7 +329,6 @@ const GoogleDriveTokenField = ({
Run the verification flow once to mint reusable tokens.
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="rounded-md border border-dashed border-muted-foreground/40 bg-muted/10 px-4 py-4 text-sm text-muted-foreground">
<div className="text-sm font-semibold text-foreground">
@ -370,7 +372,6 @@ const GoogleDriveTokenField = ({
</div>
</div>
</div>
<DialogFooter className="pt-2">
<Button variant="ghost" onClick={handleCancel}>
Cancel

View File

@ -1,6 +1,7 @@
import { FormFieldType } from '@/components/dynamic-form';
import SvgIcon from '@/components/svg-icon';
import { t } from 'i18next';
import GmailTokenField from './component/gmail-token-field';
import GoogleDriveTokenField from './component/google-drive-token-field';
export enum DataSourceKey {
@ -10,7 +11,7 @@ export enum DataSourceKey {
DISCORD = 'discord',
GOOGLE_DRIVE = 'google_drive',
MOODLE = 'moodle',
// GMAIL = 'gmail',
GMAIL = 'gmail',
JIRA = 'jira',
WEBDAV = 'webdav',
DROPBOX = 'dropbox',
@ -45,6 +46,11 @@ export const DataSourceInfo = {
description: t(`setting.${DataSourceKey.GOOGLE_DRIVE}Description`),
icon: <SvgIcon name={'data-source/google-drive'} width={38} />,
},
[DataSourceKey.GMAIL]: {
name: 'Gmail',
description: t(`setting.${DataSourceKey.GMAIL}Description`),
icon: <SvgIcon name={'data-source/gmail'} width={38} />,
},
[DataSourceKey.MOODLE]: {
name: 'Moodle',
description: t(`setting.${DataSourceKey.MOODLE}Description`),
@ -320,6 +326,38 @@ export const DataSourceFormFields = {
defaultValue: 'uploaded',
},
],
[DataSourceKey.GMAIL]: [
{
label: 'Primary Admin Email',
name: 'config.credentials.google_primary_admin',
type: FormFieldType.Text,
required: true,
placeholder: 'admin@example.com',
tooltip: t('setting.gmailPrimaryAdminTip'),
},
{
label: 'OAuth Token JSON',
name: 'config.credentials.google_tokens',
type: FormFieldType.Textarea,
required: true,
render: (fieldProps: any) => (
<GmailTokenField
value={fieldProps.value}
onChange={fieldProps.onChange}
placeholder='{ "token": "...", "refresh_token": "...", ... }'
/>
),
tooltip: t('setting.gmailTokenTip'),
},
{
label: '',
name: 'config.credentials.authentication_method',
type: FormFieldType.Text,
required: false,
hidden: true,
defaultValue: 'uploaded',
},
],
[DataSourceKey.MOODLE]: [
{
label: 'Moodle URL',
@ -550,6 +588,17 @@ export const DataSourceFormDefaultValues = {
},
},
},
[DataSourceKey.GMAIL]: {
name: '',
source: DataSourceKey.GMAIL,
config: {
credentials: {
google_primary_admin: '',
google_tokens: '',
authentication_method: 'uploaded',
},
},
},
[DataSourceKey.MOODLE]: {
name: '',
source: DataSourceKey.MOODLE,

View File

@ -34,9 +34,17 @@ export const featchDataSourceDetail = (id: string) =>
request.get(api.dataSourceDetail(id));
export const startGoogleDriveWebAuth = (payload: { credentials: string }) =>
request.post(api.googleDriveWebAuthStart, { data: payload });
request.post(api.googleWebAuthStart('google-drive'), { data: payload });
export const pollGoogleDriveWebAuthResult = (payload: { flow_id: string }) =>
request.post(api.googleDriveWebAuthResult, { data: payload });
request.post(api.googleWebAuthResult('google-drive'), { data: payload });
// Gmail web auth follows the same pattern as Google Drive, but uses
// Gmail-specific endpoints and is consumed by the GmailTokenField UI.
export const startGmailWebAuth = (payload: { credentials: string }) =>
request.post(api.googleWebAuthStart('gmail'), { data: payload });
export const pollGmailWebAuthResult = (payload: { flow_id: string }) =>
request.post(api.googleWebAuthResult('gmail'), { data: payload });
export default dataSourceService;

View File

@ -42,8 +42,10 @@ export default {
dataSourceRebuild: (id: string) => `${api_host}/connector/${id}/rebuild`,
dataSourceLogs: (id: string) => `${api_host}/connector/${id}/logs`,
dataSourceDetail: (id: string) => `${api_host}/connector/${id}`,
googleDriveWebAuthStart: `${api_host}/connector/google-drive/oauth/web/start`,
googleDriveWebAuthResult: `${api_host}/connector/google-drive/oauth/web/result`,
googleWebAuthStart: (type: 'google-drive' | 'gmail') =>
`${api_host}/connector/google/oauth/web/start?type=${type}`,
googleWebAuthResult: (type: 'google-drive' | 'gmail') =>
`${api_host}/connector/google/oauth/web/result?type=${type}`,
// plugin
llm_tools: `${api_host}/plugin/llm_tools`,