From d039d1e73d1f2e52e13a92611742695e758a40b7 Mon Sep 17 00:00:00 2001 From: chanx <1243304602@qq.com> Date: Mon, 22 Sep 2025 10:01:34 +0800 Subject: [PATCH] fix: Added dataset generation logging functionality #9869 (#10180) ### What problem does this PR solve? fix: Added dataset generation logging functionality #9869 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- .../components/data-pipeline-select/index.tsx | 7 - .../graph-rag-form-fields.tsx | 18 +++ .../raptor-form-fields.tsx | 22 +++- web/src/components/ui/modal/modal.tsx | 12 +- web/src/locales/en.ts | 31 ++++- web/src/locales/zh.ts | 29 ++++- .../pages/dataset/dataset-overview/index.tsx | 50 ++++--- .../dataset-overview/overview-table.tsx | 80 ++++++------ .../components/link-data-pipeline.tsx | 123 ++++++++++++++++-- .../components/link-data-pipline-modal.tsx | 96 ++++++++++++-- .../dataset/dataset-setting/form-schema.ts | 14 +- .../pages/dataset/dataset-setting/index.tsx | 7 +- .../dataset/generate-button/generate.tsx | 113 +++++++++++++++- web/src/pages/dataset/dataset/index.tsx | 2 +- web/src/pages/dataset/sidebar/index.tsx | 6 +- 15 files changed, 491 insertions(+), 119 deletions(-) diff --git a/web/src/components/data-pipeline-select/index.tsx b/web/src/components/data-pipeline-select/index.tsx index 976ca6272..f5ac1b373 100644 --- a/web/src/components/data-pipeline-select/index.tsx +++ b/web/src/components/data-pipeline-select/index.tsx @@ -20,17 +20,10 @@ interface IProps { isMult?: boolean; } -const data = [ - { id: '1', name: 'data-pipeline-1' }, - { id: '2', name: 'data-pipeline-2' }, - { id: '3', name: 'data-pipeline-3' }, - { id: '4', name: 'data-pipeline-4' }, -]; export function DataFlowSelect(props: IProps) { const { toDataPipeline, formFieldName, isMult = true } = props; const { t } = useTranslate('knowledgeConfiguration'); const form = useFormContext(); - console.log('data-pipline form', form); const toDataPipLine = () => { toDataPipeline?.(); }; diff --git a/web/src/components/parse-configuration/graph-rag-form-fields.tsx b/web/src/components/parse-configuration/graph-rag-form-fields.tsx index 8d40b2a66..a8bff43d0 100644 --- a/web/src/components/parse-configuration/graph-rag-form-fields.tsx +++ b/web/src/components/parse-configuration/graph-rag-form-fields.tsx @@ -1,6 +1,10 @@ import { DocumentParserType } from '@/constants/knowledge'; import { useTranslate } from '@/hooks/common-hooks'; import { cn } from '@/lib/utils'; +import { + GenerateLogButton, + GenerateType, +} from '@/pages/dataset/dataset/generate-button/generate'; import { upperFirst } from 'lodash'; import { useCallback, useMemo } from 'react'; import { useFormContext, useWatch } from 'react-hook-form'; @@ -47,6 +51,7 @@ export const showGraphRagItems = (parserId: DocumentParserType | undefined) => { type GraphRagItemsProps = { marginBottom?: boolean; className?: string; + showGenerateItem?: boolean; }; export function UseGraphRagFormField() { @@ -88,6 +93,7 @@ export function UseGraphRagFormField() { // The three types "table", "resume" and "one" do not display this configuration. const GraphRagItems = ({ marginBottom = false, + showGenerateItem = false, className = 'p-10', }: GraphRagItemsProps) => { const { t } = useTranslate('knowledgeConfiguration'); @@ -210,6 +216,18 @@ const GraphRagItems = ({ )} /> + {showGenerateItem && ( +
+
+ {t('extractKnowledgeGraph')} +
+ +
+ )} )} diff --git a/web/src/components/parse-configuration/raptor-form-fields.tsx b/web/src/components/parse-configuration/raptor-form-fields.tsx index 5f1606b92..ec22be81a 100644 --- a/web/src/components/parse-configuration/raptor-form-fields.tsx +++ b/web/src/components/parse-configuration/raptor-form-fields.tsx @@ -1,6 +1,10 @@ import { FormLayout } from '@/constants/form'; import { DocumentParserType } from '@/constants/knowledge'; import { useTranslate } from '@/hooks/common-hooks'; +import { + GenerateLogButton, + GenerateType, +} from '@/pages/dataset/dataset/generate-button/generate'; import random from 'lodash/random'; import { Shuffle } from 'lucide-react'; import { useCallback } from 'react'; @@ -52,7 +56,11 @@ const Prompt = 'parser_config.raptor.prompt'; // The three types "table", "resume" and "one" do not display this configuration. -const RaptorFormFields = () => { +const RaptorFormFields = ({ + showGenerateItem = false, +}: { + showGenerateItem?: boolean; +}) => { const form = useFormContext(); const { t } = useTranslate('knowledgeConfiguration'); const useRaptor = useWatch({ name: UseRaptorField }); @@ -211,6 +219,18 @@ const RaptorFormFields = () => { )} /> + {showGenerateItem && ( +
+
+ {t('extractRaptor')} +
+ +
+ )} )} diff --git a/web/src/components/ui/modal/modal.tsx b/web/src/components/ui/modal/modal.tsx index f086008d5..177addbaa 100644 --- a/web/src/components/ui/modal/modal.tsx +++ b/web/src/components/ui/modal/modal.tsx @@ -75,21 +75,21 @@ const Modal: ModalType = ({ const handleCancel = useCallback(() => { onOpenChange?.(false); - onCancel?.(); - }, [onOpenChange, onCancel]); + // onCancel?.(); + }, [onOpenChange]); const handleOk = useCallback(() => { onOpenChange?.(true); - onOk?.(); - }, [onOpenChange, onOk]); + // onOk?.(); + }, [onOpenChange]); const handleChange = (open: boolean) => { onOpenChange?.(open); console.log('open', open, onOpenChange); if (open) { - handleOk(); + onOk?.(); } if (!open) { - handleCancel(); + onCancel?.(); } }; const footEl = useMemo(() => { diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index eca4c1d35..e9065be8b 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -102,13 +102,15 @@ export default { noMoreData: `That's all. Nothing more.`, }, knowledgeDetails: { + notGenerated: 'Not generated', + generatedOn: 'Generated on', + subbarFiles: 'Files', generateKnowledgeGraph: 'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.', generateRaptor: 'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.', generate: 'Generate', raptor: 'Raptor', - knowledgeGraph: 'Knowledge Graph', processingType: 'Processing Type', dataPipeline: 'Data Pipeline', operations: 'Operations', @@ -138,12 +140,12 @@ export default { testing: 'Retrieval testing', files: 'files', configuration: 'Configuration', - knowledgeGraph: 'Knowledge graph', + knowledgeGraph: 'Knowledge Graph', name: 'Name', namePlaceholder: 'Please input name!', doc: 'Docs', datasetDescription: - '😉 Please wait for your files to finish parsing before starting an AI-powered chat.', + 'Please wait for your files to finish parsing before starting an AI-powered chat.', addFile: 'Add file', searchFiles: 'Search your files', localFiles: 'Local files', @@ -261,6 +263,22 @@ export default { reRankModelWaring: 'Re-rank model is very time consuming.', }, knowledgeConfiguration: { + deleteGenerateModalContent: ` +

Deleting the generated {{type}} results + will remove all derived entities and relationships from this dataset. + Your original files will remain intact.

+
+ Do you want to continue? + `, + extractRaptor: 'Extract Raptor', + extractKnowledgeGraph: 'Extract Knowledge Graph', + filterPlaceholder: 'please input filter', + fileFilterTip: '', + fileFilter: 'File Filter', + setDefaultTip: '', + setDefault: 'Set as Default', + eidtLinkDataPipeline: 'Edit Data Pipeline', + linkPipelineSetTip: 'Manage data pipeline linkage with this dataset', default: 'Default', dataPipeline: 'Data Pipeline', linkDataPipeline: 'Link Data Pipeline', @@ -1646,6 +1664,13 @@ This delimiter is used to split the input text into several text pieces echo of

To keep them, please click Rerun to re-run the current stage.

`, changeStepModalConfirmText: 'Switch Anyway', changeStepModalCancelText: 'Cancel', + unlinkPipelineModalTitle: 'Unlink data pipeline', + unlinkPipelineModalContent: ` +

Once unlinked, this Dataset will no longer be connected to the current Data Pipeline.

+

Files that are already being parsed will continue until completion

+

Files that are not yet parsed will no longer be processed


+

Are you sure you want to proceed?

`, + unlinkPipelineModalConfirmText: 'Unlink', }, dataflow: { parser: 'Parser', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 531135e2d..7ee4cab8e 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -94,9 +94,11 @@ export default { noMoreData: '没有更多数据了', }, knowledgeDetails: { + notGenerated: '未生成', + generatedOn: '生成于', + subbarFiles: '文件列表', generate: '生成', raptor: 'Raptor', - knowledgeGraph: '知识图谱', processingType: '处理类型', dataPipeline: '数据管道', operations: '操作', @@ -130,7 +132,7 @@ export default { name: '名称', namePlaceholder: '请输入名称', doc: '文档', - datasetDescription: '😉 解析成功后才能问答哦。', + datasetDescription: '解析成功后才能问答哦。', addFile: '新增文件', searchFiles: '搜索文件', localFiles: '本地文件', @@ -246,6 +248,22 @@ export default { theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除', }, knowledgeConfiguration: { + deleteGenerateModalContent: ` +

删除生成的 {{type}} 结果 + 将从此数据集中移除所有派生实体和关系。 + 您的原始文件将保持不变。

+
+ 是否要继续？ + `, + extractRaptor: '从文档中提取Raptor', + extractKnowledgeGraph: '从文档中提取知识图谱', + filterPlaceholder: '请输入', + fileFilterTip: '', + fileFilter: '正则匹配表达式', + setDefaultTip: '', + setDefault: '设置默认', + eidtLinkDataPipeline: '编辑数据流', + linkPipelineSetTip: '管理与此数据集的数据管道链接', default: '默认', dataPipeline: '数据流', linkDataPipeline: '关联数据流', @@ -1556,6 +1574,13 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于

要保留这些更改，请点击“重新运行”以重新运行当前阶段。

`, changeStepModalConfirmText: '继续切换', changeStepModalCancelText: '取消', + unlinkPipelineModalTitle: '解绑数据流', + unlinkPipelineModalContent: ` +

一旦取消链接，该数据集将不再连接到当前数据管道。

+

正在解析的文件将继续解析，直到完成。

+

尚未解析的文件将不再被处理。


+

你确定要继续吗?

`, + unlinkPipelineModalConfirmText: '解绑', }, dataflow: { parser: '解析器', diff --git a/web/src/pages/dataset/dataset-overview/index.tsx b/web/src/pages/dataset/dataset-overview/index.tsx index 13184061e..3bb673335 100644 --- a/web/src/pages/dataset/dataset-overview/index.tsx +++ b/web/src/pages/dataset/dataset-overview/index.tsx @@ -2,7 +2,7 @@ import SvgIcon from '@/components/svg-icon'; import { useIsDarkTheme } from '@/components/theme-provider'; import { parseColorToRGBA } from '@/utils/common-util'; import { CircleQuestionMark } from 'lucide-react'; -import { FC, useMemo, useState } from 'react'; +import { FC, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { LogTabs } from './dataset-common'; import { DatasetFilter } from './dataset-filter'; @@ -74,25 +74,35 @@ const FileLogsPage: FC = () => { const [active, setActive] = useState<(typeof LogTabs)[keyof typeof LogTabs]>( LogTabs.FILE_LOGS, ); - const topMockData = { + const [topAllData, setTopAllData] = useState({ totalFiles: { - value: 2827, - precent: 12.5, + value: 0, + precent: 0, }, downloads: { - value: 28, - success: 8, - failed: 2, + value: 0, + success: 0, + failed: 0, }, processing: { - value: 156, - success: 8, - failed: 2, + value: 0, + success: 0, + failed: 0, }, - }; + }); const { data: topData } = useFetchOverviewTital(); console.log('topData --> ', topData); + useEffect(() => { + setTopAllData({ + ...topAllData, + processing: { + value: topData?.processing || 0, + success: topData?.finished || 0, + failed: topData?.failed || 0, + }, + }); + }, [topData, topAllData]); const mockData = useMemo(() => { if (active === LogTabs.FILE_LOGS) { @@ -161,7 +171,7 @@ const FileLogsPage: FC = () => {
@@ -172,15 +182,15 @@ const FileLogsPage: FC = () => { >
- {topMockData.totalFiles.precent > 0 ? '+' : ''} - {topMockData.totalFiles.precent}%{' '} + {topAllData.totalFiles.precent > 0 ? '+' : ''} + {topAllData.totalFiles.precent}%{' '} from last week
@@ -190,13 +200,13 @@ const FileLogsPage: FC = () => { } > @@ -206,8 +216,8 @@ const FileLogsPage: FC = () => { } >
diff --git a/web/src/pages/dataset/dataset-overview/overview-table.tsx b/web/src/pages/dataset/dataset-overview/overview-table.tsx index 04585c403..ad81e05ab 100644 --- a/web/src/pages/dataset/dataset-overview/overview-table.tsx +++ b/web/src/pages/dataset/dataset-overview/overview-table.tsx @@ -65,25 +65,25 @@ export const getFileLogsTableColumns = ( ) => { // const { t } = useTranslate('knowledgeDetails'); const columns: ColumnDef[] = [ - { - id: 'select', - header: ({ table }) => ( - - ), - cell: ({ row }) => ( - - ), - }, + // { + // id: 'select', + // header: ({ table }) => ( + // + // ), + // cell: ({ row }) => ( + // + // ), + // }, { accessorKey: 'id', header: 'ID', @@ -156,7 +156,7 @@ export const getFileLogsTableColumns = ( id: 'operations', header: t('operations'), cell: ({ row }) => ( -
+
+ +
+ ), + }); + }; + return (
@@ -28,42 +68,89 @@ const DataPipelineItem = (props: DataPipelineItemProps) => { )}
- {!isDefault && ( - + <> + {linked && ( + + )} + )}
); }; + +export interface IDataPipelineNodeProps { + id: string; + name: string; + avatar?: string; + isDefault?: boolean; + linked?: boolean; +} const LinkDataPipeline = () => { const { t } = useTranslation(); const [openLinkModal, setOpenLinkModal] = useState(false); + const [currentDataPipeline, setCurrentDataPipeline] = + useState(); const testNode = [ { + id: '1', name: 'Data Pipeline 1', avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4', isDefault: true, linked: true, }, { + id: '2', name: 'Data Pipeline 2', avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4', linked: false, }, + { + id: '3', + name: 'Data Pipeline 3', + avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4', + linked: false, + }, + { + id: '4', + name: 'Data Pipeline 4', + avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4', + linked: true, + }, ]; - const openLinkModalFunc = (open: boolean) => { + const openLinkModalFunc = (open: boolean, data?: IDataPipelineNodeProps) => { + console.log('open', open, data); setOpenLinkModal(open); + if (data) { + setCurrentDataPipeline(data); + } else { + setCurrentDataPipeline(undefined); + } + }; + const handleLinkOrEditSubmit = ( + data: z.infer, + ) => { + console.log('handleLinkOrEditSubmit', data); }; return (
@@ -74,9 +161,15 @@ const LinkDataPipeline = () => {
- Manage data pipeline linkage with this dataset + {t('knowledgeConfiguration.linkPipelineSetTip')}
-
); diff --git a/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx b/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx index d84f1562f..bd970cd80 100644 --- a/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx +++ b/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx @@ -10,32 +10,53 @@ import { FormMessage, } from '@/components/ui/form'; import { Modal } from '@/components/ui/modal/modal'; +import { Switch } from '@/components/ui/switch'; import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks'; import { zodResolver } from '@hookform/resolvers/zod'; import { t } from 'i18next'; import { useForm } from 'react-hook-form'; import { z } from 'zod'; -import { linkPiplineFormSchema } from '../form-schema'; +import { pipelineFormSchema } from '../form-schema'; +import { IDataPipelineNodeProps } from './link-data-pipeline'; const LinkDataPipelineModal = ({ + data, open, setOpen, + onSubmit, }: { + data: IDataPipelineNodeProps | undefined; open: boolean; setOpen: (open: boolean) => void; + onSubmit?: (data: any) => void; }) => { - const form = useForm>({ - resolver: zodResolver(linkPiplineFormSchema), - defaultValues: { data_flow: ['888'], file_filter: '' }, + const isEdit = !!data; + const form = useForm>({ + resolver: zodResolver(pipelineFormSchema), + defaultValues: { + data_flow: [], + set_default: false, + file_filter: '', + }, }); // const [open, setOpen] = useState(false); const { navigateToAgents } = useNavigatePage(); const handleFormSubmit = (values: any) => { - console.log(values); + console.log(values, data); + const param = { + ...data, + ...values, + }; + onSubmit?.(param); }; return (
- + {!isEdit && ( + + )} @@ -78,11 +103,56 @@ const LinkDataPipelineModal = ({ )} /> + {isEdit && ( + ( + +
+
+ + {t('knowledgeConfiguration.setDefault')} + +
+ +
+ + + +
+
+
+
+ +
+ + )} + /> + )}
- -
diff --git a/web/src/pages/dataset/dataset-setting/form-schema.ts b/web/src/pages/dataset/dataset-setting/form-schema.ts index d0c669b14..f8381ef3b 100644 --- a/web/src/pages/dataset/dataset-setting/form-schema.ts +++ b/web/src/pages/dataset/dataset-setting/form-schema.ts @@ -72,7 +72,17 @@ export const formSchema = z.object({ // icon: z.array(z.instanceof(File)), }); -export const linkPiplineFormSchema = z.object({ - data_flow: z.array(z.string()), +export const pipelineFormSchema = z.object({ + data_flow: z.array(z.string()).optional(), + set_default: z.boolean().optional(), file_filter: z.string().optional(), }); + +export const linkPiplineFormSchema = pipelineFormSchema.pick({ + data_flow: true, + file_filter: true, +}); +export const editPiplineFormSchema = pipelineFormSchema.pick({ + set_default: true, + file_filter: true, +}); diff --git a/web/src/pages/dataset/dataset-setting/index.tsx b/web/src/pages/dataset/dataset-setting/index.tsx index 3e5172171..e6ba5bcb7 100644 --- a/web/src/pages/dataset/dataset-setting/index.tsx +++ b/web/src/pages/dataset/dataset-setting/index.tsx @@ -86,9 +86,12 @@ export default function DatasetSettings() { - + - + diff --git a/web/src/pages/dataset/dataset/generate-button/generate.tsx b/web/src/pages/dataset/dataset/generate-button/generate.tsx index 7b3fe75ae..3b846efed 100644 --- a/web/src/pages/dataset/dataset/generate-button/generate.tsx +++ b/web/src/pages/dataset/dataset/generate-button/generate.tsx @@ -6,16 +6,20 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu'; +import { Modal } from '@/components/ui/modal/modal'; +import { cn } from '@/lib/utils'; import { toFixed } from '@/utils/common-util'; import { t } from 'i18next'; import { lowerFirst } from 'lodash'; -import { CirclePause, WandSparkles } from 'lucide-react'; +import { CirclePause, Trash2, WandSparkles } from 'lucide-react'; import { useState } from 'react'; +import { useTranslation } from 'react-i18next'; import { generateStatus, useFetchGenerateData } from './hook'; - -const MenuItem: React.FC<{ name: 'KnowledgeGraph' | 'Raptor' }> = ({ - name, -}) => { +export enum GenerateType { + KnowledgeGraph = 'KnowledgeGraph', + Raptor = 'Raptor', +} +const MenuItem: React.FC<{ name: GenerateType }> = ({ name }) => { console.log(name, 'pppp'); const iconKeyMap = { KnowledgeGraph: 'knowledgegraph', @@ -111,3 +115,102 @@ const Generate: React.FC = () => { }; export default Generate; + +export type IGenerateLogProps = { + id?: string; + status: 0 | 1; + message?: string; + created_at?: string; + updated_at?: string; + type?: GenerateType; + className?: string; + onDelete?: () => void; +}; +export const GenerateLogButton = (props: IGenerateLogProps) => { + const { t } = useTranslation(); + const { + id, + status, + message, + created_at, + updated_at, + type, + className, + onDelete, + } = props; + const handleDelete = () => { + Modal.show({ + visible: true, + className: '!w-[560px]', + title: + t('common.delete') + + ' ' + + (type === GenerateType.KnowledgeGraph + ? t('knowledgeDetails.knowledgeGraph') + : t('knowledgeDetails.raptor')), + children: ( +
+ ), + onVisibleChange: () => { + Modal.hide(); + }, + footer: ( +
+ + +
+ ), + }); + }; + return ( +
+
+ {status === 1 && ( + <> +
+ {message || t('knowledgeDetails.generatedOn')} + {created_at} +
+ { + console.log('delete'); + handleDelete(); + e.stopPropagation(); + }} + /> + + )} + {status === 0 &&
{t('knowledgeDetails.notGenerated')}
} +
+
+ ); +}; diff --git a/web/src/pages/dataset/dataset/index.tsx b/web/src/pages/dataset/dataset/index.tsx index c3c63d6aa..7994d6be8 100644 --- a/web/src/pages/dataset/dataset/index.tsx +++ b/web/src/pages/dataset/dataset/index.tsx @@ -75,7 +75,7 @@ export default function Dataset() { filters={filters} leftPanel={
-
{t('knowledgeDetails.dataset')}
+
{t('knowledgeDetails.subbarFiles')}
{t('knowledgeDetails.datasetDescription')}
diff --git a/web/src/pages/dataset/sidebar/index.tsx b/web/src/pages/dataset/sidebar/index.tsx index 46daf52e2..05208016d 100644 --- a/web/src/pages/dataset/sidebar/index.tsx +++ b/web/src/pages/dataset/sidebar/index.tsx @@ -9,7 +9,7 @@ import { cn, formatBytes } from '@/lib/utils'; import { Routes } from '@/routes'; import { formatPureDate } from '@/utils/date'; import { isEmpty } from 'lodash'; -import { Banknote, Database, FileSearch2, GitGraph } from 'lucide-react'; +import { Banknote, FileSearch2, FolderOpen, GitGraph } from 'lucide-react'; import { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useHandleMenuClick } from './hooks'; @@ -34,8 +34,8 @@ export function SideBar({ refreshCount }: PropType) { // key: Routes.DataSetOverview, // }, { - icon: Database, - label: t(`knowledgeDetails.dataset`), + icon: FolderOpen, + label: t(`knowledgeDetails.subbarFiles`), key: Routes.DatasetBase, }, {