fix: Added dataset generation logging functionality #9869 (#10180)

### What problem does this PR solve?

fix: Added dataset generation logging functionality #9869

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
chanx
2025-09-22 10:01:34 +08:00
committed by GitHub
parent d050ef568d
commit d039d1e73d
15 changed files with 491 additions and 119 deletions

View File

@ -2,7 +2,7 @@ import SvgIcon from '@/components/svg-icon';
import { useIsDarkTheme } from '@/components/theme-provider';
import { parseColorToRGBA } from '@/utils/common-util';
import { CircleQuestionMark } from 'lucide-react';
import { FC, useMemo, useState } from 'react';
import { FC, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { LogTabs } from './dataset-common';
import { DatasetFilter } from './dataset-filter';
@ -74,25 +74,35 @@ const FileLogsPage: FC = () => {
const [active, setActive] = useState<(typeof LogTabs)[keyof typeof LogTabs]>(
LogTabs.FILE_LOGS,
);
const topMockData = {
const [topAllData, setTopAllData] = useState({
totalFiles: {
value: 2827,
precent: 12.5,
value: 0,
precent: 0,
},
downloads: {
value: 28,
success: 8,
failed: 2,
value: 0,
success: 0,
failed: 0,
},
processing: {
value: 156,
success: 8,
failed: 2,
value: 0,
success: 0,
failed: 0,
},
};
});
const { data: topData } = useFetchOverviewTital();
console.log('topData --> ', topData);
useEffect(() => {
setTopAllData({
...topAllData,
processing: {
value: topData?.processing || 0,
success: topData?.finished || 0,
failed: topData?.failed || 0,
},
});
}, [topData, topAllData]);
const mockData = useMemo(() => {
if (active === LogTabs.FILE_LOGS) {
@ -161,7 +171,7 @@ const FileLogsPage: FC = () => {
<div className="grid grid-cols-3 md:grid-cols-3 gap-4 mb-6">
<StatCard
title="Total Files"
value={topMockData.totalFiles.value}
value={topAllData.totalFiles.value}
icon={
isDark ? (
<SvgIcon name="data-flow/total-files-icon" width={40} />
@ -172,15 +182,15 @@ const FileLogsPage: FC = () => {
>
<div>
<span className="text-accent-primary">
{topMockData.totalFiles.precent > 0 ? '+' : ''}
{topMockData.totalFiles.precent}%{' '}
{topAllData.totalFiles.precent > 0 ? '+' : ''}
{topAllData.totalFiles.precent}%{' '}
</span>
from last week
</div>
</StatCard>
<StatCard
title="Downloading"
value={topMockData.downloads.value}
value={topAllData.downloads.value}
icon={
isDark ? (
<SvgIcon name="data-flow/data-icon" width={40} />
@ -190,13 +200,13 @@ const FileLogsPage: FC = () => {
}
>
<CardFooterProcess
success={topMockData.downloads.success}
failed={topMockData.downloads.failed}
success={topAllData.downloads.success}
failed={topAllData.downloads.failed}
/>
</StatCard>
<StatCard
title="Processing"
value={topMockData.processing.value}
value={topAllData.processing.value}
icon={
isDark ? (
<SvgIcon name="data-flow/processing-icon" width={40} />
@ -206,8 +216,8 @@ const FileLogsPage: FC = () => {
}
>
<CardFooterProcess
success={topMockData.processing.success}
failed={topMockData.processing.failed}
success={topAllData.processing.success}
failed={topAllData.processing.failed}
/>
</StatCard>
</div>

View File

@ -65,25 +65,25 @@ export const getFileLogsTableColumns = (
) => {
// const { t } = useTranslate('knowledgeDetails');
const columns: ColumnDef<DocumentLog>[] = [
{
id: 'select',
header: ({ table }) => (
<input
type="checkbox"
checked={table.getIsAllRowsSelected()}
onChange={table.getToggleAllRowsSelectedHandler()}
className="rounded bg-gray-900 text-blue-500 focus:ring-blue-500"
/>
),
cell: ({ row }) => (
<input
type="checkbox"
checked={row.getIsSelected()}
onChange={row.getToggleSelectedHandler()}
className="rounded border-gray-600 bg-gray-900 text-blue-500 focus:ring-blue-500"
/>
),
},
// {
// id: 'select',
// header: ({ table }) => (
// <input
// type="checkbox"
// checked={table.getIsAllRowsSelected()}
// onChange={table.getToggleAllRowsSelectedHandler()}
// className="rounded bg-gray-900 text-blue-500 focus:ring-blue-500"
// />
// ),
// cell: ({ row }) => (
// <input
// type="checkbox"
// checked={row.getIsSelected()}
// onChange={row.getToggleSelectedHandler()}
// className="rounded border-gray-600 bg-gray-900 text-blue-500 focus:ring-blue-500"
// />
// ),
// },
{
accessorKey: 'id',
header: 'ID',
@ -156,7 +156,7 @@ export const getFileLogsTableColumns = (
id: 'operations',
header: t('operations'),
cell: ({ row }) => (
<div className="flex justify-start space-x-2">
<div className="flex justify-start space-x-2 opacity-0 group-hover:opacity-100 transition-opacity">
<Button
variant="ghost"
size="sm"
@ -189,25 +189,25 @@ export const getDatasetLogsTableColumns = (
) => {
// const { t } = useTranslate('knowledgeDetails');
const columns: ColumnDef<DocumentLog>[] = [
{
id: 'select',
header: ({ table }) => (
<input
type="checkbox"
checked={table.getIsAllRowsSelected()}
onChange={table.getToggleAllRowsSelectedHandler()}
className="rounded bg-gray-900 text-blue-500 focus:ring-blue-500"
/>
),
cell: ({ row }) => (
<input
type="checkbox"
checked={row.getIsSelected()}
onChange={row.getToggleSelectedHandler()}
className="rounded border-gray-600 bg-gray-900 text-blue-500 focus:ring-blue-500"
/>
),
},
// {
// id: 'select',
// header: ({ table }) => (
// <input
// type="checkbox"
// checked={table.getIsAllRowsSelected()}
// onChange={table.getToggleAllRowsSelectedHandler()}
// className="rounded bg-gray-900 text-blue-500 focus:ring-blue-500"
// />
// ),
// cell: ({ row }) => (
// <input
// type="checkbox"
// checked={row.getIsSelected()}
// onChange={row.getToggleSelectedHandler()}
// className="rounded border-gray-600 bg-gray-900 text-blue-500 focus:ring-blue-500"
// />
// ),
// },
{
accessorKey: 'id',
header: 'ID',
@ -251,7 +251,7 @@ export const getDatasetLogsTableColumns = (
id: 'operations',
header: t('operations'),
cell: ({ row }) => (
<div className="flex justify-start space-x-2">
<div className="flex justify-start space-x-2 opacity-0 group-hover:opacity-100 transition-opacity">
<Button
variant="ghost"
size="sm"

View File

@ -1,21 +1,61 @@
import { IconFont } from '@/components/icon-font';
import { RAGFlowAvatar } from '@/components/ragflow-avatar';
import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { omit } from 'lodash';
import { Link, Settings2, Unlink } from 'lucide-react';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import { linkPiplineFormSchema } from '../form-schema';
import LinkDataPipelineModal from './link-data-pipline-modal';
interface DataPipelineItemProps {
id: string;
name: string;
avatar?: string;
isDefault?: boolean;
linked?: boolean;
openLinkModalFunc?: (open: boolean) => void;
openLinkModalFunc?: (open: boolean, data?: IDataPipelineNodeProps) => void;
}
const DataPipelineItem = (props: DataPipelineItemProps) => {
const { t } = useTranslation();
const { name, avatar, isDefault, linked, openLinkModalFunc } = props;
const openUnlinkModal = () => {
Modal.show({
visible: true,
className: '!w-[560px]',
title: t('dataflowParser.unlinkPipelineModalTitle'),
children: (
<div
className="text-sm text-text-secondary"
dangerouslySetInnerHTML={{
__html: t('dataflowParser.unlinkPipelineModalContent'),
}}
></div>
),
onVisibleChange: () => {
Modal.hide();
},
footer: (
<div className="flex justify-end gap-2">
<Button variant={'outline'} onClick={() => Modal.hide()}>
{t('dataflowParser.changeStepModalCancelText')}
</Button>
<Button
variant={'secondary'}
className="!bg-state-error text-bg-base"
onClick={() => {
Modal.hide();
}}
>
{t('dataflowParser.unlinkPipelineModalConfirmText')}
</Button>
</div>
),
});
};
return (
<div className="flex items-center justify-between gap-1 px-2 rounded-lg border">
<div className="flex items-center gap-1">
@ -28,42 +68,89 @@ const DataPipelineItem = (props: DataPipelineItemProps) => {
)}
</div>
<div className="flex gap-1 items-center">
<Button variant={'transparent'} className="border-none">
<Button
variant={'transparent'}
className="border-none"
type="button"
onClick={() =>
openLinkModalFunc?.(true, { ...omit(props, ['openLinkModalFunc']) })
}
>
<Settings2 />
</Button>
{!isDefault && (
<Button
variant={'transparent'}
className="border-none"
onClick={() => {
openLinkModalFunc?.(true);
}}
>
{linked ? <Link /> : <Unlink />}
</Button>
<>
{linked && (
<Button
type="button"
variant={'transparent'}
className="border-none"
onClick={() => {
openUnlinkModal();
}}
>
<Unlink />
</Button>
)}
</>
)}
</div>
</div>
);
};
export interface IDataPipelineNodeProps {
id: string;
name: string;
avatar?: string;
isDefault?: boolean;
linked?: boolean;
}
const LinkDataPipeline = () => {
const { t } = useTranslation();
const [openLinkModal, setOpenLinkModal] = useState(false);
const [currentDataPipeline, setCurrentDataPipeline] =
useState<IDataPipelineNodeProps>();
const testNode = [
{
id: '1',
name: 'Data Pipeline 1',
avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4',
isDefault: true,
linked: true,
},
{
id: '2',
name: 'Data Pipeline 2',
avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4',
linked: false,
},
{
id: '3',
name: 'Data Pipeline 3',
avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4',
linked: false,
},
{
id: '4',
name: 'Data Pipeline 4',
avatar: 'https://avatars.githubusercontent.com/u/10656201?v=4',
linked: true,
},
];
const openLinkModalFunc = (open: boolean) => {
const openLinkModalFunc = (open: boolean, data?: IDataPipelineNodeProps) => {
console.log('open', open, data);
setOpenLinkModal(open);
if (data) {
setCurrentDataPipeline(data);
} else {
setCurrentDataPipeline(undefined);
}
};
const handleLinkOrEditSubmit = (
data: z.infer<typeof linkPiplineFormSchema>,
) => {
console.log('handleLinkOrEditSubmit', data);
};
return (
<div className="flex flex-col gap-2">
@ -74,9 +161,15 @@ const LinkDataPipeline = () => {
</div>
<div className="flex justify-between items-center">
<div className="text-center text-xs text-text-secondary">
Manage data pipeline linkage with this dataset
{t('knowledgeConfiguration.linkPipelineSetTip')}
</div>
<Button variant={'transparent'}>
<Button
type="button"
variant={'transparent'}
onClick={() => {
openLinkModalFunc?.(true);
}}
>
<Link />
<span className="text-xs text-text-primary">
{t('knowledgeConfiguration.linkDataPipeline')}
@ -94,10 +187,12 @@ const LinkDataPipeline = () => {
))}
</section>
<LinkDataPipelineModal
data={currentDataPipeline}
open={openLinkModal}
setOpen={(open: boolean) => {
openLinkModalFunc(open);
}}
onSubmit={handleLinkOrEditSubmit}
/>
</div>
);

View File

@ -10,32 +10,53 @@ import {
FormMessage,
} from '@/components/ui/form';
import { Modal } from '@/components/ui/modal/modal';
import { Switch } from '@/components/ui/switch';
import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
import { zodResolver } from '@hookform/resolvers/zod';
import { t } from 'i18next';
import { useForm } from 'react-hook-form';
import { z } from 'zod';
import { linkPiplineFormSchema } from '../form-schema';
import { pipelineFormSchema } from '../form-schema';
import { IDataPipelineNodeProps } from './link-data-pipeline';
const LinkDataPipelineModal = ({
data,
open,
setOpen,
onSubmit,
}: {
data: IDataPipelineNodeProps | undefined;
open: boolean;
setOpen: (open: boolean) => void;
onSubmit?: (data: any) => void;
}) => {
const form = useForm<z.infer<typeof linkPiplineFormSchema>>({
resolver: zodResolver(linkPiplineFormSchema),
defaultValues: { data_flow: ['888'], file_filter: '' },
const isEdit = !!data;
const form = useForm<z.infer<typeof pipelineFormSchema>>({
resolver: zodResolver(pipelineFormSchema),
defaultValues: {
data_flow: [],
set_default: false,
file_filter: '',
},
});
// const [open, setOpen] = useState(false);
const { navigateToAgents } = useNavigatePage();
const handleFormSubmit = (values: any) => {
console.log(values);
console.log(values, data);
const param = {
...data,
...values,
};
onSubmit?.(param);
};
return (
<Modal
title={t('knowledgeConfiguration.linkDataPipeline')}
className="!w-[560px]"
title={
!isEdit
? t('knowledgeConfiguration.linkDataPipeline')
: t('knowledgeConfiguration.eidtLinkDataPipeline')
}
open={open}
onOpenChange={setOpen}
showfooter={false}
@ -43,10 +64,12 @@ const LinkDataPipelineModal = ({
<Form {...form}>
<form onSubmit={form.handleSubmit(handleFormSubmit)}>
<div className="flex flex-col gap-4 ">
<DataFlowSelect
toDataPipeline={navigateToAgents}
formFieldName="data_flow"
/>
{!isEdit && (
<DataFlowSelect
toDataPipeline={navigateToAgents}
formFieldName="data_flow"
/>
)}
<FormField
control={form.control}
name={'file_filter'}
@ -65,7 +88,9 @@ const LinkDataPipelineModal = ({
<div className="text-muted-foreground">
<FormControl>
<Input
placeholder={t('dataFlowPlaceholder')}
placeholder={t(
'knowledgeConfiguration.filterPlaceholder',
)}
{...field}
/>
</FormControl>
@ -78,11 +103,56 @@ const LinkDataPipelineModal = ({
</FormItem>
)}
/>
{isEdit && (
<FormField
control={form.control}
name={'set_default'}
render={({ field }) => (
<FormItem className=" items-center space-y-0 ">
<div className="flex flex-col gap-1">
<div className="flex gap-2 justify-between ">
<FormLabel
tooltip={t('knowledgeConfiguration.setDefaultTip')}
className="text-sm text-text-primary whitespace-wrap "
>
{t('knowledgeConfiguration.setDefault')}
</FormLabel>
</div>
<div className="text-muted-foreground">
<FormControl>
<Switch
value={field.value}
onCheckedChange={field.onChange}
/>
</FormControl>
</div>
</div>
<div className="flex pt-1">
<div className="w-full"></div>
<FormMessage />
</div>
</FormItem>
)}
/>
)}
<div className="flex justify-end gap-1">
<Button type="reset" variant={'outline'} className="btn-primary">
<Button
type="button"
variant={'outline'}
className="btn-primary"
onClick={() => {
setOpen(false);
}}
>
{t('modal.cancelText')}
</Button>
<Button type="submit" variant={'default'} className="btn-primary">
<Button
type="button"
variant={'default'}
className="btn-primary"
onClick={form.handleSubmit(handleFormSubmit)}
>
{t('modal.okText')}
</Button>
</div>

View File

@ -72,7 +72,17 @@ export const formSchema = z.object({
// icon: z.array(z.instanceof(File)),
});
export const linkPiplineFormSchema = z.object({
data_flow: z.array(z.string()),
export const pipelineFormSchema = z.object({
data_flow: z.array(z.string()).optional(),
set_default: z.boolean().optional(),
file_filter: z.string().optional(),
});
export const linkPiplineFormSchema = pipelineFormSchema.pick({
data_flow: true,
file_filter: true,
});
export const editPiplineFormSchema = pipelineFormSchema.pick({
set_default: true,
file_filter: true,
});

View File

@ -86,9 +86,12 @@ export default function DatasetSettings() {
<GeneralForm></GeneralForm>
<Divider />
<GraphRagItems className="border-none p-0"></GraphRagItems>
<GraphRagItems
className="border-none p-0"
showGenerateItem={true}
></GraphRagItems>
<Divider />
<RaptorFormFields></RaptorFormFields>
<RaptorFormFields showGenerateItem={true}></RaptorFormFields>
<Divider />
<LinkDataPipeline />
</MainContainer>

View File

@ -6,16 +6,20 @@ import {
DropdownMenuItem,
DropdownMenuTrigger,
} from '@/components/ui/dropdown-menu';
import { Modal } from '@/components/ui/modal/modal';
import { cn } from '@/lib/utils';
import { toFixed } from '@/utils/common-util';
import { t } from 'i18next';
import { lowerFirst } from 'lodash';
import { CirclePause, WandSparkles } from 'lucide-react';
import { CirclePause, Trash2, WandSparkles } from 'lucide-react';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { generateStatus, useFetchGenerateData } from './hook';
const MenuItem: React.FC<{ name: 'KnowledgeGraph' | 'Raptor' }> = ({
name,
}) => {
export enum GenerateType {
KnowledgeGraph = 'KnowledgeGraph',
Raptor = 'Raptor',
}
const MenuItem: React.FC<{ name: GenerateType }> = ({ name }) => {
console.log(name, 'pppp');
const iconKeyMap = {
KnowledgeGraph: 'knowledgegraph',
@ -111,3 +115,102 @@ const Generate: React.FC = () => {
};
export default Generate;
export type IGenerateLogProps = {
id?: string;
status: 0 | 1;
message?: string;
created_at?: string;
updated_at?: string;
type?: GenerateType;
className?: string;
onDelete?: () => void;
};
export const GenerateLogButton = (props: IGenerateLogProps) => {
const { t } = useTranslation();
const {
id,
status,
message,
created_at,
updated_at,
type,
className,
onDelete,
} = props;
const handleDelete = () => {
Modal.show({
visible: true,
className: '!w-[560px]',
title:
t('common.delete') +
' ' +
(type === GenerateType.KnowledgeGraph
? t('knowledgeDetails.knowledgeGraph')
: t('knowledgeDetails.raptor')),
children: (
<div
className="text-sm text-text-secondary"
dangerouslySetInnerHTML={{
__html: t('knowledgeConfiguration.deleteGenerateModalContent', {
type:
type === GenerateType.KnowledgeGraph
? t('knowledgeDetails.knowledgeGraph')
: t('knowledgeDetails.raptor'),
}),
}}
></div>
),
onVisibleChange: () => {
Modal.hide();
},
footer: (
<div className="flex justify-end gap-2">
<Button
type="button"
variant={'outline'}
onClick={() => Modal.hide()}
>
{t('dataflowParser.changeStepModalCancelText')}
</Button>
<Button
type="button"
variant={'secondary'}
className="!bg-state-error text-text-primary"
onClick={() => {
Modal.hide();
}}
>
{t('common.delete')}
</Button>
</div>
),
});
};
return (
<div
className={cn('flex bg-bg-card rounded-md py-1 px-3', props.className)}
>
<div className="flex items-center justify-between w-full">
{status === 1 && (
<>
<div>
{message || t('knowledgeDetails.generatedOn')}
{created_at}
</div>
<Trash2
size={14}
className="cursor-pointer"
onClick={(e) => {
console.log('delete');
handleDelete();
e.stopPropagation();
}}
/>
</>
)}
{status === 0 && <div>{t('knowledgeDetails.notGenerated')}</div>}
</div>
</div>
);
};

View File

@ -75,7 +75,7 @@ export default function Dataset() {
filters={filters}
leftPanel={
<div className="items-start">
<div className="pb-1">{t('knowledgeDetails.dataset')}</div>
<div className="pb-1">{t('knowledgeDetails.subbarFiles')}</div>
<div className="text-text-sub-title-invert text-sm">
{t('knowledgeDetails.datasetDescription')}
</div>

View File

@ -9,7 +9,7 @@ import { cn, formatBytes } from '@/lib/utils';
import { Routes } from '@/routes';
import { formatPureDate } from '@/utils/date';
import { isEmpty } from 'lodash';
import { Banknote, Database, FileSearch2, GitGraph } from 'lucide-react';
import { Banknote, FileSearch2, FolderOpen, GitGraph } from 'lucide-react';
import { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useHandleMenuClick } from './hooks';
@ -34,8 +34,8 @@ export function SideBar({ refreshCount }: PropType) {
// key: Routes.DataSetOverview,
// },
{
icon: Database,
label: t(`knowledgeDetails.dataset`),
icon: FolderOpen,
label: t(`knowledgeDetails.subbarFiles`),
key: Routes.DatasetBase,
},
{