Fix: Optimized knowledge base file parsing and display #9869 (#10292)

### What problem does this PR solve?

Fix: Optimized knowledge base file parsing and display #9869

- Optimized the ChunkMethodDialog component logic and adjusted
FormSchema validation rules
- Updated the document information interface definition, adding
pipeline_id, pipeline_name, and suffix fields
- Refactored the ChunkResultBar component, removing filter-related logic
and simplifying the input box and chunk creation functionality
- Improved FormatPreserveEditor to support text mode switching
(full/omitted) display control
- Updated timeline node titles to more accurate semantic descriptions
(e.g., character splitters)
- Optimized the data flow result page structure and style, dynamically
adjusting height and content display
- Fixed the table sorting function on the dataset overview page and
enhanced the display of task type icons and status mapping.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
chanx
2025-09-25 19:53:49 +08:00
committed by GitHub
parent abe7132630
commit 14273b4595
27 changed files with 674 additions and 386 deletions

View File

@ -51,7 +51,7 @@ import RaptorFormFields, {
import { ButtonLoading } from '../ui/button';
import { Input } from '../ui/input';
import { DynamicPageRange } from './dynamic-page-range';
import { useFetchParserListOnMount, useShowAutoKeywords } from './hooks';
import { useShowAutoKeywords } from './hooks';
import {
useDefaultParserValues,
useFillDefaultValueOnMount,
@ -93,8 +93,6 @@ export function ChunkMethodDialog({
}: IProps) {
const { t } = useTranslation();
const { parserList } = useFetchParserListOnMount(documentExtension);
const { data: knowledgeDetails } = useFetchKnowledgeBaseConfiguration();
const useGraphRag = useMemo(() => {
@ -105,7 +103,8 @@ export function ChunkMethodDialog({
const fillDefaultParserValue = useFillDefaultValueOnMount();
const FormSchema = z.object({
const FormSchema = z
.object({
parseType: z.number(),
parser_id: z
.string()
@ -140,13 +139,22 @@ export function ChunkMethodDialog({
.array(z.object({ from: z.coerce.number(), to: z.coerce.number() }))
.optional(),
}),
})
.superRefine((data, ctx) => {
if (data.parseType === 2 && !data.pipeline_id) {
ctx.addIssue({
path: ['pipeline_id'],
message: t('common.pleaseSelect'),
code: 'custom',
});
}
});
const form = useForm<z.infer<typeof FormSchema>>({
resolver: zodResolver(FormSchema),
defaultValues: {
parser_id: parserId,
pipeline_id: pipelineId,
parser_id: parserId || '',
pipeline_id: pipelineId || '',
parseType: pipelineId ? 2 : 1,
parser_config: defaultParserValues,
},
@ -209,8 +217,8 @@ export function ChunkMethodDialog({
const pages =
parserConfig?.pages?.map((x) => ({ from: x[0], to: x[1] })) ?? [];
form.reset({
parser_id: parserId,
pipeline_id: pipelineId,
parser_id: parserId || '',
pipeline_id: pipelineId || '',
parseType: pipelineId ? 2 : 1,
parser_config: fillDefaultParserValue({
pages: pages.length > 0 ? pages : [{ from: 1, to: 1024 }],
@ -231,13 +239,14 @@ export function ChunkMethodDialog({
knowledgeDetails.parser_config,
parserConfig,
parserId,
pipelineId,
useGraphRag,
visible,
]);
const parseType = useWatch({
control: form.control,
name: 'parseType',
defaultValue: 1,
defaultValue: pipelineId ? 2 : 1,
});
return (
<Dialog open onOpenChange={hideModal}>

View File

@ -49,7 +49,7 @@ function Radio({ value, checked, disabled, onChange, children }: RadioProps) {
>
<span
className={cn(
'flex h-4 w-4 items-center justify-center rounded-full border border-input transition-colors',
'flex h-4 w-4 items-center justify-center rounded-full border border-border transition-colors',
'peer ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2',
isChecked && 'border-primary bg-primary/10',
mergedDisabled && 'border-muted',

View File

@ -11,6 +11,8 @@ export interface IDocumentInfo {
name: string;
parser_config: IParserConfig;
parser_id: string;
pipeline_id: string;
pipeline_name: string;
process_begin_at?: string;
process_duration: number;
progress: number;
@ -19,6 +21,7 @@ export interface IDocumentInfo {
size: number;
source_type: string;
status: string;
suffix: string;
thumbnail: string;
token_num: number;
type: string;

View File

@ -102,6 +102,9 @@ export default {
noMoreData: `That's all. Nothing more.`,
},
knowledgeDetails: {
fileSize: 'File Size',
fileType: 'File Type',
uploadedBy: 'Uploaded by',
notGenerated: 'Not generated',
generatedOn: 'Generated on',
subbarFiles: 'Files',
@ -128,7 +131,7 @@ export default {
success: 'Success',
failed: 'Failed',
completed: 'Completed',
processLog: 'Process Log',
datasetLog: 'Dataset Log',
created: 'Created',
learnMore: 'Learn More',
general: 'General',

View File

@ -94,6 +94,9 @@ export default {
noMoreData: '没有更多数据了',
},
knowledgeDetails: {
fileSize: '文件大小',
fileType: '文件类型',
uploadedBy: '创建者',
notGenerated: '未生成',
generatedOn: '生成于',
subbarFiles: '文件列表',
@ -116,7 +119,7 @@ export default {
success: '成功',
failed: '失败',
completed: '已完成',
processLog: '处理进度日志',
datasetLog: '知识库日志',
created: '创建于',
learnMore: '了解更多',
general: '通用',

View File

@ -1,55 +1,34 @@
import { Input } from '@/components/originui/input';
import { Button } from '@/components/ui/button';
import {
Popover,
PopoverContent,
PopoverTrigger,
} from '@/components/ui/popover';
import { Radio } from '@/components/ui/radio';
import { useTranslate } from '@/hooks/common-hooks';
import { cn } from '@/lib/utils';
import { SearchOutlined } from '@ant-design/icons';
import { ListFilter, Plus } from 'lucide-react';
import { Plus } from 'lucide-react';
import { useState } from 'react';
import { ChunkTextMode } from '../../constant';
interface ChunkResultBarProps {
changeChunkTextMode: React.Dispatch<React.SetStateAction<string | number>>;
available: number | undefined;
selectAllChunk: (value: boolean) => void;
handleSetAvailable: (value: number | undefined) => void;
createChunk: () => void;
handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
searchString: string;
createChunk: (text: string) => void;
}
export default ({
changeChunkTextMode,
available,
selectAllChunk,
handleSetAvailable,
createChunk,
handleInputChange,
searchString,
}: ChunkResultBarProps) => {
export default ({ changeChunkTextMode, createChunk }: ChunkResultBarProps) => {
const { t } = useTranslate('chunk');
const [textSelectValue, setTextSelectValue] = useState<string | number>(
ChunkTextMode.Full,
);
const handleFilterChange = (e: string | number) => {
const value = e === -1 ? undefined : (e as number);
selectAllChunk(false);
handleSetAvailable(value);
};
const filterContent = (
<div className="w-[200px]">
<Radio.Group onChange={handleFilterChange} value={available}>
<div className="flex flex-col gap-2 p-4">
<Radio value={-1}>{t('all')}</Radio>
<Radio value={1}>{t('enabled')}</Radio>
<Radio value={0}>{t('disabled')}</Radio>
</div>
</Radio.Group>
</div>
);
// const handleFilterChange = (e: string | number) => {
// const value = e === -1 ? undefined : (e as number);
// selectAllChunk(false);
// handleSetAvailable(value);
// };
// const filterContent = (
// <div className="w-[200px]">
// <Radio.Group onChange={handleFilterChange} value={available}>
// <div className="flex flex-col gap-2 p-4">
// <Radio value={-1}>{t('all')}</Radio>
// <Radio value={1}>{t('enabled')}</Radio>
// <Radio value={0}>{t('disabled')}</Radio>
// </div>
// </Radio.Group>
// </div>
// );
const textSelectOptions = [
{ label: t(ChunkTextMode.Full), value: ChunkTextMode.Full },
{ label: t(ChunkTextMode.Ellipse), value: ChunkTextMode.Ellipse },
@ -78,7 +57,7 @@ export default ({
</div>
))}
</div>
<Input
{/* <Input
className="bg-bg-card text-muted-foreground"
style={{ width: 200 }}
placeholder={t('search')}
@ -95,9 +74,9 @@ export default ({
<PopoverContent className="p-0 w-[200px]">
{filterContent}
</PopoverContent>
</Popover>
</Popover> */}
<Button
onClick={() => createChunk()}
onClick={() => createChunk('')}
variant={'secondary'}
className="bg-bg-card text-muted-foreground hover:bg-card"
>

View File

@ -2,8 +2,9 @@ import { Checkbox } from '@/components/ui/checkbox';
import { Textarea } from '@/components/ui/textarea';
import { cn } from '@/lib/utils';
import { CheckedState } from '@radix-ui/react-checkbox';
import { useState } from 'react';
import { useEffect, useState } from 'react';
import { ChunkTextMode } from '../../constant';
import styles from '../../index.less';
interface FormatPreserveEditorProps {
initialValue: {
key: string;
@ -17,6 +18,7 @@ interface FormatPreserveEditorProps {
isChunck?: boolean;
handleCheckboxClick?: (id: string | number, checked: boolean) => void;
selectedChunkIds?: string[];
textMode?: ChunkTextMode;
}
const FormatPreserveEditor = ({
initialValue,
@ -25,6 +27,7 @@ const FormatPreserveEditor = ({
isChunck,
handleCheckboxClick,
selectedChunkIds,
textMode,
}: FormatPreserveEditorProps) => {
const [content, setContent] = useState(initialValue);
// const [isEditing, setIsEditing] = useState(false);
@ -32,6 +35,10 @@ const FormatPreserveEditor = ({
undefined,
);
console.log('initialValue', initialValue);
useEffect(() => {
setContent(initialValue);
}, [initialValue]);
const handleEdit = (e?: any, index?: number) => {
console.log(e, index, content);
if (content.key === 'json') {
@ -143,7 +150,12 @@ const FormatPreserveEditor = ({
)}
{activeEditIndex !== index && (
<div
className="text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap"
className={cn(
'text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap w-full',
{
[styles.contentEllipsis]: textMode === ChunkTextMode.Ellipse,
},
)}
key={index}
onClick={(e) => {
handleEdit(e, index);

View File

@ -37,11 +37,11 @@ export const TimelineNodeObj = {
icon: <Heading size={13} />,
},
[TimelineNodeType.characterSplitter]: {
title: 'Title Splitter',
title: 'Character Splitter',
icon: <Heading size={13} />,
},
[TimelineNodeType.splitter]: {
title: 'Character Splitter',
title: 'Splitter',
icon: <Blocks size={13} />,
},
[TimelineNodeType.tokenizer]: {
@ -50,40 +50,6 @@ export const TimelineNodeObj = {
clickable: false,
},
};
// export const TimelineNodeArr = [
// {
// id: 1,
// title: 'File',
// icon: <PlayIcon size={13} />,
// clickable: false,
// type: TimelineNodeType.begin,
// },
// {
// id: 2,
// title: 'Context Generator',
// icon: <PlayIcon size={13} />,
// type: TimelineNodeType.contextGenerator,
// },
// {
// id: 3,
// title: 'Title Splitter',
// icon: <PlayIcon size={13} />,
// type: TimelineNodeType.titleSplitter,
// },
// {
// id: 4,
// title: 'Character Splitter',
// icon: <PlayIcon size={13} />,
// type: TimelineNodeType.characterSplitter,
// },
// {
// id: 5,
// title: 'Tokenizer',
// icon: <CheckLine size={13} />,
// clickable: false,
// type: TimelineNodeType.tokenizer,
// },
// ]
export interface TimelineDataFlowProps {
activeId: number | string;
activeFunc: (id: number | string, step: TimelineNode) => void;

View File

@ -249,7 +249,6 @@ export const useTimelineDataFlow = (data: IPipelineFileLogDetail) => {
}
const timeNode = {
...TimelineNodeObj[name],
clickable: true,
id: index,
className: 'w-32',
completed: false,

View File

@ -82,15 +82,6 @@
}
}
.card {
:global {
.ant-card-body {
padding: 10px;
margin: 0;
}
margin-bottom: 10px;
}
cursor: pointer;
.contentEllipsis {
.multipleLineEllipsis(3);
}

View File

@ -42,7 +42,7 @@ const Chunk = () => {
data: { documentInfo },
} = useFetchNextChunkList();
const { selectedChunkId } = useHandleChunkCardClick();
const [activeStepId, setActiveStepId] = useState<number | string>(0);
const [activeStepId, setActiveStepId] = useState<number | string>(2);
const { data: dataset } = useFetchPipelineFileLogDetail();
const { t } = useTranslation();

View File

@ -1,14 +1,16 @@
import { TimelineNode } from '@/components/originui/timeline';
import Spotlight from '@/components/spotlight';
import { Spin } from '@/components/ui/spin';
import { cn } from '@/lib/utils';
import classNames from 'classnames';
import { useCallback, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import ChunkResultBar from './components/chunk-result-bar';
import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
import FormatPreserEditor from './components/parse-editer';
import RerunButton from './components/rerun-button';
import { TimelineNodeType } from './constant';
import { useFetchParserList } from './hooks';
import { useChangeChunkTextMode, useFetchParserList } from './hooks';
import { IDslComponent } from './interface';
interface IProps {
isChange: boolean;
@ -23,6 +25,7 @@ const ParserContainer = (props: IProps) => {
const { t } = useTranslation();
const { loading } = useFetchParserList();
const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
const { changeChunkTextMode, textMode } = useChangeChunkTextMode();
const initialValue = useMemo(() => {
const outputs = data?.value?.obj?.params?.outputs;
const key = outputs?.output_format?.value;
@ -108,6 +111,19 @@ const ParserContainer = (props: IProps) => {
step?.type === TimelineNodeType.characterSplitter ||
step?.type === TimelineNodeType.titleSplitter ||
step?.type === TimelineNodeType.splitter;
const handleCreateChunk = useCallback(
(text: string) => {
console.log('handleCreateChunk', text);
const newText = [...initialText.value, { text: text || ' ' }];
setInitialText({
...initialText,
value: newText,
});
console.log('newText', newText, initialText);
},
[initialText],
);
return (
<>
{isChange && (
@ -122,6 +138,7 @@ const ParserContainer = (props: IProps) => {
<div className={classNames('flex flex-col w-full')}>
<Spin spinning={loading} className="" size="large">
<div className="h-[50px] flex flex-col justify-end pb-[5px]">
{!isChunck && (
<div>
<h2 className="text-[16px]">
{t('dataflowParser.parseSummary')}
@ -130,20 +147,41 @@ const ParserContainer = (props: IProps) => {
{t('dataflowParser.parseSummaryTip')}
</div>
</div>
)}
{isChunck && (
<div>
<h2 className="text-[16px]">{t('chunk.chunkResult')}</h2>
<div className="text-[12px] text-text-secondary italic">
{t('chunk.chunkResultTip')}
</div>
</div>
)}
</div>
{isChunck && (
<div className="pt-[5px] pb-[5px]">
<div className="pt-[5px] pb-[5px] flex justify-between items-center">
<CheckboxSets
selectAllChunk={selectAllChunk}
removeChunk={handleRemoveChunk}
checked={selectedChunkIds.length === initialText.value.length}
selectedChunkIds={selectedChunkIds}
/>
<ChunkResultBar
changeChunkTextMode={changeChunkTextMode}
createChunk={handleCreateChunk}
/>
</div>
)}
<div className=" border rounded-lg p-[20px] box-border h-[calc(100vh-180px)] w-[calc(100%-20px)] overflow-auto scrollbar-none">
<div
className={cn(
' border rounded-lg p-[20px] box-border w-[calc(100%-20px)] overflow-auto scrollbar-none',
{
'h-[calc(100vh-240px)]': isChunck,
'h-[calc(100vh-180px)]': !isChunck,
},
)}
>
<FormatPreserEditor
initialValue={initialText}
onSave={handleSave}
@ -151,6 +189,7 @@ const ParserContainer = (props: IProps) => {
initialText.key !== 'json' ? '!h-[calc(100vh-220px)]' : ''
}
isChunck={isChunck}
textMode={textMode}
isDelete={
step?.type === TimelineNodeType.characterSplitter ||
step?.type === TimelineNodeType.titleSplitter ||

View File

@ -4,6 +4,11 @@ export enum LogTabs {
}
export enum ProcessingType {
knowledgeGraph = 'knowledgeGraph',
raptor = 'raptor',
knowledgeGraph = 'GraphRAG',
raptor = 'RAPTOR',
}
export const ProcessingTypeMap = {
[ProcessingType.knowledgeGraph]: 'Knowledge Graph',
[ProcessingType.raptor]: 'Raptor',
};

View File

@ -10,13 +10,8 @@ import { useQuery } from '@tanstack/react-query';
import { useCallback, useState } from 'react';
import { useParams, useSearchParams } from 'umi';
import { LogTabs } from './dataset-common';
import { IFileLogList, IOverviewTital } from './interface';
export interface IOverviewTital {
cancelled: number;
failed: number;
finished: number;
processing: number;
}
const useFetchOverviewTital = () => {
const [searchParams] = useSearchParams();
const { id } = useParams();
@ -33,40 +28,6 @@ const useFetchOverviewTital = () => {
return { data };
};
export interface IFileLogItem {
create_date: string;
create_time: number;
document_id: string;
document_name: string;
document_suffix: string;
document_type: string;
dsl: any;
path: string[];
task_id: string;
id: string;
name: string;
kb_id: string;
operation_status: string;
parser_id: string;
pipeline_id: string;
pipeline_title: string;
avatar: string;
process_begin_at: null | string;
process_duration: number;
progress: number;
progress_msg: string;
source_from: string;
status: string;
task_type: string;
tenant_id: string;
update_date: string;
update_time: number;
}
export interface IFileLogList {
logs: IFileLogItem[];
total: number;
}
const useFetchFileLogList = () => {
const [searchParams] = useSearchParams();
const { searchString, handleInputChange } = useHandleSearchChange();

View File

@ -124,7 +124,7 @@ const FileLogsPage: FC = () => {
};
});
}
}, [tableOriginData, active]);
}, [tableOriginData]);
const changeActiveLogs = (active: (typeof LogTabs)[keyof typeof LogTabs]) => {
setActive(active);

View File

@ -0,0 +1,62 @@
import { RunningStatus, RunningStatusMap } from '../dataset/constant';
import { LogTabs } from './dataset-common';
export interface DocumentLog {
fileName: string;
status: RunningStatus;
statusName: typeof RunningStatusMap;
}
export interface FileLogsTableProps {
data: Array<IFileLogItem & DocumentLog>;
pageCount: number;
pagination: {
current: number;
pageSize: number;
total: number;
};
setPagination: (pagination: { page: number; pageSize: number }) => void;
loading?: boolean;
active: (typeof LogTabs)[keyof typeof LogTabs];
}
export interface IOverviewTital {
cancelled: number;
failed: number;
finished: number;
processing: number;
}
export interface IFileLogItem {
create_date: string;
create_time: number;
document_id: string;
document_name: string;
document_suffix: string;
document_type: string;
dsl: any;
path: string[];
task_id: string;
id: string;
name: string;
kb_id: string;
operation_status: string;
parser_id: string;
pipeline_id: string;
pipeline_title: string;
avatar: string;
process_begin_at: null | string;
process_duration: number;
progress: number;
progress_msg: string;
source_from: string;
status: string;
task_type: string;
tenant_id: string;
update_date: string;
update_time: number;
}
export interface IFileLogList {
logs: IFileLogItem[];
total: number;
}

View File

@ -1,7 +1,6 @@
import FileStatusBadge from '@/components/file-status-badge';
import { FileIcon } from '@/components/icon-font';
import { FileIcon, IconFontFill } from '@/components/icon-font';
import { RAGFlowAvatar } from '@/components/ragflow-avatar';
import SvgIcon from '@/components/svg-icon';
import { Button } from '@/components/ui/button';
import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import {
@ -15,7 +14,7 @@ import {
import { RunningStatusMap } from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
import { formatDate } from '@/utils/date';
import { formatDate, formatSecondsToHumanReadable } from '@/utils/date';
import {
ColumnDef,
ColumnFiltersState,
@ -29,32 +28,13 @@ import {
useReactTable,
} from '@tanstack/react-table';
import { TFunction } from 'i18next';
import { ClipboardList, Eye } from 'lucide-react';
import { ArrowUpDown, ClipboardList, Eye } from 'lucide-react';
import { FC, useMemo, useState } from 'react';
import { useParams } from 'umi';
import { RunningStatus } from '../dataset/constant';
import ProcessLogModal from '../process-log-modal';
import { LogTabs, ProcessingType } from './dataset-common';
import { IFileLogItem } from './hook';
interface DocumentLog {
fileName: string;
status: RunningStatus;
statusName: typeof RunningStatusMap;
}
interface FileLogsTableProps {
data: DocumentLog[];
pageCount: number;
pagination: {
current: number;
pageSize: number;
total: number;
};
setPagination: (pagination: { page: number; pageSize: number }) => void;
loading?: boolean;
active: (typeof LogTabs)[keyof typeof LogTabs];
}
import { LogTabs, ProcessingType, ProcessingTypeMap } from './dataset-common';
import { DocumentLog, FileLogsTableProps, IFileLogItem } from './interface';
export const getFileLogsTableColumns = (
t: TFunction<'translation', string>,
@ -133,7 +113,18 @@ export const getFileLogsTableColumns = (
},
{
accessorKey: 'process_begin_at',
header: t('startDate'),
header: ({ column }) => {
return (
<Button
variant="transparent"
className="border-none"
onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
>
{t('startDate')}
<ArrowUpDown />
</Button>
);
},
cell: ({ row }) => (
<div className="text-text-primary">
{formatDate(row.original.process_begin_at)}
@ -227,34 +218,60 @@ export const getDatasetLogsTableColumns = (
),
},
{
accessorKey: 'startDate',
header: t('startDate'),
cell: ({ row }) => (
<div className="text-text-primary">{row.original.startDate}</div>
),
accessorKey: 'process_begin_at',
header: ({ column }) => {
return (
<Button
variant="transparent"
className="border-none"
onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
>
{t('startDate')}
<ArrowUpDown />
</Button>
);
},
{
accessorKey: 'processingType',
header: t('processingType'),
cell: ({ row }) => (
<div className="flex items-center gap-2 text-text-primary">
{ProcessingType.knowledgeGraph === row.original.processingType && (
<SvgIcon name={`data-flow/knowledgegraph`} width={24}></SvgIcon>
)}
{ProcessingType.raptor === row.original.processingType && (
<SvgIcon name={`data-flow/raptor`} width={24}></SvgIcon>
)}
{row.original.processingType}
<div className="text-text-primary">
{formatDate(row.original.process_begin_at)}
</div>
),
},
{
accessorKey: 'status',
accessorKey: 'task_type',
header: t('processingType'),
cell: ({ row }) => (
<div className="flex items-center gap-2 text-text-primary">
{ProcessingType.knowledgeGraph === row.original.task_type && (
<IconFontFill
name={`knowledgegraph`}
className="text-text-secondary"
></IconFontFill>
)}
{ProcessingType.raptor === row.original.task_type && (
<IconFontFill
name={`dataflow-01`}
className="text-text-secondary"
></IconFontFill>
)}
{ProcessingTypeMap[row.original.task_type as ProcessingType] ||
row.original.task_type}
</div>
),
},
{
accessorKey: 'operation_status',
header: t('status'),
cell: ({ row }) => (
// <FileStatusBadge
// status={row.original.status}
// name={row.original.statusName}
// />
<FileStatusBadge
status={row.original.status}
name={row.original.statusName}
status={row.original.operation_status as RunningStatus}
name={
RunningStatusMap[row.original.operation_status as RunningStatus]
}
/>
),
},
@ -294,17 +311,19 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
const { t } = useTranslate('knowledgeDetails');
const [isModalVisible, setIsModalVisible] = useState(false);
const { navigateToDataflowResult } = useNavigatePage();
const [logInfo, setLogInfo] = useState<IFileLogItem>({});
const [logInfo, setLogInfo] = useState<IFileLogItem>();
const kowledgeId = useParams().id;
const showLog = (row: Row<IFileLogItem & DocumentLog>) => {
const logDetail = {
taskId: row.original.id,
taskId: row.original?.dsl?.task_id,
fileName: row.original.document_name,
source: row.original.source_from,
task: row.original.dsl.task_id,
task: row.original?.task_type,
status: row.original.statusName,
startDate: formatDate(row.original.process_begin_at),
duration: (row.original.process_duration || 0) + 's',
duration: formatSecondsToHumanReadable(
row.original.process_duration || 0,
),
details: row.original.progress_msg,
};
console.log('logDetail', logDetail);
@ -331,7 +350,7 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
[pagination],
);
const table = useReactTable({
const table = useReactTable<IFileLogItem & DocumentLog>({
data: data || [],
columns,
manualPagination: true,
@ -405,11 +424,14 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
/>
</div>
</div>
{isModalVisible && (
<ProcessLogModal
title={active === LogTabs.FILE_LOGS ? t('fileLogs') : t('datasetLog')}
visible={isModalVisible}
onCancel={() => setIsModalVisible(false)}
logInfo={logInfo}
/>
)}
</div>
);
};

View File

@ -27,6 +27,7 @@ import {
import { UseRowSelectionType } from '@/hooks/logic-hooks/use-row-selection';
import { useFetchDocumentList } from '@/hooks/use-document-request';
import { getExtension } from '@/utils/document-util';
import { t } from 'i18next';
import { pick } from 'lodash';
import { useMemo } from 'react';
import ProcessLogModal from '../process-log-modal';
@ -184,6 +185,7 @@ export function DatasetTable({
<ChunkMethodDialog
documentId={changeParserRecord.id}
parserId={changeParserRecord.parser_id}
pipelineId={changeParserRecord.pipeline_id}
parserConfig={changeParserRecord.parser_config}
documentExtension={getExtension(changeParserRecord.name)}
onOk={onChangeParserOk}
@ -213,6 +215,7 @@ export function DatasetTable({
)}
{logVisible && (
<ProcessLogModal
title={t('knowledgeDetails.fileLogs')}
visible={logVisible}
onCancel={() => hideLog()}
logInfo={logInfo}

View File

@ -9,30 +9,94 @@ import {
import { Modal } from '@/components/ui/modal/modal';
import { cn } from '@/lib/utils';
import { toFixed } from '@/utils/common-util';
import { UseMutateAsyncFunction } from '@tanstack/react-query';
import { t } from 'i18next';
import { lowerFirst } from 'lodash';
import { CirclePause, Trash2, WandSparkles } from 'lucide-react';
import { useState } from 'react';
import { useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { generateStatus, useFetchGenerateData } from './hook';
import { replaceText } from '../../process-log-modal';
import {
ITraceInfo,
generateStatus,
useDatasetGenerate,
useTraceGenerate,
} from './hook';
export enum GenerateType {
KnowledgeGraph = 'KnowledgeGraph',
Raptor = 'Raptor',
}
const MenuItem: React.FC<{ name: GenerateType }> = ({ name }) => {
console.log(name, 'pppp');
const MenuItem: React.FC<{
name: GenerateType;
data: ITraceInfo;
pauseGenerate: () => void;
runGenerate: UseMutateAsyncFunction<
any,
Error,
{
type: GenerateType;
},
unknown
>;
}> = ({ name, runGenerate, data, pauseGenerate }) => {
console.log(name, 'pppp', data);
const iconKeyMap = {
KnowledgeGraph: 'knowledgegraph',
Raptor: 'dataflow-01',
};
const {
data: { percent, type },
pauseGenerate,
} = useFetchGenerateData();
const type = useMemo(() => {
if (!data) {
return generateStatus.start;
}
if (data.progress >= 1) {
return generateStatus.completed;
} else if (!data.progress && data.progress !== 0) {
return generateStatus.start;
} else if (data.progress < 0) {
return generateStatus.failed;
} else if (data.progress < 1) {
return generateStatus.running;
}
}, [data]);
const percent =
type === generateStatus.failed
? 100
: type === generateStatus.running
? data.progress * 100
: 0;
return (
<div className="flex items-start gap-2 flex-col w-full">
<DropdownMenuItem
className={cn(
'border cursor-pointer p-2 rounded-md focus:bg-transparent',
{
'hover:border-accent-primary hover:bg-[rgba(59,160,92,0.1)]':
type === generateStatus.start,
'hover:border-border hover:bg-[rgba(59,160,92,0)]':
type !== generateStatus.start,
},
)}
onSelect={(e) => {
e.preventDefault();
}}
onClick={(e) => {
e.stopPropagation();
}}
>
<div
className="flex items-start gap-2 flex-col w-full"
onClick={() => {
if (type === generateStatus.start) {
runGenerate({ type: name });
}
}}
>
<div className="flex justify-start text-text-primary items-center gap-2">
<IconFontFill name={iconKeyMap[name]} className="text-accent-primary" />
<IconFontFill
name={iconKeyMap[name]}
className="text-accent-primary"
/>
{t(`knowledgeDetails.${lowerFirst(name)}`)}
</div>
{type === generateStatus.start && (
@ -40,32 +104,53 @@ const MenuItem: React.FC<{ name: GenerateType }> = ({ name }) => {
{t(`knowledgeDetails.generate${name}`)}
</div>
)}
{type === generateStatus.running && (
<div className="flex justify-between items-center w-full">
<div className="w-[calc(100%-100px)] bg-border-button h-1 rounded-full">
{(type === generateStatus.running ||
type === generateStatus.failed) && (
<div className="flex justify-between items-center w-full px-2.5 py-1">
<div
className="h-1 bg-accent-primary rounded-full"
className={cn(' bg-border-button h-1 rounded-full', {
'w-[calc(100%-100px)]': type === generateStatus.running,
'w-[calc(100%-50px)]': type === generateStatus.failed,
})}
>
<div
className={cn('h-1 rounded-full', {
'bg-state-error': type === generateStatus.failed,
'bg-accent-primary': type === generateStatus.running,
})}
style={{ width: `${toFixed(percent)}%` }}
></div>
</div>
<span>{toFixed(percent) as string}%</span>
{type === generateStatus.running && (
<span>{(toFixed(percent) as string) + '%'}</span>
)}
<span
className="text-state-error"
onClick={() => {
onClick={(e) => {
e.stopPropagation();
pauseGenerate();
}}
>
{type === generateStatus.failed ? (
<IconFontFill name="reparse" className="text-accent-primary" />
) : (
<CirclePause />
)}
</span>
</div>
)}
<div className="w-full whitespace-pre-line text-wrap rounded-lg h-fit max-h-[350px] overflow-y-auto scrollbar-auto px-2.5 py-1">
{replaceText(data?.progress_msg || '')}
</div>
</div>
</DropdownMenuItem>
);
};
const Generate: React.FC = () => {
const [open, setOpen] = useState(false);
const { graphRunData, raptorRunData } = useTraceGenerate({ open });
const { runGenerate, pauseGenerate } = useDatasetGenerate();
const handleOpenChange = (isOpen: boolean) => {
setOpen(isOpen);
console.log('Dropdown is now', isOpen ? 'open' : 'closed');
@ -85,29 +170,30 @@ const Generate: React.FC = () => {
{t('knowledgeDetails.generate')}
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent className="w-[380px] p-5 ">
<DropdownMenuItem
className="border cursor-pointer p-2 rounded-md hover:border-accent-primary hover:bg-[rgba(59,160,92,0.1)]"
onSelect={(e) => {
e.preventDefault();
}}
onClick={(e) => {
e.stopPropagation();
}}
>
<MenuItem name="KnowledgeGraph" />
</DropdownMenuItem>
<DropdownMenuItem
className="border cursor-pointer p-2 rounded-md mt-3 hover:border-accent-primary hover:bg-[rgba(59,160,92,0.1)]"
onSelect={(e) => {
e.preventDefault();
}}
onClick={(e) => {
e.stopPropagation();
}}
>
<MenuItem name="Raptor" />
</DropdownMenuItem>
<DropdownMenuContent className="w-[380px] p-5 flex flex-col gap-2 ">
{Object.values(GenerateType).map((name) => {
const data = (
name === GenerateType.KnowledgeGraph
? graphRunData
: raptorRunData
) as ITraceInfo;
console.log(
name,
'data',
data,
!data || (!data.progress && data.progress !== 0),
);
return (
<div key={name}>
<MenuItem
name={name}
runGenerate={runGenerate}
data={data}
pauseGenerate={pauseGenerate}
/>
</div>
);
})}
</DropdownMenuContent>
</DropdownMenu>
</div>

View File

@ -1,25 +1,128 @@
import { useQuery } from '@tanstack/react-query';
import { useCallback } from 'react';
import message from '@/components/ui/message';
import kbService from '@/services/knowledge-service';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { t } from 'i18next';
import { useCallback, useEffect, useState } from 'react';
import { useParams } from 'umi';
import { GenerateType } from './generate';
export const generateStatus = {
running: 'running',
completed: 'completed',
start: 'start',
failed: 'failed',
};
const useFetchGenerateData = () => {
let number = 10;
// TODO: 获取数据
const { data, isFetching: loading } = useQuery({
queryKey: ['generateData', 'id'],
initialData: { id: 0, percent: 0, type: 'running' },
enum DatasetKey {
generate = 'generate',
}
export interface ITraceInfo {
begin_at: string;
chunk_ids: string;
create_date: string;
create_time: number;
digest: string;
doc_id: string;
from_page: number;
id: string;
priority: number;
process_duration: number;
progress: number;
progress_msg: string;
retry_count: number;
task_type: string;
to_page: number;
update_date: string;
update_time: number;
}
export const useTraceGenerate = ({ open }: { open: boolean }) => {
const { id } = useParams();
const [isLoopGraphRun, setLoopGraphRun] = useState(false);
const [isLoopRaptorRun, setLoopRaptorRun] = useState(false);
const { data: graphRunData, isFetching: graphRunloading } =
useQuery<ITraceInfo>({
queryKey: [GenerateType.KnowledgeGraph, id, open],
// initialData: {},
gcTime: 0,
refetchInterval: 3000,
refetchInterval: isLoopGraphRun ? 5000 : false,
retry: 3,
retryDelay: 1000,
enabled: open,
queryFn: async () => {
number += Math.random() * 10;
const data = {
id: Math.random(),
percent: number,
type: generateStatus.running,
const { data } = await kbService.traceGraphRag({
kb_id: id,
});
return data?.data || {};
},
});
const { data: raptorRunData, isFetching: raptorRunloading } =
useQuery<ITraceInfo>({
queryKey: [GenerateType.Raptor, id, open],
// initialData: {},
gcTime: 0,
refetchInterval: isLoopRaptorRun ? 5000 : false,
retry: 3,
retryDelay: 1000,
enabled: open,
queryFn: async () => {
const { data } = await kbService.traceRaptor({
kb_id: id,
});
return data?.data || {};
},
});
useEffect(() => {
setLoopGraphRun(
!!(
(graphRunData?.progress || graphRunData?.progress === 0) &&
graphRunData?.progress < 1 &&
graphRunData?.progress >= 0
),
);
}, [graphRunData?.progress]);
useEffect(() => {
setLoopRaptorRun(
!!(
(raptorRunData?.progress || raptorRunData?.progress === 0) &&
raptorRunData?.progress < 1 &&
raptorRunData?.progress >= 0
),
);
}, [raptorRunData?.progress]);
return {
graphRunData,
graphRunloading,
raptorRunData,
raptorRunloading,
};
};
export const useDatasetGenerate = () => {
const queryClient = useQueryClient();
const { id } = useParams();
const {
data,
isPending: loading,
mutateAsync,
} = useMutation({
mutationKey: [DatasetKey.generate],
mutationFn: async ({ type }: { type: GenerateType }) => {
const func =
type === GenerateType.KnowledgeGraph
? kbService.runGraphRag
: kbService.runRaptor;
const { data } = await func({
kb_id: id,
});
if (data.code === 0) {
message.success(t('message.operated'));
queryClient.invalidateQueries({
queryKey: [type],
});
}
return data;
},
});
@ -27,6 +130,5 @@ const useFetchGenerateData = () => {
// TODO: pause generate
console.log('pause generate');
}, []);
return { data, loading, pauseGenerate };
return { runGenerate: mutateAsync, pauseGenerate, data, loading };
};
export { useFetchGenerateData };

View File

@ -2,9 +2,12 @@ import { useSetModalState } from '@/hooks/common-hooks';
import { useNextWebCrawl } from '@/hooks/document-hooks';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { IDocumentInfo } from '@/interfaces/database/document';
import { formatDate, formatSecondsToHumanReadable } from '@/utils/date';
import { formatBytes } from '@/utils/file-util';
import { useCallback, useMemo, useState } from 'react';
import { useNavigate } from 'umi';
import { ILogInfo } from '../process-log-modal';
import { RunningStatus } from './constant';
export const useNavigateToOtherPage = () => {
const navigate = useNavigate();
@ -75,15 +78,17 @@ export const useShowLog = (documents: IDocumentInfo[]) => {
};
if (findRecord) {
log = {
taskId: findRecord.id,
fileName: findRecord.name,
fileSize: findRecord.size + '',
source: findRecord.source_type,
task: findRecord.status,
status: findRecord.run,
startTime: findRecord.process_begin_at,
endTime: findRecord.process_begin_at,
duration: findRecord.process_duration + 's',
fileType: findRecord?.suffix,
uploadedBy: findRecord?.created_by,
fileName: findRecord?.name,
uploadDate: formatDate(findRecord.create_date),
fileSize: formatBytes(findRecord.size || 0),
processBeginAt: formatDate(findRecord.process_begin_at),
chunkNumber: findRecord.chunk_num,
duration: formatSecondsToHumanReadable(
findRecord.process_duration || 0,
),
status: findRecord.run as RunningStatus,
details: findRecord.progress_msg,
};
}

View File

@ -1,9 +1,4 @@
import { Button } from '@/components/ui/button';
import {
HoverCard,
HoverCardContent,
HoverCardTrigger,
} from '@/components/ui/hover-card';
import { IDocumentInfo } from '@/interfaces/database/document';
import { useTranslation } from 'react-i18next';
import reactStringReplace from 'react-string-replace';
@ -88,8 +83,6 @@ export const PopoverContent = ({ record }: IProps) => {
export function ParsingCard({ record, handleShowLog }: IProps) {
return (
<HoverCard>
<HoverCardTrigger asChild>
<Button
variant={'transparent'}
className="border-none"
@ -98,10 +91,5 @@ export function ParsingCard({ record, handleShowLog }: IProps) {
>
<Dot run={record.run}></Dot>
</Button>
</HoverCardTrigger>
<HoverCardContent className="w-[40vw]">
<PopoverContent record={record}></PopoverContent>
</HoverCardContent>
</HoverCard>
);
}

View File

@ -46,7 +46,15 @@ export function ParsingStatusCell({
} & UseChangeDocumentParserShowType &
UseSaveMetaShowType) {
const { t } = useTranslation();
const { run, parser_id, progress, chunk_num, id } = record;
const {
run,
parser_id,
pipeline_id,
pipeline_name,
progress,
chunk_num,
id,
} = record;
const operationIcon = IconMap[run];
const p = Number((progress * 100).toFixed(2));
const { handleRunDocumentByIds } = useHandleRunDocumentByIds(id);
@ -80,7 +88,11 @@ export function ParsingStatusCell({
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant={'transparent'} className="border-none" size={'sm'}>
{parser_id === 'naive' ? 'general' : parser_id}
{pipeline_id
? pipeline_name || pipeline_id
: parser_id === 'naive'
? 'general'
: parser_id}
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent>

View File

@ -3,10 +3,16 @@ import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { RunningStatusMap } from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
import React from 'react';
import React, { useMemo } from 'react';
import reactStringReplace from 'react-string-replace';
import { RunningStatus } from './dataset/constant';
export interface ILogInfo {
fileType?: string;
uploadedBy?: string;
uploadDate?: string;
processBeginAt?: string;
chunkNumber?: number;
taskId?: string;
fileName: string;
fileSize?: string;
@ -23,6 +29,7 @@ interface ProcessLogModalProps {
visible: boolean;
onCancel: () => void;
logInfo: ILogInfo;
title: string;
}
const InfoItem: React.FC<{
@ -37,15 +44,7 @@ const InfoItem: React.FC<{
</div>
);
};
const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
visible,
onCancel,
logInfo,
}) => {
const { t } = useTranslate('knowledgeDetails');
const blackKeyList = [''];
const replaceText = (text: string) => {
export const replaceText = (text: string) => {
// Remove duplicate \n
const nextText = text.replace(/(\n)\1+/g, '$1');
@ -62,10 +61,24 @@ const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
);
return replacedText;
};
};
const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
visible,
onCancel,
logInfo: initData,
title,
}) => {
const { t } = useTranslate('knowledgeDetails');
const blackKeyList = [''];
console.log('logInfo', initData);
const logInfo = useMemo(() => {
console.log('logInfo', initData);
return initData;
}, [initData]);
return (
<Modal
title={t('processLog')}
title={title || 'log'}
open={visible}
onCancel={onCancel}
footer={
@ -77,7 +90,7 @@ const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
>
<div className=" rounded-lg">
<div className="flex flex-wrap ">
{Object.keys(logInfo).map((key) => {
{Object?.keys(logInfo).map((key) => {
if (
blackKeyList.includes(key) ||
!logInfo[key as keyof typeof logInfo]
@ -86,7 +99,7 @@ const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
}
if (key === 'details') {
return (
<div className="w-full" key={key}>
<div className="w-full mt-2" key={key}>
<InfoItem
label={t(key)}
value={

View File

@ -42,6 +42,10 @@ const {
getKnowledgeBasicInfo,
fetchDataPipelineLog,
fetchPipelineDatasetLogs,
runGraphRag,
traceGraphRag,
runRaptor,
traceRaptor,
} = api;
const methods = {
@ -188,6 +192,23 @@ const methods = {
url: api.get_pipeline_detail,
method: 'get',
},
runGraphRag: {
url: runGraphRag,
method: 'post',
},
traceGraphRag: {
url: traceGraphRag,
method: 'get',
},
runRaptor: {
url: runRaptor,
method: 'post',
},
traceRaptor: {
url: traceRaptor,
method: 'get',
},
};
const kbService = registerServer<keyof typeof methods>(methods, request);

View File

@ -50,6 +50,10 @@ export default {
fetchDataPipelineLog: `${api_host}/kb/list_pipeline_logs`,
get_pipeline_detail: `${api_host}/kb/pipeline_log_detail`,
fetchPipelineDatasetLogs: `${api_host}/kb/list_pipeline_dataset_logs`,
runGraphRag: `${api_host}/kb/run_graphrag`,
traceGraphRag: `${api_host}/kb/trace_graphrag`,
runRaptor: `${api_host}/kb/run_raptor`,
traceRaptor: `${api_host}/kb/trace_raptor`,
// tags
listTag: (knowledgeId: string) => `${api_host}/kb/${knowledgeId}/tags`,

View File

@ -1,5 +1,4 @@
import dayjs from 'dayjs';
import { toFixed } from './common-util';
export function formatDate(date: any) {
if (!date) {
@ -52,12 +51,13 @@ export function formatSecondsToHumanReadable(seconds: number): string {
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
const s = toFixed(seconds % 60, 3);
// const s = toFixed(seconds % 60, 3);
const s = seconds % 60;
const formattedSeconds = s === 0 ? '0' : s.toFixed(3).replace(/\.?0+$/, '');
const parts = [];
if (h > 0) parts.push(`${h}h`);
if (m > 0) parts.push(`${m}m`);
if (s || parts.length === 0) parts.push(`${s}s`);
if (h > 0) parts.push(`${h}h `);
if (m > 0) parts.push(`${m}m `);
if (s || parts.length === 0) parts.push(`${formattedSeconds}s`);
return parts.join('');
}