pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-25 16:26:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

Browse Source 
### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>
This commit is contained in:
Kevin Hu
2025-10-09 12:36:19 +08:00
committed by GitHub
parent ef0aecea3b
commit cbf04ee470
490 changed files with 10630 additions and 30688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
web/src/pages/dataflow-result/chunker.tsx
View File

@ -1,3 +1,4 @@
import { TimelineNode } from '@/components/originui/timeline';
import message from '@/components/ui/message';
import {
RAGFlowPagination,
@ -23,9 +24,16 @@ import {
useUpdateChunk,
} from './hooks';
import styles from './index.less';
const ChunkerContainer = () => {
interface IProps {
isChange: boolean;
setIsChange: (isChange: boolean) => void;
step?: TimelineNode;
}
const ChunkerContainer = (props: IProps) => {
const { isChange, setIsChange, step } = props;
const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
const [isChange, setIsChange] = useState(false);
const { t } = useTranslation();
const {
data: { documentInfo, data = [], total },
@ -135,19 +143,18 @@ const ChunkerContainer = () => {
setIsChange(true);
onChunkUpdatingOk(e);
};
const handleReRunFunc = () => {
setIsChange(false);
};
return (
<>
<div className="w-full h-full">
{isChange && (
<div className=" absolute top-2 right-6">
<RerunButton />
<RerunButton step={step} onRerun={handleReRunFunc} />
</div>
)}
<div
className={classNames(
{ [styles.pagePdfWrapper]: isPdf },
'flex flex-col w-3/5',
)}
>
<div className={classNames('flex flex-col w-full')}>
<Spin spinning={loading} className={styles.spin} size="large">
<div className="h-[50px] flex flex-row justify-between items-end pb-[5px]">
<div>
@ -176,7 +183,7 @@ const ChunkerContainer = () => {
selectedChunkIds={selectedChunkIds}
/>
</div>
<div className="h-[calc(100vh-280px)] overflow-y-auto pr-2 scrollbar-thin">
<div className="h-[calc(100vh-280px)] overflow-y-auto pr-2 scrollbar-auto">
<div
className={classNames(
styles.chunkContainer,
@ -227,7 +234,7 @@ const ChunkerContainer = () => {
parserId={documentInfo.parser_id}
/>
)}
</>
</div>
);
};

33
web/src/pages/dataflow-result/components/chunk-result-bar/checkbox-sets.tsx
View File

@ -1,24 +1,17 @@
import { Checkbox } from '@/components/ui/checkbox';
import { Label } from '@/components/ui/label';
import { Ban, CircleCheck, Trash2 } from 'lucide-react';
import { Trash2 } from 'lucide-react';
import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
type ICheckboxSetProps = {
selectAllChunk: (e: any) => void;
removeChunk: (e?: any) => void;
switchChunk: (available: number) => void;
checked: boolean;
selectedChunkIds: string[];
};
export default (props: ICheckboxSetProps) => {
const {
selectAllChunk,
removeChunk,
switchChunk,
checked,
selectedChunkIds,
} = props;
const { selectAllChunk, removeChunk, checked, selectedChunkIds } = props;
const { t } = useTranslation();
const handleSelectAllCheck = useCallback(
(e: any) => {
@ -32,14 +25,6 @@ export default (props: ICheckboxSetProps) => {
removeChunk();
}, [removeChunk]);
const handleEnabledClick = useCallback(() => {
switchChunk(1);
}, [switchChunk]);
const handleDisabledClick = useCallback(() => {
switchChunk(0);
}, [switchChunk]);
const isSelected = useMemo(() => {
return selectedChunkIds?.length > 0;
}, [selectedChunkIds]);
@ -57,20 +42,6 @@ export default (props: ICheckboxSetProps) => {
</div>
{isSelected && (
<>
<div
className="flex items-center cursor-pointer text-muted-foreground hover:text-text-primary"
onClick={handleEnabledClick}
>
<CircleCheck size={16} />
<span className="block ml-1">{t('chunk.enable')}</span>
</div>
<div
className="flex items-center cursor-pointer text-muted-foreground hover:text-text-primary"
onClick={handleDisabledClick}
>
<Ban size={16} />
<span className="block ml-1">{t('chunk.disable')}</span>
</div>
<div
className="flex items-center cursor-pointer text-red-400 hover:text-red-500"
onClick={handleDeleteClick}

74
web/src/pages/dataflow-result/components/chunk-result-bar/index.tsx
View File

@ -1,55 +1,23 @@
import { Input } from '@/components/originui/input';
import { Button } from '@/components/ui/button';
import {
Popover,
PopoverContent,
PopoverTrigger,
} from '@/components/ui/popover';
import { Radio } from '@/components/ui/radio';
import { useTranslate } from '@/hooks/common-hooks';
import { cn } from '@/lib/utils';
import { SearchOutlined } from '@ant-design/icons';
import { ListFilter, Plus } from 'lucide-react';
import { Plus } from 'lucide-react';
import { useState } from 'react';
import { ChunkTextMode } from '../../constant';
interface ChunkResultBarProps {
changeChunkTextMode: React.Dispatch<React.SetStateAction<string | number>>;
available: number | undefined;
selectAllChunk: (value: boolean) => void;
handleSetAvailable: (value: number | undefined) => void;
createChunk: () => void;
handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
searchString: string;
createChunk: (text: string) => void;
isReadonly: boolean;
}
export default ({
changeChunkTextMode,
available,
selectAllChunk,
handleSetAvailable,
createChunk,
handleInputChange,
searchString,
isReadonly,
}: ChunkResultBarProps) => {
const { t } = useTranslate('chunk');
const [textSelectValue, setTextSelectValue] = useState<string | number>(
ChunkTextMode.Full,
);
const handleFilterChange = (e: string | number) => {
const value = e === -1 ? undefined : (e as number);
selectAllChunk(false);
handleSetAvailable(value);
};
const filterContent = (
<div className="w-[200px]">
<Radio.Group onChange={handleFilterChange} value={available}>
<div className="flex flex-col gap-2 p-4">
<Radio value={-1}>{t('all')}</Radio>
<Radio value={1}>{t('enabled')}</Radio>
<Radio value={0}>{t('disabled')}</Radio>
</div>
</Radio.Group>
</div>
);
const textSelectOptions = [
{ label: t(ChunkTextMode.Full), value: ChunkTextMode.Full },
{ label: t(ChunkTextMode.Ellipse), value: ChunkTextMode.Ellipse },
@ -78,31 +46,15 @@ export default ({
</div>
))}
</div>
<Input
className="bg-bg-card text-muted-foreground"
style={{ width: 200 }}
placeholder={t('search')}
icon={<SearchOutlined />}
onChange={handleInputChange}
value={searchString}
/>
<Popover>
<PopoverTrigger asChild>
<Button className="bg-bg-card text-muted-foreground hover:bg-card">
<ListFilter />
</Button>
</PopoverTrigger>
<PopoverContent className="p-0 w-[200px]">
{filterContent}
</PopoverContent>
</Popover>
<Button
onClick={() => createChunk()}
variant={'secondary'}
className="bg-bg-card text-muted-foreground hover:bg-card"
>
<Plus size={44} />
</Button>
{!isReadonly && (
<Button
onClick={() => createChunk('')}
variant={'secondary'}
className="bg-bg-card text-muted-foreground hover:bg-card"
>
<Plus size={44} />
</Button>
)}
</div>
);
};

11
web/src/pages/dataflow-result/components/document-preview/hooks.ts
View File

@ -1,8 +1,9 @@
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { api_host } from '@/utils/api';
import api, { api_host } from '@/utils/api';
import { useSize } from 'ahooks';
import { CustomTextRenderer } from 'node_modules/react-pdf/dist/esm/shared/types';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useGetPipelineResultSearchParams } from '../../hooks';
export const useDocumentResizeObserver = () => {
const [containerWidth, setContainerWidth] = useState<number>();
@ -44,12 +45,16 @@ export const useHighlightText = (searchText: string = '') => {
return textRenderer;
};
export const useGetDocumentUrl = () => {
export const useGetDocumentUrl = (isAgent: boolean) => {
const { documentId } = useGetKnowledgeSearchParams();
const { createdBy, documentId: id } = useGetPipelineResultSearchParams();
const url = useMemo(() => {
if (isAgent) {
return api.downloadFile + `?id=${id}&created_by=${createdBy}`;
}
return `${api_host}/document/get/${documentId}`;
}, [documentId]);
}, [createdBy, documentId, id, isAgent]);
return url;
};

30
web/src/pages/dataflow-result/components/parse-editer/hook.ts Normal file
View File

@ -0,0 +1,30 @@
import { useEffect, useRef, useState } from 'react';
import { IJsonContainerProps, IObjContainerProps } from './interface';
export const useParserInit = ({
initialValue,
}: {
initialValue:
| IJsonContainerProps['initialValue']
| IObjContainerProps['initialValue'];
}) => {
const [content, setContent] = useState(initialValue);
useEffect(() => {
setContent(initialValue);
console.log('initialValue json parse', initialValue);
}, [initialValue]);
const [activeEditIndex, setActiveEditIndex] = useState<number | undefined>(
undefined,
);
const editDivRef = useRef<HTMLDivElement>(null);
return {
content,
setContent,
activeEditIndex,
setActiveEditIndex,
editDivRef,
};
};

78
web/src/pages/dataflow-result/components/parse-editer/index.tsx
View File

@ -1,45 +1,63 @@
import { Textarea } from '@/components/ui/textarea';
import { cn } from '@/lib/utils';
import { useState } from 'react';
import { CheckedState } from '@radix-ui/react-checkbox';
import { FormatPreserveEditorProps } from './interface';
import { ArrayContainer } from './json-parser';
import { ObjectContainer } from './object-parser';
interface FormatPreserveEditorProps {
initialValue: string;
onSave: (value: string) => void;
className?: string;
}
const FormatPreserveEditor = ({
initialValue,
onSave,
className,
isChunck,
handleCheckboxClick,
selectedChunkIds,
textMode,
clickChunk,
isReadonly,
}: FormatPreserveEditorProps) => {
const [content, setContent] = useState(initialValue);
const [isEditing, setIsEditing] = useState(false);
console.log('initialValue', initialValue);
const handleEdit = () => setIsEditing(true);
const handleSave = () => {
onSave(content);
setIsEditing(false);
const escapeNewlines = (text: string) => {
return text.replace(/\n/g, '\\n');
};
const unescapeNewlines = (text: string) => {
return text.replace(/\\n/g, '\n');
};
const handleCheck = (e: CheckedState, id: string | number) => {
handleCheckboxClick?.(id, e === 'indeterminate' ? false : e);
};
return (
<div className="editor-container">
{isEditing ? (
<Textarea
className={cn(
'w-full h-full bg-transparent text-text-secondary',
className,
)}
value={content}
onChange={(e) => setContent(e.target.value)}
onBlur={handleSave}
autoSize={{ maxRows: 100 }}
autoFocus
{['json', 'chunks'].includes(initialValue.key) && (
<ArrayContainer
isReadonly={isReadonly}
className={className}
initialValue={initialValue}
handleCheck={handleCheck}
selectedChunkIds={selectedChunkIds}
onSave={onSave}
escapeNewlines={escapeNewlines}
unescapeNewlines={unescapeNewlines}
textMode={textMode}
isChunck={isChunck}
clickChunk={clickChunk}
/>
)}
{['text', 'html'].includes(initialValue.key) && (
<ObjectContainer
isReadonly={isReadonly}
className={className}
initialValue={initialValue}
handleCheck={handleCheck}
selectedChunkIds={selectedChunkIds}
onSave={onSave}
escapeNewlines={escapeNewlines}
unescapeNewlines={unescapeNewlines}
textMode={textMode}
isChunck={isChunck}
clickChunk={clickChunk}
/>
) : (
<pre className="text-text-secondary" onClick={handleEdit}>
{content}
</pre>
)}
</div>
);

65
web/src/pages/dataflow-result/components/parse-editer/interface.ts Normal file
View File

@ -0,0 +1,65 @@
import { CheckedState } from '@radix-ui/react-checkbox';
import { ChunkTextMode } from '../../constant';
import { IChunk } from '../../interface';
import { parserKeyMap } from './json-parser';
export interface FormatPreserveEditorProps {
initialValue: {
key: keyof typeof parserKeyMap | 'text' | 'html';
type: string;
value: Array<{ [key: string]: string }>;
};
onSave: (value: any) => void;
className?: string;
isSelect?: boolean;
isDelete?: boolean;
isChunck?: boolean;
handleCheckboxClick?: (id: string | number, checked: boolean) => void;
selectedChunkIds?: string[];
textMode?: ChunkTextMode;
clickChunk: (chunk: IChunk) => void;
isReadonly: boolean;
}
export type IJsonContainerProps = {
initialValue: {
key: keyof typeof parserKeyMap;
type: string;
value: {
[key: string]: string;
}[];
};
isChunck?: boolean;
handleCheck: (e: CheckedState, index: number) => void;
selectedChunkIds: string[] | undefined;
unescapeNewlines: (text: string) => string;
escapeNewlines: (text: string) => string;
onSave: (data: {
value: {
text: string;
}[];
key: string;
type: string;
}) => void;
className?: string;
textMode?: ChunkTextMode;
clickChunk: (chunk: IChunk) => void;
isReadonly: boolean;
};
export type IObjContainerProps = {
initialValue: {
key: string;
type: string;
value: string;
};
isChunck?: boolean;
handleCheck: (e: CheckedState, index: number) => void;
unescapeNewlines: (text: string) => string;
escapeNewlines: (text: string) => string;
onSave: (data: { value: string; key: string; type: string }) => void;
className?: string;
textMode?: ChunkTextMode;
clickChunk: (chunk: IChunk) => void;
isReadonly: boolean;
};

139
web/src/pages/dataflow-result/components/parse-editer/json-parser.tsx Normal file
View File

@ -0,0 +1,139 @@
import { Checkbox } from '@/components/ui/checkbox';
import { cn } from '@/lib/utils';
import { useCallback, useEffect } from 'react';
import { ChunkTextMode } from '../../constant';
import styles from '../../index.less';
import { useParserInit } from './hook';
import { IJsonContainerProps } from './interface';
export const parserKeyMap = {
json: 'text',
chunks: 'text',
} as const;
export const ArrayContainer = (props: IJsonContainerProps) => {
const {
initialValue,
isChunck,
handleCheck,
selectedChunkIds,
unescapeNewlines,
escapeNewlines,
onSave,
className,
textMode,
clickChunk,
isReadonly,
} = props;
const {
content,
setContent,
activeEditIndex,
setActiveEditIndex,
editDivRef,
} = useParserInit({ initialValue });
const parserKey = parserKeyMap[content.key as keyof typeof parserKeyMap];
const handleEdit = useCallback(
(e?: any, index?: number) => {
setActiveEditIndex(index);
},
[setContent, setActiveEditIndex],
);
const handleSave = useCallback(
(e: any) => {
const saveData = {
...content,
value: content.value?.map((item, index) => {
if (index === activeEditIndex) {
return {
...item,
[parserKey]: e.target.textContent || '',
};
} else {
return item;
}
}),
};
onSave(saveData);
setActiveEditIndex(undefined);
},
[content, onSave],
);
useEffect(() => {
if (activeEditIndex !== undefined && editDivRef.current) {
editDivRef.current.focus();
editDivRef.current.textContent =
content.value[activeEditIndex][parserKey];
}
}, [editDivRef, activeEditIndex, content, parserKey]);
return (
<>
{content.value?.map((item, index) => {
if (
item[parserKeyMap[content.key as keyof typeof parserKeyMap]] === ''
) {
return null;
}
return (
<section
key={index}
className={cn(
isChunck
? 'bg-bg-card my-2 p-2 rounded-lg flex gap-1 items-start'
: '',
activeEditIndex === index && isChunck ? 'bg-bg-title' : '',
)}
>
{isChunck && !isReadonly && (
<Checkbox
onCheckedChange={(e) => {
handleCheck(e, index);
}}
checked={selectedChunkIds?.some(
(id) => id.toString() === index.toString(),
)}
></Checkbox>
)}
{activeEditIndex === index && (
<div
ref={editDivRef}
contentEditable={!isReadonly}
onBlur={handleSave}
className={cn(
'w-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none p-0',
className,
)}
></div>
)}
{activeEditIndex !== index && (
<div
className={cn(
'text-text-secondary overflow-auto scrollbar-auto w-full',
{
[styles.contentEllipsis]:
textMode === ChunkTextMode.Ellipse,
},
)}
key={index}
onClick={(e) => {
clickChunk(item);
if (!isReadonly) {
handleEdit(e, index);
}
}}
>
{item[parserKeyMap[content.key]]}
</div>
)}
</section>
);
})}
</>
);
};

96
web/src/pages/dataflow-result/components/parse-editer/object-parser.tsx Normal file
View File

@ -0,0 +1,96 @@
import { cn } from '@/lib/utils';
import { useCallback, useEffect } from 'react';
import { ChunkTextMode } from '../../constant';
import styles from '../../index.less';
import { useParserInit } from './hook';
import { IObjContainerProps } from './interface';
export const ObjectContainer = (props: IObjContainerProps) => {
const {
initialValue,
isChunck,
unescapeNewlines,
escapeNewlines,
onSave,
className,
textMode,
clickChunk,
isReadonly,
} = props;
const {
content,
setContent,
activeEditIndex,
setActiveEditIndex,
editDivRef,
} = useParserInit({ initialValue });
const handleEdit = useCallback(() => {
// setContent((pre) => ({
// ...pre,
// value: escapeNewlines(e.target.innerText),
// }));
setActiveEditIndex(1);
}, [setContent, setActiveEditIndex]);
const handleSave = useCallback(
(e: any) => {
const saveData = {
...content,
value: e.target.textContent,
};
onSave(saveData);
setActiveEditIndex(undefined);
},
[content, onSave],
);
useEffect(() => {
if (activeEditIndex !== undefined && editDivRef.current) {
editDivRef.current.focus();
editDivRef.current.textContent = content.value;
}
}, [activeEditIndex, content]);
return (
<>
<section
className={
isChunck
? 'bg-bg-card my-2 p-2 rounded-lg flex gap-1 items-start'
: ''
}
>
{activeEditIndex && (
<div
ref={editDivRef}
contentEditable={!isReadonly}
onBlur={handleSave}
className={cn(
'w-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none p-0',
className,
)}
/>
)}
{!activeEditIndex && (
<div
className={cn(
'text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap w-full',
{
[styles.contentEllipsis]: textMode === ChunkTextMode.Ellipse,
},
)}
onClick={(e) => {
clickChunk(content);
if (!isReadonly) {
handleEdit(e);
}
}}
>
{content.value}
</div>
)}
</section>
</>
);
};

32
web/src/pages/dataflow-result/components/rerun-button/index.tsx
View File

@ -1,16 +1,44 @@
import { TimelineNode } from '@/components/originui/timeline';
import SvgIcon from '@/components/svg-icon';
import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { CircleAlert } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { useRerunDataflow } from '../../hooks';
interface RerunButtonProps {
className?: string;
step?: TimelineNode;
onRerun?: () => void;
loading?: boolean;
}
const RerunButton = (props: RerunButtonProps) => {
const { className, step, onRerun, loading } = props;
const { t } = useTranslation();
const { loading } = useRerunDataflow();
const clickFunc = () => {
console.log('click rerun button');
Modal.show({
visible: true,
className: '!w-[560px]',
title: t('dataflowParser.confirmRerun'),
children: (
<div
dangerouslySetInnerHTML={{
__html: t('dataflowParser.confirmRerunModalContent', {
step: step?.title,
}),
}}
></div>
),
okText: t('modal.okText'),
cancelText: t('modal.cancelText'),
onVisibleChange: (visible: boolean) => {
if (!visible) {
Modal.destroy();
} else {
onRerun?.();
Modal.destroy();
}
},
});
};
return (
<div className="flex flex-col gap-2">

91
web/src/pages/dataflow-result/components/time-line/index.tsx
View File

@ -1,61 +1,74 @@
import { CustomTimeline, TimelineNode } from '@/components/originui/timeline';
import {
CheckLine,
FilePlayIcon,
Grid3x2,
Blocks,
File,
FilePlay,
FileStack,
Heading,
ListPlus,
PlayIcon,
} from 'lucide-react';
import { useMemo } from 'react';
import { TimelineNodeType } from '../../constant';
import { IPipelineFileLogDetail } from '../../interface';
export type ITimelineNodeObj = {
title: string;
icon: JSX.Element;
clickable?: boolean;
type: TimelineNodeType;
};
export const TimelineNodeObj = {
begin: {
id: 1,
title: 'Begin',
icon: <PlayIcon size={13} />,
[TimelineNodeType.begin]: {
title: 'File',
icon: <File size={13} />,
clickable: false,
},
parser: { id: 2, title: 'Parser', icon: <FilePlayIcon size={13} /> },
chunker: { id: 3, title: 'Chunker', icon: <Grid3x2 size={13} /> },
indexer: {
id: 4,
title: 'Indexer',
[TimelineNodeType.parser]: {
title: 'Parser',
icon: <FilePlay size={13} />,
},
[TimelineNodeType.contextGenerator]: {
title: 'Context Generator',
icon: <FileStack size={13} />,
},
[TimelineNodeType.titleSplitter]: {
title: 'Title Splitter',
icon: <Heading size={13} />,
},
[TimelineNodeType.characterSplitter]: {
title: 'Character Splitter',
icon: <Blocks size={13} />,
},
[TimelineNodeType.tokenizer]: {
title: 'Tokenizer',
icon: <ListPlus size={13} />,
clickable: false,
},
complete: {
id: 5,
title: 'Complete',
icon: <CheckLine size={13} />,
clickable: false,
},
};
export interface TimelineDataFlowProps {
activeId: number | string;
activeFunc: (id: number | string) => void;
activeFunc: (id: number | string, step: TimelineNode) => void;
data: IPipelineFileLogDetail;
timelineNodes: TimelineNode[];
}
const TimelineDataFlow = ({ activeFunc, activeId }: TimelineDataFlowProps) => {
// const [activeStep, setActiveStep] = useState(2);
const timelineNodes: TimelineNode[] = useMemo(() => {
const nodes: TimelineNode[] = [];
Object.keys(TimelineNodeObj).forEach((key) => {
nodes.push({
...TimelineNodeObj[key as keyof typeof TimelineNodeObj],
className: 'w-32',
completed: false,
});
});
return nodes;
}, []);
const TimelineDataFlow = ({
activeFunc,
activeId,
data,
timelineNodes,
}: TimelineDataFlowProps) => {
// const [timelineNodeArr,setTimelineNodeArr] = useState<ITimelineNodeObj & {id: number | string}>()
const activeStep = useMemo(() => {
const index = timelineNodes.findIndex((node) => node.id === activeId);
return index > -1 ? index + 1 : 0;
}, [activeId, timelineNodes]);
const handleStepChange = (step: number, id: string | number) => {
// setActiveStep(step);
activeFunc?.(id);
console.log(step, id);
activeFunc?.(
id,
timelineNodes.find((node) => node.id === activeStep) as TimelineNode,
);
};
return (
@ -70,8 +83,8 @@ const TimelineDataFlow = ({ activeFunc, activeId }: TimelineDataFlowProps) => {
nodeSize={24}
activeStyle={{
nodeSize: 30,
iconColor: 'var(--accent-primary)',
textColor: 'var(--accent-primary)',
iconColor: 'rgb(var(--accent-primary))',
textColor: 'rgb(var(--accent-primary))',
}}
/>
</div>

21
web/src/pages/dataflow-result/constant.ts
View File

@ -2,3 +2,24 @@ export enum ChunkTextMode {
Full = 'full',
Ellipse = 'ellipse',
}
export enum TimelineNodeType {
begin = 'file',
parser = 'parser',
contextGenerator = 'extractor',
titleSplitter = 'hierarchicalMerger',
characterSplitter = 'splitter',
tokenizer = 'tokenizer',
end = 'end',
}
export enum PipelineResultSearchParams {
DocumentId = 'doc_id',
KnowledgeId = 'knowledgeId',
Type = 'type',
IsReadOnly = 'is_read_only',
AgentId = 'agent_id',
AgentTitle = 'agent_title',
CreatedBy = 'created_by', // Who uploaded the file
DocumentExtension = 'extension',
}

284
web/src/pages/dataflow-result/hooks.ts
View File

@ -1,42 +1,81 @@
import { TimelineNode } from '@/components/originui/timeline';
import message from '@/components/ui/message';
import {
useCreateChunk,
useDeleteChunk,
useSelectChunkList,
} from '@/hooks/chunk-hooks';
import { useCreateChunk, useDeleteChunk } from '@/hooks/chunk-hooks';
import { useSetModalState, useShowDeleteConfirm } from '@/hooks/common-hooks';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { useFetchMessageTrace } from '@/hooks/use-agent-request';
import { IChunk } from '@/interfaces/database/knowledge';
import kbService from '@/services/knowledge-service';
import { formatSecondsToHumanReadable } from '@/utils/date';
import { buildChunkHighlights } from '@/utils/document-util';
import { useMutation, useQueryClient } from '@tanstack/react-query';
import { useCallback, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useMutation, useQuery } from '@tanstack/react-query';
import { t } from 'i18next';
import { camelCase, upperFirst } from 'lodash';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { IHighlight } from 'react-pdf-highlighter';
import { ChunkTextMode } from './constant';
import { useParams, useSearchParams } from 'umi';
import { ITimelineNodeObj, TimelineNodeObj } from './components/time-line';
import {
ChunkTextMode,
PipelineResultSearchParams,
TimelineNodeType,
} from './constant';
import { IDslComponent, IPipelineFileLogDetail } from './interface';
export const useFetchPipelineFileLogDetail = ({
isAgent = false,
isEdit = true,
refreshCount,
}: {
isEdit?: boolean;
refreshCount?: number;
isAgent: boolean;
}) => {
const { id } = useParams();
const [searchParams] = useSearchParams();
const logId = searchParams.get('id') || id;
let queryKey: (string | number)[] = [];
if (typeof refreshCount === 'number') {
queryKey = ['fetchLogDetail', refreshCount];
}
const { data, isFetching: loading } = useQuery<IPipelineFileLogDetail>({
queryKey,
initialData: {} as IPipelineFileLogDetail,
gcTime: 0,
enabled: !isAgent,
queryFn: async () => {
if (isEdit) {
const { data } = await kbService.get_pipeline_detail({
log_id: logId,
});
return data?.data ?? {};
} else {
return {};
}
},
});
return { data, loading };
};
export const useHandleChunkCardClick = () => {
const [selectedChunkId, setSelectedChunkId] = useState<string>('');
const [selectedChunk, setSelectedChunk] = useState<IChunk>();
const handleChunkCardClick = useCallback((chunkId: string) => {
setSelectedChunkId(chunkId);
const handleChunkCardClick = useCallback((chunk: IChunk) => {
console.log('click-chunk-->', chunk);
setSelectedChunk(chunk);
}, []);
return { handleChunkCardClick, selectedChunkId };
return { handleChunkCardClick, selectedChunk };
};
export const useGetSelectedChunk = (selectedChunkId: string) => {
const data = useSelectChunkList();
return (
data?.data?.find((x) => x.chunk_id === selectedChunkId) ?? ({} as IChunk)
);
};
export const useGetChunkHighlights = (selectedChunkId: string) => {
export const useGetChunkHighlights = (selectedChunk?: IChunk) => {
const [size, setSize] = useState({ width: 849, height: 1200 });
const selectedChunk: IChunk = useGetSelectedChunk(selectedChunkId);
const highlights: IHighlight[] = useMemo(() => {
return buildChunkHighlights(selectedChunk, size);
return selectedChunk ? buildChunkHighlights(selectedChunk, size) : [];
}, [selectedChunk, size]);
const setWidthAndHeight = useCallback((width: number, height: number) => {
@ -131,55 +170,162 @@ export const useUpdateChunk = () => {
};
};
export const useFetchParserList = () => {
const [loading, setLoading] = useState(false);
return {
loading,
};
};
export const useRerunDataflow = ({
data,
}: {
data: IPipelineFileLogDetail;
}) => {
const [isChange, setIsChange] = useState(false);
export const useRerunDataflow = () => {
const [loading, setLoading] = useState(false);
return {
loading,
};
};
const { mutateAsync: handleReRunFunc, isPending: loading } = useMutation({
mutationKey: ['pipelineRerun', data],
mutationFn: async (newData: { value: IDslComponent; key: string }) => {
const newDsl = {
...data.dsl,
components: {
...data.dsl.components,
[newData.key]: newData.value,
},
};
export const useFetchPaserText = () => {
const initialText =
'第一行文本\n\t第二行缩进文本\n第三行 多个空格 第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 ' +
'多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格第一行文本\n\t第二行缩进文本\n第三行 多个空格';
const [loading, setLoading] = useState(false);
const [data, setData] = useState<string>(initialText);
const { t } = useTranslation();
const queryClient = useQueryClient();
const {
// data,
// isPending: loading,
mutateAsync,
} = useMutation({
mutationKey: ['createChunk'],
mutationFn: async (payload: any) => {
// let service = kbService.create_chunk;
// if (payload.chunk_id) {
// service = kbService.set_chunk;
// }
// const { data } = await service(payload);
// if (data.code === 0) {
message.success(t('message.created'));
setTimeout(() => {
queryClient.invalidateQueries({ queryKey: ['fetchChunkList'] });
}, 1000); // Delay to ensure the list is updated
// }
// return data?.code;
// this Data provided to the interface
const params = {
id: data.id,
dsl: newDsl,
component_id: newData.key,
};
const { data: result } = await kbService.pipelineRerun(params);
if (result.code === 0) {
message.success(t('message.operated'));
// queryClient.invalidateQueries({
// queryKey: [type],
// });
}
return result;
},
});
return { data, loading, rerun: mutateAsync };
return {
loading,
isChange,
setIsChange,
handleReRunFunc,
};
};
export const useTimelineDataFlow = (data: IPipelineFileLogDetail) => {
const timelineNodes: TimelineNode[] = useMemo(() => {
const nodes: Array<ITimelineNodeObj & { id: number | string }> = [];
console.log('time-->', data);
const times = data?.dsl?.components;
if (times) {
const getNode = (
key: string,
index: number,
type:
| TimelineNodeType.begin
| TimelineNodeType.parser
| TimelineNodeType.tokenizer
| TimelineNodeType.characterSplitter
| TimelineNodeType.titleSplitter,
) => {
const node = times[key].obj;
const name = camelCase(
node.component_name,
) as keyof typeof TimelineNodeObj;
let tempType = type;
if (name === TimelineNodeType.parser) {
tempType = TimelineNodeType.parser;
} else if (name === TimelineNodeType.tokenizer) {
tempType = TimelineNodeType.tokenizer;
} else if (
name === TimelineNodeType.characterSplitter ||
name === TimelineNodeType.titleSplitter
) {
tempType = TimelineNodeType.characterSplitter;
}
const timeNode = {
...TimelineNodeObj[name],
id: index,
className: 'w-32',
completed: false,
date: formatSecondsToHumanReadable(
node.params?.outputs?._elapsed_time?.value || 0,
),
type: tempType,
detail: { value: times[key], key: key },
};
console.log('timeNodetype-->', type);
nodes.push(timeNode);
if (times[key].downstream && times[key].downstream.length > 0) {
const nextKey = times[key].downstream[0];
// nodes.push(timeNode);
getNode(nextKey, index + 1, tempType);
}
};
getNode(upperFirst(TimelineNodeType.begin), 1, TimelineNodeType.begin);
// setTimelineNodeArr(nodes as unknown as ITimelineNodeObj & {id: number | string})
}
return nodes;
}, [data]);
return {
timelineNodes,
};
};
export const useGetPipelineResultSearchParams = () => {
const [currentQueryParameters] = useSearchParams();
const is_read_only = currentQueryParameters.get(
PipelineResultSearchParams.IsReadOnly,
) as 'true' | 'false';
console.log('is_read_only', is_read_only);
return {
type: currentQueryParameters.get(PipelineResultSearchParams.Type) || '',
documentId:
currentQueryParameters.get(PipelineResultSearchParams.DocumentId) || '',
knowledgeId:
currentQueryParameters.get(PipelineResultSearchParams.KnowledgeId) || '',
isReadOnly: is_read_only === 'true',
agentId:
currentQueryParameters.get(PipelineResultSearchParams.AgentId) || '',
agentTitle:
currentQueryParameters.get(PipelineResultSearchParams.AgentTitle) || '',
documentExtension:
currentQueryParameters.get(
PipelineResultSearchParams.DocumentExtension,
) || '',
createdBy:
currentQueryParameters.get(PipelineResultSearchParams.CreatedBy) || '',
};
};
export function useFetchPipelineResult({
agentId,
}: Pick<ReturnType<typeof useGetPipelineResultSearchParams>, 'agentId'>) {
const [searchParams] = useSearchParams();
const messageId = searchParams.get('id');
const { data, setMessageId, setISStopFetchTrace } =
useFetchMessageTrace(agentId);
useEffect(() => {
if (messageId) {
setMessageId(messageId);
setISStopFetchTrace(true);
}
}, [agentId, messageId, setISStopFetchTrace, setMessageId]);
const pipelineResult = useMemo(() => {
if (Array.isArray(data)) {
const latest = data?.at(-1);
if (latest?.component_id === 'END' && Array.isArray(latest.trace)) {
return latest.trace.at(0);
}
}
}, [data]);
return { pipelineResult };
}

13
web/src/pages/dataflow-result/index.less
View File

@ -82,15 +82,6 @@
}
}
.card {
:global {
.ant-card-body {
padding: 10px;
margin: 0;
}
margin-bottom: 10px;
}
cursor: pointer;
.contentEllipsis {
.multipleLineEllipsis(3);
}

205
web/src/pages/dataflow-result/index.tsx
View File

@ -2,11 +2,21 @@ import { useFetchNextChunkList } from '@/hooks/use-chunk-request';
import { useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import DocumentPreview from './components/document-preview';
import { useGetChunkHighlights, useHandleChunkCardClick } from './hooks';
import {
useFetchPipelineFileLogDetail,
useFetchPipelineResult,
useGetChunkHighlights,
useGetPipelineResultSearchParams,
useHandleChunkCardClick,
useRerunDataflow,
useTimelineDataFlow,
} from './hooks';
import DocumentHeader from './components/document-preview/document-header';
import { TimelineNode } from '@/components/originui/timeline';
import { PageHeader } from '@/components/page-header';
import Spotlight from '@/components/spotlight';
import {
Breadcrumb,
BreadcrumbItem,
@ -15,35 +25,58 @@ import {
BreadcrumbPage,
BreadcrumbSeparator,
} from '@/components/ui/breadcrumb';
import {
QueryStringMap,
useNavigatePage,
} from '@/hooks/logic-hooks/navigate-hooks';
import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request';
import { ChunkerContainer } from './chunker';
import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { Images } from '@/constants/common';
import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
import { useGetDocumentUrl } from './components/document-preview/hooks';
import TimelineDataFlow, { TimelineNodeObj } from './components/time-line';
import TimelineDataFlow from './components/time-line';
import { TimelineNodeType } from './constant';
import styles from './index.less';
import { IDslComponent, IPipelineFileLogDetail } from './interface';
import ParserContainer from './parser';
const Chunk = () => {
const { isReadOnly, knowledgeId, agentId, agentTitle, documentExtension } =
useGetPipelineResultSearchParams();
const isAgent = !!agentId;
const { pipelineResult } = useFetchPipelineResult({ agentId });
const {
data: { documentInfo },
} = useFetchNextChunkList();
const { selectedChunkId } = useHandleChunkCardClick();
const [activeStepId, setActiveStepId] = useState<number | string>(0);
const { data: dataset } = useFetchKnowledgeBaseConfiguration();
} = useFetchNextChunkList(!isAgent);
const { selectedChunk, handleChunkCardClick } = useHandleChunkCardClick();
const [activeStepId, setActiveStepId] = useState<number | string>(2);
const { data: dataset } = useFetchPipelineFileLogDetail({
isAgent,
});
const { t } = useTranslation();
const { navigateToDataset, getQueryString, navigateToDatasetList } =
useNavigatePage();
const fileUrl = useGetDocumentUrl();
const { timelineNodes } = useTimelineDataFlow(
agentId ? (pipelineResult as IPipelineFileLogDetail) : dataset,
);
const {
navigateToDataset,
navigateToDatasetList,
navigateToAgents,
navigateToDataflow,
} = useNavigatePage();
let fileUrl = useGetDocumentUrl(isAgent);
const { highlights, setWidthAndHeight } =
useGetChunkHighlights(selectedChunkId);
useGetChunkHighlights(selectedChunk);
const fileType = useMemo(() => {
if (isAgent) {
return Images.some((x) => x === documentExtension)
? 'visual'
: documentExtension;
}
switch (documentInfo?.type) {
case 'doc':
return documentInfo?.name.split('.').pop() || 'doc';
@ -55,29 +88,101 @@ const Chunk = () => {
return documentInfo?.type;
}
return 'unknown';
}, [documentInfo]);
}, [documentExtension, documentInfo?.name, documentInfo?.type, isAgent]);
const handleStepChange = (id: number | string) => {
setActiveStepId(id);
const {
handleReRunFunc,
isChange,
setIsChange,
loading: reRunLoading,
} = useRerunDataflow({
data: dataset,
});
const handleStepChange = (id: number | string, step: TimelineNode) => {
if (isChange) {
Modal.show({
visible: true,
className: '!w-[560px]',
title: t('dataflowParser.changeStepModalTitle'),
children: (
<div
className="text-sm text-text-secondary"
dangerouslySetInnerHTML={{
__html: t('dataflowParser.changeStepModalContent', {
step: step?.title,
}),
}}
></div>
),
onVisibleChange: () => {
Modal.destroy();
},
footer: (
<div className="flex justify-end gap-2">
<Button variant={'outline'} onClick={() => Modal.destroy()}>
{t('dataflowParser.changeStepModalCancelText')}
</Button>
<Button
variant={'secondary'}
className="!bg-state-error text-text-primary"
onClick={() => {
Modal.destroy();
setActiveStepId(id);
setIsChange(false);
}}
>
{t('dataflowParser.changeStepModalConfirmText')}
</Button>
</div>
),
});
} else {
setActiveStepId(id);
}
};
const { type } = useGetKnowledgeSearchParams();
const currentTimeNode: TimelineNode = useMemo(() => {
return (
timelineNodes.find((node) => node.id === activeStepId) ||
({} as TimelineNode)
);
}, [activeStepId, timelineNodes]);
return (
<>
<PageHeader>
<Breadcrumb>
<BreadcrumbList>
<BreadcrumbItem>
<BreadcrumbLink onClick={navigateToDatasetList}>
{t('knowledgeDetails.dataset')}
<BreadcrumbLink
onClick={() => {
if (knowledgeId) {
navigateToDatasetList();
}
if (agentId) {
navigateToAgents();
}
}}
>
{knowledgeId ? t('knowledgeDetails.dataset') : t('header.flow')}
</BreadcrumbLink>
</BreadcrumbItem>
<BreadcrumbSeparator />
<BreadcrumbItem>
<BreadcrumbLink
onClick={navigateToDataset(
getQueryString(QueryStringMap.id) as string,
)}
onClick={() => {
if (knowledgeId) {
navigateToDataset(knowledgeId)();
}
if (agentId) {
navigateToDataflow(agentId)();
}
}}
>
{dataset.name}
{knowledgeId ? t('knowledgeDetails.overview') : agentTitle}
</BreadcrumbLink>
</BreadcrumbItem>
<BreadcrumbSeparator />
@ -87,12 +192,16 @@ const Chunk = () => {
</BreadcrumbList>
</Breadcrumb>
</PageHeader>
<div className=" absolute ml-[50%] translate-x-[-50%] top-4 flex justify-center">
<TimelineDataFlow
activeFunc={handleStepChange}
activeId={activeStepId}
/>
</div>
{type === 'dataflow' && (
<div className=" absolute ml-[50%] translate-x-[-50%] top-4 flex justify-center">
<TimelineDataFlow
activeFunc={handleStepChange}
activeId={activeStepId}
data={dataset}
timelineNodes={timelineNodes}
/>
</div>
)}
<div className={styles.chunkPage}>
<div className="flex flex-none gap-8 border border-border mt-[26px] p-3 rounded-lg h-[calc(100vh-100px)]">
<div className="w-2/5">
@ -110,8 +219,38 @@ const Chunk = () => {
</section>
</div>
<div className="h-dvh border-r -mt-3"></div>
{activeStepId === TimelineNodeObj.chunker.id && <ChunkerContainer />}
{activeStepId === TimelineNodeObj.parser.id && <ParserContainer />}
<div className="w-3/5 h-full">
{/* {currentTimeNode?.type === TimelineNodeType.splitter && (
<ChunkerContainer
isChange={isChange}
setIsChange={setIsChange}
step={currentTimeNode as TimelineNode}
/>
)} */}
{/* {currentTimeNode?.type === TimelineNodeType.parser && ( */}
{(currentTimeNode?.type === TimelineNodeType.parser ||
currentTimeNode?.type === TimelineNodeType.characterSplitter ||
currentTimeNode?.type === TimelineNodeType.titleSplitter ||
currentTimeNode?.type === TimelineNodeType.contextGenerator) && (
<ParserContainer
isReadonly={isReadOnly}
isChange={isChange}
reRunLoading={reRunLoading}
setIsChange={setIsChange}
step={currentTimeNode as TimelineNode}
data={
currentTimeNode.detail as {
value: IDslComponent;
key: string;
}
}
clickChunk={handleChunkCardClick}
reRunFunc={handleReRunFunc}
/>
)}
{/* )} */}
<Spotlight opcity={0.6} coverage={60} />
</div>
</div>
</div>
</>

82
web/src/pages/dataflow-result/interface.ts Normal file
View File

@ -0,0 +1,82 @@
import { PipelineResultSearchParams } from './constant';
interface ComponentParams {
debug_inputs: Record<string, any>;
delay_after_error: number;
description: string;
exception_default_value: any;
exception_goto: any;
exception_method: any;
inputs: Record<string, any>;
max_retries: number;
message_history_window_size: number;
outputs: {
_created_time: Record<string, any>;
_elapsed_time: Record<string, any>;
name: Record<string, any>;
output_format: { type: string; value: string };
json: { type: string; value: string };
};
persist_logs: boolean;
timeout: number;
}
interface ComponentObject {
component_name: string;
params: ComponentParams;
}
export interface IDslComponent {
downstream: Array<string>;
obj: ComponentObject;
upstream: Array<string>;
}
export interface IPipelineFileLogDetail {
avatar: string;
create_date: string;
create_time: number;
document_id: string;
document_name: string;
document_suffix: string;
document_type: string;
dsl: {
components: {
[key: string]: IDslComponent;
};
task_id: string;
path: Array<string>;
};
id: string;
kb_id: string;
operation_status: string;
parser_id: string;
pipeline_id: string;
pipeline_title: string;
process_begin_at: string;
process_duration: number;
progress: number;
progress_msg: string;
source_from: string;
status: string;
task_type: string;
tenant_id: string;
update_date: string;
update_time: number;
}
export interface IChunk {
positions: number[][];
image_id: string;
text: string;
}
export interface NavigateToDataflowResultProps {
id: string;
[PipelineResultSearchParams.KnowledgeId]?: string;
[PipelineResultSearchParams.DocumentId]: string;
[PipelineResultSearchParams.AgentId]?: string;
[PipelineResultSearchParams.AgentTitle]?: string;
[PipelineResultSearchParams.IsReadOnly]?: string;
[PipelineResultSearchParams.Type]: string;
[PipelineResultSearchParams.CreatedBy]: string;
[PipelineResultSearchParams.DocumentExtension]: string;
}

211
web/src/pages/dataflow-result/parser.tsx
View File

@ -1,38 +1,156 @@
import { TimelineNode } from '@/components/originui/timeline';
import Spotlight from '@/components/spotlight';
import { Spin } from '@/components/ui/spin';
import { cn } from '@/lib/utils';
import classNames from 'classnames';
import { useState } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import FormatPreserveEditor from './components/parse-editer';
import ChunkResultBar from './components/chunk-result-bar';
import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
import FormatPreserEditor from './components/parse-editer';
import RerunButton from './components/rerun-button';
import { useFetchParserList, useFetchPaserText } from './hooks';
const ParserContainer = () => {
const { data: initialValue, rerun: onSave } = useFetchPaserText();
import { TimelineNodeType } from './constant';
import { useChangeChunkTextMode } from './hooks';
import { IChunk, IDslComponent } from './interface';
interface IProps {
isReadonly: boolean;
isChange: boolean;
setIsChange: (isChange: boolean) => void;
step?: TimelineNode;
data: { value: IDslComponent; key: string };
reRunLoading: boolean;
clickChunk: (chunk: IChunk) => void;
reRunFunc: (data: { value: IDslComponent; key: string }) => void;
}
const ParserContainer = (props: IProps) => {
const {
isChange,
setIsChange,
step,
data,
reRunFunc,
reRunLoading,
clickChunk,
isReadonly,
} = props;
const { t } = useTranslation();
const { loading } = useFetchParserList();
const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
const { changeChunkTextMode, textMode } = useChangeChunkTextMode();
const initialValue = useMemo(() => {
const outputs = data?.value?.obj?.params?.outputs;
const key = outputs?.output_format?.value;
if (!outputs || !key) return { key: '', type: '', value: [] };
const value = outputs[key]?.value;
const type = outputs[key]?.type;
console.log('outputs-->', outputs, data, key, value);
return {
key,
type,
value,
};
}, [data]);
const [initialText, setInitialText] = useState(initialValue);
const [isChange, setIsChange] = useState(false);
const handleSave = (newContent: string) => {
console.log('保存内容:', newContent);
if (newContent !== initialText) {
useEffect(() => {
setInitialText(initialValue);
}, [initialValue]);
const handleSave = (newContent: any) => {
console.log('newContent-change-->', newContent, initialValue);
if (JSON.stringify(newContent) !== JSON.stringify(initialValue)) {
setIsChange(true);
onSave(newContent);
setInitialText(newContent);
} else {
setIsChange(false);
}
// Here, the API is called to send newContent to the backend
};
const handleReRunFunc = useCallback(() => {
const newData: { value: IDslComponent; key: string } = {
...data,
value: {
...data.value,
obj: {
...data.value.obj,
params: {
...(data.value?.obj?.params || {}),
outputs: {
...(data.value?.obj?.params?.outputs || {}),
[initialText.key]: {
type: initialText.type,
value: initialText.value,
},
},
},
},
},
};
reRunFunc(newData);
setIsChange(false);
}, [data, initialText, reRunFunc, setIsChange]);
const handleRemoveChunk = useCallback(async () => {
if (selectedChunkIds.length > 0) {
initialText.value = initialText.value.filter(
(item: any, index: number) => !selectedChunkIds.includes(index + ''),
);
setIsChange(true);
setSelectedChunkIds([]);
}
}, [selectedChunkIds, initialText, setIsChange]);
const handleCheckboxClick = useCallback(
(id: string | number, checked: boolean) => {
setSelectedChunkIds((prev) => {
if (checked) {
return [...prev, id.toString()];
} else {
return prev.filter((item) => item.toString() !== id.toString());
}
});
},
[],
);
const selectAllChunk = useCallback(
(checked: boolean) => {
setSelectedChunkIds(
checked ? initialText.value.map((x, index: number) => index) : [],
);
},
[initialText.value],
);
const isChunck =
step?.type === TimelineNodeType.characterSplitter ||
step?.type === TimelineNodeType.titleSplitter;
const handleCreateChunk = useCallback(
(text: string) => {
const newText = [...initialText.value, { text: text || ' ' }];
setInitialText({
...initialText,
value: newText,
});
},
[initialText],
);
return (
<>
{isChange && (
{isChange && !isReadonly && (
<div className=" absolute top-2 right-6">
<RerunButton />
<RerunButton
step={step}
onRerun={handleReRunFunc}
loading={reRunLoading}
/>
</div>
)}
<div className={classNames('flex flex-col w-3/5')}>
<Spin spinning={loading} className="" size="large">
<div className="h-[50px] flex flex-col justify-end pb-[5px]">
<div className={classNames('flex flex-col w-full')}>
{/* <Spin spinning={false} className="" size="large"> */}
<div className="h-[50px] flex flex-col justify-end pb-[5px]">
{!isChunck && (
<div>
<h2 className="text-[16px]">
{t('dataflowParser.parseSummary')}
@ -41,16 +159,63 @@ const ParserContainer = () => {
{t('dataflowParser.parseSummaryTip')}
</div>
</div>
)}
{isChunck && (
<div>
<h2 className="text-[16px]">{t('chunk.chunkResult')}</h2>
<div className="text-[12px] text-text-secondary italic">
{t('chunk.chunkResultTip')}
</div>
</div>
)}
</div>
{isChunck && (
<div className="pt-[5px] pb-[5px] flex justify-between items-center">
{!isReadonly && (
<CheckboxSets
selectAllChunk={selectAllChunk}
removeChunk={handleRemoveChunk}
checked={selectedChunkIds.length === initialText.value.length}
selectedChunkIds={selectedChunkIds}
/>
)}
<ChunkResultBar
isReadonly={isReadonly}
changeChunkTextMode={changeChunkTextMode}
createChunk={handleCreateChunk}
/>
</div>
<div className=" border rounded-lg p-[20px] box-border h-[calc(100vh-180px)] overflow-auto scrollbar-none">
<FormatPreserveEditor
)}
<div
className={cn(
' border rounded-lg p-[20px] box-border w-[calc(100%-20px)] overflow-auto scrollbar-none',
{
'h-[calc(100vh-240px)]': isChunck,
'h-[calc(100vh-180px)]': !isChunck,
},
)}
>
{initialText && (
<FormatPreserEditor
initialValue={initialText}
onSave={handleSave}
className="!h-[calc(100vh-220px)]"
isReadonly={isReadonly}
isChunck={isChunck}
textMode={textMode}
isDelete={
step?.type === TimelineNodeType.characterSplitter ||
step?.type === TimelineNodeType.titleSplitter
}
clickChunk={clickChunk}
handleCheckboxClick={handleCheckboxClick}
selectedChunkIds={selectedChunkIds}
/>
<Spotlight opcity={0.6} coverage={60} />
</div>
</Spin>
)}
<Spotlight opcity={0.6} coverage={60} />
</div>
{/* </Spin> */}
</div>
</>
);