pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-17 11:09:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

Browse Source 
### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>
This commit is contained in:
Kevin Hu
2025-10-09 12:36:19 +08:00
committed by GitHub
parent ef0aecea3b
commit cbf04ee470
490 changed files with 10630 additions and 30688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
web/src/pages/data-flow/hooks/use-add-node.ts
View File

@ -1,7 +1,6 @@
import { useFetchModelId } from '@/hooks/logic-hooks';
import { Connection, Node, Position, ReactFlowInstance } from '@xyflow/react';
import humanId from 'human-id';
import { t } from 'i18next';
import { lowerFirst } from 'lodash';
import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
@ -9,30 +8,13 @@ import {
NodeHandleId,
NodeMap,
Operator,
initialAgentValues,
initialBeginValues,
initialCategorizeValues,
initialChunkerValues,
initialCodeValues,
initialConcentratorValues,
initialCrawlerValues,
initialEmailValues,
initialExeSqlValues,
initialInvokeValues,
initialIterationStartValues,
initialIterationValues,
initialKeywordExtractValues,
initialMessageValues,
initialExtractorValues,
initialHierarchicalMergerValues,
initialNoteValues,
initialParserValues,
initialRelevantValues,
initialRetrievalValues,
initialRewriteQuestionValues,
initialStringTransformValues,
initialSwitchValues,
initialSplitterValues,
initialTokenizerValues,
initialUserFillUpValues,
initialWaitingDialogueValues,
} from '../constant';
import useGraphStore from '../store';
import {
@ -40,61 +22,30 @@ import {
getNodeDragHandle,
} from '../utils';
function isBottomSubAgent(type: string, position: Position) {
return (
(type === Operator.Agent && position === Position.Bottom) ||
type === Operator.Tool
);
}
export const useInitializeOperatorParams = () => {
const llmId = useFetchModelId();
const { t } = useTranslation();
const initialFormValuesMap = useMemo(() => {
return {
[Operator.Begin]: initialBeginValues,
[Operator.Retrieval]: initialRetrievalValues,
[Operator.Categorize]: { ...initialCategorizeValues, llm_id: llmId },
[Operator.Relevant]: { ...initialRelevantValues, llm_id: llmId },
[Operator.RewriteQuestion]: {
...initialRewriteQuestionValues,
llm_id: llmId,
},
[Operator.Message]: initialMessageValues,
[Operator.KeywordExtract]: {
...initialKeywordExtractValues,
llm_id: llmId,
},
[Operator.ExeSQL]: initialExeSqlValues,
[Operator.Switch]: initialSwitchValues,
[Operator.Concentrator]: initialConcentratorValues,
[Operator.Note]: initialNoteValues,
[Operator.Crawler]: initialCrawlerValues,
[Operator.Invoke]: initialInvokeValues,
[Operator.Email]: initialEmailValues,
[Operator.Iteration]: initialIterationValues,
[Operator.IterationStart]: initialIterationStartValues,
[Operator.Code]: initialCodeValues,
[Operator.WaitingDialogue]: initialWaitingDialogueValues,
[Operator.Agent]: { ...initialAgentValues, llm_id: llmId },
[Operator.Tool]: {},
[Operator.UserFillUp]: initialUserFillUpValues,
[Operator.StringTransform]: initialStringTransformValues,
[Operator.Parser]: initialParserValues,
[Operator.Chunker]: initialChunkerValues,
[Operator.Tokenizer]: initialTokenizerValues,
[Operator.Splitter]: initialSplitterValues,
[Operator.HierarchicalMerger]: initialHierarchicalMergerValues,
[Operator.Extractor]: {
...initialExtractorValues,
llm_id: llmId,
sys_prompt: t('dataflow.prompts.system.summary'),
prompts: t('dataflow.prompts.user.summary'),
},
};
}, [llmId]);
}, [llmId, t]);
const initializeOperatorParams = useCallback(
(operatorName: Operator, position: Position) => {
(operatorName: Operator) => {
const initialValues = initialFormValuesMap[operatorName];
if (isBottomSubAgent(operatorName, position)) {
return {
...initialValues,
description: t('flow.descriptionMessage'),
user_prompt: t('flow.userPromptDefaultValue'),
};
}
return initialValues;
},
@ -171,95 +122,17 @@ function useAddChildEdge() {
return { addChildEdge };
}
function useAddToolNode() {
const { nodes, edges, addEdge, getNode, addNode } = useGraphStore(
(state) => state,
);
const addToolNode = useCallback(
(newNode: Node<any>, nodeId?: string): boolean => {
const agentNode = getNode(nodeId);
if (agentNode) {
const childToolNodeIds = edges
.filter(
(x) => x.source === nodeId && x.sourceHandle === NodeHandleId.Tool,
)
.map((x) => x.target);
if (
childToolNodeIds.length > 0 &&
nodes.some((x) => x.id === childToolNodeIds[0])
) {
return false;
}
newNode.position = {
x: agentNode.position.x - 82,
y: agentNode.position.y + 140,
};
addNode(newNode);
if (nodeId) {
addEdge({
source: nodeId,
target: newNode.id,
sourceHandle: NodeHandleId.Tool,
targetHandle: NodeHandleId.End,
});
}
return true;
}
return false;
},
[addEdge, addNode, edges, getNode, nodes],
);
return { addToolNode };
}
function useResizeIterationNode() {
const { getNode, nodes, updateNode } = useGraphStore((state) => state);
const resizeIterationNode = useCallback(
(type: string, position: Position, parentId?: string) => {
const parentNode = getNode(parentId);
if (parentNode && !isBottomSubAgent(type, position)) {
const MoveRightDistance = 310;
const childNodeList = nodes.filter((x) => x.parentId === parentId);
const maxX = Math.max(...childNodeList.map((x) => x.position.x));
if (maxX + MoveRightDistance > parentNode.position.x) {
updateNode({
...parentNode,
width: (parentNode.width || 0) + MoveRightDistance,
position: {
x: parentNode.position.x + MoveRightDistance / 2,
y: parentNode.position.y,
},
});
}
}
},
[getNode, nodes, updateNode],
);
return { resizeIterationNode };
}
type CanvasMouseEvent = Pick<
React.MouseEvent<HTMLElement>,
'clientX' | 'clientY'
>;
export function useAddNode(reactFlowInstance?: ReactFlowInstance<any, any>) {
const { edges, nodes, addEdge, addNode, getNode } = useGraphStore(
(state) => state,
);
const { nodes, addNode } = useGraphStore((state) => state);
const getNodeName = useGetNodeName();
const { initializeOperatorParams } = useInitializeOperatorParams();
const { calculateNewlyBackChildPosition } = useCalculateNewlyChildPosition();
const { addChildEdge } = useAddChildEdge();
const { addToolNode } = useAddToolNode();
const { resizeIterationNode } = useResizeIterationNode();
// const [reactFlowInstance, setReactFlowInstance] =
// useState<ReactFlowInstance<any, any>>();
@ -277,7 +150,6 @@ export function useAddNode(reactFlowInstance?: ReactFlowInstance<any, any>) {
) =>
(event?: CanvasMouseEvent): string | undefined => {
const nodeId = params.nodeId;
const node = getNode(nodeId);
// reactFlowInstance.project was renamed to reactFlowInstance.screenToFlowPosition
// and you don't need to subtract the reactFlowBounds.left/top anymore
@ -308,112 +180,30 @@ export function useAddNode(reactFlowInstance?: ReactFlowInstance<any, any>) {
getNodeName(type),
nodes,
),
form: initializeOperatorParams(type as Operator, params.position),
form: initializeOperatorParams(type as Operator),
},
sourcePosition: Position.Right,
targetPosition: Position.Left,
dragHandle: getNodeDragHandle(type),
};
if (node && node.parentId) {
newNode.parentId = node.parentId;
newNode.extent = 'parent';
const parentNode = getNode(node.parentId);
if (parentNode && !isBottomSubAgent(type, params.position)) {
resizeIterationNode(type, params.position, node.parentId);
}
}
if (type === Operator.Iteration) {
newNode.width = 500;
newNode.height = 250;
const iterationStartNode: Node<any> = {
id: `${Operator.IterationStart}:${humanId()}`,
type: 'iterationStartNode',
position: { x: 50, y: 100 },
// draggable: false,
data: {
label: Operator.IterationStart,
name: Operator.IterationStart,
form: initialIterationStartValues,
},
parentId: newNode.id,
extent: 'parent',
};
addNode(newNode);
addNode(iterationStartNode);
if (nodeId) {
addEdge({
source: nodeId,
target: newNode.id,
sourceHandle: NodeHandleId.Start,
targetHandle: NodeHandleId.End,
});
}
return newNode.id;
} else if (
type === Operator.Agent &&
params.position === Position.Bottom
) {
const agentNode = getNode(nodeId);
if (agentNode) {
// Calculate the coordinates of child nodes to prevent newly added child nodes from covering other child nodes
const allChildAgentNodeIds = edges
.filter(
(x) =>
x.source === nodeId &&
x.sourceHandle === NodeHandleId.AgentBottom,
)
.map((x) => x.target);
const xAxises = nodes
.filter((x) => allChildAgentNodeIds.some((y) => y === x.id))
.map((x) => x.position.x);
const maxX = Math.max(...xAxises);
newNode.position = {
x: xAxises.length > 0 ? maxX + 262 : agentNode.position.x + 82,
y: agentNode.position.y + 140,
};
}
addNode(newNode);
if (nodeId) {
addEdge({
source: nodeId,
target: newNode.id,
sourceHandle: NodeHandleId.AgentBottom,
targetHandle: NodeHandleId.AgentTop,
});
}
return newNode.id;
} else if (type === Operator.Tool) {
const toolNodeAdded = addToolNode(newNode, params.nodeId);
return toolNodeAdded ? newNode.id : undefined;
} else {
addNode(newNode);
addChildEdge(params.position, {
source: params.nodeId,
target: newNode.id,
sourceHandle: params.id,
});
}
addNode(newNode);
addChildEdge(params.position, {
source: params.nodeId,
target: newNode.id,
sourceHandle: params.id,
});
return newNode.id;
},
[
addChildEdge,
addEdge,
addNode,
addToolNode,
calculateNewlyBackChildPosition,
edges,
getNode,
getNodeName,
initializeOperatorParams,
nodes,
reactFlowInstance,
resizeIterationNode,
],
);

39
web/src/pages/data-flow/hooks/use-before-delete.tsx
View File

@ -1,17 +1,12 @@
import { RAGFlowNodeType } from '@/interfaces/database/flow';
import { Node, OnBeforeDelete } from '@xyflow/react';
import { OnBeforeDelete } from '@xyflow/react';
import { Operator } from '../constant';
import useGraphStore from '../store';
import { deleteAllDownstreamAgentsAndTool } from '../utils/delete-node';
const UndeletableNodes = [Operator.Begin, Operator.IterationStart];
const UndeletableNodes = [Operator.Begin];
export function useBeforeDelete() {
const { getOperatorTypeFromId, getNode } = useGraphStore((state) => state);
const agentPredicate = (node: Node) => {
return getOperatorTypeFromId(node.id) === Operator.Agent;
};
const { getOperatorTypeFromId } = useGraphStore((state) => state);
const handleBeforeDelete: OnBeforeDelete<RAGFlowNodeType> = async ({
nodes, // Nodes to be deleted
@ -23,13 +18,6 @@ export function useBeforeDelete() {
return false;
}
if (
operatorType === Operator.IterationStart &&
!nodes.some((x) => x.id === node.parentId)
) {
return false;
}
return true;
});
@ -51,27 +39,6 @@ export function useBeforeDelete() {
return true;
});
// Delete the agent and tool nodes downstream of the agent node
if (nodes.some(agentPredicate)) {
nodes.filter(agentPredicate).forEach((node) => {
const { downstreamAgentAndToolEdges, downstreamAgentAndToolNodeIds } =
deleteAllDownstreamAgentsAndTool(node.id, edges);
downstreamAgentAndToolNodeIds.forEach((nodeId) => {
const currentNode = getNode(nodeId);
if (toBeDeletedNodes.every((x) => x.id !== nodeId) && currentNode) {
toBeDeletedNodes.push(currentNode);
}
});
downstreamAgentAndToolEdges.forEach((edge) => {
if (toBeDeletedEdges.every((x) => x.id !== edge.id)) {
toBeDeletedEdges.push(edge);
}
});
}, []);
}
return {
nodes: toBeDeletedNodes,
edges: toBeDeletedEdges,

19
web/src/pages/data-flow/hooks/use-build-options.tsx Normal file
View File

@ -0,0 +1,19 @@
import { buildNodeOutputOptions } from '@/utils/canvas-util';
import { useMemo } from 'react';
import { Operator } from '../constant';
import OperatorIcon from '../operator-icon';
import useGraphStore from '../store';
export function useBuildNodeOutputOptions(nodeId?: string) {
const nodes = useGraphStore((state) => state.nodes);
const edges = useGraphStore((state) => state.edges);
return useMemo(() => {
return buildNodeOutputOptions({
nodes,
edges,
nodeId,
Icon: ({ name }) => <OperatorIcon name={name as Operator}></OperatorIcon>,
});
}, [edges, nodeId, nodes]);
}

21
web/src/pages/data-flow/hooks/use-cancel-dataflow.ts Normal file
View File

@ -0,0 +1,21 @@
import { useCancelDataflow } from '@/hooks/use-agent-request';
import { useCallback } from 'react';
export function useCancelCurrentDataflow({
messageId,
stopFetchTrace,
}: {
messageId: string;
stopFetchTrace(): void;
}) {
const { cancelDataflow } = useCancelDataflow();
const handleCancel = useCallback(async () => {
const code = await cancelDataflow(messageId);
if (code === 0) {
stopFetchTrace();
}
}, [cancelDataflow, messageId, stopFetchTrace]);
return { handleCancel };
}

14
web/src/pages/data-flow/hooks/use-change-node-name.ts
View File

@ -9,7 +9,6 @@ import {
useMemo,
useState,
} from 'react';
import { Operator } from '../constant';
import useGraphStore from '../store';
import { getAgentNodeTools } from '../utils';
@ -77,13 +76,10 @@ export const useHandleNodeNameChange = ({
data: any;
}) => {
const [name, setName] = useState<string>('');
const { updateNodeName, nodes, getOperatorTypeFromId } = useGraphStore(
(state) => state,
);
const { updateNodeName, nodes } = useGraphStore((state) => state);
const previousName = data?.name;
const isToolNode = getOperatorTypeFromId(id) === Operator.Tool;
const { handleToolNameBlur, previousToolName } = useHandleTooNodeNameChange({
const { previousToolName } = useHandleTooNodeNameChange({
id,
name,
setName,
@ -109,12 +105,12 @@ export const useHandleNodeNameChange = ({
}, []);
useEffect(() => {
setName(isToolNode ? previousToolName : previousName);
}, [isToolNode, previousName, previousToolName]);
setName(previousName);
}, [previousName, previousToolName]);
return {
name,
handleNameBlur: isToolNode ? handleToolNameBlur : handleNameBlur,
handleNameBlur: handleNameBlur,
handleNameChange,
};
};

60
web/src/pages/data-flow/hooks/use-chat-logic.ts
View File

@ -1,60 +0,0 @@
import { MessageType } from '@/constants/chat';
import { Message } from '@/interfaces/database/chat';
import { IMessage } from '@/pages/chat/interface';
import { get } from 'lodash';
import { useCallback, useMemo } from 'react';
import { BeginQuery } from '../interface';
import { buildBeginQueryWithObject } from '../utils';
type IAwaitCompentData = {
derivedMessages: IMessage[];
sendFormMessage: (params: {
inputs: Record<string, BeginQuery>;
id: string;
}) => void;
canvasId: string;
};
const useAwaitCompentData = (props: IAwaitCompentData) => {
const { derivedMessages, sendFormMessage, canvasId } = props;
const getInputs = useCallback((message: Message) => {
return get(message, 'data.inputs', {}) as Record<string, BeginQuery>;
}, []);
const buildInputList = useCallback(
(message: Message) => {
return Object.entries(getInputs(message)).map(([key, val]) => {
return {
...val,
key,
};
});
},
[getInputs],
);
const handleOk = useCallback(
(message: Message) => (values: BeginQuery[]) => {
const inputs = getInputs(message);
const nextInputs = buildBeginQueryWithObject(inputs, values);
sendFormMessage({
inputs: nextInputs,
id: canvasId,
});
},
[getInputs, sendFormMessage, canvasId],
);
const isWaitting = useMemo(() => {
const temp = derivedMessages?.some((message, i) => {
const flag =
message.role === MessageType.Assistant &&
derivedMessages.length - 1 === i &&
message.data;
return flag;
});
return temp;
}, [derivedMessages]);
return { getInputs, buildInputList, handleOk, isWaitting };
};
export { useAwaitCompentData };

38
web/src/pages/data-flow/hooks/use-download-output.ts Normal file
View File

@ -0,0 +1,38 @@
import { useFetchAgent } from '@/hooks/use-agent-request';
import { ITraceData } from '@/interfaces/database/agent';
import { downloadJsonFile } from '@/utils/file-util';
import { get, isEmpty } from 'lodash';
import { useCallback } from 'react';
export function findEndOutput(list?: ITraceData[]) {
if (Array.isArray(list)) {
const trace = list.find((x) => x.component_id === 'END')?.trace;
const str = get(trace, '0.message');
try {
if (!isEmpty(str)) {
const json = JSON.parse(str);
return json;
}
} catch (error) {}
}
}
export function isEndOutputEmpty(list?: ITraceData[]) {
return isEmpty(findEndOutput(list));
}
export function useDownloadOutput(data?: ITraceData[]) {
const { data: agent } = useFetchAgent();
const handleDownloadJson = useCallback(() => {
const output = findEndOutput(data);
if (!isEndOutputEmpty(data)) {
downloadJsonFile(output, `${agent.title}.json`);
}
}, [agent.title, data]);
return {
handleDownloadJson,
};
}

54
web/src/pages/data-flow/hooks/use-export-json.ts
View File

@ -1,71 +1,17 @@
import { useToast } from '@/components/hooks/use-toast';
import { FileMimeType, Platform } from '@/constants/common';
import { useSetModalState } from '@/hooks/common-hooks';
import { useFetchAgent } from '@/hooks/use-agent-request';
import { IGraph } from '@/interfaces/database/flow';
import { downloadJsonFile } from '@/utils/file-util';
import { message } from 'antd';
import isEmpty from 'lodash/isEmpty';
import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { useBuildDslData } from './use-build-dsl';
import { useSetGraphInfo } from './use-set-graph';
export const useHandleExportOrImportJsonFile = () => {
const { buildDslData } = useBuildDslData();
const {
visible: fileUploadVisible,
hideModal: hideFileUploadModal,
showModal: showFileUploadModal,
} = useSetModalState();
const setGraphInfo = useSetGraphInfo();
const { data } = useFetchAgent();
const { t } = useTranslation();
const { toast } = useToast();
const onFileUploadOk = useCallback(
async ({
fileList,
platform,
}: {
fileList: File[];
platform: Platform;
}) => {
console.log('🚀 ~ useHandleExportOrImportJsonFile ~ platform:', platform);
if (fileList.length > 0) {
const file = fileList[0];
if (file.type !== FileMimeType.Json) {
toast({ title: t('flow.jsonUploadTypeErrorMessage') });
return;
}
const graphStr = await file.text();
const errorMessage = t('flow.jsonUploadContentErrorMessage');
try {
const graph = JSON.parse(graphStr);
if (graphStr && !isEmpty(graph) && Array.isArray(graph?.nodes)) {
setGraphInfo(graph ?? ({} as IGraph));
hideFileUploadModal();
} else {
message.error(errorMessage);
}
} catch (error) {
message.error(errorMessage);
}
}
},
[hideFileUploadModal, setGraphInfo, t, toast],
);
const handleExportJson = useCallback(() => {
downloadJsonFile(buildDslData().graph, `${data.title}.json`);
}, [buildDslData, data.title]);
return {
fileUploadVisible,
handleExportJson,
handleImportJson: showFileUploadModal,
hideFileUploadModal,
onFileUploadOk,
};
};

56
web/src/pages/data-flow/hooks/use-fetch-log.ts Normal file
View File

@ -0,0 +1,56 @@
import { useFetchMessageTrace } from '@/hooks/use-agent-request';
import { isEmpty } from 'lodash';
import { useCallback, useEffect, useMemo } from 'react';
export function useFetchLog(logSheetVisible: boolean) {
const {
setMessageId,
data,
loading,
messageId,
setISStopFetchTrace,
isStopFetchTrace,
} = useFetchMessageTrace();
const isCompleted = useMemo(() => {
if (Array.isArray(data)) {
const latest = data?.at(-1);
return (
latest?.component_id === 'END' && !isEmpty(latest?.trace[0].message)
);
}
return false;
}, [data]);
const isLogEmpty = !data || !data.length;
const stopFetchTrace = useCallback(() => {
setISStopFetchTrace(true);
}, [setISStopFetchTrace]);
// cancel request
useEffect(() => {
if (isCompleted) {
stopFetchTrace();
}
}, [isCompleted, stopFetchTrace]);
useEffect(() => {
if (logSheetVisible) {
setISStopFetchTrace(false);
}
}, [logSheetVisible, setISStopFetchTrace]);
return {
logs: data,
isLogEmpty,
isCompleted,
loading,
isParsing: !isLogEmpty && !isCompleted && !isStopFetchTrace,
messageId,
setMessageId,
stopFetchTrace,
};
}
export type UseFetchLogReturnType = ReturnType<typeof useFetchLog>;

12
web/src/pages/data-flow/hooks/use-find-mcp-by-id.ts
View File

@ -1,12 +0,0 @@
import { useListMcpServer } from '@/hooks/use-mcp-request';
export function useFindMcpById() {
const { data } = useListMcpServer();
const findMcpById = (id: string) =>
data.mcp_servers.find((item) => item.id === id);
return {
findMcpById,
};
}

317
web/src/pages/data-flow/hooks/use-get-begin-query.tsx
View File

@ -1,317 +0,0 @@
import { AgentGlobals } from '@/constants/agent';
import { useFetchAgent } from '@/hooks/use-agent-request';
import { RAGFlowNodeType } from '@/interfaces/database/flow';
import { Edge } from '@xyflow/react';
import { DefaultOptionType } from 'antd/es/select';
import { t } from 'i18next';
import { isEmpty } from 'lodash';
import get from 'lodash/get';
import {
ReactNode,
useCallback,
useContext,
useEffect,
useMemo,
useState,
} from 'react';
import {
AgentDialogueMode,
BeginId,
BeginQueryType,
Operator,
VariableType,
} from '../constant';
import { AgentFormContext } from '../context';
import { buildBeginInputListFromObject } from '../form/begin-form/utils';
import { BeginQuery } from '../interface';
import OperatorIcon from '../operator-icon';
import useGraphStore from '../store';
export function useSelectBeginNodeDataInputs() {
const getNode = useGraphStore((state) => state.getNode);
return buildBeginInputListFromObject(
getNode(BeginId)?.data?.form?.inputs ?? {},
);
}
export function useIsTaskMode() {
const getNode = useGraphStore((state) => state.getNode);
return useMemo(() => {
const node = getNode(BeginId);
return node?.data?.form?.mode === AgentDialogueMode.Task;
}, [getNode]);
}
export const useGetBeginNodeDataQuery = () => {
const getNode = useGraphStore((state) => state.getNode);
const getBeginNodeDataQuery = useCallback(() => {
return buildBeginInputListFromObject(
get(getNode(BeginId), 'data.form.inputs', {}),
);
}, [getNode]);
return getBeginNodeDataQuery;
};
export const useGetBeginNodeDataInputs = () => {
const getNode = useGraphStore((state) => state.getNode);
const inputs = get(getNode(BeginId), 'data.form.inputs', {});
const beginNodeDataInputs = useMemo(() => {
return buildBeginInputListFromObject(inputs);
}, [inputs]);
return beginNodeDataInputs;
};
export const useGetBeginNodeDataQueryIsSafe = () => {
const [isBeginNodeDataQuerySafe, setIsBeginNodeDataQuerySafe] =
useState(false);
const inputs = useSelectBeginNodeDataInputs();
const nodes = useGraphStore((state) => state.nodes);
useEffect(() => {
const query: BeginQuery[] = inputs;
const isSafe = !query.some((q) => !q.optional && q.type === 'file');
setIsBeginNodeDataQuerySafe(isSafe);
}, [inputs, nodes]);
return isBeginNodeDataQuerySafe;
};
function filterAllUpstreamNodeIds(edges: Edge[], nodeIds: string[]) {
return nodeIds.reduce<string[]>((pre, nodeId) => {
const currentEdges = edges.filter((x) => x.target === nodeId);
const upstreamNodeIds: string[] = currentEdges.map((x) => x.source);
const ids = upstreamNodeIds.concat(
filterAllUpstreamNodeIds(edges, upstreamNodeIds),
);
ids.forEach((x) => {
if (pre.every((y) => y !== x)) {
pre.push(x);
}
});
return pre;
}, []);
}
export function buildOutputOptions(
outputs: Record<string, any> = {},
nodeId?: string,
parentLabel?: string | ReactNode,
icon?: ReactNode,
) {
return Object.keys(outputs).map((x) => ({
label: x,
value: `${nodeId}@${x}`,
parentLabel,
icon,
type: outputs[x]?.type,
}));
}
export function useBuildNodeOutputOptions(nodeId?: string) {
const nodes = useGraphStore((state) => state.nodes);
const edges = useGraphStore((state) => state.edges);
const nodeOutputOptions = useMemo(() => {
if (!nodeId) {
return [];
}
const upstreamIds = filterAllUpstreamNodeIds(edges, [nodeId]);
const nodeWithOutputList = nodes.filter(
(x) =>
upstreamIds.some((y) => y === x.id) && !isEmpty(x.data?.form?.outputs),
);
return nodeWithOutputList
.filter((x) => x.id !== nodeId)
.map((x) => ({
label: x.data.name,
value: x.id,
title: x.data.name,
options: buildOutputOptions(
x.data.form.outputs,
x.id,
x.data.name,
<OperatorIcon name={x.data.label as Operator} />,
),
}));
}, [edges, nodeId, nodes]);
return nodeOutputOptions;
}
// exclude nodes with branches
const ExcludedNodes = [
Operator.Categorize,
Operator.Relevant,
Operator.Begin,
Operator.Note,
];
const StringList = [
BeginQueryType.Line,
BeginQueryType.Paragraph,
BeginQueryType.Options,
];
function transferToVariableType(type: string) {
if (StringList.some((x) => x === type)) {
return VariableType.String;
}
return type;
}
export function useBuildBeginVariableOptions() {
const inputs = useSelectBeginNodeDataInputs();
const options = useMemo(() => {
return [
{
label: <span>{t('flow.beginInput')}</span>,
title: t('flow.beginInput'),
options: inputs.map((x) => ({
label: x.name,
parentLabel: <span>{t('flow.beginInput')}</span>,
icon: <OperatorIcon name={Operator.Begin} className="block" />,
value: `begin@${x.key}`,
type: transferToVariableType(x.type),
})),
},
];
}, [inputs]);
return options;
}
export const useBuildVariableOptions = (nodeId?: string, parentId?: string) => {
const nodeOutputOptions = useBuildNodeOutputOptions(nodeId);
const parentNodeOutputOptions = useBuildNodeOutputOptions(parentId);
const beginOptions = useBuildBeginVariableOptions();
const options = useMemo(() => {
return [...beginOptions, ...nodeOutputOptions, ...parentNodeOutputOptions];
}, [beginOptions, nodeOutputOptions, parentNodeOutputOptions]);
return options;
};
export function useBuildQueryVariableOptions(n?: RAGFlowNodeType) {
const { data } = useFetchAgent();
const node = useContext(AgentFormContext) || n;
const options = useBuildVariableOptions(node?.id, node?.parentId);
const nextOptions = useMemo(() => {
const globals = data?.dsl?.globals ?? {};
const globalOptions = Object.entries(globals).map(([key, value]) => ({
label: key,
value: key,
icon: <OperatorIcon name={Operator.Begin} className="block" />,
parentLabel: <span>{t('flow.beginInput')}</span>,
type: Array.isArray(value)
? `${VariableType.Array}${key === AgentGlobals.SysFiles ? '<file>' : ''}`
: typeof value,
}));
return [
{ ...options[0], options: [...options[0]?.options, ...globalOptions] },
...options.slice(1),
];
}, [data.dsl?.globals, options]);
return nextOptions;
}
export function useBuildComponentIdOptions(nodeId?: string, parentId?: string) {
const nodes = useGraphStore((state) => state.nodes);
// Limit the nodes inside iteration to only reference peer nodes with the same parentId and other external nodes other than their parent nodes
const filterChildNodesToSameParentOrExternal = useCallback(
(node: RAGFlowNodeType) => {
// Node inside iteration
if (parentId) {
return (
(node.parentId === parentId || node.parentId === undefined) &&
node.id !== parentId
);
}
return node.parentId === undefined; // The outermost node
},
[parentId],
);
const componentIdOptions = useMemo(() => {
return nodes
.filter(
(x) =>
x.id !== nodeId &&
!ExcludedNodes.some((y) => y === x.data.label) &&
filterChildNodesToSameParentOrExternal(x),
)
.map((x) => ({ label: x.data.name, value: x.id }));
}, [nodes, nodeId, filterChildNodesToSameParentOrExternal]);
return [
{
label: <span>Component Output</span>,
title: 'Component Output',
options: componentIdOptions,
},
];
}
export function useBuildComponentIdAndBeginOptions(
nodeId?: string,
parentId?: string,
) {
const componentIdOptions = useBuildComponentIdOptions(nodeId, parentId);
const beginOptions = useBuildBeginVariableOptions();
return [...beginOptions, ...componentIdOptions];
}
export const useGetComponentLabelByValue = (nodeId: string) => {
const options = useBuildComponentIdAndBeginOptions(nodeId);
const flattenOptions = useMemo(() => {
return options.reduce<DefaultOptionType[]>((pre, cur) => {
return [...pre, ...cur.options];
}, []);
}, [options]);
const getLabel = useCallback(
(val?: string) => {
return flattenOptions.find((x) => x.value === val)?.label;
},
[flattenOptions],
);
return getLabel;
};
export function useGetVariableLabelByValue(nodeId: string) {
const { getNode } = useGraphStore((state) => state);
const nextOptions = useBuildQueryVariableOptions(getNode(nodeId));
const flattenOptions = useMemo(() => {
return nextOptions.reduce<DefaultOptionType[]>((pre, cur) => {
return [...pre, ...cur.options];
}, []);
}, [nextOptions]);
const getLabel = useCallback(
(val?: string) => {
return flattenOptions.find((x) => x.value === val)?.label;
},
[flattenOptions],
);
return getLabel;
}

0
web/src/pages/data-flow/hooks/use-iteration.ts
View File

59
web/src/pages/data-flow/hooks/use-run-dataflow.ts Normal file
View File

@ -0,0 +1,59 @@
import message from '@/components/ui/message';
import { useSendMessageBySSE } from '@/hooks/use-send-message';
import api from '@/utils/api';
import { get } from 'lodash';
import { useCallback, useContext } from 'react';
import { useParams } from 'umi';
import { LogContext } from '../context';
import { useSaveGraphBeforeOpeningDebugDrawer } from './use-save-graph';
export function useRunDataflow(
showLogSheet: () => void,
hideRunOrChatDrawer: () => void,
) {
const { send } = useSendMessageBySSE(api.runCanvas);
const { id } = useParams();
const { setMessageId, setUploadedFileData } = useContext(LogContext);
const { handleRun: saveGraph, loading } =
useSaveGraphBeforeOpeningDebugDrawer(showLogSheet!);
const run = useCallback(
async (fileResponseData: Record<string, any>) => {
const success = await saveGraph();
if (!success) return;
const res = await send({
id,
query: '',
session_id: null,
files: [fileResponseData.file],
});
if (res && res?.response.status === 200 && get(res, 'data.code') === 0) {
// fetch canvas
hideRunOrChatDrawer();
setUploadedFileData(fileResponseData.file);
const msgId = get(res, 'data.data.message_id');
if (msgId) {
setMessageId(msgId);
}
return msgId;
} else {
message.error(get(res, 'data.message', ''));
}
},
[
hideRunOrChatDrawer,
id,
saveGraph,
send,
setMessageId,
setUploadedFileData,
],
);
return { run, loading: loading };
}
export type RunDataflowType = ReturnType<typeof useRunDataflow>;

20
web/src/pages/data-flow/hooks/use-save-graph.ts
View File

@ -1,8 +1,4 @@
import {
useFetchAgent,
useResetAgent,
useSetAgent,
} from '@/hooks/use-agent-request';
import { useFetchAgent, useSetAgent } from '@/hooks/use-agent-request';
import { RAGFlowNodeType } from '@/interfaces/database/flow';
import { formatDate } from '@/utils/date';
import { useDebounceEffect } from 'ahooks';
@ -22,6 +18,7 @@ export const useSaveGraph = (showMessage: boolean = true) => {
return setAgent({
id,
title: data.title,
canvas_category: data.canvas_category,
dsl: buildDslData(currentNodes),
});
},
@ -33,21 +30,22 @@ export const useSaveGraph = (showMessage: boolean = true) => {
export const useSaveGraphBeforeOpeningDebugDrawer = (show: () => void) => {
const { saveGraph, loading } = useSaveGraph();
const { resetAgent } = useResetAgent();
// const { resetAgent } = useResetAgent();
const handleRun = useCallback(
async (nextNodes?: RAGFlowNodeType[]) => {
const saveRet = await saveGraph(nextNodes);
if (saveRet?.code === 0) {
// Call the reset api before opening the run drawer each time
const resetRet = await resetAgent();
// const resetRet = await resetAgent();
// After resetting, all previous messages will be cleared.
if (resetRet?.code === 0) {
show();
}
// if (resetRet?.code === 0) {
show();
// }
}
return saveRet?.code === 0;
},
[saveGraph, resetAgent, show],
[saveGraph, show],
);
return { handleRun, loading };

28
web/src/pages/data-flow/hooks/use-show-drawer.tsx
View File

@ -2,10 +2,9 @@ import { useSetModalState } from '@/hooks/common-hooks';
import { NodeMouseHandler } from '@xyflow/react';
import get from 'lodash/get';
import React, { useCallback, useEffect } from 'react';
import { Operator } from '../constant';
import { BeginId, Operator } from '../constant';
import useGraphStore from '../store';
import { useCacheChatLog } from './use-cache-chat-log';
import { useGetBeginNodeDataInputs } from './use-get-begin-query';
import { useSaveGraph } from './use-save-graph';
export const useShowFormDrawer = () => {
@ -13,7 +12,6 @@ export const useShowFormDrawer = () => {
clickedNodeId: clickNodeId,
setClickedNodeId,
getNode,
setClickedToolId,
} = useGraphStore((state) => state);
const {
visible: formDrawerVisible,
@ -23,16 +21,14 @@ export const useShowFormDrawer = () => {
const handleShow = useCallback(
(e: React.MouseEvent<Element>, nodeId: string) => {
const tool = get(e.target, 'dataset.tool');
// TODO: Operator type judgment should be used
if (nodeId.startsWith(Operator.Tool) && !tool) {
if (nodeId === BeginId) {
return;
}
setClickedNodeId(nodeId);
setClickedToolId(tool);
showFormDrawer();
},
[setClickedNodeId, setClickedToolId, showFormDrawer],
[setClickedNodeId, showFormDrawer],
);
return {
@ -87,26 +83,12 @@ export function useShowDrawer({
} = useShowSingleDebugDrawer();
const { formDrawerVisible, hideFormDrawer, showFormDrawer, clickedNode } =
useShowFormDrawer();
const inputs = useGetBeginNodeDataInputs();
useEffect(() => {
if (drawerVisible) {
if (inputs.length > 0) {
showRunModal();
hideChatModal();
} else {
showChatModal();
hideRunModal();
}
showRunModal();
}
}, [
hideChatModal,
hideRunModal,
showChatModal,
showRunModal,
drawerVisible,
inputs,
]);
}, [hideChatModal, hideRunModal, showChatModal, showRunModal, drawerVisible]);
const hideRunOrChatDrawer = useCallback(() => {
hideChatModal();

9
web/src/pages/data-flow/hooks/use-watch-form-change.ts
View File

@ -4,15 +4,12 @@ import useGraphStore from '../store';
export function useWatchFormChange(id?: string, form?: UseFormReturn<any>) {
let values = useWatch({ control: form?.control });
const updateNodeForm = useGraphStore((state) => state.updateNodeForm);
useEffect(() => {
// Manually triggered form updates are synchronized to the canvas
if (id) {
values = form?.getValues() || {};
let nextValues: any = values;
updateNodeForm(id, nextValues);
updateNodeForm(id, values);
}
}, [form?.formState.isDirty, id, updateNodeForm, values]);
}, [id, updateNodeForm, values]);
}