From 93cf0258c3771bacb1056a2acd9a8c2813dfca7c Mon Sep 17 00:00:00 2001 From: balibabu Date: Tue, 16 Sep 2025 17:53:48 +0800 Subject: [PATCH] Feat: Add splitter node component #9869 (#10114) ### What problem does this PR solve? Feat: Add splitter node component #9869 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- web/src/hooks/use-dataflow-request.ts | 127 ++++++++++++++++++ .../pages/agents/hooks/use-create-agent.ts | 16 ++- web/src/pages/data-flow/canvas/index.tsx | 2 + .../node/dropdown/next-step-dropdown.tsx | 2 + .../canvas/node/hierarchical-merger-node.tsx | 1 + .../data-flow/canvas/node/splitter-node.tsx | 1 + web/src/pages/data-flow/constant.tsx | 22 ++- web/src/pages/data-flow/hooks/use-add-node.ts | 4 + web/src/services/dataflow-service.ts | 37 +++++ web/src/utils/api.ts | 8 ++ 10 files changed, 201 insertions(+), 19 deletions(-) create mode 100644 web/src/hooks/use-dataflow-request.ts create mode 100644 web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx create mode 100644 web/src/pages/data-flow/canvas/node/splitter-node.tsx create mode 100644 web/src/services/dataflow-service.ts diff --git a/web/src/hooks/use-dataflow-request.ts b/web/src/hooks/use-dataflow-request.ts new file mode 100644 index 000000000..ec01b764c --- /dev/null +++ b/web/src/hooks/use-dataflow-request.ts @@ -0,0 +1,127 @@ +import message from '@/components/ui/message'; +import { IFlow } from '@/interfaces/database/agent'; +import { Operator } from '@/pages/data-flow/constant'; +import dataflowService from '@/services/dataflow-service'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useTranslation } from 'react-i18next'; +import { useParams } from 'umi'; + +export const enum DataflowApiAction { + ListDataflow = 'listDataflow', + RemoveDataflow = 'removeDataflow', + FetchDataflow = 'fetchDataflow', + RunDataflow = 'runDataflow', + SetDataflow = 'setDataflow', +} + +export const EmptyDsl = { + graph: { + nodes: [ + { + id: Operator.Begin, + type: 'beginNode', + position: { + x: 50, + y: 200, + }, + data: { + label: 'Begin', + name: Operator.Begin, + }, + sourcePosition: 'left', + targetPosition: 'right', + }, + ], + edges: [], + }, + components: { + begin: { + obj: { + component_name: 'Begin', + params: {}, + }, + downstream: [], // other edge target is downstream, edge source is current node id + upstream: [], // edge source is upstream, edge target is current node id + }, + }, + retrieval: [], // reference + history: [], + path: [], +}; + +export const useRemoveDataflow = () => { + const queryClient = useQueryClient(); + const { t } = useTranslation(); + + const { + data, + isPending: loading, + mutateAsync, + } = useMutation({ + mutationKey: [DataflowApiAction.RemoveDataflow], + mutationFn: async (ids: string[]) => { + const { data } = await dataflowService.removeDataflow({ + canvas_ids: ids, + }); + if (data.code === 0) { + queryClient.invalidateQueries({ + queryKey: [DataflowApiAction.ListDataflow], + }); + + message.success(t('message.deleted')); + } + return data.code; + }, + }); + + return { data, loading, removeDataflow: mutateAsync }; +}; + +export const useSetDataflow = () => { + const queryClient = useQueryClient(); + const { t } = useTranslation(); + + const { + data, + isPending: loading, + mutateAsync, + } = useMutation({ + mutationKey: [DataflowApiAction.SetDataflow], + mutationFn: async (params: Partial) => { + const { data } = await dataflowService.setDataflow(params); + if (data.code === 0) { + queryClient.invalidateQueries({ + queryKey: [DataflowApiAction.FetchDataflow], + }); + + message.success(t(`message.${params.id ? 'modified' : 'created'}`)); + } + return data?.code; + }, + }); + + return { data, loading, setDataflow: mutateAsync }; +}; + +export const useFetchDataflow = () => { + const { id } = useParams(); + + const { + data, + isFetching: loading, + refetch, + } = useQuery({ + queryKey: [DataflowApiAction.FetchDataflow, id], + gcTime: 0, + initialData: {} as IFlow, + enabled: !!id, + refetchOnWindowFocus: false, + queryFn: async () => { + const { data } = await dataflowService.fetchDataflow(id); + + return data?.data ?? ({} as IFlow); + }, + }); + + return { data, loading, refetch }; +}; diff --git a/web/src/pages/agents/hooks/use-create-agent.ts b/web/src/pages/agents/hooks/use-create-agent.ts index 3693ce4ec..41b6ce424 100644 --- a/web/src/pages/agents/hooks/use-create-agent.ts +++ b/web/src/pages/agents/hooks/use-create-agent.ts @@ -1,12 +1,13 @@ import { useSetModalState } from '@/hooks/common-hooks'; -import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request'; -import { DSL } from '@/interfaces/database/agent'; +import { useSetAgent } from '@/hooks/use-agent-request'; +import { EmptyDsl, useSetDataflow } from '@/hooks/use-dataflow-request'; import { useCallback } from 'react'; import { FlowType } from '../constant'; import { FormSchemaType } from '../create-agent-form'; export function useCreateAgentOrPipeline() { const { loading, setAgent } = useSetAgent(); + const { loading: dataflowLoading, setDataflow } = useSetDataflow(); const { visible: creatingVisible, hideModal: hideCreatingModal, @@ -15,7 +16,7 @@ export function useCreateAgentOrPipeline() { const createAgent = useCallback( async (name: string) => { - return setAgent({ title: name, dsl: EmptyDsl as DSL }); + return setAgent({ title: name, dsl: EmptyDsl }); }, [setAgent], ); @@ -27,13 +28,18 @@ export function useCreateAgentOrPipeline() { if (ret.code === 0) { hideCreatingModal(); } + } else { + setDataflow({ + title: data.name, + dsl: EmptyDsl, + }); } }, - [createAgent, hideCreatingModal], + [createAgent, hideCreatingModal, setDataflow], ); return { - loading, + loading: loading || dataflowLoading, creatingVisible, hideCreatingModal, showCreatingModal, diff --git a/web/src/pages/data-flow/canvas/index.tsx b/web/src/pages/data-flow/canvas/index.tsx index 73ce3cb98..d48bfd68a 100644 --- a/web/src/pages/data-flow/canvas/index.tsx +++ b/web/src/pages/data-flow/canvas/index.tsx @@ -54,6 +54,7 @@ import ParserNode from './node/parser-node'; import { RelevantNode } from './node/relevant-node'; import { RetrievalNode } from './node/retrieval-node'; import { RewriteNode } from './node/rewrite-node'; +import { SplitterNode } from './node/splitter-node'; import { SwitchNode } from './node/switch-node'; import { TemplateNode } from './node/template-node'; import TokenizerNode from './node/tokenizer-node'; @@ -82,6 +83,7 @@ export const nodeTypes: NodeTypes = { parserNode: ParserNode, chunkerNode: ChunkerNode, tokenizerNode: TokenizerNode, + splitterNode: SplitterNode, }; const edgeTypes = { diff --git a/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx b/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx index 641eec7ae..2a2d9ab96 100644 --- a/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx +++ b/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx @@ -141,6 +141,8 @@ function AccordionOperators({ Operator.Parser, Operator.Chunker, Operator.Tokenizer, + Operator.Splitter, + Operator.HierarchicalMerger, ]} isCustomDropdown={isCustomDropdown} mousePosition={mousePosition} diff --git a/web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx b/web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx new file mode 100644 index 000000000..ba47dccfb --- /dev/null +++ b/web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx @@ -0,0 +1 @@ +export { RagNode as HierarchicalMergerNode } from './index'; diff --git a/web/src/pages/data-flow/canvas/node/splitter-node.tsx b/web/src/pages/data-flow/canvas/node/splitter-node.tsx new file mode 100644 index 000000000..c9948d3c6 --- /dev/null +++ b/web/src/pages/data-flow/canvas/node/splitter-node.tsx @@ -0,0 +1 @@ +export { RagNode as SplitterNode } from './index'; diff --git a/web/src/pages/data-flow/constant.tsx b/web/src/pages/data-flow/constant.tsx index d260d4c4c..96569cde1 100644 --- a/web/src/pages/data-flow/constant.tsx +++ b/web/src/pages/data-flow/constant.tsx @@ -66,6 +66,8 @@ export enum Operator { Parser = 'Parser', Chunker = 'Chunker', Tokenizer = 'Tokenizer', + Splitter = 'Splitter', + HierarchicalMerger = 'HierarchicalMerger', } export const SwitchLogicOperatorOptions = ['and', 'or']; @@ -74,20 +76,6 @@ export const CommonOperatorList = Object.values(Operator).filter( (x) => x !== Operator.Note, ); -export const AgentOperatorList = [ - Operator.Retrieval, - Operator.Categorize, - Operator.Message, - Operator.RewriteQuestion, - Operator.KeywordExtract, - Operator.Switch, - Operator.Concentrator, - Operator.Iteration, - Operator.WaitingDialogue, - Operator.Note, - Operator.Agent, -]; - export const SwitchOperatorOptions = [ { value: '=', label: 'equal', icon: 'equal' }, { value: '≠', label: 'notEqual', icon: 'not-equals' }, @@ -390,6 +378,10 @@ export const initialStringTransformValues = { export const initialParserValues = { outputs: {}, parser: [] }; +export const initialSplitterValues = {}; + +export const initialHierarchicalMergerValues = {}; + export const CategorizeAnchorPointPositions = [ { top: 1, right: 34 }, { top: 8, right: 18 }, @@ -473,6 +465,8 @@ export const NodeMap = { [Operator.Parser]: 'parserNode', [Operator.Chunker]: 'chunkerNode', [Operator.Tokenizer]: 'tokenizerNode', + [Operator.Splitter]: 'splitterNode', + [Operator.HierarchicalMerger]: 'hierarchicalMergerrNode', }; export enum BeginQueryType { diff --git a/web/src/pages/data-flow/hooks/use-add-node.ts b/web/src/pages/data-flow/hooks/use-add-node.ts index 7e0ddf584..e398a671f 100644 --- a/web/src/pages/data-flow/hooks/use-add-node.ts +++ b/web/src/pages/data-flow/hooks/use-add-node.ts @@ -18,6 +18,7 @@ import { initialCrawlerValues, initialEmailValues, initialExeSqlValues, + initialHierarchicalMergerValues, initialInvokeValues, initialIterationStartValues, initialIterationValues, @@ -28,6 +29,7 @@ import { initialRelevantValues, initialRetrievalValues, initialRewriteQuestionValues, + initialSplitterValues, initialStringTransformValues, initialSwitchValues, initialTokenizerValues, @@ -82,6 +84,8 @@ export const useInitializeOperatorParams = () => { [Operator.Parser]: initialParserValues, [Operator.Chunker]: initialChunkerValues, [Operator.Tokenizer]: initialTokenizerValues, + [Operator.Splitter]: initialSplitterValues, + [Operator.HierarchicalMerger]: initialHierarchicalMergerValues, }; }, [llmId]); diff --git a/web/src/services/dataflow-service.ts b/web/src/services/dataflow-service.ts new file mode 100644 index 000000000..6c3b21f3e --- /dev/null +++ b/web/src/services/dataflow-service.ts @@ -0,0 +1,37 @@ +import api from '@/utils/api'; +import { registerNextServer } from '@/utils/register-server'; + +const { + listDataflow, + removeDataflow, + fetchDataflow, + runDataflow, + setDataflow, +} = api; + +const methods = { + listDataflow: { + url: listDataflow, + method: 'get', + }, + removeDataflow: { + url: removeDataflow, + method: 'post', + }, + fetchDataflow: { + url: fetchDataflow, + method: 'get', + }, + runDataflow: { + url: runDataflow, + method: 'post', + }, + setDataflow: { + url: setDataflow, + method: 'post', + }, +} as const; + +const dataflowService = registerNextServer(methods); + +export default dataflowService; diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index cf294a67c..1fe0e73d0 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -190,4 +190,12 @@ export default { mindmapShare: `${ExternalApi}${api_host}/searchbots/mindmap`, getRelatedQuestionsShare: `${ExternalApi}${api_host}/searchbots/related_questions`, retrievalTestShare: `${ExternalApi}${api_host}/searchbots/retrieval_test`, + + // data pipeline + + fetchDataflow: (id: string) => `${api_host}/dataflow/get/${id}`, + setDataflow: `${api_host}/dataflow/set`, + removeDataflow: `${api_host}/dataflow/rm`, + listDataflow: `${api_host}/dataflow/list`, + runDataflow: `${api_host}/dataflow/run`, };