Feat: Add splitter node component #9869 (#10114)

### What problem does this PR solve?

Feat: Add splitter node component #9869
### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/hooks/use-dataflow-request.ts

@@ -0,0 +1,127 @@
+import message from '@/components/ui/message';
+import { IFlow } from '@/interfaces/database/agent';
+import { Operator } from '@/pages/data-flow/constant';
+import dataflowService from '@/services/dataflow-service';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { useTranslation } from 'react-i18next';
+import { useParams } from 'umi';
+
+export const enum DataflowApiAction {
+  ListDataflow = 'listDataflow',
+  RemoveDataflow = 'removeDataflow',
+  FetchDataflow = 'fetchDataflow',
+  RunDataflow = 'runDataflow',
+  SetDataflow = 'setDataflow',
+}
+
+export const EmptyDsl = {
+  graph: {
+    nodes: [
+      {
+        id: Operator.Begin,
+        type: 'beginNode',
+        position: {
+          x: 50,
+          y: 200,
+        },
+        data: {
+          label: 'Begin',
+          name: Operator.Begin,
+        },
+        sourcePosition: 'left',
+        targetPosition: 'right',
+      },
+    ],
+    edges: [],
+  },
+  components: {
+    begin: {
+      obj: {
+        component_name: 'Begin',
+        params: {},
+      },
+      downstream: [], // other edge target is downstream, edge source is current node id
+      upstream: [], // edge source is upstream, edge target is current node id
+    },
+  },
+  retrieval: [], // reference
+  history: [],
+  path: [],
+};
+
+export const useRemoveDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.RemoveDataflow],
+    mutationFn: async (ids: string[]) => {
+      const { data } = await dataflowService.removeDataflow({
+        canvas_ids: ids,
+      });
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.ListDataflow],
+        });
+
+        message.success(t('message.deleted'));
+      }
+      return data.code;
+    },
+  });
+
+  return { data, loading, removeDataflow: mutateAsync };
+};
+
+export const useSetDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.SetDataflow],
+    mutationFn: async (params: Partial<IFlow>) => {
+      const { data } = await dataflowService.setDataflow(params);
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.FetchDataflow],
+        });
+
+        message.success(t(`message.${params.id ? 'modified' : 'created'}`));
+      }
+      return data?.code;
+    },
+  });
+
+  return { data, loading, setDataflow: mutateAsync };
+};
+
+export const useFetchDataflow = () => {
+  const { id } = useParams();
+
+  const {
+    data,
+    isFetching: loading,
+    refetch,
+  } = useQuery<IFlow>({
+    queryKey: [DataflowApiAction.FetchDataflow, id],
+    gcTime: 0,
+    initialData: {} as IFlow,
+    enabled: !!id,
+    refetchOnWindowFocus: false,
+    queryFn: async () => {
+      const { data } = await dataflowService.fetchDataflow(id);
+
+      return data?.data ?? ({} as IFlow);
+    },
+  });
+
+  return { data, loading, refetch };
+};

web/src/pages/agents/hooks/use-create-agent.ts

@@ -1,12 +1,13 @@
 import { useSetModalState } from '@/hooks/common-hooks';
-import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request';
+import { useSetAgent } from '@/hooks/use-agent-request';
-import { DSL } from '@/interfaces/database/agent';
+import { EmptyDsl, useSetDataflow } from '@/hooks/use-dataflow-request';
 import { useCallback } from 'react';
 import { FlowType } from '../constant';
 import { FormSchemaType } from '../create-agent-form';
 
 export function useCreateAgentOrPipeline() {
   const { loading, setAgent } = useSetAgent();
+  const { loading: dataflowLoading, setDataflow } = useSetDataflow();
   const {
     visible: creatingVisible,
     hideModal: hideCreatingModal,
@@ -15,7 +16,7 @@ export function useCreateAgentOrPipeline() {
 
   const createAgent = useCallback(
     async (name: string) => {
-      return setAgent({ title: name, dsl: EmptyDsl as DSL });
+      return setAgent({ title: name, dsl: EmptyDsl });
     },
     [setAgent],
   );
@@ -27,13 +28,18 @@ export function useCreateAgentOrPipeline() {
         if (ret.code === 0) {
           hideCreatingModal();
         }
+      } else {
+        setDataflow({
+          title: data.name,
+          dsl: EmptyDsl,
+        });
       }
     },
-    [createAgent, hideCreatingModal],
+    [createAgent, hideCreatingModal, setDataflow],
   );
 
   return {
-    loading,
+    loading: loading || dataflowLoading,
     creatingVisible,
     hideCreatingModal,
     showCreatingModal,

web/src/pages/data-flow/canvas/index.tsx

@@ -54,6 +54,7 @@ import ParserNode from './node/parser-node';
 import { RelevantNode } from './node/relevant-node';
 import { RetrievalNode } from './node/retrieval-node';
 import { RewriteNode } from './node/rewrite-node';
+import { SplitterNode } from './node/splitter-node';
 import { SwitchNode } from './node/switch-node';
 import { TemplateNode } from './node/template-node';
 import TokenizerNode from './node/tokenizer-node';
@@ -82,6 +83,7 @@ export const nodeTypes: NodeTypes = {
   parserNode: ParserNode,
   chunkerNode: ChunkerNode,
   tokenizerNode: TokenizerNode,
+  splitterNode: SplitterNode,
 };
 
 const edgeTypes = {

web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx

@@ -141,6 +141,8 @@ function AccordionOperators({
               Operator.Parser,
               Operator.Chunker,
               Operator.Tokenizer,
+              Operator.Splitter,
+              Operator.HierarchicalMerger,
             ]}
             isCustomDropdown={isCustomDropdown}
             mousePosition={mousePosition}

web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx

@@ -0,0 +1 @@
+export { RagNode as HierarchicalMergerNode } from './index';

web/src/pages/data-flow/canvas/node/splitter-node.tsx

@@ -0,0 +1 @@
+export { RagNode as SplitterNode } from './index';

web/src/pages/data-flow/constant.tsx

@@ -66,6 +66,8 @@ export enum Operator {
   Parser = 'Parser',
   Chunker = 'Chunker',
   Tokenizer = 'Tokenizer',
+  Splitter = 'Splitter',
+  HierarchicalMerger = 'HierarchicalMerger',
 }
 
 export const SwitchLogicOperatorOptions = ['and', 'or'];
@@ -74,20 +76,6 @@ export const CommonOperatorList = Object.values(Operator).filter(
   (x) => x !== Operator.Note,
 );
 
-export const AgentOperatorList = [
-  Operator.Retrieval,
-  Operator.Categorize,
-  Operator.Message,
-  Operator.RewriteQuestion,
-  Operator.KeywordExtract,
-  Operator.Switch,
-  Operator.Concentrator,
-  Operator.Iteration,
-  Operator.WaitingDialogue,
-  Operator.Note,
-  Operator.Agent,
-];
-
 export const SwitchOperatorOptions = [
   { value: '=', label: 'equal', icon: 'equal' },
   { value: '≠', label: 'notEqual', icon: 'not-equals' },
@@ -390,6 +378,10 @@ export const initialStringTransformValues = {
 
 export const initialParserValues = { outputs: {}, parser: [] };
 
+export const initialSplitterValues = {};
+
+export const initialHierarchicalMergerValues = {};
+
 export const CategorizeAnchorPointPositions = [
   { top: 1, right: 34 },
   { top: 8, right: 18 },
@@ -473,6 +465,8 @@ export const NodeMap = {
   [Operator.Parser]: 'parserNode',
   [Operator.Chunker]: 'chunkerNode',
   [Operator.Tokenizer]: 'tokenizerNode',
+  [Operator.Splitter]: 'splitterNode',
+  [Operator.HierarchicalMerger]: 'hierarchicalMergerrNode',
 };
 
 export enum BeginQueryType {

web/src/pages/data-flow/hooks/use-add-node.ts

@@ -18,6 +18,7 @@ import {
   initialCrawlerValues,
   initialEmailValues,
   initialExeSqlValues,
+  initialHierarchicalMergerValues,
   initialInvokeValues,
   initialIterationStartValues,
   initialIterationValues,
@@ -28,6 +29,7 @@ import {
   initialRelevantValues,
   initialRetrievalValues,
   initialRewriteQuestionValues,
+  initialSplitterValues,
   initialStringTransformValues,
   initialSwitchValues,
   initialTokenizerValues,
@@ -82,6 +84,8 @@ export const useInitializeOperatorParams = () => {
       [Operator.Parser]: initialParserValues,
       [Operator.Chunker]: initialChunkerValues,
       [Operator.Tokenizer]: initialTokenizerValues,
+      [Operator.Splitter]: initialSplitterValues,
+      [Operator.HierarchicalMerger]: initialHierarchicalMergerValues,
     };
   }, [llmId]);
 

web/src/services/dataflow-service.ts

@@ -0,0 +1,37 @@
+import api from '@/utils/api';
+import { registerNextServer } from '@/utils/register-server';
+
+const {
+  listDataflow,
+  removeDataflow,
+  fetchDataflow,
+  runDataflow,
+  setDataflow,
+} = api;
+
+const methods = {
+  listDataflow: {
+    url: listDataflow,
+    method: 'get',
+  },
+  removeDataflow: {
+    url: removeDataflow,
+    method: 'post',
+  },
+  fetchDataflow: {
+    url: fetchDataflow,
+    method: 'get',
+  },
+  runDataflow: {
+    url: runDataflow,
+    method: 'post',
+  },
+  setDataflow: {
+    url: setDataflow,
+    method: 'post',
+  },
+} as const;
+
+const dataflowService = registerNextServer<keyof typeof methods>(methods);
+
+export default dataflowService;

web/src/utils/api.ts

@@ -190,4 +190,12 @@ export default {
   mindmapShare: `${ExternalApi}${api_host}/searchbots/mindmap`,
   getRelatedQuestionsShare: `${ExternalApi}${api_host}/searchbots/related_questions`,
   retrievalTestShare: `${ExternalApi}${api_host}/searchbots/retrieval_test`,
+
+  // data pipeline
+
+  fetchDataflow: (id: string) => `${api_host}/dataflow/get/${id}`,
+  setDataflow: `${api_host}/dataflow/set`,
+  removeDataflow: `${api_host}/dataflow/rm`,
+  listDataflow: `${api_host}/dataflow/list`,
+  runDataflow: `${api_host}/dataflow/run`,
 };