Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/data-flow/canvas/index.tsx

@@ -12,6 +12,7 @@ import {
   ControlButton,
   Controls,
   NodeTypes,
+  OnConnectEnd,
   Position,
   ReactFlow,
   ReactFlowInstance,
@@ -30,58 +31,34 @@ import { useBeforeDelete } from '../hooks/use-before-delete';
 import { useMoveNote } from '../hooks/use-move-note';
 import { useDropdownManager } from './context';
 
+import { useRunDataflow } from '../hooks/use-run-dataflow';
 import {
   useHideFormSheetOnNodeDeletion,
   useShowDrawer,
 } from '../hooks/use-show-drawer';
 import RunSheet from '../run-sheet';
+import useGraphStore from '../store';
 import { ButtonEdge } from './edge';
 import styles from './index.less';
 import { RagNode } from './node';
-import { AgentNode } from './node/agent-node';
 import { BeginNode } from './node/begin-node';
-import { CategorizeNode } from './node/categorize-node';
-import ChunkerNode from './node/chunker-node';
-import { InnerNextStepDropdown } from './node/dropdown/next-step-dropdown';
-import { GenerateNode } from './node/generate-node';
-import { InvokeNode } from './node/invoke-node';
-import { IterationNode, IterationStartNode } from './node/iteration-node';
-import { KeywordNode } from './node/keyword-node';
-import { LogicNode } from './node/logic-node';
-import { MessageNode } from './node/message-node';
+import { NextStepDropdown } from './node/dropdown/next-step-dropdown';
+import { ExtractorNode } from './node/extractor-node';
+import { HierarchicalMergerNode } from './node/hierarchical-merger-node';
 import NoteNode from './node/note-node';
 import ParserNode from './node/parser-node';
-import { RelevantNode } from './node/relevant-node';
-import { RetrievalNode } from './node/retrieval-node';
-import { RewriteNode } from './node/rewrite-node';
-import { SwitchNode } from './node/switch-node';
-import { TemplateNode } from './node/template-node';
+import { SplitterNode } from './node/splitter-node';
 import TokenizerNode from './node/tokenizer-node';
-import { ToolNode } from './node/tool-node';
 
 export const nodeTypes: NodeTypes = {
   ragNode: RagNode,
-  categorizeNode: CategorizeNode,
   beginNode: BeginNode,
-  relevantNode: RelevantNode,
-  logicNode: LogicNode,
   noteNode: NoteNode,
-  switchNode: SwitchNode,
-  generateNode: GenerateNode,
-  retrievalNode: RetrievalNode,
-  messageNode: MessageNode,
-  rewriteNode: RewriteNode,
-  keywordNode: KeywordNode,
-  invokeNode: InvokeNode,
-  templateNode: TemplateNode,
-  // emailNode: EmailNode,
-  group: IterationNode,
-  iterationStartNode: IterationStartNode,
-  agentNode: AgentNode,
-  toolNode: ToolNode,
   parserNode: ParserNode,
-  chunkerNode: ChunkerNode,
   tokenizerNode: TokenizerNode,
+  splitterNode: SplitterNode,
+  hierarchicalMergerNode: HierarchicalMergerNode,
+  contextNode: ExtractorNode,
 };
 
 const edgeTypes = {
@@ -91,9 +68,10 @@ const edgeTypes = {
 interface IProps {
   drawerVisible: boolean;
   hideDrawer(): void;
+  showLogSheet(): void;
 }
 
-function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
+function DataFlowCanvas({ drawerVisible, hideDrawer, showLogSheet }: IProps) {
   const { t } = useTranslation();
   const {
     nodes,
@@ -121,7 +99,6 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
     chatVisible,
     runVisible,
     hideRunOrChatDrawer,
-    showChatModal,
     showFormDrawer,
   } = useShowDrawer({
     drawerVisible,
@@ -141,6 +118,7 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
   useHideFormSheetOnNodeDeletion({ hideFormDrawer });
 
   const { visible, hideModal, showModal } = useSetModalState();
+
   const [dropdownPosition, setDropdownPosition] = useState({ x: 0, y: 0 });
 
   const isConnectedRef = useRef(false);
@@ -153,6 +131,8 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
 
   const { setActiveDropdown, clearActiveDropdown } = useDropdownManager();
 
+  const { hasChildNode } = useGraphStore((state) => state);
+
   const onPaneClick = useCallback(() => {
     hideFormDrawer();
     if (visible && !preventCloseRef.current) {
@@ -174,12 +154,17 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
     clearActiveDropdown,
   ]);
 
+  const { run, loading: running } = useRunDataflow(
+    showLogSheet!,
+    hideRunOrChatDrawer,
+  );
+
   const onConnect = (connection: Connection) => {
     originalOnConnect(connection);
     isConnectedRef.current = true;
   };
 
-  const OnConnectStart = (event: any, params: any) => {
+  const onConnectStart = (event: any, params: any) => {
     isConnectedRef.current = false;
 
     if (params && params.nodeId && params.handleId) {
@@ -192,7 +177,12 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
     }
   };
 
-  const OnConnectEnd = (event: MouseEvent | TouchEvent) => {
+  const onConnectEnd: OnConnectEnd = (event, connectionState) => {
+    const nodeId = connectionState.fromNode?.id;
+    // Events triggered by Handle are directly interrupted
+    if (connectionState.toNode !== null || (nodeId && hasChildNode(nodeId))) {
+      return;
+    }
     if ('clientX' in event && 'clientY' in event) {
       const { clientX, clientY } = event;
       setDropdownPosition({ x: clientX, y: clientY });
@@ -240,8 +230,8 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
           onConnect={onConnect}
           nodeTypes={nodeTypes}
           edgeTypes={edgeTypes}
-          onConnectStart={OnConnectStart}
-          onConnectEnd={OnConnectEnd}
+          onConnectStart={onConnectStart}
+          onConnectEnd={onConnectEnd}
           onNodeClick={onNodeClick}
           onPaneClick={onPaneClick}
           onInit={setReactFlowInstance}
@@ -288,7 +278,7 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
               isFromConnectionDrag: true,
             }}
           >
-            <InnerNextStepDropdown
+            <NextStepDropdown
               hideModal={() => {
                 hideModal();
                 clearActiveDropdown();
@@ -296,7 +286,7 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
               position={dropdownPosition}
             >
               <span></span>
-            </InnerNextStepDropdown>
+            </NextStepDropdown>
           </HandleContext.Provider>
         )}
       </AgentInstanceContext.Provider>
@@ -322,7 +312,8 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
       {runVisible && (
         <RunSheet
           hideModal={hideRunOrChatDrawer}
-          showModal={showChatModal}
+          run={run}
+          loading={running}
         ></RunSheet>
       )}
     </div>