Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/data-flow/index.tsx

@@ -21,26 +21,27 @@ import { ReactFlowProvider } from '@xyflow/react';
 import {
   ChevronDown,
   CirclePlay,
-  Download,
   History,
   LaptopMinimalCheck,
   Settings,
   Upload,
 } from 'lucide-react';
-import { ComponentPropsWithoutRef, useCallback } from 'react';
+import { ComponentPropsWithoutRef, useCallback, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import DataFlowCanvas from './canvas';
 import { DropdownProvider } from './canvas/context';
+import { LogContext } from './context';
+import { useCancelCurrentDataflow } from './hooks/use-cancel-dataflow';
 import { useHandleExportOrImportJsonFile } from './hooks/use-export-json';
 import { useFetchDataOnMount } from './hooks/use-fetch-data';
-import { useGetBeginNodeDataInputs } from './hooks/use-get-begin-query';
+import { useFetchLog } from './hooks/use-fetch-log';
 import {
   useSaveGraph,
   useSaveGraphBeforeOpeningDebugDrawer,
   useWatchAgentChange,
 } from './hooks/use-save-graph';
+import { LogSheet } from './log-sheet';
 import { SettingDialog } from './setting-dialog';
-import { UploadAgentDialog } from './upload-agent-dialog';
 import { useAgentHistoryManager } from './use-agent-history-manager';
 import { VersionDialog } from './version-dialog';
 
@@ -64,24 +65,12 @@ export default function DataFlow() {
   } = useSetModalState();
   const { t } = useTranslation();
   useAgentHistoryManager();
-  const {
-    handleExportJson,
-    handleImportJson,
-    fileUploadVisible,
-    onFileUploadOk,
-    hideFileUploadModal,
-  } = useHandleExportOrImportJsonFile();
+  const { handleExportJson } = useHandleExportOrImportJsonFile();
   const { saveGraph, loading } = useSaveGraph();
   const { flowDetail: agentDetail } = useFetchDataOnMount();
-  const inputs = useGetBeginNodeDataInputs();
-  const { handleRun } = useSaveGraphBeforeOpeningDebugDrawer(showChatDrawer);
-  const handleRunAgent = useCallback(() => {
-    if (inputs.length > 0) {
-      showChatDrawer();
-    } else {
-      handleRun();
-    }
-  }, [handleRun, inputs, showChatDrawer]);
+  const { handleRun, loading: running } =
+    useSaveGraphBeforeOpeningDebugDrawer(showChatDrawer);
+
   const {
     visible: versionDialogVisible,
     hideModal: hideVersionDialog,
@@ -94,6 +83,40 @@ export default function DataFlow() {
     showModal: showSettingDialog,
   } = useSetModalState();
 
+  const {
+    visible: logSheetVisible,
+    showModal: showLogSheet,
+    hideModal: hideLogSheet,
+  } = useSetModalState();
+
+  const {
+    isParsing,
+    logs,
+    messageId,
+    setMessageId,
+    isCompleted,
+    stopFetchTrace,
+    isLogEmpty,
+  } = useFetchLog(logSheetVisible);
+
+  const [uploadedFileData, setUploadedFileData] =
+    useState<Record<string, any>>();
+
+  const handleRunAgent = useCallback(() => {
+    if (isParsing) {
+      // show log sheet
+      showLogSheet();
+    } else {
+      hideLogSheet();
+      handleRun();
+    }
+  }, [handleRun, hideLogSheet, isParsing, showLogSheet]);
+
+  const { handleCancel } = useCancelCurrentDataflow({
+    messageId,
+    stopFetchTrace,
+  });
+
   const time = useWatchAgentChange(chatDrawerVisible);
 
   return (
@@ -125,15 +148,25 @@ export default function DataFlow() {
           >
             <LaptopMinimalCheck /> {t('flow.save')}
           </ButtonLoading>
-          <Button variant={'secondary'} onClick={handleRunAgent}>
-            <CirclePlay />
-            {t('flow.run')}
-          </Button>
+          <ButtonLoading
+            variant={'secondary'}
+            onClick={handleRunAgent}
+            loading={running}
+          >
+            {running || (
+              <CirclePlay className={isParsing ? 'animate-spin' : ''} />
+            )}
+
+            {isParsing || running ? t('dataflow.running') : t('flow.run')}
+          </ButtonLoading>
           <Button variant={'secondary'} onClick={showVersionDialog}>
             <History />
             {t('flow.historyversion')}
           </Button>
-
+          {/* <Button variant={'secondary'}>
+            <Send />
+            {t('flow.release')}
+          </Button> */}
           <DropdownMenu>
             <DropdownMenuTrigger asChild>
               <Button variant={'secondary'}>
@@ -141,11 +174,6 @@ export default function DataFlow() {
               </Button>
             </DropdownMenuTrigger>
             <DropdownMenuContent>
-              <AgentDropdownMenuItem onClick={handleImportJson}>
-                <Download />
-                {t('flow.import')}
-              </AgentDropdownMenuItem>
-              <DropdownMenuSeparator />
               <AgentDropdownMenuItem onClick={handleExportJson}>
                 <Upload />
                 {t('flow.export')}
@@ -159,21 +187,19 @@ export default function DataFlow() {
           </DropdownMenu>
         </div>
       </PageHeader>
-      <ReactFlowProvider>
-        <DropdownProvider>
-          <DataFlowCanvas
-            drawerVisible={chatDrawerVisible}
-            hideDrawer={hideChatDrawer}
-          ></DataFlowCanvas>
-        </DropdownProvider>
-      </ReactFlowProvider>
-      {fileUploadVisible && (
-        <UploadAgentDialog
-          hideModal={hideFileUploadModal}
-          onOk={onFileUploadOk}
-        ></UploadAgentDialog>
-      )}
-
+      <LogContext.Provider
+        value={{ messageId, setMessageId, setUploadedFileData }}
+      >
+        <ReactFlowProvider>
+          <DropdownProvider>
+            <DataFlowCanvas
+              drawerVisible={chatDrawerVisible}
+              hideDrawer={hideChatDrawer}
+              showLogSheet={showLogSheet}
+            ></DataFlowCanvas>
+          </DropdownProvider>
+        </ReactFlowProvider>
+      </LogContext.Provider>
       {versionDialogVisible && (
         <DropdownProvider>
           <VersionDialog hideModal={hideVersionDialog}></VersionDialog>
@@ -182,6 +208,18 @@ export default function DataFlow() {
       {settingDialogVisible && (
         <SettingDialog hideModal={hideSettingDialog}></SettingDialog>
       )}
+      {logSheetVisible && (
+        <LogSheet
+          hideModal={hideLogSheet}
+          isParsing={isParsing}
+          isCompleted={isCompleted}
+          isLogEmpty={isLogEmpty}
+          logs={logs}
+          handleCancel={handleCancel}
+          messageId={messageId}
+          uploadedFileData={uploadedFileData}
+        ></LogSheet>
+      )}
     </section>
   );
 }