Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/hooks/flow-hooks.ts

@@ -1,79 +1,14 @@
-import { ResponseType } from '@/interfaces/database/base';
-import { DSL, IFlow, IFlowTemplate } from '@/interfaces/database/flow';
+import { DSL, IFlow } from '@/interfaces/database/flow';
 import { IDebugSingleRequestBody } from '@/interfaces/request/flow';
 import i18n from '@/locales/config';
 import { useGetSharedChatSearchParams } from '@/pages/chat/shared-hooks';
-import { BeginId } from '@/pages/flow/constant';
 import flowService from '@/services/flow-service';
 import { buildMessageListWithUuid } from '@/utils/chat';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { message } from 'antd';
 import { set } from 'lodash';
 import get from 'lodash/get';
-import { useTranslation } from 'react-i18next';
 import { useParams } from 'umi';
-import { v4 as uuid } from 'uuid';
-
-export const EmptyDsl = {
-  graph: {
-    nodes: [
-      {
-        id: BeginId,
-        type: 'beginNode',
-        position: {
-          x: 50,
-          y: 200,
-        },
-        data: {
-          label: 'Begin',
-          name: 'begin',
-        },
-        sourcePosition: 'left',
-        targetPosition: 'right',
-      },
-    ],
-    edges: [],
-  },
-  components: {
-    begin: {
-      obj: {
-        component_name: 'Begin',
-        params: {},
-      },
-      downstream: ['Answer:China'], // other edge target is downstream, edge source is current node id
-      upstream: [], // edge source is upstream, edge target is current node id
-    },
-  },
-  messages: [],
-  reference: [],
-  history: [],
-  path: [],
-  answer: [],
-};
-
-export const useFetchFlowTemplates = (): ResponseType<IFlowTemplate[]> => {
-  const { t } = useTranslation();
-
-  const { data } = useQuery({
-    queryKey: ['fetchFlowTemplates'],
-    initialData: [],
-    queryFn: async () => {
-      const { data } = await flowService.listTemplates();
-      if (Array.isArray(data?.data)) {
-        data.data.unshift({
-          id: uuid(),
-          title: t('flow.blank'),
-          description: t('flow.createFromNothing'),
-          dsl: EmptyDsl,
-        });
-      }
-
-      return data;
-    },
-  });
-
-  return data;
-};
 
 export const useFetchFlowList = (): { data: IFlow[]; loading: boolean } => {
   const { data, isFetching: loading } = useQuery({

web/src/hooks/logic-hooks/navigate-hooks.ts

@@ -1,3 +1,4 @@
+import { NavigateToDataflowResultProps } from '@/pages/dataflow-result/interface';
 import { Routes } from '@/routes';
 import { useCallback } from 'react';
 import { useNavigate, useParams, useSearchParams } from 'umi';
@@ -18,7 +19,14 @@ export const useNavigatePage = () => {
 
   const navigateToDataset = useCallback(
     (id: string) => () => {
-      navigate(`${Routes.Dataset}/${id}`);
+      navigate(`${Routes.DatasetBase}${Routes.DataSetOverview}/${id}`);
+    },
+    [navigate],
+  );
+
+  const navigateToDataFile = useCallback(
+    (id: string) => () => {
+      navigate(`${Routes.DatasetBase}${Routes.DatasetBase}/${id}`);
     },
     [navigate],
   );
@@ -61,6 +69,13 @@ export const useNavigatePage = () => {
     [navigate],
   );
 
+  const navigateToDataflow = useCallback(
+    (id: string) => () => {
+      navigate(`${Routes.DataFlow}/${id}`);
+    },
+    [navigate],
+  );
+
   const navigateToAgentLogs = useCallback(
     (id: string) => () => {
       navigate(`${Routes.AgentLogPage}/${id}`);
@@ -86,8 +101,8 @@ export const useNavigatePage = () => {
   const navigateToChunkParsedResult = useCallback(
     (id: string, knowledgeId?: string) => () => {
       navigate(
-        // `${Routes.ParsedResult}/${id}?${QueryStringMap.KnowledgeId}=${knowledgeId}`,
         `${Routes.ParsedResult}/chunks?id=${knowledgeId}&doc_id=${id}`,
+        // `${Routes.DataflowResult}?id=${knowledgeId}&doc_id=${id}&type=chunk`,
       );
     },
     [navigate],
@@ -126,10 +141,16 @@ export const useNavigatePage = () => {
   );
 
   const navigateToDataflowResult = useCallback(
-    (id: string, knowledgeId?: string) => () => {
+    (props: NavigateToDataflowResultProps) => () => {
+      let params: string[] = [];
+      Object.keys(props).forEach((key) => {
+        if (props[key]) {
+          params.push(`${key}=${props[key]}`);
+        }
+      });
       navigate(
         // `${Routes.ParsedResult}/${id}?${QueryStringMap.KnowledgeId}=${knowledgeId}`,
-        `${Routes.DataflowResult}/${id}`,
+        `${Routes.DataflowResult}?${params.join('&')}`,
       );
     },
     [navigate],
@@ -155,5 +176,7 @@ export const useNavigatePage = () => {
     navigateToAgentList,
     navigateToOldProfile,
     navigateToDataflowResult,
+    navigateToDataflow,
+    navigateToDataFile,
   };
 };

web/src/hooks/route-hook.ts

@@ -29,6 +29,7 @@ export const useGetKnowledgeSearchParams = () => {
   const [currentQueryParameters] = useSearchParams();
 
   return {
+    type: currentQueryParameters.get(KnowledgeSearchParams.Type) || '',
     documentId:
       currentQueryParameters.get(KnowledgeSearchParams.DocumentId) || '',
     knowledgeId:

web/src/hooks/use-agent-request.ts

@@ -1,4 +1,5 @@
 import { FileUploadProps } from '@/components/file-upload';
+import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';
 import message from '@/components/ui/message';
 import { AgentGlobals } from '@/constants/agent';
 import {
@@ -7,6 +8,7 @@ import {
   IAgentLogsResponse,
   IFlow,
   IFlowTemplate,
+  IPipeLineListRequest,
   ITraceData,
 } from '@/interfaces/database/agent';
 import { IDebugSingleRequestBody } from '@/interfaces/request/agent';
@@ -16,6 +18,7 @@ import { IInputs } from '@/pages/agent/interface';
 import { useGetSharedChatSearchParams } from '@/pages/chat/shared-hooks';
 import agentService, {
   fetchAgentLogsByCanvasId,
+  fetchPipeLineList,
   fetchTrace,
 } from '@/services/agent-service';
 import api from '@/utils/api';
@@ -31,6 +34,7 @@ import {
 } from './logic-hooks';
 
 export const enum AgentApiAction {
+  FetchAgentListByPage = 'fetchAgentListByPage',
   FetchAgentList = 'fetchAgentList',
   UpdateAgentSetting = 'updateAgentSetting',
   DeleteAgent = 'deleteAgent',
@@ -50,6 +54,7 @@ export const enum AgentApiAction {
   FetchExternalAgentInputs = 'fetchExternalAgentInputs',
   SetAgentSetting = 'setAgentSetting',
   FetchPrompt = 'fetchPrompt',
+  CancelDataflow = 'cancelDataflow',
 }
 
 export const EmptyDsl = {
@@ -111,28 +116,47 @@ export const useFetchAgentListByPage = () => {
   const { searchString, handleInputChange } = useHandleSearchChange();
   const { pagination, setPagination } = useGetPaginationWithRouter();
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const { filterValue, handleFilterSubmit } = useHandleFilterSubmit();
+  const canvasCategory = Array.isArray(filterValue.canvasCategory)
+    ? filterValue.canvasCategory
+    : [];
+  const owner = filterValue.owner;
+
+  const requestParams: Record<string, any> = {
+    keywords: debouncedSearchString,
+    page_size: pagination.pageSize,
+    page: pagination.current,
+    canvas_category:
+      canvasCategory.length === 1 ? canvasCategory[0] : undefined,
+  };
+
+  if (Array.isArray(owner) && owner.length > 0) {
+    requestParams.owner_ids = owner.join(',');
+  }
 
   const { data, isFetching: loading } = useQuery<{
     canvas: IFlow[];
     total: number;
   }>({
     queryKey: [
-      AgentApiAction.FetchAgentList,
+      AgentApiAction.FetchAgentListByPage,
       {
         debouncedSearchString,
         ...pagination,
+        filterValue,
       },
     ],
-    initialData: { canvas: [], total: 0 },
+    placeholderData: (previousData) => {
+      if (previousData === undefined) {
+        return { canvas: [], total: 0 };
+      }
+      return previousData;
+    },
     gcTime: 0,
     queryFn: async () => {
-      const { data } = await agentService.listCanvasTeam(
+      const { data } = await agentService.listCanvas(
         {
-          params: {
-            keywords: debouncedSearchString,
-            page_size: pagination.pageSize,
-            page: pagination.current,
-          },
+          params: requestParams,
         },
         true,
       );
@@ -150,12 +174,14 @@ export const useFetchAgentListByPage = () => {
   );
 
   return {
-    data: data.canvas,
+    data: data?.canvas ?? [],
     loading,
     searchString,
     handleInputChange: onInputChange,
     pagination: { ...pagination, total: data?.total },
     setPagination,
+    filterValue,
+    handleFilterSubmit,
   };
 };
 
@@ -173,7 +199,7 @@ export const useUpdateAgentSetting = () => {
       if (ret?.data?.code === 0) {
         message.success('success');
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       } else {
         message.error(ret?.data?.data);
@@ -197,7 +223,7 @@ export const useDeleteAgent = () => {
       const { data } = await agentService.removeCanvas({ canvasIds });
       if (data.code === 0) {
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       }
       return data?.data ?? [];
@@ -271,6 +297,7 @@ export const useSetAgent = (showMessage: boolean = true) => {
       title?: string;
       dsl?: DSL;
       avatar?: string;
+      canvas_category?: string;
     }) => {
       const { data = {} } = await agentService.setCanvas(params);
       if (data.code === 0) {
@@ -280,7 +307,7 @@ export const useSetAgent = (showMessage: boolean = true) => {
           );
         }
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       }
       return data;
@@ -379,7 +406,7 @@ export const useUploadCanvasFileWithProgress = (
         files.forEach((file) => {
           onError(file, error as Error);
         });
-        message.error(error?.message);
+        message.error((error as Error)?.message || 'Upload failed');
       }
     },
   });
@@ -387,13 +414,11 @@ export const useUploadCanvasFileWithProgress = (
   return { data, loading, uploadCanvasFile: mutateAsync };
 };
 
-export const useFetchMessageTrace = (
-  isStopFetchTrace: boolean,
-  canvasId?: string,
-) => {
+export const useFetchMessageTrace = (canvasId?: string) => {
   const { id } = useParams();
   const queryId = id || canvasId;
   const [messageId, setMessageId] = useState('');
+  const [isStopFetchTrace, setISStopFetchTrace] = useState(false);
 
   const {
     data,
@@ -413,11 +438,19 @@ export const useFetchMessageTrace = (
         message_id: messageId,
       });
 
-      return data?.data ?? [];
+      return Array.isArray(data?.data) ? data?.data : [];
     },
   });
 
-  return { data, loading, refetch, setMessageId };
+  return {
+    data,
+    loading,
+    refetch,
+    setMessageId,
+    messageId,
+    isStopFetchTrace,
+    setISStopFetchTrace,
+  };
 };
 
 export const useTestDbConnect = () => {
@@ -563,7 +596,6 @@ export const useFetchAgentLog = (searchParams: IAgentLogsRequest) => {
     initialData: {} as IAgentLogsResponse,
     gcTime: 0,
     queryFn: async () => {
-      console.log('useFetchAgentLog', searchParams);
       const { data } = await fetchAgentLogsByCanvasId(id as string, {
         ...searchParams,
       });
@@ -647,3 +679,59 @@ export const useFetchPrompt = () => {
 
   return { data, loading, refetch };
 };
+
+export const useFetchAgentList = ({
+  canvas_category,
+}: IPipeLineListRequest) => {
+  const { data, isFetching: loading } = useQuery<{
+    canvas: IFlow[];
+    total: number;
+  }>({
+    queryKey: [AgentApiAction.FetchAgentList],
+    initialData: { canvas: [], total: 0 },
+    gcTime: 0,
+    queryFn: async () => {
+      const { data } = await fetchPipeLineList({ canvas_category });
+
+      return data?.data ?? [];
+    },
+  });
+
+  return { data, loading };
+};
+
+export const useCancelDataflow = () => {
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [AgentApiAction.CancelDataflow],
+    mutationFn: async (taskId: string) => {
+      const ret = await agentService.cancelDataflow(taskId);
+      if (ret?.data?.code === 0) {
+        message.success('success');
+      } else {
+        message.error(ret?.data?.data);
+      }
+      return ret?.data?.code;
+    },
+  });
+
+  return { data, loading, cancelDataflow: mutateAsync };
+};
+
+// export const useFetchKnowledgeList = () => {
+//   const { data, isFetching: loading } = useQuery<IFlow[]>({
+//     queryKey: [AgentApiAction.FetchAgentList],
+//     initialData: [],
+//     gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
+//     queryFn: async () => {
+//       const { data } = await agentService.listCanvas();
+
+//       return data?.data ?? [];
+//     },
+//   });
+
+//   return { list: data, loading };
+// };

web/src/hooks/use-chunk-request.ts

@@ -13,7 +13,9 @@ import {
 } from './logic-hooks';
 import { useGetKnowledgeSearchParams } from './route-hook';
 
-export const useFetchNextChunkList = (): ResponseGetType<{
+export const useFetchNextChunkList = (
+  enabled = true,
+): ResponseGetType<{
   data: IChunk[];
   total: number;
   documentInfo: IKnowledgeFile;
@@ -37,6 +39,7 @@ export const useFetchNextChunkList = (): ResponseGetType<{
     placeholderData: (previousData: any) =>
       previousData ?? { data: [], total: 0, documentInfo: {} }, // https://github.com/TanStack/query/issues/8183
     gcTime: 0,
+    enabled,
     queryFn: async () => {
       const { data } = await kbService.chunk_list({
         doc_id: documentId,

web/src/hooks/use-dataflow-request.ts

@@ -0,0 +1,91 @@
+import message from '@/components/ui/message';
+import { IFlow } from '@/interfaces/database/agent';
+import dataflowService from '@/services/dataflow-service';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { useTranslation } from 'react-i18next';
+import { useParams } from 'umi';
+
+export const enum DataflowApiAction {
+  ListDataflow = 'listDataflow',
+  RemoveDataflow = 'removeDataflow',
+  FetchDataflow = 'fetchDataflow',
+  RunDataflow = 'runDataflow',
+  SetDataflow = 'setDataflow',
+}
+
+export const useRemoveDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.RemoveDataflow],
+    mutationFn: async (ids: string[]) => {
+      const { data } = await dataflowService.removeDataflow({
+        canvas_ids: ids,
+      });
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.ListDataflow],
+        });
+
+        message.success(t('message.deleted'));
+      }
+      return data.code;
+    },
+  });
+
+  return { data, loading, removeDataflow: mutateAsync };
+};
+
+export const useSetDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.SetDataflow],
+    mutationFn: async (params: Partial<IFlow>) => {
+      const { data } = await dataflowService.setDataflow(params);
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.FetchDataflow],
+        });
+
+        message.success(t(`message.${params.id ? 'modified' : 'created'}`));
+      }
+      return data?.code;
+    },
+  });
+
+  return { data, loading, setDataflow: mutateAsync };
+};
+
+export const useFetchDataflow = () => {
+  const { id } = useParams();
+
+  const {
+    data,
+    isFetching: loading,
+    refetch,
+  } = useQuery<IFlow>({
+    queryKey: [DataflowApiAction.FetchDataflow, id],
+    gcTime: 0,
+    initialData: {} as IFlow,
+    enabled: !!id,
+    refetchOnWindowFocus: false,
+    queryFn: async () => {
+      const { data } = await dataflowService.fetchDataflow(id);
+
+      return data?.data ?? ({} as IFlow);
+    },
+  });
+
+  return { data, loading, refetch };
+};

web/src/hooks/use-document-request.ts

@@ -335,15 +335,18 @@ export const useSetDocumentParser = () => {
     mutationKey: [DocumentApiAction.SetDocumentParser],
     mutationFn: async ({
       parserId,
+      pipelineId,
       documentId,
       parserConfig,
     }: {
       parserId: string;
+      pipelineId: string;
       documentId: string;
       parserConfig: IChangeParserConfigRequestBody;
     }) => {
       const { data } = await kbService.document_change_parser({
         parser_id: parserId,
+        pipeline_id: pipelineId,
         doc_id: documentId,
         parser_config: parserConfig,
       });

web/src/hooks/use-knowledge-request.ts

@@ -31,6 +31,7 @@ export const enum KnowledgeApiAction {
   FetchKnowledgeDetail = 'fetchKnowledgeDetail',
   FetchKnowledgeGraph = 'fetchKnowledgeGraph',
   FetchMetadata = 'fetchMetadata',
+  FetchKnowledgeList = 'fetchKnowledgeList',
   RemoveKnowledgeGraph = 'removeKnowledgeGraph',
 }
 
@@ -238,7 +239,11 @@ export const useUpdateKnowledge = (shouldFetchList = false) => {
   return { data, loading, saveKnowledgeConfiguration: mutateAsync };
 };
 
-export const useFetchKnowledgeBaseConfiguration = (refreshCount?: number) => {
+export const useFetchKnowledgeBaseConfiguration = (props?: {
+  isEdit?: boolean;
+  refreshCount?: number;
+}) => {
+  const { isEdit = true, refreshCount } = props || { isEdit: true };
   const { id } = useParams();
   const [searchParams] = useSearchParams();
   const knowledgeBaseId = searchParams.get('id') || id;
@@ -255,10 +260,14 @@ export const useFetchKnowledgeBaseConfiguration = (refreshCount?: number) => {
     initialData: {} as IKnowledge,
     gcTime: 0,
     queryFn: async () => {
-      const { data } = await kbService.get_kb_detail({
-        kb_id: knowledgeBaseId,
-      });
-      return data?.data ?? {};
+      if (isEdit) {
+        const { data } = await kbService.get_kb_detail({
+          kb_id: knowledgeBaseId,
+        });
+        return data?.data ?? {};
+      } else {
+        return {};
+      }
     },
   });
 
@@ -323,3 +332,25 @@ export const useRemoveKnowledgeGraph = () => {
 
   return { data, loading, removeKnowledgeGraph: mutateAsync };
 };
+
+export const useFetchKnowledgeList = (
+  shouldFilterListWithoutDocument: boolean = false,
+): {
+  list: IKnowledge[];
+  loading: boolean;
+} => {
+  const { data, isFetching: loading } = useQuery({
+    queryKey: [KnowledgeApiAction.FetchKnowledgeList],
+    initialData: [],
+    gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
+    queryFn: async () => {
+      const { data } = await listDataset();
+      const list = data?.data?.kbs ?? [];
+      return shouldFilterListWithoutDocument
+        ? list.filter((x: IKnowledge) => x.chunk_num > 0)
+        : list;
+    },
+  });
+
+  return { list: data, loading };
+};