Feat: Displays the loading status of the data flow log #9869 (#10347)

### What problem does this PR solve?

Feat: Displays the loading status of the data flow log #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

api/db/services/pipeline_operation_log_service.py

@@ -15,6 +15,7 @@
 #
 import json
 import logging
+import os
 from datetime import datetime, timedelta
 
 from peewee import fn
@@ -81,9 +82,17 @@ class PipelineOperationLogService(CommonService):
             cls.model.update_date,
         ]
 
+    @classmethod
+    def save(cls, **kwargs):
+        """
+        wrap this function in a transaction
+        """
+        sample_obj = cls.model(**kwargs).save(force_insert=True)
+        return sample_obj
+
     @classmethod
     @DB.connection_context()
-    def create(cls, document_id, pipeline_id, task_type, fake_document_ids=[], dsl:str="{}"):
+    def create(cls, document_id, pipeline_id, task_type, fake_document_ids=[], dsl: str = "{}"):
         referred_document_id = document_id
 
         if referred_document_id == GRAPH_RAPTOR_FAKE_DOC_ID and fake_document_ids:
@@ -163,7 +172,19 @@ class PipelineOperationLogService(CommonService):
         log["create_date"] = datetime_format(datetime.now())
         log["update_time"] = current_timestamp()
         log["update_date"] = datetime_format(datetime.now())
-        obj = cls.save(**log)
+
+        with DB.atomic():
+            obj = cls.save(**log)
+
+            limit = int(os.getenv("PIPELINE_OPERATION_LOG_LIMIT", 1))
+            total = cls.model.select().where(cls.model.kb_id == document.kb_id).count()
+
+            if total > limit:
+                keep_ids = [m.id for m in cls.model.select(cls.model.id).where(cls.model.kb_id == document.kb_id).order_by(cls.model.create_time.desc()).limit(limit)]
+
+                deleted = cls.model.delete().where(cls.model.kb_id == document.kb_id, cls.model.id.not_in(keep_ids)).execute()
+                logging.info(f"[PipelineOperationLogService] Cleaned {deleted} old logs, kept latest {limit} for {document.kb_id}")
+
         return obj
 
     @classmethod

web/src/components/data-pipeline-select/index.tsx

@@ -1,3 +1,4 @@
+import { AgentCategory } from '@/constants/agent';
 import { useTranslate } from '@/hooks/common-hooks';
 import { useFetchAgentList } from '@/hooks/use-agent-request';
 import { buildSelectOptions } from '@/utils/component-util';
@@ -33,8 +34,8 @@ export function DataFlowSelect(props: IProps) {
   const toDataPipLine = () => {
     toDataPipeline?.();
   };
-  const { data: dataPipelineOptions, loading } = useFetchAgentList({
-    canvas_category: 'dataflow_canvas',
+  const { data: dataPipelineOptions } = useFetchAgentList({
+    canvas_category: AgentCategory.DataflowCanvas,
   });
   const options = useMemo(() => {
     const option = buildSelectOptions(

web/src/components/parse-configuration/graph-rag-form-fields.tsx

@@ -53,12 +53,15 @@ type GraphRagItemsProps = {
   marginBottom?: boolean;
   className?: string;
   data: IGenerateLogButtonProps;
+  onDelete?: () => void;
 };
 
 export function UseGraphRagFormField({
   data,
+  onDelete,
 }: {
   data: IGenerateLogButtonProps;
+  onDelete?: () => void;
 }) {
   const form = useFormContext();
   const { t } = useTranslate('knowledgeConfiguration');
@@ -84,6 +87,7 @@ export function UseGraphRagFormField({
                 ></Switch> */}
                 <GenerateLogButton
                   {...data}
+                  onDelete={onDelete}
                   className="w-full text-text-secondary"
                   status={1}
                   type={GenerateType.KnowledgeGraph}
@@ -106,6 +110,7 @@ const GraphRagItems = ({
   marginBottom = false,
   className = 'p-10',
   data,
+  onDelete,
 }: GraphRagItemsProps) => {
   const { t } = useTranslate('knowledgeConfiguration');
   const form = useFormContext();
@@ -131,7 +136,10 @@ const GraphRagItems = ({
 
   return (
     <FormContainer className={cn({ 'mb-4': marginBottom }, className)}>
-      <UseGraphRagFormField data={data}></UseGraphRagFormField>
+      <UseGraphRagFormField
+        data={data}
+        onDelete={onDelete}
+      ></UseGraphRagFormField>
       {useRaptor && (
         <>
           <EntityTypesFormField name="parser_config.graphrag.entity_types"></EntityTypesFormField>

web/src/components/parse-configuration/raptor-form-fields.tsx

@@ -56,7 +56,13 @@ const Prompt = 'parser_config.raptor.prompt';
 
 // The three types "table", "resume" and "one" do not display this configuration.
 
-const RaptorFormFields = ({ data }: { data: IGenerateLogButtonProps }) => {
+const RaptorFormFields = ({
+  data,
+  onDelete,
+}: {
+  data: IGenerateLogButtonProps;
+  onDelete: () => void;
+}) => {
   const form = useFormContext();
   const { t } = useTranslate('knowledgeConfiguration');
   const useRaptor = useWatch({ name: UseRaptorField });
@@ -106,6 +112,7 @@ const RaptorFormFields = ({ data }: { data: IGenerateLogButtonProps }) => {
                   <FormControl>
                     <GenerateLogButton
                       {...data}
+                      onDelete={onDelete}
                       className="w-full text-text-secondary"
                       status={1}
                       type={GenerateType.Raptor}

web/src/constants/agent.ts

@@ -47,3 +47,8 @@ export const initialLlmBaseValues = {
   presence_penalty: 0.4,
   max_tokens: 256,
 };
+
+export enum AgentCategory {
+  AgentCanvas = 'agent_canvas',
+  DataflowCanvas = 'dataflow_canvas',
+}

web/src/hooks/use-agent-request.ts

@@ -1,4 +1,5 @@
 import { FileUploadProps } from '@/components/file-upload';
+import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';
 import message from '@/components/ui/message';
 import { AgentGlobals } from '@/constants/agent';
 import {
@@ -33,6 +34,7 @@ import {
 } from './logic-hooks';
 
 export const enum AgentApiAction {
+  FetchAgentListByPage = 'fetchAgentListByPage',
   FetchAgentList = 'fetchAgentList',
   UpdateAgentSetting = 'updateAgentSetting',
   DeleteAgent = 'deleteAgent',
@@ -114,16 +116,36 @@ export const useFetchAgentListByPage = () => {
   const { searchString, handleInputChange } = useHandleSearchChange();
   const { pagination, setPagination } = useGetPaginationWithRouter();
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const { filterValue, handleFilterSubmit } = useHandleFilterSubmit();
+  const canvasCategory = Array.isArray(filterValue.canvasCategory)
+    ? filterValue.canvasCategory
+    : [];
+  const owner = filterValue.owner;
+
+  const requestParams = {
+    keywords: debouncedSearchString,
+    page_size: pagination.pageSize,
+    page: pagination.current,
+    canvas_category:
+      canvasCategory.length === 1 ? canvasCategory[0] : undefined,
+    owner_ids: '',
+  };
+
+  if (Array.isArray(owner) && owner.length > 0) {
+    requestParams.owner_ids =
+      `${owner[0]}` + owner.slice(1).map((id) => `&owner_ids=${id}`);
+  }
 
   const { data, isFetching: loading } = useQuery<{
     canvas: IFlow[];
     total: number;
   }>({
     queryKey: [
-      AgentApiAction.FetchAgentList,
+      AgentApiAction.FetchAgentListByPage,
       {
         debouncedSearchString,
         ...pagination,
+        filterValue,
       },
     ],
     placeholderData: (previousData) => {
@@ -134,13 +156,9 @@ export const useFetchAgentListByPage = () => {
     },
     gcTime: 0,
     queryFn: async () => {
-      const { data } = await agentService.listCanvasTeam(
+      const { data } = await agentService.listCanvas(
         {
-          params: {
-            keywords: debouncedSearchString,
-            page_size: pagination.pageSize,
-            page: pagination.current,
-          },
+          params: requestParams,
         },
         true,
       );
@@ -164,6 +182,8 @@ export const useFetchAgentListByPage = () => {
     handleInputChange: onInputChange,
     pagination: { ...pagination, total: data?.total },
     setPagination,
+    filterValue,
+    handleFilterSubmit,
   };
 };
 
@@ -181,7 +201,7 @@ export const useUpdateAgentSetting = () => {
       if (ret?.data?.code === 0) {
         message.success('success');
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       } else {
         message.error(ret?.data?.data);
@@ -205,7 +225,7 @@ export const useDeleteAgent = () => {
       const { data } = await agentService.removeCanvas({ canvasIds });
       if (data.code === 0) {
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       }
       return data?.data ?? [];
@@ -289,7 +309,7 @@ export const useSetAgent = (showMessage: boolean = true) => {
           );
         }
         queryClient.invalidateQueries({
-          queryKey: [AgentApiAction.FetchAgentList],
+          queryKey: [AgentApiAction.FetchAgentListByPage],
         });
       }
       return data;
@@ -396,13 +416,11 @@ export const useUploadCanvasFileWithProgress = (
   return { data, loading, uploadCanvasFile: mutateAsync };
 };
 
-export const useFetchMessageTrace = (
-  isStopFetchTrace: boolean,
-  canvasId?: string,
-) => {
+export const useFetchMessageTrace = (canvasId?: string) => {
   const { id } = useParams();
   const queryId = id || canvasId;
   const [messageId, setMessageId] = useState('');
+  const [isStopFetchTrace, setISStopFetchTrace] = useState(false);
 
   const {
     data,
@@ -422,11 +440,19 @@ export const useFetchMessageTrace = (
         message_id: messageId,
       });
 
-      return data?.data ?? [];
+      return Array.isArray(data?.data) ? data?.data : [];
     },
   });
 
-  return { data, loading, refetch, setMessageId, messageId };
+  return {
+    data,
+    loading,
+    refetch,
+    setMessageId,
+    messageId,
+    isStopFetchTrace,
+    setISStopFetchTrace,
+  };
 };
 
 export const useTestDbConnect = () => {
@@ -657,17 +683,14 @@ export const useFetchPrompt = () => {
 };
 
 export const useFetchAgentList = ({
-  canvas_category = 'agent_canvas',
-}: IPipeLineListRequest): {
-  data: {
+  canvas_category,
+}: IPipeLineListRequest) => {
+  const { data, isFetching: loading } = useQuery<{
     canvas: IFlow[];
     total: number;
-  };
-  loading: boolean;
-} => {
-  const { data, isFetching: loading } = useQuery({
-    queryKey: ['fetchPipeLineList'],
-    initialData: [],
+  }>({
+    queryKey: [AgentApiAction.FetchAgentList],
+    initialData: { canvas: [], total: 0 },
     gcTime: 0,
     queryFn: async () => {
       const { data } = await fetchPipeLineList({ canvas_category });
@@ -699,3 +722,18 @@ export const useCancelDataflow = () => {
 
   return { data, loading, cancelDataflow: mutateAsync };
 };
+
+// export const useFetchKnowledgeList = () => {
+//   const { data, isFetching: loading } = useQuery<IFlow[]>({
+//     queryKey: [AgentApiAction.FetchAgentList],
+//     initialData: [],
+//     gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
+//     queryFn: async () => {
+//       const { data } = await agentService.listCanvas();
+
+//       return data?.data ?? [];
+//     },
+//   });
+
+//   return { list: data, loading };
+// };

web/src/hooks/use-knowledge-request.ts

@@ -30,6 +30,7 @@ export const enum KnowledgeApiAction {
   FetchKnowledgeDetail = 'fetchKnowledgeDetail',
   FetchKnowledgeGraph = 'fetchKnowledgeGraph',
   FetchMetadata = 'fetchMetadata',
+  FetchKnowledgeList = 'fetchKnowledgeList',
 }
 
 export const useKnowledgeBaseId = (): string => {
@@ -304,3 +305,25 @@ export function useFetchKnowledgeMetadata(kbIds: string[] = []) {
 
   return { data, loading };
 }
+
+export const useFetchKnowledgeList = (
+  shouldFilterListWithoutDocument: boolean = false,
+): {
+  list: IKnowledge[];
+  loading: boolean;
+} => {
+  const { data, isFetching: loading } = useQuery({
+    queryKey: [KnowledgeApiAction.FetchKnowledgeList],
+    initialData: [],
+    gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
+    queryFn: async () => {
+      const { data } = await listDataset();
+      const list = data?.data?.kbs ?? [];
+      return shouldFilterListWithoutDocument
+        ? list.filter((x: IKnowledge) => x.chunk_num > 0)
+        : list;
+    },
+  });
+
+  return { list: data, loading };
+};

web/src/interfaces/database/agent.ts

@@ -30,6 +30,7 @@ export interface ISwitchForm {
   no: string;
 }
 
+import { AgentCategory } from '@/constants/agent';
 import { Edge, Node } from '@xyflow/react';
 import { IReference, Message } from './chat';
 
@@ -273,5 +274,5 @@ export interface IPipeLineListRequest {
   keywords?: string;
   orderby?: string;
   desc?: boolean;
-  canvas_category?: 'agent_canvas' | 'dataflow_canvas';
+  canvas_category?: AgentCategory;
 }

web/src/interfaces/database/document.ts

@@ -5,6 +5,7 @@ export interface IDocumentInfo {
   create_date: string;
   create_time: number;
   created_by: string;
+  nickname: string;
   id: string;
   kb_id: string;
   location: string;

web/src/locales/en.ts

@@ -1771,6 +1771,12 @@ Important structured information may include: names, dates, locations, events, k
       cancel: 'Cancel',
       swicthPromptMessage:
         'The prompt word will change. Please confirm whether to abandon the existing prompt word?',
+      tokenizerFieldsOptions: {
+        text: 'Text',
+        keywords: 'Keywords',
+        questions: 'Questions',
+        summary: 'Augmented Context',
+      },
     },
   },
 };

web/src/pages/agent/constant.tsx

@@ -924,8 +924,3 @@ export enum AgentExceptionMethod {
   Comment = 'comment',
   Goto = 'goto',
 }
-
-export enum AgentCategory {
-  AgentCanvas = 'agent_canvas',
-  DataflowCanvas = 'dataflow_canvas',
-}

web/src/pages/agent/log-sheet/workflow-timeline.tsx

@@ -23,7 +23,7 @@ import { ITraceData } from '@/interfaces/database/agent';
 import { cn } from '@/lib/utils';
 import { t } from 'i18next';
 import { get, isEmpty, isEqual, uniqWith } from 'lodash';
-import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo } from 'react';
 import JsonView from 'react18-json-view';
 import { Operator } from '../constant';
 import { useCacheChatLog } from '../hooks/use-cache-chat-log';
@@ -116,12 +116,12 @@ export const WorkFlowTimeline = ({
   isShare,
 }: LogFlowTimelineProps) => {
   // const getNode = useGraphStore((state) => state.getNode);
-  const [isStopFetchTrace, setISStopFetchTrace] = useState(false);
 
-  const { data: traceData, setMessageId } = useFetchMessageTrace(
-    isStopFetchTrace,
-    canvasId,
-  );
+  const {
+    data: traceData,
+    setMessageId,
+    setISStopFetchTrace,
+  } = useFetchMessageTrace(canvasId);
 
   useEffect(() => {
     setMessageId(currentMessageId);
@@ -133,7 +133,7 @@ export const WorkFlowTimeline = ({
 
   useEffect(() => {
     setISStopFetchTrace(!sendLoading);
-  }, [sendLoading]);
+  }, [sendLoading, setISStopFetchTrace]);
 
   const startedNodeList = useMemo(() => {
     const finish = currentEventListWithoutMessage?.some(
@@ -151,7 +151,7 @@ export const WorkFlowTimeline = ({
       }
       return pre;
     }, []);
-  }, [currentEventListWithoutMessage, sendLoading]);
+  }, [currentEventListWithoutMessage, sendLoading, setISStopFetchTrace]);
 
   const getElapsedTime = (nodeId: string) => {
     if (nodeId === 'begin') {

web/src/pages/agents/agent-card.tsx

@@ -2,10 +2,10 @@ import { HomeCard } from '@/components/home-card';
 import { MoreButton } from '@/components/more-button';
 import { SharedBadge } from '@/components/shared-badge';
 import { Button } from '@/components/ui/button';
+import { AgentCategory } from '@/constants/agent';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { IFlow } from '@/interfaces/database/agent';
-import { DatabaseZap } from 'lucide-react';
-import { AgentCategory } from '../agent/constant';
+import { Route } from 'lucide-react';
 import { AgentDropdown } from './agent-dropdown';
 import { useRenameAgent } from './use-rename-agent';
 
@@ -33,7 +33,7 @@ export function AgentCard({ data, showAgentRenameModal }: DatasetCardProps) {
       icon={
         data.canvas_category === AgentCategory.DataflowCanvas && (
           <Button variant={'ghost'} size={'sm'}>
-            <DatabaseZap />
+            <Route />
           </Button>
         )
       }

web/src/pages/agents/hooks/use-create-agent.ts

@@ -1,7 +1,7 @@
+import { AgentCategory } from '@/constants/agent';
 import { useSetModalState } from '@/hooks/common-hooks';
 import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request';
 import { DSL } from '@/interfaces/database/agent';
-import { AgentCategory } from '@/pages/agent/constant';
 import {
   BeginId,
   Operator,

web/src/pages/agents/hooks/use-selelct-filters.ts

@@ -0,0 +1,23 @@
+import { FilterCollection } from '@/components/list-filter-bar/interface';
+import { useFetchAgentList } from '@/hooks/use-agent-request';
+import { buildOwnersFilter, groupListByType } from '@/utils/list-filter-util';
+import { useMemo } from 'react';
+
+export function useSelectFilters() {
+  const { data } = useFetchAgentList({});
+
+  const canvasCategory = useMemo(() => {
+    return groupListByType(data.canvas, 'canvas_category', 'canvas_category');
+  }, [data.canvas]);
+
+  const filters: FilterCollection[] = [
+    buildOwnersFilter(data.canvas),
+    {
+      field: 'canvasCategory',
+      list: canvasCategory,
+      label: 'Canvas category',
+    },
+  ];
+
+  return filters;
+}

web/src/pages/agents/index.tsx

@@ -17,13 +17,21 @@ import { useCallback } from 'react';
 import { AgentCard } from './agent-card';
 import { CreateAgentDialog } from './create-agent-dialog';
 import { useCreateAgentOrPipeline } from './hooks/use-create-agent';
+import { useSelectFilters } from './hooks/use-selelct-filters';
 import { UploadAgentDialog } from './upload-agent-dialog';
 import { useHandleImportJsonFile } from './use-import-json';
 import { useRenameAgent } from './use-rename-agent';
 
 export default function Agents() {
-  const { data, pagination, setPagination, searchString, handleInputChange } =
-    useFetchAgentListByPage();
+  const {
+    data,
+    pagination,
+    setPagination,
+    searchString,
+    handleInputChange,
+    filterValue,
+    handleFilterSubmit,
+  } = useFetchAgentListByPage();
   const { navigateToAgentTemplates } = useNavigatePage();
 
   const {
@@ -50,6 +58,8 @@ export default function Agents() {
     hideFileUploadModal,
   } = useHandleImportJsonFile();
 
+  const filters = useSelectFilters();
+
   const handlePageChange = useCallback(
     (page: number, pageSize?: number) => {
       setPagination({ page, pageSize });
@@ -65,6 +75,9 @@ export default function Agents() {
           searchString={searchString}
           onSearchChange={handleInputChange}
           icon="agent"
+          filters={filters}
+          onChange={handleFilterSubmit}
+          value={filterValue}
         >
           <DropdownMenu>
             <DropdownMenuTrigger>

web/src/pages/data-flow/form/tokenizer-form/index.tsx

@@ -31,12 +31,16 @@ export const FormSchema = z.object({
 
 const SearchMethodOptions = buildOptions(TokenizerSearchMethod);
 
-const FieldsOptions = buildOptions(TokenizerFields);
-
 const TokenizerForm = ({ node }: INextOperatorForm) => {
   const { t } = useTranslation();
   const defaultValues = useFormValues(initialTokenizerValues, node);
 
+  const FieldsOptions = buildOptions(
+    TokenizerFields,
+    t,
+    'dataflow.tokenizerFieldsOptions',
+  );
+
   const form = useForm<z.infer<typeof FormSchema>>({
     defaultValues,
     resolver: zodResolver(FormSchema),

web/src/pages/data-flow/hooks/use-cancel-dataflow.ts

@@ -3,19 +3,19 @@ import { useCallback } from 'react';
 
 export function useCancelCurrentDataflow({
   messageId,
-  setMessageId,
+  stopFetchTrace,
 }: {
   messageId: string;
-  setMessageId: (messageId: string) => void;
+  stopFetchTrace(): void;
 }) {
   const { cancelDataflow } = useCancelDataflow();
 
   const handleCancel = useCallback(async () => {
     const code = await cancelDataflow(messageId);
     if (code === 0) {
-      setMessageId('');
+      stopFetchTrace();
     }
-  }, [cancelDataflow, messageId, setMessageId]);
+  }, [cancelDataflow, messageId, stopFetchTrace]);
 
   return { handleCancel };
 }

web/src/pages/data-flow/hooks/use-fetch-log.ts

@@ -1,10 +1,16 @@
 import { useFetchMessageTrace } from '@/hooks/use-agent-request';
 import { isEmpty } from 'lodash';
-import { useMemo } from 'react';
+import { useCallback, useEffect, useMemo } from 'react';
 
-export function useFetchLog() {
-  const { setMessageId, data, loading, messageId } =
-    useFetchMessageTrace(false);
+export function useFetchLog(logSheetVisible: boolean) {
+  const {
+    setMessageId,
+    data,
+    loading,
+    messageId,
+    setISStopFetchTrace,
+    isStopFetchTrace,
+  } = useFetchMessageTrace();
 
   const isCompleted = useMemo(() => {
     if (Array.isArray(data)) {
@@ -13,18 +19,38 @@ export function useFetchLog() {
         latest?.component_id === 'END' && !isEmpty(latest?.trace[0].message)
       );
     }
-    return true;
+    return false;
   }, [data]);
 
   const isLogEmpty = !data || !data.length;
 
+  const stopFetchTrace = useCallback(() => {
+    setISStopFetchTrace(true);
+  }, [setISStopFetchTrace]);
+
+  // cancel request
+  useEffect(() => {
+    if (isCompleted) {
+      stopFetchTrace();
+    }
+  }, [isCompleted, stopFetchTrace]);
+
+  useEffect(() => {
+    if (logSheetVisible) {
+      setISStopFetchTrace(false);
+    }
+  }, [logSheetVisible, setISStopFetchTrace]);
+
   return {
-    data,
+    logs: data,
     isLogEmpty,
     isCompleted,
     loading,
-    isParsing: !isLogEmpty && !isCompleted,
+    isParsing: !isLogEmpty && !isCompleted && !isStopFetchTrace,
     messageId,
     setMessageId,
+    stopFetchTrace,
   };
 }
+
+export type UseFetchLogReturnType = ReturnType<typeof useFetchLog>;

web/src/pages/data-flow/index.tsx

@@ -89,20 +89,29 @@ export default function DataFlow() {
     hideModal: hideLogSheet,
   } = useSetModalState();
 
-  const { isParsing, data, messageId, setMessageId } = useFetchLog();
+  const {
+    isParsing,
+    logs,
+    messageId,
+    setMessageId,
+    isCompleted,
+    stopFetchTrace,
+    isLogEmpty,
+  } = useFetchLog(logSheetVisible);
 
   const handleRunAgent = useCallback(() => {
     if (isParsing) {
       // show log sheet
       showLogSheet();
     } else {
+      hideLogSheet();
       handleRun();
     }
-  }, [handleRun, isParsing, showLogSheet]);
+  }, [handleRun, hideLogSheet, isParsing, showLogSheet]);
 
   const { handleCancel } = useCancelCurrentDataflow({
     messageId,
-    setMessageId,
+    stopFetchTrace,
   });
 
   const time = useWatchAgentChange(chatDrawerVisible);
@@ -139,7 +148,6 @@ export default function DataFlow() {
           <ButtonLoading
             variant={'secondary'}
             onClick={handleRunAgent}
-            disabled={isParsing}
             loading={running}
           >
             {running || (
@@ -199,7 +207,9 @@ export default function DataFlow() {
         <LogSheet
           hideModal={hideLogSheet}
           isParsing={isParsing}
-          logs={data}
+          isCompleted={isCompleted}
+          isLogEmpty={isLogEmpty}
+          logs={logs}
           handleCancel={handleCancel}
         ></LogSheet>
       )}

web/src/pages/data-flow/log-sheet/dataflow-timeline.tsx

@@ -10,6 +10,7 @@ import {
 import { Progress } from '@/components/ui/progress';
 import { ITraceData } from '@/interfaces/database/agent';
 import { cn } from '@/lib/utils';
+import { isEmpty } from 'lodash';
 import { File } from 'lucide-react';
 import { useCallback } from 'react';
 import { Operator } from '../constant';
@@ -82,25 +83,29 @@ export function DataflowTimeline({ traceList }: DataflowTimelineProps) {
                       </div>
                     </section>
                     <div className="divide-y space-y-1">
-                      {traces.map((x, idx) => (
-                        <section
-                          key={idx}
-                          className="text-text-secondary text-xs space-x-2 py-2.5 !m-0"
-                        >
-                          <span>{x.datetime}</span>
-                          {item.component_id !== 'END' && (
-                            <span
-                              className={cn({
-                                'text-state-error':
-                                  x.message.startsWith('[ERROR]'),
-                              })}
-                            >
-                              {x.message}
+                      {traces
+                        .filter((x) => !isEmpty(x.message))
+                        .map((x, idx) => (
+                          <section
+                            key={idx}
+                            className="text-text-secondary text-xs space-x-2 py-2.5 !m-0"
+                          >
+                            <span>{x.datetime}</span>
+                            {item.component_id !== 'END' && (
+                              <span
+                                className={cn({
+                                  'text-state-error':
+                                    x.message.startsWith('[ERROR]'),
+                                })}
+                              >
+                                {x.message}
+                              </span>
+                            )}
+                            <span>
+                              {x.elapsed_time.toString().slice(0, 6)}s
                             </span>
-                          )}
-                          <span>{x.elapsed_time.toString().slice(0, 6)}s</span>
-                        </section>
-                      ))}
+                          </section>
+                        ))}
                     </div>
                   </TimelineContent>
                 </TimelineTitle>

web/src/pages/data-flow/log-sheet/index.tsx

@@ -1,3 +1,4 @@
+import { SkeletonCard } from '@/components/skeleton-card';
 import { Button } from '@/components/ui/button';
 import {
   Sheet,
@@ -6,7 +7,6 @@ import {
   SheetTitle,
 } from '@/components/ui/sheet';
 import { IModalProps } from '@/interfaces/common';
-import { ITraceData } from '@/interfaces/database/agent';
 import { cn } from '@/lib/utils';
 import {
   ArrowUpRight,
@@ -20,19 +20,23 @@ import {
   isEndOutputEmpty,
   useDownloadOutput,
 } from '../hooks/use-download-output';
+import { UseFetchLogReturnType } from '../hooks/use-fetch-log';
 import { DataflowTimeline } from './dataflow-timeline';
 
 type LogSheetProps = IModalProps<any> & {
-  isParsing: boolean;
   handleCancel(): void;
-  logs?: ITraceData[];
-};
+} & Pick<
+    UseFetchLogReturnType,
+    'isCompleted' | 'isLogEmpty' | 'isParsing' | 'logs'
+  >;
 
 export function LogSheet({
   hideModal,
   isParsing,
   logs,
   handleCancel,
+  isCompleted,
+  isLogEmpty,
 }: LogSheetProps) {
   const { t } = useTranslation();
 
@@ -47,13 +51,17 @@ export function LogSheet({
         <SheetHeader>
           <SheetTitle className="flex items-center gap-2.5">
             <Logs className="size-4" /> {t('flow.log')}
-            <Button variant={'ghost'}>
+            <Button variant={'ghost'} disabled={!isCompleted}>
               {t('dataflow.viewResult')} <ArrowUpRight />
             </Button>
           </SheetTitle>
         </SheetHeader>
         <section className="max-h-[82vh] overflow-auto mt-6">
-          <DataflowTimeline traceList={logs}></DataflowTimeline>
+          {isLogEmpty ? (
+            <SkeletonCard className="mt-2" />
+          ) : (
+            <DataflowTimeline traceList={logs}></DataflowTimeline>
+          )}
         </section>
         {isParsing ? (
           <Button

web/src/pages/dataflow-result/components/parse-editer/index.tsx

@@ -1,13 +1,10 @@
-import { Checkbox } from '@/components/ui/checkbox';
-import { Textarea } from '@/components/ui/textarea';
-import { cn } from '@/lib/utils';
 import { CheckedState } from '@radix-ui/react-checkbox';
-import { useEffect, useState } from 'react';
 import { ChunkTextMode } from '../../constant';
-import styles from '../../index.less';
+import { ArrayContainer, parserKeyMap } from './json-parser';
+import { ObjectContainer } from './object-parser';
 interface FormatPreserveEditorProps {
   initialValue: {
-    key: string;
+    key: keyof typeof parserKeyMap | 'text' | 'html';
     type: string;
     value: Array<{ [key: string]: string }>;
   };
@@ -29,152 +26,47 @@ const FormatPreserveEditor = ({
   selectedChunkIds,
   textMode,
 }: FormatPreserveEditorProps) => {
-  const [content, setContent] = useState(initialValue);
-  // const [isEditing, setIsEditing] = useState(false);
-  const [activeEditIndex, setActiveEditIndex] = useState<number | undefined>(
-    undefined,
-  );
   console.log('initialValue', initialValue);
 
-  useEffect(() => {
-    setContent(initialValue);
-  }, [initialValue]);
-  const handleEdit = (e?: any, index?: number) => {
-    console.log(e, index, content);
-    if (content.key === 'json') {
-      console.log(e, e.target.innerText);
-      setContent((pre) => ({
-        ...pre,
-        value: pre.value.map((item, i) => {
-          if (i === index) {
-            return {
-              ...item,
-              [Object.keys(item)[0]]: e.target.innerText,
-            };
-          }
-          return item;
-        }),
-      }));
-      setActiveEditIndex(index);
-    }
-  };
-
-  const handleChange = (e: any) => {
-    if (content.key === 'json') {
-      setContent((pre) => ({
-        ...pre,
-        value: pre.value.map((item, i) => {
-          if (i === activeEditIndex) {
-            return {
-              ...item,
-              [Object.keys(item)[0]]: e.target.value,
-            };
-          }
-          return item;
-        }),
-      }));
-    } else {
-      setContent(e.target.value);
-    }
-  };
-
   const escapeNewlines = (text: string) => {
     return text.replace(/\n/g, '\\n');
   };
   const unescapeNewlines = (text: string) => {
     return text.replace(/\\n/g, '\n');
   };
-
-  const handleSave = () => {
-    const saveData = {
-      ...content,
-      value: content.value?.map((item) => {
-        return { ...item, text: unescapeNewlines(item.text) };
-      }),
-    };
-    onSave(saveData);
-    setActiveEditIndex(undefined);
-  };
   const handleCheck = (e: CheckedState, id: string | number) => {
     handleCheckboxClick?.(id, e === 'indeterminate' ? false : e);
   };
+
   return (
     <div className="editor-container">
-      {/* {isEditing && content.key === 'json' ? (
-        <Textarea
-          className={cn(
-            'w-full h-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none min-h-6 p-0',
-            className,
-          )}
-          value={content.value}
-          onChange={handleChange}
-          onBlur={handleSave}
-          autoSize={{ maxRows: 100 }}
-          autoFocus
+      {['json', 'chunks'].includes(initialValue.key) && (
+        <ArrayContainer
+          className={className}
+          initialValue={initialValue}
+          handleCheck={handleCheck}
+          selectedChunkIds={selectedChunkIds}
+          onSave={onSave}
+          escapeNewlines={escapeNewlines}
+          unescapeNewlines={unescapeNewlines}
+          textMode={textMode}
+          isChunck={isChunck}
         />
-      ) : (
-        <>
-          {content.key === 'json' && */}
-      {content.value?.map((item, index) => (
-        <section
-          key={index}
-          className={
-            isChunck
-              ? 'bg-bg-card my-2 p-2 rounded-lg flex gap-1 items-start'
-              : ''
-          }
-        >
-          {isChunck && (
-            <Checkbox
-              onCheckedChange={(e) => {
-                handleCheck(e, index);
-              }}
-              checked={selectedChunkIds?.some(
-                (id) => id.toString() === index.toString(),
-              )}
-            ></Checkbox>
-          )}
-          {activeEditIndex === index && (
-            <Textarea
-              key={'t' + index}
-              className={cn(
-                'w-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none !h-6 min-h-6 p-0',
-                className,
-              )}
-              value={escapeNewlines(content.value[index].text)}
-              onChange={handleChange}
-              onBlur={handleSave}
-              autoSize={{ maxRows: 100, minRows: 1 }}
-              autoFocus
-            />
-          )}
-          {activeEditIndex !== index && (
-            <div
-              className={cn(
-                'text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap w-full',
-                {
-                  [styles.contentEllipsis]: textMode === ChunkTextMode.Ellipse,
-                },
-              )}
-              key={index}
-              onClick={(e) => {
-                handleEdit(e, index);
-              }}
-            >
-              {escapeNewlines(item.text)}
-            </div>
-          )}
-        </section>
-      ))}
-      {/* {content.key !== 'json' && (
-            <pre
-              className="text-text-secondary overflow-auto scrollbar-auto"
-              onClick={handleEdit}
-            >
-            </pre>
-          )} 
-        </>
-      )}*/}
+      )}
+
+      {['text', 'html'].includes(initialValue.key) && (
+        <ObjectContainer
+          className={className}
+          initialValue={initialValue}
+          handleCheck={handleCheck}
+          selectedChunkIds={selectedChunkIds}
+          onSave={onSave}
+          escapeNewlines={escapeNewlines}
+          unescapeNewlines={unescapeNewlines}
+          textMode={textMode}
+          isChunck={isChunck}
+        />
+      )}
     </div>
   );
 };

web/src/pages/dataflow-result/components/parse-editer/json-parser.tsx

@@ -0,0 +1,173 @@
+import { Checkbox } from '@/components/ui/checkbox';
+import { cn } from '@/lib/utils';
+import { CheckedState } from '@radix-ui/react-checkbox';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { ChunkTextMode } from '../../constant';
+import styles from '../../index.less';
+export const parserKeyMap = {
+  json: 'text',
+  chunks: 'content_with_weight',
+};
+type IProps = {
+  initialValue: {
+    key: keyof typeof parserKeyMap;
+    type: string;
+    value: {
+      [key: string]: string;
+    }[];
+  };
+  isChunck?: boolean;
+  handleCheck: (e: CheckedState, index: number) => void;
+  selectedChunkIds: string[] | undefined;
+  unescapeNewlines: (text: string) => string;
+  escapeNewlines: (text: string) => string;
+  onSave: (data: {
+    value: {
+      text: string;
+    }[];
+    key: string;
+    type: string;
+  }) => void;
+  className?: string;
+  textMode?: ChunkTextMode;
+};
+export const ArrayContainer = (props: IProps) => {
+  const {
+    initialValue,
+    isChunck,
+    handleCheck,
+    selectedChunkIds,
+    unescapeNewlines,
+    escapeNewlines,
+    onSave,
+    className,
+    textMode,
+  } = props;
+
+  const [content, setContent] = useState(initialValue);
+
+  useEffect(() => {
+    setContent(initialValue);
+    console.log('initialValue json parse', initialValue);
+  }, [initialValue]);
+
+  const [activeEditIndex, setActiveEditIndex] = useState<number | undefined>(
+    undefined,
+  );
+  const editDivRef = useRef<HTMLDivElement>(null);
+  const handleEdit = useCallback(
+    (e?: any, index?: number) => {
+      console.log(e, e.target.innerText);
+      setContent((pre) => ({
+        ...pre,
+        value: pre.value.map((item, i) => {
+          if (i === index) {
+            return {
+              ...item,
+              [parserKeyMap[content.key]]: e.target.innerText,
+            };
+          }
+          return item;
+        }),
+      }));
+      setActiveEditIndex(index);
+    },
+    [setContent, setActiveEditIndex],
+  );
+  const handleSave = useCallback(
+    (e: any) => {
+      console.log(e, e.target.innerText);
+      const saveData = {
+        ...content,
+        value: content.value?.map((item, index) => {
+          if (index === activeEditIndex) {
+            return {
+              ...item,
+              [parserKeyMap[content.key]]: unescapeNewlines(e.target.innerText),
+            };
+          } else {
+            return item;
+          }
+        }),
+      };
+      onSave(saveData);
+      setActiveEditIndex(undefined);
+    },
+    [content, onSave],
+  );
+
+  useEffect(() => {
+    if (activeEditIndex !== undefined && editDivRef.current) {
+      editDivRef.current.focus();
+      editDivRef.current.textContent =
+        content.value[activeEditIndex][parserKeyMap[content.key]];
+    }
+  }, [activeEditIndex, content]);
+
+  return (
+    <>
+      {content.value?.map((item, index) => {
+        if (item[parserKeyMap[content.key]] === '') {
+          return null;
+        }
+        return (
+          <section
+            key={index}
+            className={
+              isChunck
+                ? 'bg-bg-card my-2 p-2 rounded-lg flex gap-1 items-start'
+                : ''
+            }
+          >
+            {isChunck && (
+              <Checkbox
+                onCheckedChange={(e) => {
+                  handleCheck(e, index);
+                }}
+                checked={selectedChunkIds?.some(
+                  (id) => id.toString() === index.toString(),
+                )}
+              ></Checkbox>
+            )}
+            {activeEditIndex === index && (
+              <div
+                ref={editDivRef}
+                contentEditable={true}
+                onBlur={handleSave}
+                //   onKeyUp={handleChange}
+                // dangerouslySetInnerHTML={{
+                //   __html: DOMPurify.sanitize(
+                //     escapeNewlines(
+                //       content.value[index][parserKeyMap[content.key]],
+                //     ),
+                //   ),
+                // }}
+                className={cn(
+                  'w-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none p-0',
+                  className,
+                )}
+              ></div>
+            )}
+            {activeEditIndex !== index && (
+              <div
+                className={cn(
+                  'text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap w-full',
+                  {
+                    [styles.contentEllipsis]:
+                      textMode === ChunkTextMode.Ellipse,
+                  },
+                )}
+                key={index}
+                onClick={(e) => {
+                  handleEdit(e, index);
+                }}
+              >
+                {escapeNewlines(item[parserKeyMap[content.key]])}
+              </div>
+            )}
+          </section>
+        );
+      })}
+    </>
+  );
+};

web/src/pages/dataflow-result/components/parse-editer/object-parser.tsx

@@ -0,0 +1,112 @@
+import { cn } from '@/lib/utils';
+import { CheckedState } from '@radix-ui/react-checkbox';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { ChunkTextMode } from '../../constant';
+import styles from '../../index.less';
+
+type IProps = {
+  initialValue: {
+    key: string;
+    type: string;
+    value: string;
+  };
+  isChunck?: boolean;
+  handleCheck: (e: CheckedState, index: number) => void;
+  unescapeNewlines: (text: string) => string;
+  escapeNewlines: (text: string) => string;
+  onSave: (data: { value: string; key: string; type: string }) => void;
+  className?: string;
+  textMode?: ChunkTextMode;
+};
+export const ObjectContainer = (props: IProps) => {
+  const {
+    initialValue,
+    isChunck,
+    unescapeNewlines,
+    escapeNewlines,
+    onSave,
+    className,
+    textMode,
+  } = props;
+
+  const [content, setContent] = useState(initialValue);
+
+  useEffect(() => {
+    setContent(initialValue);
+    console.log('initialValue object parse', initialValue);
+  }, [initialValue]);
+
+  const [activeEditIndex, setActiveEditIndex] = useState<number | undefined>(
+    undefined,
+  );
+  const editDivRef = useRef<HTMLDivElement>(null);
+  const handleEdit = useCallback(
+    (e?: any) => {
+      console.log(e, e.target.innerText);
+      setContent((pre) => ({
+        ...pre,
+        value: e.target.innerText,
+      }));
+      setActiveEditIndex(1);
+    },
+    [setContent, setActiveEditIndex],
+  );
+  const handleSave = useCallback(
+    (e: any) => {
+      console.log(e, e.target.innerText);
+      const saveData = {
+        ...content,
+        value: unescapeNewlines(e.target.innerText),
+      };
+      onSave(saveData);
+      setActiveEditIndex(undefined);
+    },
+    [content, onSave],
+  );
+
+  useEffect(() => {
+    if (activeEditIndex !== undefined && editDivRef.current) {
+      editDivRef.current.focus();
+      editDivRef.current.textContent = escapeNewlines(content.value);
+    }
+  }, [activeEditIndex, content, escapeNewlines]);
+
+  return (
+    <>
+      <section
+        className={
+          isChunck
+            ? 'bg-bg-card my-2 p-2 rounded-lg flex gap-1 items-start'
+            : ''
+        }
+      >
+        {activeEditIndex && (
+          <div
+            ref={editDivRef}
+            contentEditable={true}
+            onBlur={handleSave}
+            className={cn(
+              'w-full bg-transparent text-text-secondary border-none focus-visible:border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus-visible:outline-none p-0',
+              className,
+            )}
+          />
+        )}
+        {!activeEditIndex && (
+          <div
+            className={cn(
+              'text-text-secondary overflow-auto scrollbar-auto whitespace-pre-wrap w-full',
+              {
+                [styles.contentEllipsis]: textMode === ChunkTextMode.Ellipse,
+              },
+            )}
+            onClick={(e) => {
+              handleEdit(e);
+            }}
+          >
+            {escapeNewlines(content.value)}
+          </div>
+        )}
+      </section>
+    </>
+  );
+};

web/src/pages/dataflow-result/components/time-line/index.tsx

@@ -38,10 +38,6 @@ export const TimelineNodeObj = {
   },
   [TimelineNodeType.characterSplitter]: {
     title: 'Character Splitter',
-    icon: <Heading size={13} />,
-  },
-  [TimelineNodeType.splitter]: {
-    title: 'Splitter',
     icon: <Blocks size={13} />,
   },
   [TimelineNodeType.tokenizer]: {

web/src/pages/dataflow-result/constant.ts

@@ -6,10 +6,9 @@ export enum ChunkTextMode {
 export enum TimelineNodeType {
   begin = 'file',
   parser = 'parser',
-  splitter = 'splitter',
-  contextGenerator = 'contextGenerator',
-  titleSplitter = 'titleSplitter',
-  characterSplitter = 'characterSplitter',
-  tokenizer = 'tokenizer',
+  contextGenerator = 'extractor',
+  titleSplitter = 'hierarchicalMerger',
+  characterSplitter = 'splitter',
+  tokenizer = 'indexer',
   end = 'end',
 }

web/src/pages/dataflow-result/hooks.ts

@@ -1,4 +1,5 @@
 import { TimelineNode } from '@/components/originui/timeline';
+import message from '@/components/ui/message';
 import {
   useCreateChunk,
   useDeleteChunk,
@@ -10,7 +11,8 @@ import { IChunk } from '@/interfaces/database/knowledge';
 import kbService from '@/services/knowledge-service';
 import { formatSecondsToHumanReadable } from '@/utils/date';
 import { buildChunkHighlights } from '@/utils/document-util';
-import { useQuery } from '@tanstack/react-query';
+import { useMutation, useQuery } from '@tanstack/react-query';
+import { t } from 'i18next';
 import { camelCase, upperFirst } from 'lodash';
 import { useCallback, useMemo, useState } from 'react';
 import { IHighlight } from 'react-pdf-highlighter';
@@ -169,22 +171,16 @@ export const useUpdateChunk = () => {
   };
 };
 
-export const useFetchParserList = () => {
-  const [loading, setLoading] = useState(false);
-  return {
-    loading,
-  };
-};
-
 export const useRerunDataflow = ({
   data,
 }: {
   data: IPipelineFileLogDetail;
 }) => {
-  const [loading, setLoading] = useState(false);
   const [isChange, setIsChange] = useState(false);
-  const handleReRunFunc = useCallback(
-    (newData: { value: IDslComponent; key: string }) => {
+
+  const { mutateAsync: handleReRunFunc, isPending: loading } = useMutation({
+    mutationKey: ['pipelineRerun', data],
+    mutationFn: async (newData: { value: IDslComponent; key: string }) => {
       const newDsl = {
         ...data.dsl,
         components: {
@@ -197,16 +193,21 @@ export const useRerunDataflow = ({
       const params = {
         id: data.id,
         dsl: newDsl,
-        compenent_id: newData.key,
+        component_id: newData.key,
       };
-      console.log('newDsl', newDsl, params);
+      const { data: result } = await kbService.pipelineRerun(params);
+      if (result.code === 0) {
+        message.success(t('message.operated'));
+        // queryClient.invalidateQueries({
+        //   queryKey: [type],
+        // });
+      }
+      return result;
     },
-    [data],
-  );
+  });
 
   return {
     loading,
-    setLoading,
     isChange,
     setIsChange,
     handleReRunFunc,
@@ -225,7 +226,6 @@ export const useTimelineDataFlow = (data: IPipelineFileLogDetail) => {
         type:
           | TimelineNodeType.begin
           | TimelineNodeType.parser
-          | TimelineNodeType.splitter
           | TimelineNodeType.tokenizer
           | TimelineNodeType.characterSplitter
           | TimelineNodeType.titleSplitter,
@@ -242,10 +242,9 @@ export const useTimelineDataFlow = (data: IPipelineFileLogDetail) => {
           tempType = TimelineNodeType.tokenizer;
         } else if (
           name === TimelineNodeType.characterSplitter ||
-          name === TimelineNodeType.titleSplitter ||
-          name === TimelineNodeType.splitter
+          name === TimelineNodeType.titleSplitter
         ) {
-          tempType = TimelineNodeType.splitter;
+          tempType = TimelineNodeType.characterSplitter;
         }
         const timeNode = {
           ...TimelineNodeObj[name],

web/src/pages/dataflow-result/index.tsx

@@ -94,18 +94,18 @@ const Chunk = () => {
           ></div>
         ),
         onVisibleChange: () => {
-          Modal.hide();
+          Modal.destroy();
         },
         footer: (
           <div className="flex justify-end gap-2">
-            <Button variant={'outline'} onClick={() => Modal.hide()}>
+            <Button variant={'outline'} onClick={() => Modal.destroy()}>
               {t('dataflowParser.changeStepModalCancelText')}
             </Button>
             <Button
               variant={'secondary'}
               className="!bg-state-error text-text-primary"
               onClick={() => {
-                Modal.hide();
+                Modal.destroy();
                 setActiveStepId(id);
                 setIsChange(false);
               }}
@@ -193,7 +193,8 @@ const Chunk = () => {
             )} */}
             {/* {currentTimeNode?.type === TimelineNodeType.parser && ( */}
             {(currentTimeNode?.type === TimelineNodeType.parser ||
-              currentTimeNode?.type === TimelineNodeType.splitter) && (
+              currentTimeNode?.type === TimelineNodeType.characterSplitter ||
+              currentTimeNode?.type === TimelineNodeType.titleSplitter) && (
               <ParserContainer
                 isChange={isChange}
                 reRunLoading={reRunLoading}

web/src/pages/dataflow-result/parser.tsx

@@ -1,16 +1,15 @@
 import { TimelineNode } from '@/components/originui/timeline';
 import Spotlight from '@/components/spotlight';
-import { Spin } from '@/components/ui/spin';
 import { cn } from '@/lib/utils';
 import classNames from 'classnames';
-import { useCallback, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import ChunkResultBar from './components/chunk-result-bar';
 import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
 import FormatPreserEditor from './components/parse-editer';
 import RerunButton from './components/rerun-button';
 import { TimelineNodeType } from './constant';
-import { useChangeChunkTextMode, useFetchParserList } from './hooks';
+import { useChangeChunkTextMode } from './hooks';
 import { IDslComponent } from './interface';
 interface IProps {
   isChange: boolean;
@@ -23,15 +22,15 @@ interface IProps {
 const ParserContainer = (props: IProps) => {
   const { isChange, setIsChange, step, data, reRunFunc, reRunLoading } = props;
   const { t } = useTranslation();
-  const { loading } = useFetchParserList();
   const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
   const { changeChunkTextMode, textMode } = useChangeChunkTextMode();
   const initialValue = useMemo(() => {
     const outputs = data?.value?.obj?.params?.outputs;
     const key = outputs?.output_format?.value;
+    if (!outputs || !key) return { key: '', type: '', value: [] };
     const value = outputs[key]?.value;
     const type = outputs[key]?.type;
-    console.log('outputs-->', outputs);
+    console.log('outputs-->', outputs, data, key, value);
     return {
       key,
       type,
@@ -40,6 +39,10 @@ const ParserContainer = (props: IProps) => {
   }, [data]);
 
   const [initialText, setInitialText] = useState(initialValue);
+
+  useEffect(() => {
+    setInitialText(initialValue);
+  }, [initialValue]);
   const handleSave = (newContent: any) => {
     console.log('newContent-change-->', newContent, initialValue);
     if (JSON.stringify(newContent) !== JSON.stringify(initialValue)) {
@@ -109,8 +112,7 @@ const ParserContainer = (props: IProps) => {
 
   const isChunck =
     step?.type === TimelineNodeType.characterSplitter ||
-    step?.type === TimelineNodeType.titleSplitter ||
-    step?.type === TimelineNodeType.splitter;
+    step?.type === TimelineNodeType.titleSplitter;
 
   const handleCreateChunk = useCallback(
     (text: string) => {
@@ -124,6 +126,7 @@ const ParserContainer = (props: IProps) => {
     },
     [initialText],
   );
+
   return (
     <>
       {isChange && (
@@ -136,71 +139,72 @@ const ParserContainer = (props: IProps) => {
         </div>
       )}
       <div className={classNames('flex flex-col w-full')}>
-        <Spin spinning={loading} className="" size="large">
-          <div className="h-[50px] flex flex-col justify-end pb-[5px]">
-            {!isChunck && (
-              <div>
-                <h2 className="text-[16px]">
-                  {t('dataflowParser.parseSummary')}
-                </h2>
-                <div className="text-[12px] text-text-secondary italic ">
-                  {t('dataflowParser.parseSummaryTip')}
-                </div>
+        {/* <Spin spinning={false} className="" size="large"> */}
+        <div className="h-[50px] flex flex-col justify-end pb-[5px]">
+          {!isChunck && (
+            <div>
+              <h2 className="text-[16px]">
+                {t('dataflowParser.parseSummary')}
+              </h2>
+              <div className="text-[12px] text-text-secondary italic ">
+                {t('dataflowParser.parseSummaryTip')}
               </div>
-            )}
-            {isChunck && (
-              <div>
-                <h2 className="text-[16px]">{t('chunk.chunkResult')}</h2>
-                <div className="text-[12px] text-text-secondary italic">
-                  {t('chunk.chunkResultTip')}
-                </div>
-              </div>
-            )}
-          </div>
-
-          {isChunck && (
-            <div className="pt-[5px] pb-[5px] flex justify-between items-center">
-              <CheckboxSets
-                selectAllChunk={selectAllChunk}
-                removeChunk={handleRemoveChunk}
-                checked={selectedChunkIds.length === initialText.value.length}
-                selectedChunkIds={selectedChunkIds}
-              />
-              <ChunkResultBar
-                changeChunkTextMode={changeChunkTextMode}
-                createChunk={handleCreateChunk}
-              />
             </div>
           )}
+          {isChunck && (
+            <div>
+              <h2 className="text-[16px]">{t('chunk.chunkResult')}</h2>
+              <div className="text-[12px] text-text-secondary italic">
+                {t('chunk.chunkResultTip')}
+              </div>
+            </div>
+          )}
+        </div>
 
-          <div
-            className={cn(
-              ' border rounded-lg p-[20px] box-border w-[calc(100%-20px)] overflow-auto scrollbar-none',
-              {
-                'h-[calc(100vh-240px)]': isChunck,
-                'h-[calc(100vh-180px)]': !isChunck,
-              },
-            )}
-          >
+        {isChunck && (
+          <div className="pt-[5px] pb-[5px] flex justify-between items-center">
+            <CheckboxSets
+              selectAllChunk={selectAllChunk}
+              removeChunk={handleRemoveChunk}
+              checked={selectedChunkIds.length === initialText.value.length}
+              selectedChunkIds={selectedChunkIds}
+            />
+            <ChunkResultBar
+              changeChunkTextMode={changeChunkTextMode}
+              createChunk={handleCreateChunk}
+            />
+          </div>
+        )}
+
+        <div
+          className={cn(
+            ' border rounded-lg p-[20px] box-border w-[calc(100%-20px)] overflow-auto scrollbar-none',
+            {
+              'h-[calc(100vh-240px)]': isChunck,
+              'h-[calc(100vh-180px)]': !isChunck,
+            },
+          )}
+        >
+          {initialText && (
             <FormatPreserEditor
               initialValue={initialText}
               onSave={handleSave}
-              className={
-                initialText.key !== 'json' ? '!h-[calc(100vh-220px)]' : ''
-              }
+              // className={
+              //   initialText.key !== 'json' ? '!h-[calc(100vh-220px)]' : ''
+              // }
               isChunck={isChunck}
               textMode={textMode}
               isDelete={
                 step?.type === TimelineNodeType.characterSplitter ||
-                step?.type === TimelineNodeType.titleSplitter ||
-                step?.type === TimelineNodeType.splitter
+                step?.type === TimelineNodeType.titleSplitter
               }
               handleCheckboxClick={handleCheckboxClick}
               selectedChunkIds={selectedChunkIds}
             />
-            <Spotlight opcity={0.6} coverage={60} />
-          </div>
-        </Spin>
+          )}
+          <Spotlight opcity={0.6} coverage={60} />
+        </div>
+        {/* </Spin> */}
       </div>
     </>
   );

web/src/pages/dataset/dataset-setting/index.tsx

@@ -12,7 +12,10 @@ import { useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
 import { TopTitle } from '../dataset-title';
-import { IGenerateLogButtonProps } from '../dataset/generate-button/generate';
+import {
+  GenerateType,
+  IGenerateLogButtonProps,
+} from '../dataset/generate-button/generate';
 import LinkDataPipeline, {
   IDataPipelineNodeProps,
 } from './components/link-data-pipeline';
@@ -120,6 +123,20 @@ export default function DatasetSettings() {
       // form.setValue('pipeline_avatar', data.avatar || '');
     }
   };
+
+  const handleDeletePipelineTask = (type: GenerateType) => {
+    if (type === GenerateType.KnowledgeGraph) {
+      setGraphRagGenerateData({
+        finish_at: '',
+        task_id: '',
+      } as IGenerateLogButtonProps);
+    } else if (type === GenerateType.Raptor) {
+      setRaptorGenerateData({
+        finish_at: '',
+        task_id: '',
+      } as IGenerateLogButtonProps);
+    }
+  };
   return (
     <section className="p-5 h-full flex flex-col">
       <TopTitle
@@ -140,10 +157,14 @@ export default function DatasetSettings() {
                 <GraphRagItems
                   className="border-none p-0"
                   data={graphRagGenerateData as IGenerateLogButtonProps}
+                  onDelete={() =>
+                    handleDeletePipelineTask(GenerateType.KnowledgeGraph)
+                  }
                 ></GraphRagItems>
                 <Divider />
                 <RaptorFormFields
                   data={raptorGenerateData as IGenerateLogButtonProps}
+                  onDelete={() => handleDeletePipelineTask(GenerateType.Raptor)}
                 ></RaptorFormFields>
                 <Divider />
                 <LinkDataPipeline

web/src/pages/dataset/dataset/generate-button/generate.tsx

@@ -16,17 +16,23 @@ import { lowerFirst } from 'lodash';
 import { CirclePause, Trash2, WandSparkles } from 'lucide-react';
 import { useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
+import { ProcessingType } from '../../dataset-overview/dataset-common';
 import { replaceText } from '../../process-log-modal';
 import {
   ITraceInfo,
   generateStatus,
   useDatasetGenerate,
   useTraceGenerate,
+  useUnBindTask,
 } from './hook';
 export enum GenerateType {
   KnowledgeGraph = 'KnowledgeGraph',
   Raptor = 'Raptor',
 }
+export const GenerateTypeMap = {
+  [GenerateType.KnowledgeGraph]: ProcessingType.knowledgeGraph,
+  [GenerateType.Raptor]: ProcessingType.raptor,
+};
 const MenuItem: React.FC<{
   name: GenerateType;
   data: ITraceInfo;
@@ -78,9 +84,11 @@ const MenuItem: React.FC<{
         'border cursor-pointer p-2 rounded-md focus:bg-transparent',
         {
           'hover:border-accent-primary hover:bg-[rgba(59,160,92,0.1)]':
-            status === generateStatus.start,
+            status === generateStatus.start ||
+            status === generateStatus.completed,
           'hover:border-border hover:bg-[rgba(59,160,92,0)]':
-            status !== generateStatus.start,
+            status !== generateStatus.start &&
+            status !== generateStatus.completed,
         },
       )}
       onSelect={(e) => {
@@ -93,7 +101,10 @@ const MenuItem: React.FC<{
       <div
         className="flex items-start gap-2 flex-col w-full"
         onClick={() => {
-          if (status === generateStatus.start) {
+          if (
+            status === generateStatus.start ||
+            status === generateStatus.completed
+          ) {
             runGenerate({ type });
           }
         }}
@@ -234,7 +245,21 @@ export type IGenerateLogProps = IGenerateLogButtonProps & {
 };
 export const GenerateLogButton = (props: IGenerateLogProps) => {
   const { t } = useTranslation();
-  const { task_id, message, finish_at, type, onDelete } = props;
+  const { message, finish_at, type, onDelete } = props;
+
+  const { handleUnbindTask } = useUnBindTask();
+
+  const handleDeleteFunc = async () => {
+    const data = await handleUnbindTask({
+      type: GenerateTypeMap[type as GenerateType],
+    });
+    Modal.destroy();
+    console.log('handleUnbindTask', data);
+    if (data.code === 0) {
+      onDelete?.();
+    }
+  };
+
   const handleDelete = () => {
     Modal.show({
       visible: true,
@@ -259,14 +284,14 @@ export const GenerateLogButton = (props: IGenerateLogProps) => {
         ></div>
       ),
       onVisibleChange: () => {
-        Modal.hide();
+        Modal.destroy();
       },
       footer: (
         <div className="flex justify-end gap-2">
           <Button
             type="button"
             variant={'outline'}
-            onClick={() => Modal.hide()}
+            onClick={() => Modal.destroy()}
           >
             {t('dataflowParser.changeStepModalCancelText')}
           </Button>
@@ -275,7 +300,7 @@ export const GenerateLogButton = (props: IGenerateLogProps) => {
             variant={'secondary'}
             className="!bg-state-error text-text-primary"
             onClick={() => {
-              Modal.hide();
+              handleDeleteFunc();
             }}
           >
             {t('common.delete')}
@@ -284,6 +309,7 @@ export const GenerateLogButton = (props: IGenerateLogProps) => {
       ),
     });
   };
+
   return (
     <div
       className={cn('flex bg-bg-card rounded-md py-1 px-3', props.className)}

web/src/pages/dataset/dataset/generate-button/hook.ts

@@ -1,10 +1,11 @@
 import message from '@/components/ui/message';
 import agentService from '@/services/agent-service';
-import kbService from '@/services/knowledge-service';
+import kbService, { deletePipelineTask } from '@/services/knowledge-service';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { t } from 'i18next';
 import { useEffect, useState } from 'react';
 import { useParams } from 'umi';
+import { ProcessingType } from '../../dataset-overview/dataset-common';
 import { GenerateType } from './generate';
 export const generateStatus = {
   running: 'running',
@@ -153,3 +154,21 @@ export const useDatasetGenerate = () => {
   });
   return { runGenerate: mutateAsync, pauseGenerate, data, loading };
 };
+
+export const useUnBindTask = () => {
+  const { id } = useParams();
+  const { mutateAsync: handleUnbindTask } = useMutation({
+    mutationKey: [DatasetKey.pauseGenerate],
+    mutationFn: async ({ type }: { type: ProcessingType }) => {
+      const { data } = await deletePipelineTask({ kb_id: id as string, type });
+      if (data.code === 0) {
+        message.success(t('message.operated'));
+        // queryClient.invalidateQueries({
+        //   queryKey: [type],
+        // });
+      }
+      return data;
+    },
+  });
+  return { handleUnbindTask };
+};

web/src/pages/dataset/dataset/hooks.ts

@@ -79,7 +79,7 @@ export const useShowLog = (documents: IDocumentInfo[]) => {
     if (findRecord) {
       log = {
         fileType: findRecord?.suffix,
-        uploadedBy: findRecord?.created_by,
+        uploadedBy: findRecord?.nickname,
         fileName: findRecord?.name,
         uploadDate: formatDate(findRecord.create_date),
         fileSize: formatBytes(findRecord.size || 0),

web/src/pages/datasets/use-select-owners.ts

@@ -1,18 +1,11 @@
 import { FilterCollection } from '@/components/list-filter-bar/interface';
-import { useFetchKnowledgeList } from '@/hooks/knowledge-hooks';
-import { groupListByType } from '@/utils/dataset-util';
-import { useMemo } from 'react';
+import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
+import { buildOwnersFilter } from '@/utils/list-filter-util';
 
 export function useSelectOwners() {
   const { list } = useFetchKnowledgeList();
 
-  const owners = useMemo(() => {
-    return groupListByType(list, 'tenant_id', 'nickname');
-  }, [list]);
-
-  const filters: FilterCollection[] = [
-    { field: 'owner', list: owners, label: 'Owner' },
-  ];
+  const filters: FilterCollection[] = [buildOwnersFilter(list)];
 
   return filters;
 }

web/src/services/agent-service.ts

@@ -17,7 +17,6 @@ const {
   testDbConnect,
   getInputElements,
   debug,
-  listCanvasTeam,
   settingCanvas,
   uploadCanvasFile,
   trace,
@@ -85,10 +84,6 @@ const methods = {
     url: debug,
     method: 'post',
   },
-  listCanvasTeam: {
-    url: listCanvasTeam,
-    method: 'get',
-  },
   settingCanvas: {
     url: settingCanvas,
     method: 'post',

web/src/services/knowledge-service.ts

@@ -4,6 +4,7 @@ import {
   IFetchKnowledgeListRequestBody,
   IFetchKnowledgeListRequestParams,
 } from '@/interfaces/request/knowledge';
+import { ProcessingType } from '@/pages/dataset/dataset-overview/dataset-common';
 import api from '@/utils/api';
 import registerServer from '@/utils/register-server';
 import request, { post } from '@/utils/request';
@@ -254,4 +255,14 @@ export const listPipelineDatasetLogs = (
   body?: IFetchDocumentListRequestBody,
 ) => request.post(api.fetchPipelineDatasetLogs, { data: body || {}, params });
 
+export function deletePipelineTask({
+  kb_id,
+  type,
+}: {
+  kb_id: string;
+  type: ProcessingType;
+}) {
+  return request.delete(api.unbindPipelineTask({ kb_id, type }));
+}
+
 export default kbService;

web/src/utils/api.ts

@@ -54,6 +54,9 @@ export default {
   traceGraphRag: `${api_host}/kb/trace_graphrag`,
   runRaptor: `${api_host}/kb/run_raptor`,
   traceRaptor: `${api_host}/kb/trace_raptor`,
+  unbindPipelineTask: ({ kb_id, type }: { kb_id: string; type: string }) =>
+    `${api_host}/kb/unbind_task?kb_id=${kb_id}&pipeline_task_type=${type}`,
+  pipelineRerun: `${api_host}/canvas/rerun`,
 
   // tags
   listTag: (knowledgeId: string) => `${api_host}/kb/${knowledgeId}/tags`,
@@ -147,7 +150,6 @@ export default {
   // flow
   listTemplates: `${api_host}/canvas/templates`,
   listCanvas: `${api_host}/canvas/list`,
-  listCanvasTeam: `${api_host}/canvas/list`,
   getCanvas: `${api_host}/canvas/get`,
   getCanvasSSE: `${api_host}/canvas/getsse`,
   removeCanvas: `${api_host}/canvas/rm`,

web/src/utils/dataset-util.ts

@@ -7,27 +7,3 @@ export function isKnowledgeGraphParser(parserId: DocumentParserType) {
 export function isNaiveParser(parserId: DocumentParserType) {
   return parserId === DocumentParserType.Naive;
 }
-
-export type FilterType = {
-  id: string;
-  label: string;
-  count: number;
-};
-
-export function groupListByType<T extends Record<string, any>>(
-  list: T[],
-  idField: string,
-  labelField: string,
-) {
-  const fileTypeList: FilterType[] = [];
-  list.forEach((x) => {
-    const item = fileTypeList.find((y) => y.id === x[idField]);
-    if (!item) {
-      fileTypeList.push({ id: x[idField], label: x[labelField], count: 1 });
-    } else {
-      item.count += 1;
-    }
-  });
-
-  return fileTypeList;
-}

web/src/utils/list-filter-util.ts

@@ -0,0 +1,29 @@
+export type FilterType = {
+  id: string;
+  label: string;
+  count: number;
+};
+
+export function groupListByType<T extends Record<string, any>>(
+  list: T[],
+  idField: string,
+  labelField: string,
+) {
+  const fileTypeList: FilterType[] = [];
+  list.forEach((x) => {
+    const item = fileTypeList.find((y) => y.id === x[idField]);
+    if (!item) {
+      fileTypeList.push({ id: x[idField], label: x[labelField], count: 1 });
+    } else {
+      item.count += 1;
+    }
+  });
+
+  return fileTypeList;
+}
+
+export function buildOwnersFilter<T extends Record<string, any>>(list: T[]) {
+  const owners = groupListByType(list, 'tenant_id', 'nickname');
+
+  return { field: 'owner', list: owners, label: 'Owner' };
+}