Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/agents/agent-card.tsx

@@ -1,8 +1,11 @@
 import { HomeCard } from '@/components/home-card';
 import { MoreButton } from '@/components/more-button';
 import { SharedBadge } from '@/components/shared-badge';
+import { Button } from '@/components/ui/button';
+import { AgentCategory } from '@/constants/agent';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { IFlow } from '@/interfaces/database/agent';
+import { Route } from 'lucide-react';
 import { AgentDropdown } from './agent-dropdown';
 import { useRenameAgent } from './use-rename-agent';
 
@@ -11,7 +14,7 @@ export type DatasetCardProps = {
 } & Pick<ReturnType<typeof useRenameAgent>, 'showAgentRenameModal'>;
 
 export function AgentCard({ data, showAgentRenameModal }: DatasetCardProps) {
-  const { navigateToAgent } = useNavigatePage();
+  const { navigateToAgent, navigateToDataflow } = useNavigatePage();
 
   return (
     <HomeCard
@@ -22,7 +25,18 @@ export function AgentCard({ data, showAgentRenameModal }: DatasetCardProps) {
         </AgentDropdown>
       }
       sharedBadge={<SharedBadge>{data.nickname}</SharedBadge>}
-      onClick={navigateToAgent(data?.id)}
+      onClick={
+        data.canvas_category === AgentCategory.DataflowCanvas
+          ? navigateToDataflow(data.id)
+          : navigateToAgent(data?.id)
+      }
+      icon={
+        data.canvas_category === AgentCategory.DataflowCanvas && (
+          <Button variant={'ghost'} size={'sm'}>
+            <Route />
+          </Button>
+        )
+      }
     />
   );
 }

web/src/pages/agents/create-agent-dialog.tsx

@@ -22,7 +22,7 @@ export function CreateAgentDialog({
 
   return (
     <Dialog open onOpenChange={hideModal}>
-      <DialogContent className="sm:max-w-[425px]">
+      <DialogContent>
         <DialogHeader>
           <DialogTitle>{t('flow.createGraph')}</DialogTitle>
         </DialogHeader>

web/src/pages/agents/create-agent-form.tsx

@@ -25,6 +25,7 @@ type FlowTypeCardProps = {
   onChange?: (value: FlowType) => void;
 };
 function FlowTypeCards({ value, onChange }: FlowTypeCardProps) {
+  const { t } = useTranslation();
   const handleChange = useCallback(
     (value: FlowType) => () => {
       onChange?.(value);
@@ -59,7 +60,11 @@ function FlowTypeCards({ value, onChange }: FlowTypeCardProps) {
                 ) : (
                   <Route className="size-6" />
                 )}
-                <p>{val}</p>
+                <p>
+                  {t(
+                    `flow.${val === FlowType.Agent ? 'createAgent' : 'createPipeline'}`,
+                  )}
+                </p>
               </div>
               {isActive && <Check />}
             </CardContent>
@@ -106,7 +111,11 @@ export function CreateAgentForm({
         id={TagRenameId}
       >
         {shouldChooseAgent && (
-          <RAGFlowFormItem required name="type" label={t('common.type')}>
+          <RAGFlowFormItem
+            required
+            name="type"
+            label={t('flow.chooseAgentType')}
+          >
             <FlowTypeCards></FlowTypeCards>
           </RAGFlowFormItem>
         )}

web/src/pages/agents/hooks/use-create-agent.ts

@@ -1,10 +1,81 @@
+import { AgentCategory } from '@/constants/agent';
 import { useSetModalState } from '@/hooks/common-hooks';
 import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request';
 import { DSL } from '@/interfaces/database/agent';
+import {
+  BeginId,
+  Operator,
+  initialParserValues,
+} from '@/pages/data-flow/constant';
 import { useCallback } from 'react';
 import { FlowType } from '../constant';
 import { FormSchemaType } from '../create-agent-form';
 
+export const DataflowEmptyDsl = {
+  graph: {
+    nodes: [
+      {
+        id: BeginId,
+        type: 'beginNode',
+        position: {
+          x: 50,
+          y: 200,
+        },
+        data: {
+          label: Operator.Begin,
+          name: Operator.Begin,
+        },
+        sourcePosition: 'left',
+        targetPosition: 'right',
+      },
+      {
+        data: {
+          form: initialParserValues,
+          label: 'Parser',
+          name: 'Parser_0',
+        },
+        dragging: false,
+        id: 'Parser:HipSignsRhyme',
+        measured: {
+          height: 57,
+          width: 200,
+        },
+        position: {
+          x: 316.99524094206413,
+          y: 195.39629819663406,
+        },
+        selected: true,
+        sourcePosition: 'right',
+        targetPosition: 'left',
+        type: 'parserNode',
+      },
+    ],
+    edges: [
+      {
+        id: 'xy-edge__Filestart-Parser:HipSignsRhymeend',
+        source: BeginId,
+        sourceHandle: 'start',
+        target: 'Parser:HipSignsRhyme',
+        targetHandle: 'end',
+      },
+    ],
+  },
+  components: {
+    [Operator.Begin]: {
+      obj: {
+        component_name: Operator.Begin,
+        params: {},
+      },
+      downstream: [], // other edge target is downstream, edge source is current node id
+      upstream: [], // edge source is upstream, edge target is current node id
+    },
+  },
+  retrieval: [], // reference
+  history: [],
+  path: [],
+  globals: {},
+};
+
 export function useCreateAgentOrPipeline() {
   const { loading, setAgent } = useSetAgent();
   const {
@@ -13,27 +84,26 @@ export function useCreateAgentOrPipeline() {
     showModal: showCreatingModal,
   } = useSetModalState();
 
-  const createAgent = useCallback(
-    async (name: string) => {
-      return setAgent({ title: name, dsl: EmptyDsl as DSL });
-    },
-    [setAgent],
-  );
-
   const handleCreateAgentOrPipeline = useCallback(
     async (data: FormSchemaType) => {
-      if (data.type === FlowType.Agent) {
-        const ret = await createAgent(data.name);
-        if (ret.code === 0) {
-          hideCreatingModal();
-        }
+      const isAgent = data.type === FlowType.Agent;
+      const ret = await setAgent({
+        title: data.name,
+        dsl: isAgent ? (EmptyDsl as DSL) : (DataflowEmptyDsl as DSL),
+        canvas_category: isAgent
+          ? AgentCategory.AgentCanvas
+          : AgentCategory.DataflowCanvas,
+      });
+
+      if (ret.code === 0) {
+        hideCreatingModal();
       }
     },
-    [createAgent, hideCreatingModal],
+    [hideCreatingModal, setAgent],
   );
 
   return {
-    loading,
+    loading: loading,
     creatingVisible,
     hideCreatingModal,
     showCreatingModal,

web/src/pages/agents/hooks/use-selelct-filters.ts

@@ -0,0 +1,23 @@
+import { FilterCollection } from '@/components/list-filter-bar/interface';
+import { useFetchAgentList } from '@/hooks/use-agent-request';
+import { buildOwnersFilter, groupListByType } from '@/utils/list-filter-util';
+import { useMemo } from 'react';
+
+export function useSelectFilters() {
+  const { data } = useFetchAgentList({});
+
+  const canvasCategory = useMemo(() => {
+    return groupListByType(data.canvas, 'canvas_category', 'canvas_category');
+  }, [data.canvas]);
+
+  const filters: FilterCollection[] = [
+    buildOwnersFilter(data.canvas),
+    {
+      field: 'canvasCategory',
+      list: canvasCategory,
+      label: 'Canvas category',
+    },
+  ];
+
+  return filters;
+}

web/src/pages/agents/index.tsx

@@ -17,13 +17,21 @@ import { useCallback } from 'react';
 import { AgentCard } from './agent-card';
 import { CreateAgentDialog } from './create-agent-dialog';
 import { useCreateAgentOrPipeline } from './hooks/use-create-agent';
+import { useSelectFilters } from './hooks/use-selelct-filters';
 import { UploadAgentDialog } from './upload-agent-dialog';
 import { useHandleImportJsonFile } from './use-import-json';
 import { useRenameAgent } from './use-rename-agent';
 
 export default function Agents() {
-  const { data, pagination, setPagination, searchString, handleInputChange } =
-    useFetchAgentListByPage();
+  const {
+    data,
+    pagination,
+    setPagination,
+    searchString,
+    handleInputChange,
+    filterValue,
+    handleFilterSubmit,
+  } = useFetchAgentListByPage();
   const { navigateToAgentTemplates } = useNavigatePage();
 
   const {
@@ -50,6 +58,8 @@ export default function Agents() {
     hideFileUploadModal,
   } = useHandleImportJsonFile();
 
+  const filters = useSelectFilters();
+
   const handlePageChange = useCallback(
     (page: number, pageSize?: number) => {
       setPagination({ page, pageSize });
@@ -65,6 +75,9 @@ export default function Agents() {
           searchString={searchString}
           onSearchChange={handleInputChange}
           icon="agent"
+          filters={filters}
+          onChange={handleFilterSubmit}
+          value={filterValue}
         >
           <DropdownMenu>
             <DropdownMenuTrigger>
@@ -79,21 +92,21 @@ export default function Agents() {
                 onClick={showCreatingModal}
               >
                 <Clipboard />
-                Create from Blank
+                {t('flow.createFromBlank')}
               </DropdownMenuItem>
               <DropdownMenuItem
                 justifyBetween={false}
                 onClick={navigateToAgentTemplates}
               >
                 <ClipboardPlus />
-                Create from Template
+                {t('flow.createFromTemplate')}
               </DropdownMenuItem>
               <DropdownMenuItem
                 justifyBetween={false}
                 onClick={handleImportJson}
               >
                 <FileInput />
-                Import json file
+                {t('flow.importJsonFile')}
               </DropdownMenuItem>
             </DropdownMenuContent>
           </DropdownMenu>

web/src/pages/agents/name-form-field.tsx

@@ -16,12 +16,7 @@ export const NameFormSchema = {
 export function NameFormField() {
   const { t } = useTranslation();
   return (
-    <RAGFlowFormItem
-      name="name"
-      required
-      label={t('common.name')}
-      tooltip={t('flow.sqlStatementTip')}
-    >
+    <RAGFlowFormItem name="name" required label={t('common.name')}>
       <Input placeholder={t('common.namePlaceholder')} autoComplete="off" />
     </RAGFlowFormItem>
   );