Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/datasets/dataset-creating-dialog.tsx

@@ -1,3 +1,4 @@
+import { DataFlowSelect } from '@/components/data-pipeline-select';
 import { ButtonLoading } from '@/components/ui/button';
 import {
   Dialog,
@@ -15,38 +16,94 @@ import {
   FormMessage,
 } from '@/components/ui/form';
 import { Input } from '@/components/ui/input';
+import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { IModalProps } from '@/interfaces/common';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { useForm } from 'react-hook-form';
+import { useEffect } from 'react';
+import { useForm, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
+import {
+  ChunkMethodItem,
+  EmbeddingModelItem,
+  ParseTypeItem,
+} from '../dataset/dataset-setting/configuration/common-item';
 
 const FormId = 'dataset-creating-form';
 
 export function InputForm({ onOk }: IModalProps<any>) {
   const { t } = useTranslation();
 
-  const FormSchema = z.object({
-    name: z
-      .string()
-      .min(1, {
-        message: t('knowledgeList.namePlaceholder'),
-      })
-      .trim(),
-    parseType: z.number().optional(),
-  });
+  const FormSchema = z
+    .object({
+      name: z
+        .string()
+        .min(1, {
+          message: t('knowledgeList.namePlaceholder'),
+        })
+        .trim(),
+      parseType: z.number().optional(),
+      embd_id: z
+        .string()
+        .min(1, {
+          message: t('knowledgeConfiguration.embeddingModelPlaceholder'),
+        })
+        .trim(),
+      parser_id: z.string().optional(),
+      pipeline_id: z.string().optional(),
+    })
+    .superRefine((data, ctx) => {
+      // When parseType === 1, parser_id is required
+      if (
+        data.parseType === 1 &&
+        (!data.parser_id || data.parser_id.trim() === '')
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: t('knowledgeList.parserRequired'),
+          path: ['parser_id'],
+        });
+      }
+
+      console.log('form-data', data);
+      // When parseType === 1, pipline_id required
+      if (data.parseType === 2 && !data.pipeline_id) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: t('knowledgeList.dataFlowRequired'),
+          path: ['pipeline_id'],
+        });
+      }
+    });
 
   const form = useForm<z.infer<typeof FormSchema>>({
     resolver: zodResolver(FormSchema),
     defaultValues: {
       name: '',
       parseType: 1,
+      parser_id: '',
+      embd_id: '',
     },
   });
 
   function onSubmit(data: z.infer<typeof FormSchema>) {
-    onOk?.(data.name);
+    console.log('submit', data);
+    onOk?.(data);
   }
+
+  const parseType = useWatch({
+    control: form.control,
+    name: 'parseType',
+  });
+
+  useEffect(() => {
+    console.log('parseType', parseType);
+    if (parseType === 1) {
+      form.setValue('pipeline_id', '');
+    }
+  }, [parseType, form]);
+  const { navigateToAgents } = useNavigatePage();
+
   return (
     <Form {...form}>
       <form
@@ -73,6 +130,17 @@ export function InputForm({ onOk }: IModalProps<any>) {
             </FormItem>
           )}
         />
+
+        <EmbeddingModelItem line={2} isEdit={false} />
+        <ParseTypeItem />
+        {parseType === 1 && <ChunkMethodItem></ChunkMethodItem>}
+        {parseType === 2 && (
+          <DataFlowSelect
+            isMult={false}
+            toDataPipeline={navigateToAgents}
+            formFieldName="pipeline_id"
+          />
+        )}
       </form>
     </Form>
   );
@@ -87,7 +155,7 @@ export function DatasetCreatingDialog({
 
   return (
     <Dialog open onOpenChange={hideModal}>
-      <DialogContent className="sm:max-w-[425px]">
+      <DialogContent className="sm:max-w-[425px] focus-visible:!outline-none">
         <DialogHeader>
           <DialogTitle>{t('knowledgeList.createKnowledgeBase')}</DialogTitle>
         </DialogHeader>

web/src/pages/datasets/hooks.ts

@@ -2,7 +2,6 @@ import { useSetModalState } from '@/hooks/common-hooks';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { useCreateKnowledge } from '@/hooks/use-knowledge-request';
 import { useCallback, useState } from 'react';
-
 export const useSearchKnowledge = () => {
   const [searchString, setSearchString] = useState<string>('');
 
@@ -15,16 +14,19 @@ export const useSearchKnowledge = () => {
   };
 };
 
+export interface Iknowledge {
+  name: string;
+  embd_id: string;
+  parser_id: string;
+}
 export const useSaveKnowledge = () => {
   const { visible: visible, hideModal, showModal } = useSetModalState();
   const { loading, createKnowledge } = useCreateKnowledge();
   const { navigateToDataset } = useNavigatePage();
 
   const onCreateOk = useCallback(
-    async (name: string) => {
-      const ret = await createKnowledge({
-        name,
-      });
+    async (data: Iknowledge) => {
+      const ret = await createKnowledge(data);
 
       if (ret?.code === 0) {
         hideModal();

web/src/pages/datasets/process-log-modal.tsx

@@ -1,116 +0,0 @@
-import { Button } from '@/components/ui/button';
-import { Modal } from '@/components/ui/modal/modal';
-import { useTranslate } from '@/hooks/common-hooks';
-import React from 'react';
-
-interface ProcessLogModalProps {
-  visible: boolean;
-  onCancel: () => void;
-  taskInfo: {
-    taskId: string;
-    fileName: string;
-    fileSize: string;
-    source: string;
-    task: string;
-    state: 'Running' | 'Completed' | 'Failed' | 'Pending';
-    startTime: string;
-    endTime?: string;
-    duration?: string;
-    details: string;
-  };
-}
-
-const StatusTag: React.FC<{ state: string }> = ({ state }) => {
-  const getTagStyle = () => {
-    switch (state) {
-      case 'Running':
-        return 'bg-green-500 text-green-100';
-      case 'Completed':
-        return 'bg-blue-500 text-blue-100';
-      case 'Failed':
-        return 'bg-red-500 text-red-100';
-      case 'Pending':
-        return 'bg-yellow-500 text-yellow-100';
-      default:
-        return 'bg-gray-500 text-gray-100';
-    }
-  };
-
-  return (
-    <span
-      className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium ${getTagStyle()}`}
-    >
-      <span className="w-1.5 h-1.5 rounded-full mr-1 bg-current"></span>
-      {state}
-    </span>
-  );
-};
-
-const InfoItem: React.FC<{
-  label: string;
-  value: string | React.ReactNode;
-  className?: string;
-}> = ({ label, value, className = '' }) => {
-  return (
-    <div className={`flex flex-col mb-4 ${className}`}>
-      <span className="text-text-secondary text-sm">{label}</span>
-      <span className="text-text-primary mt-1">{value}</span>
-    </div>
-  );
-};
-
-const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
-  visible,
-  onCancel,
-  taskInfo,
-}) => {
-  const { t } = useTranslate('knowledgeDetails');
-  return (
-    <Modal
-      title={t('processLog')}
-      open={visible}
-      onCancel={onCancel}
-      footer={
-        <div className="flex justify-end">
-          <Button onClick={onCancel}>{t('close')}</Button>
-        </div>
-      }
-      className="process-log-modal"
-    >
-      <div className="p-6 rounded-lg">
-        <div className="grid grid-cols-1 md:grid-cols-2 gap-6">
-          {/* Left Column */}
-          <div className="space-y-4">
-            <InfoItem label="Task ID" value={taskInfo.taskId} />
-            <InfoItem label="File Name" value={taskInfo.fileName} />
-            <InfoItem label="File Size" value={taskInfo.fileSize} />
-            <InfoItem label="Source" value={taskInfo.source} />
-            <InfoItem label="Task" value={taskInfo.task} />
-            <InfoItem label="Details" value={taskInfo.details} />
-          </div>
-
-          {/* Right Column */}
-          <div className="space-y-4">
-            <div className="flex flex-col">
-              <span className="text-text-secondary text-sm">States</span>
-              <div className="mt-1">
-                <StatusTag state={taskInfo.state} />
-              </div>
-            </div>
-
-            <InfoItem label="Start Time" value={taskInfo.startTime} />
-
-            <InfoItem label="End Time" value={taskInfo.endTime || '-'} />
-
-            <InfoItem
-              label="Duration"
-              value={taskInfo.duration ? `${taskInfo.duration}s` : '-'}
-            />
-          </div>
-        </div>
-      </div>
-    </Modal>
-  );
-};
-
-export default ProcessLogModal;

web/src/pages/datasets/use-select-owners.ts

@@ -1,18 +1,11 @@
 import { FilterCollection } from '@/components/list-filter-bar/interface';
-import { useFetchKnowledgeList } from '@/hooks/knowledge-hooks';
-import { groupListByType } from '@/utils/dataset-util';
-import { useMemo } from 'react';
+import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
+import { buildOwnersFilter } from '@/utils/list-filter-util';
 
 export function useSelectOwners() {
   const { list } = useFetchKnowledgeList();
 
-  const owners = useMemo(() => {
-    return groupListByType(list, 'tenant_id', 'nickname');
-  }, [list]);
-
-  const filters: FilterCollection[] = [
-    { field: 'owner', list: owners, label: 'Owner' },
-  ];
+  const filters: FilterCollection[] = [buildOwnersFilter(list)];
 
   return filters;
 }