feat(dataset): Added data pipeline configuration functionality #9869 (#10132)

### What problem does this PR solve? feat(dataset): Added data pipeline configuration functionality #9869 - Added a data pipeline selection component to link data pipelines with knowledge bases - Added file filtering functionality, supporting custom file filtering rules - Optimized the configuration interface layout, adjusting style and spacing - Introduced new icons and buttons to enhance the user experience ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
2025-12-26 08:56:47 +08:00 · 2025-09-18 09:31:57 +08:00
parent a7abc57f68
commit e82617f6de
25 changed files with 397 additions and 134 deletions
--- a/web/src/pages/dataset/dataset-setting/components/link-data-pipeline.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/link-data-pipeline.tsx
@ -1,17 +1,21 @@
+import { IconFont } from '@/components/icon-font';
 import { RAGFlowAvatar } from '@/components/ragflow-avatar';
 import { Button } from '@/components/ui/button';
-import { Link, Route, Settings2, Unlink } from 'lucide-react';
+import { Link, Settings2, Unlink } from 'lucide-react';
+import { useState } from 'react';
 import { useTranslation } from 'react-i18next';
+import LinkDataPipelineModal from './link-data-pipline-modal';

 interface DataPipelineItemProps {
  name: string;
  avatar?: string;
  isDefault?: boolean;
  linked?: boolean;
+  openLinkModalFunc?: (open: boolean) => void;
 }
 const DataPipelineItem = (props: DataPipelineItemProps) => {
  const { t } = useTranslation();
-  const { name, avatar, isDefault, linked } = props;
+  const { name, avatar, isDefault, linked, openLinkModalFunc } = props;
  return (
    <div className="flex items-center justify-between gap-1 px-2 rounded-lg border">
      <div className="flex items-center gap-1">
@ -27,15 +31,24 @@ const DataPipelineItem = (props: DataPipelineItemProps) => {
        <Button variant={'transparent'} className="border-none">
          <Settings2 />
        </Button>
-        <Button variant={'transparent'} className="border-none">
-          {linked ? <Link /> : <Unlink />}
-        </Button>
+        {!isDefault && (
+          <Button
+            variant={'transparent'}
+            className="border-none"
+            onClick={() => {
+              openLinkModalFunc?.(true);
+            }}
+          >
+            {linked ? <Link /> : <Unlink />}
+          </Button>
+        )}
      </div>
    </div>
  );
 };
 const LinkDataPipeline = () => {
  const { t } = useTranslation();
+  const [openLinkModal, setOpenLinkModal] = useState(false);
  const testNode = [
    {
      name: 'Data Pipeline 1',
@ -49,11 +62,14 @@ const LinkDataPipeline = () => {
      linked: false,
    },
  ];
+  const openLinkModalFunc = (open: boolean) => {
+    setOpenLinkModal(open);
+  };
  return (
    <div className="flex flex-col gap-2">
      <section className="flex flex-col">
        <div className="flex items-center gap-1 text-text-primary text-sm">
-          <Route className="size-4" />
+          <IconFont name="Pipeline" />
          {t('knowledgeConfiguration.dataPipeline')}
        </div>
        <div className="flex justify-between items-center">
@ -70,9 +86,19 @@ const LinkDataPipeline = () => {
      </section>
      <section className="flex flex-col gap-2">
        {testNode.map((item) => (
-          <DataPipelineItem key={item.name} {...item} />
+          <DataPipelineItem
+            key={item.name}
+            openLinkModalFunc={openLinkModalFunc}
+            {...item}
+          />
        ))}
      </section>
+      <LinkDataPipelineModal
+        open={openLinkModal}
+        setOpen={(open: boolean) => {
+          openLinkModalFunc(open);
+        }}
+      />
    </div>
  );
 };
--- a/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/link-data-pipline-modal.tsx
@ -0,0 +1,95 @@
+import { DataFlowSelect } from '@/components/data-pipeline-select';
+import Input from '@/components/originui/input';
+import { Button } from '@/components/ui/button';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { Modal } from '@/components/ui/modal/modal';
+import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { t } from 'i18next';
+import { useForm } from 'react-hook-form';
+import { z } from 'zod';
+import { linkPiplineFormSchema } from '../form-schema';
+
+const LinkDataPipelineModal = ({
+  open,
+  setOpen,
+}: {
+  open: boolean;
+  setOpen: (open: boolean) => void;
+}) => {
+  const form = useForm<z.infer<typeof linkPiplineFormSchema>>({
+    resolver: zodResolver(linkPiplineFormSchema),
+    defaultValues: { data_flow: ['888'], file_filter: '' },
+  });
+  //   const [open, setOpen] = useState(false);
+  const { navigateToAgents } = useNavigatePage();
+  const handleFormSubmit = (values: any) => {
+    console.log(values);
+  };
+  return (
+    <Modal
+      title={t('knowledgeConfiguration.linkDataPipeline')}
+      open={open}
+      onOpenChange={setOpen}
+      showfooter={false}
+    >
+      <Form {...form}>
+        <form onSubmit={form.handleSubmit(handleFormSubmit)}>
+          <div className="flex flex-col gap-4 ">
+            <DataFlowSelect
+              toDataPipeline={navigateToAgents}
+              formFieldName="data_flow"
+            />
+            <FormField
+              control={form.control}
+              name={'file_filter'}
+              render={({ field }) => (
+                <FormItem className=" items-center space-y-0 ">
+                  <div className="flex flex-col gap-1">
+                    <div className="flex gap-2 justify-between ">
+                      <FormLabel
+                        tooltip={t('knowledgeConfiguration.fileFilterTip')}
+                        className="text-sm text-text-primary whitespace-wrap "
+                      >
+                        {t('knowledgeConfiguration.fileFilter')}
+                      </FormLabel>
+                    </div>
+
+                    <div className="text-muted-foreground">
+                      <FormControl>
+                        <Input
+                          placeholder={t('dataFlowPlaceholder')}
+                          {...field}
+                        />
+                      </FormControl>
+                    </div>
+                  </div>
+                  <div className="flex pt-1">
+                    <div className="w-full"></div>
+                    <FormMessage />
+                  </div>
+                </FormItem>
+              )}
+            />
+            <div className="flex justify-end gap-1">
+              <Button type="reset" variant={'outline'} className="btn-primary">
+                {t('modal.cancelText')}
+              </Button>
+              <Button type="submit" variant={'default'} className="btn-primary">
+                {t('modal.okText')}
+              </Button>
+            </div>
+          </div>
+        </form>
+      </Form>
+    </Modal>
+  );
+};
+export default LinkDataPipelineModal;
--- a/web/src/pages/dataset/dataset-setting/configuration-form-container.tsx
+++ b/web/src/pages/dataset/dataset-setting/configuration-form-container.tsx
@ -9,6 +9,9 @@ export function ConfigurationFormContainer({
  return <section className={cn('space-y-4', className)}>{children}</section>;
 }

-export function MainContainer({ children }: PropsWithChildren) {
-  return <section className="space-y-5">{children}</section>;
+export function MainContainer({
+  children,
+  className,
+}: PropsWithChildren & { className?: string }) {
+  return <section className={cn('space-y-5', className)}>{children}</section>;
 }
--- a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx
+++ b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx
@ -1,3 +1,4 @@
+import { SelectWithSearch } from '@/components/originui/select-with-search';
 import {
  FormControl,
  FormField,
@ -16,8 +17,12 @@ import {
  useSelectChunkMethodList,
  useSelectEmbeddingModelOptions,
 } from '../hooks';
-
-export function ChunkMethodItem() {
+interface IProps {
+  line?: 1 | 2;
+  isEdit?: boolean;
+}
+export function ChunkMethodItem(props: IProps) {
+  const { line } = props;
  const { t } = useTranslate('knowledgeConfiguration');
  const form = useFormContext();
  // const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
@ -28,28 +33,29 @@ export function ChunkMethodItem() {
      control={form.control}
      name={'parser_id'}
      render={({ field }) => (
-        <FormItem className=" items-center space-y-0 ">
-          <div className="flex items-center">
+        <FormItem className=" items-center space-y-1">
+          <div className={line === 1 ? 'flex items-center' : ''}>
            <FormLabel
              required
              tooltip={t('chunkMethodTip')}
-              className="text-sm text-muted-foreground whitespace-wrap w-1/4"
+              className={cn('text-sm', {
+                'w-1/4 whitespace-pre-wrap': line === 1,
+              })}
            >
              {t('chunkMethod')}
            </FormLabel>
-            <div className="w-3/4 ">
+            <div className={line === 1 ? 'w-3/4 ' : 'w-full'}>
              <FormControl>
                <RAGFlowSelect
                  {...field}
                  options={parserList}
                  placeholder={t('chunkMethodPlaceholder')}
-                  // onChange={handleChunkMethodSelectChange}
                />
              </FormControl>
            </div>
          </div>
          <div className="flex pt-1">
-            <div className="w-1/4"></div>
+            <div className={line === 1 ? 'w-1/4' : ''}></div>
            <FormMessage />
          </div>
        </FormItem>
@ -57,54 +63,55 @@ export function ChunkMethodItem() {
    />
  );
 }
-
-export function EmbeddingModelItem({ line = 1 }: { line?: 1 | 2 }) {
+export function EmbeddingModelItem({ line = 1, isEdit = true }: IProps) {
  const { t } = useTranslate('knowledgeConfiguration');
  const form = useFormContext();
  const embeddingModelOptions = useSelectEmbeddingModelOptions();
-  const disabled = useHasParsedDocument();
-
+  const disabled = useHasParsedDocument(isEdit);
  return (
-    <FormField
-      control={form.control}
-      name={'embd_id'}
-      render={({ field }) => (
-        <FormItem className={cn(' items-center space-y-0 ')}>
-          <div
-            className={cn('flex', {
-              ' items-center': line === 1,
-              'flex-col gap-1': line === 2,
-            })}
-          >
-            <FormLabel
-              required
-              tooltip={t('embeddingModelTip')}
-              className={cn('text-sm  whitespace-wrap ', {
-                'w-1/4': line === 1,
+    <>
+      <FormField
+        control={form.control}
+        name={'embd_id'}
+        render={({ field }) => (
+          <FormItem className={cn(' items-center space-y-0 ')}>
+            <div
+              className={cn('flex', {
+                ' items-center': line === 1,
+                'flex-col gap-1': line === 2,
              })}
            >
-              {t('embeddingModel')}
-            </FormLabel>
-            <div
-              className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
-            >
-              <FormControl>
-                <RAGFlowSelect
-                  {...field}
-                  options={embeddingModelOptions}
-                  disabled={disabled}
-                  placeholder={t('embeddingModelPlaceholder')}
-                />
-              </FormControl>
+              <FormLabel
+                required
+                tooltip={t('embeddingModelTip')}
+                className={cn('text-sm  whitespace-wrap ', {
+                  'w-1/4': line === 1,
+                })}
+              >
+                {t('embeddingModel')}
+              </FormLabel>
+              <div
+                className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
+              >
+                <FormControl>
+                  <SelectWithSearch
+                    onChange={field.onChange}
+                    value={field.value}
+                    options={embeddingModelOptions}
+                    disabled={isEdit ? disabled : false}
+                    placeholder={t('embeddingModelPlaceholder')}
+                  />
+                </FormControl>
+              </div>
            </div>
-          </div>
-          <div className="flex pt-1">
-            <div className="w-1/4"></div>
-            <FormMessage />
-          </div>
-        </FormItem>
-      )}
-    />
+            <div className="flex pt-1">
+              <div className={line === 1 ? 'w-1/4' : ''}></div>
+              <FormMessage />
+            </div>
+          </FormItem>
+        )}
+      />
+    </>
  );
 }

--- a/web/src/pages/dataset/dataset-setting/form-schema.ts
+++ b/web/src/pages/dataset/dataset-setting/form-schema.ts
@ -71,3 +71,8 @@ export const formSchema = z.object({
  pagerank: z.number(),
  // icon: z.array(z.instanceof(File)),
 });
+
+export const linkPiplineFormSchema = z.object({
+  data_flow: z.array(z.string()),
+  file_filter: z.string().optional(),
+});
--- a/web/src/pages/dataset/dataset-setting/general-form.tsx
+++ b/web/src/pages/dataset/dataset-setting/general-form.tsx
@ -24,7 +24,7 @@ export function GeneralForm() {
        render={({ field }) => (
          <FormItem className="items-center space-y-0">
            <div className="flex">
-              <FormLabel className="text-sm text-muted-foreground whitespace-nowrap w-1/4">
+              <FormLabel className="text-sm whitespace-nowrap w-1/4">
                <span className="text-red-600">*</span>
                {t('common.name')}
              </FormLabel>
@ -45,7 +45,7 @@ export function GeneralForm() {
        render={({ field }) => (
          <FormItem className="items-center space-y-0">
            <div className="flex">
-              <FormLabel className="text-sm text-muted-foreground whitespace-nowrap w-1/4">
+              <FormLabel className="text-sm  whitespace-nowrap w-1/4">
                {t('setting.avatar')}
              </FormLabel>
              <FormControl className="w-3/4">
@ -70,7 +70,7 @@ export function GeneralForm() {
          return (
            <FormItem className="items-center space-y-0">
              <div className="flex">
-                <FormLabel className="text-sm text-muted-foreground whitespace-nowrap w-1/4">
+                <FormLabel className="text-sm  whitespace-nowrap w-1/4">
                  {t('flow.description')}
                </FormLabel>
                <FormControl className="w-3/4">
--- a/web/src/pages/dataset/dataset-setting/hooks.ts
+++ b/web/src/pages/dataset/dataset-setting/hooks.ts
@ -25,8 +25,10 @@ export function useSelectEmbeddingModelOptions() {
  return allOptions[LlmModelType.Embedding];
 }

-export function useHasParsedDocument() {
-  const { data: knowledgeDetails } = useFetchKnowledgeBaseConfiguration();
+export function useHasParsedDocument(isEdit?: boolean) {
+  const { data: knowledgeDetails } = useFetchKnowledgeBaseConfiguration({
+    isEdit,
+  });
  return knowledgeDetails.chunk_num > 0;
 }

@ -52,7 +54,7 @@ export const useFetchKnowledgeConfigurationOnMount = (
        'pagerank',
        'avatar',
      ]),
-    };
+    } as z.infer<typeof formSchema>;
    form.reset(formValues);
  }, [form, knowledgeDetails]);

--- a/web/src/pages/dataset/dataset-setting/index.tsx
+++ b/web/src/pages/dataset/dataset-setting/index.tsx
@ -82,7 +82,7 @@ export default function DatasetSettings() {
            className="space-y-6 flex-1"
          >
            <div className="w-[768px] h-[calc(100vh-240px)] pr-1 overflow-y-auto scrollbar-auto">
-              <MainContainer>
+              <MainContainer className="text-text-secondary">
                <GeneralForm></GeneralForm>
                <Divider />

--- a/web/src/pages/dataset/sidebar/index.tsx
+++ b/web/src/pages/dataset/sidebar/index.tsx
@ -28,7 +28,7 @@ export function SideBar({ refreshCount }: PropType) {
  const pathName = useSecondPathName();
  const { handleMenuClick } = useHandleMenuClick();
  // refreshCount: be for avatar img sync update on top left
-  const { data } = useFetchKnowledgeBaseConfiguration(refreshCount);
+  const { data } = useFetchKnowledgeBaseConfiguration({ refreshCount });
  const { data: routerData } = useFetchKnowledgeGraph();
  const { t } = useTranslation();

@ -52,7 +52,7 @@ export function SideBar({ refreshCount }: PropType) {
      {
        icon: Banknote,
        label: t(`knowledgeDetails.configuration`),
-        key: Routes.DatasetSetting,
+        key: Routes.DataSetSetting,
      },
    ];
    if (!isEmpty(routerData?.graph)) {
--- a/web/src/pages/datasets/dataset-creating-dialog.tsx
+++ b/web/src/pages/datasets/dataset-creating-dialog.tsx
@ -1,4 +1,4 @@
-import { DataFlowItem } from '@/components/data-pipeline-select';
+import { DataFlowSelect } from '@/components/data-pipeline-select';
 import { ButtonLoading } from '@/components/ui/button';
 import {
  Dialog,
@ -23,6 +23,7 @@ import { useForm, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
 import {
+  ChunkMethodItem,
  EmbeddingModelItem,
  ParseTypeItem,
 } from '../dataset/dataset-setting/configuration/common-item';
@ -32,32 +33,68 @@ const FormId = 'dataset-creating-form';
 export function InputForm({ onOk }: IModalProps<any>) {
  const { t } = useTranslation();

-  const FormSchema = z.object({
-    name: z
-      .string()
-      .min(1, {
-        message: t('knowledgeList.namePlaceholder'),
-      })
-      .trim(),
-    parseType: z.number().optional(),
-  });
+  const FormSchema = z
+    .object({
+      name: z
+        .string()
+        .min(1, {
+          message: t('knowledgeList.namePlaceholder'),
+        })
+        .trim(),
+      parseType: z.number().optional(),
+      embd_id: z
+        .string()
+        .min(1, {
+          message: t('knowledgeConfiguration.embeddingModelPlaceholder'),
+        })
+        .trim(),
+      parser_id: z.string().optional(),
+      pipline_id: z.string().optional(),
+    })
+    .superRefine((data, ctx) => {
+      // When parseType === 1, parser_id is required
+      if (
+        data.parseType === 1 &&
+        (!data.parser_id || data.parser_id.trim() === '')
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: t('knowledgeList.parserRequired'),
+          path: ['parser_id'],
+        });
+      }
+
+      console.log('form-data', data);
+      // When parseType === 1, pipline_id required
+      if (data.parseType === 2 && !data.pipline_id) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: t('knowledgeList.dataFlowRequired'),
+          path: ['pipline_id'],
+        });
+      }
+    });

  const form = useForm<z.infer<typeof FormSchema>>({
    resolver: zodResolver(FormSchema),
    defaultValues: {
      name: '',
      parseType: 1,
+      parser_id: '',
+      embd_id: '',
    },
  });

  function onSubmit(data: z.infer<typeof FormSchema>) {
-    onOk?.(data.name);
+    console.log('submit', data);
+    onOk?.(data);
  }
  const parseType = useWatch({
    control: form.control,
    name: 'parseType',
  });
  const { navigateToAgents } = useNavigatePage();
+
  return (
    <Form {...form}>
      <form
@ -84,13 +121,19 @@ export function InputForm({ onOk }: IModalProps<any>) {
            </FormItem>
          )}
        />
-        <EmbeddingModelItem line={2} />
+        <EmbeddingModelItem line={2} isEdit={false} />
        <ParseTypeItem />
+        {parseType === 1 && (
+          <>
+            <ChunkMethodItem></ChunkMethodItem>
+          </>
+        )}
        {parseType === 2 && (
          <>
-            <DataFlowItem
+            <DataFlowSelect
+              isMult={false}
              toDataPipeline={navigateToAgents}
-              formFieldName="data_flow"
+              formFieldName="pipline_id"
            />
          </>
        )}
@ -108,7 +151,7 @@ export function DatasetCreatingDialog({

  return (
    <Dialog open onOpenChange={hideModal}>
-      <DialogContent className="sm:max-w-[425px]">
+      <DialogContent className="sm:max-w-[425px] focus-visible:!outline-none">
        <DialogHeader>
          <DialogTitle>{t('knowledgeList.createKnowledgeBase')}</DialogTitle>
        </DialogHeader>
--- a/web/src/pages/datasets/hooks.ts
+++ b/web/src/pages/datasets/hooks.ts
@ -2,7 +2,6 @@ import { useSetModalState } from '@/hooks/common-hooks';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { useCreateKnowledge } from '@/hooks/use-knowledge-request';
 import { useCallback, useState } from 'react';
-
 export const useSearchKnowledge = () => {
  const [searchString, setSearchString] = useState<string>('');

@ -15,16 +14,19 @@ export const useSearchKnowledge = () => {
  };
 };

+export interface Iknowledge {
+  name: string;
+  embd_id: string;
+  parser_id: string;
+}
 export const useSaveKnowledge = () => {
  const { visible: visible, hideModal, showModal } = useSetModalState();
  const { loading, createKnowledge } = useCreateKnowledge();
  const { navigateToDataset } = useNavigatePage();

  const onCreateOk = useCallback(
-    async (name: string) => {
-      const ret = await createKnowledge({
-        name,
-      });
+    async (data: Iknowledge) => {
+      const ret = await createKnowledge(data);

      if (ret?.code === 0) {
        hideModal();