feat(dataset): Added data pipeline configuration functionality #9869 (#10132)

### What problem does this PR solve? feat(dataset): Added data pipeline configuration functionality #9869 - Added a data pipeline selection component to link data pipelines with knowledge bases - Added file filtering functionality, supporting custom file filtering rules - Optimized the configuration interface layout, adjusting style and spacing - Introduced new icons and buttons to enhance the user experience ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
2026-01-04 03:25:30 +08:00 · 2025-09-18 09:31:57 +08:00
parent a7abc57f68
commit e82617f6de
25 changed files with 397 additions and 134 deletions
--- a/web/src/components/data-pipeline-select/index.tsx
+++ b/web/src/components/data-pipeline-select/index.tsx
@ -1,7 +1,10 @@
 import { useTranslate } from '@/hooks/common-hooks';
+import { useFetchAgentList } from '@/hooks/use-agent-request';
 import { buildSelectOptions } from '@/utils/component-util';
 import { ArrowUpRight } from 'lucide-react';
+import { useMemo } from 'react';
 import { useFormContext } from 'react-hook-form';
+import { SelectWithSearch } from '../originui/select-with-search';
 import {
  FormControl,
  FormField,
@ -9,11 +12,12 @@ import {
  FormLabel,
  FormMessage,
 } from '../ui/form';
-import { RAGFlowSelect } from '../ui/select';
+import { MultiSelect } from '../ui/multi-select';

 interface IProps {
  toDataPipeline?: () => void;
  formFieldName: string;
+  isMult?: boolean;
 }

 const data = [
@ -22,16 +26,25 @@ const data = [
  { id: '3', name: 'data-pipeline-3' },
  { id: '4', name: 'data-pipeline-4' },
 ];
-export function DataFlowItem(props: IProps) {
-  const { toDataPipeline, formFieldName } = props;
+export function DataFlowSelect(props: IProps) {
+  const { toDataPipeline, formFieldName, isMult = true } = props;
  const { t } = useTranslate('knowledgeConfiguration');
  const form = useFormContext();
+  console.log('data-pipline form', form);
  const toDataPipLine = () => {
-    // window.open('/data-pipeline');
-
    toDataPipeline?.();
  };
-  const options = buildSelectOptions(data, 'id', 'name');
+  const { data: dataPipelineOptions, loading } = useFetchAgentList({
+    canvas_category: 'dataflow_canvas',
+  });
+  const options = useMemo(() => {
+    const option = buildSelectOptions(
+      dataPipelineOptions?.canvas,
+      'id',
+      'title',
+    );
+    return option || [];
+  }, [dataPipelineOptions]);
  return (
    <FormField
      control={form.control}
@ -57,16 +70,27 @@ export function DataFlowItem(props: IProps) {

            <div className="text-muted-foreground">
              <FormControl>
-                <RAGFlowSelect
-                  {...field}
-                  placeholder={t('dataFlowPlaceholder')}
-                  options={options}
-                />
+                <>
+                  {!isMult && (
+                    <SelectWithSearch
+                      {...field}
+                      placeholder={t('dataFlowPlaceholder')}
+                      options={options}
+                    />
+                  )}
+                  {isMult && (
+                    <MultiSelect
+                      {...field}
+                      onValueChange={field.onChange}
+                      placeholder={t('dataFlowPlaceholder')}
+                      options={options}
+                    />
+                  )}
+                </>
              </FormControl>
            </div>
          </div>
          <div className="flex pt-1">
-            <div className="w-1/4"></div>
            <FormMessage />
          </div>
        </FormItem>
--- a/web/src/components/entity-types-form-field.tsx
+++ b/web/src/components/entity-types-form-field.tsx
@ -26,7 +26,7 @@ export function EntityTypesFormField({
        return (
          <FormItem className=" items-center space-y-0 ">
            <div className="flex items-center">
-              <FormLabel className="text-sm text-muted-foreground whitespace-nowrap w-1/4">
+              <FormLabel className="text-sm whitespace-nowrap w-1/4">
                <span className="text-red-600">*</span> {t('entityTypes')}
              </FormLabel>
              <div className="w-3/4">
--- a/web/src/components/home-card.tsx
+++ b/web/src/components/home-card.tsx
@ -39,8 +39,8 @@ export function HomeCard({
          />
        </div>
        <div className="flex flex-col justify-between gap-1 flex-1 h-full w-[calc(100%-50px)]">
-          <section className="flex justify-between">
-            <section className="flex gap-1 items-center">
+          <section className="flex justify-between w-full">
+            <section className="flex gap-1 items-center w-full">
              <div className="text-[20px] font-bold w-80% leading-5 text-ellipsis overflow-hidden">
                {data.name}
              </div>
--- a/web/src/components/originui/select-with-search.tsx
+++ b/web/src/components/originui/select-with-search.tsx
@ -45,6 +45,8 @@ export type SelectWithSearchFlagProps = {
  onChange?(value: string): void;
  triggerClassName?: string;
  allowClear?: boolean;
+  disabled?: boolean;
+  placeholder?: string;
 };

 export const SelectWithSearch = forwardRef<
@ -58,6 +60,8 @@ export const SelectWithSearch = forwardRef<
      options = [],
      triggerClassName,
      allowClear = false,
+      disabled = false,
+      placeholder = t('common.selectPlaceholder'),
    },
    ref,
  ) => {
@ -105,6 +109,7 @@ export const SelectWithSearch = forwardRef<
            role="combobox"
            aria-expanded={open}
            ref={ref}
+            disabled={disabled}
            className={cn(
              'bg-background hover:bg-background border-input w-full  justify-between px-3 font-normal outline-offset-0 outline-none focus-visible:outline-[3px] [&_svg]:pointer-events-auto',
              triggerClassName,
@ -115,9 +120,7 @@ export const SelectWithSearch = forwardRef<
                <span className="leading-none truncate">{selectLabel}</span>
              </span>
            ) : (
-              <span className="text-muted-foreground">
-                {t('common.selectPlaceholder')}
-              </span>
+              <span className="text-muted-foreground">{placeholder}</span>
            )}
            <div className="flex items-center justify-between">
              {value && allowClear && (
--- a/web/src/components/parse-configuration/graph-rag-form-fields.tsx
+++ b/web/src/components/parse-configuration/graph-rag-form-fields.tsx
@ -62,7 +62,7 @@ export function UseGraphRagFormField() {
          <div className="flex items-center gap-1">
            <FormLabel
              tooltip={t('useGraphRagTip')}
-              className="text-sm text-muted-foreground whitespace-break-spaces w-1/4"
+              className="text-sm whitespace-break-spaces w-1/4"
            >
              {t('useGraphRag')}
            </FormLabel>
@ -125,7 +125,7 @@ const GraphRagItems = ({
              <FormItem className=" items-center space-y-0 ">
                <div className="flex items-center">
                  <FormLabel
-                    className="text-sm text-muted-foreground whitespace-nowrap w-1/4"
+                    className="text-sm whitespace-nowrap w-1/4"
                    tooltip={renderWideTooltip(
                      <div
                        dangerouslySetInnerHTML={{
@ -161,7 +161,7 @@ const GraphRagItems = ({
                <div className="flex items-center">
                  <FormLabel
                    tooltip={renderWideTooltip('resolutionTip')}
-                    className="text-sm text-muted-foreground whitespace-nowrap w-1/4"
+                    className="text-sm whitespace-nowrap w-1/4"
                  >
                    {t('resolution')}
                  </FormLabel>
@ -190,7 +190,7 @@ const GraphRagItems = ({
                <div className="flex items-center">
                  <FormLabel
                    tooltip={renderWideTooltip('communityTip')}
-                    className="text-sm text-muted-foreground whitespace-nowrap w-1/4"
+                    className="text-sm whitespace-nowrap w-1/4"
                  >
                    {t('community')}
                  </FormLabel>
--- a/web/src/components/parse-configuration/raptor-form-fields.tsx
+++ b/web/src/components/parse-configuration/raptor-form-fields.tsx
@ -2,11 +2,10 @@ import { FormLayout } from '@/constants/form';
 import { DocumentParserType } from '@/constants/knowledge';
 import { useTranslate } from '@/hooks/common-hooks';
 import random from 'lodash/random';
-import { Plus } from 'lucide-react';
+import { Shuffle } from 'lucide-react';
 import { useCallback } from 'react';
 import { useFormContext, useWatch } from 'react-hook-form';
 import { SliderInputFormField } from '../slider-input-form-field';
-import { Button } from '../ui/button';
 import {
  FormControl,
  FormField,
@ -14,7 +13,7 @@ import {
  FormLabel,
  FormMessage,
 } from '../ui/form';
-import { Input } from '../ui/input';
+import { ExpandedInput } from '../ui/input';
 import { Switch } from '../ui/switch';
 import { Textarea } from '../ui/textarea';

@ -93,7 +92,7 @@ const RaptorFormFields = () => {
              <div className="flex items-center gap-1">
                <FormLabel
                  tooltip={t('useRaptorTip')}
-                  className="text-sm text-muted-foreground w-1/4 whitespace-break-spaces"
+                  className="text-sm  w-1/4 whitespace-break-spaces"
                >
                  <div className="w-auto xl:w-20 2xl:w-24 3xl:w-28 4xl:w-auto ">
                    {t('useRaptor')}
@ -130,7 +129,7 @@ const RaptorFormFields = () => {
                  <div className="flex items-start">
                    <FormLabel
                      tooltip={t('promptTip')}
-                      className="text-sm text-muted-foreground whitespace-nowrap w-1/4"
+                      className="text-sm  whitespace-nowrap w-1/4"
                    >
                      {t('prompt')}
                    </FormLabel>
@ -185,21 +184,23 @@ const RaptorFormFields = () => {
            render={({ field }) => (
              <FormItem className=" items-center space-y-0 ">
                <div className="flex items-center">
-                  <FormLabel className="text-sm text-muted-foreground whitespace-wrap w-1/4">
+                  <FormLabel className="text-sm  whitespace-wrap w-1/4">
                    {t('randomSeed')}
                  </FormLabel>
                  <div className="w-3/4">
                    <FormControl defaultValue={0}>
-                      <div className="flex gap-4 items-center">
-                        <Input {...field} defaultValue={0} type="number" />
-                        <Button
-                          size={'sm'}
-                          onClick={handleGenerate}
-                          type={'button'}
-                        >
-                          <Plus />
-                        </Button>
-                      </div>
+                      <ExpandedInput
+                        {...field}
+                        className="w-full"
+                        defaultValue={0}
+                        type="number"
+                        suffix={
+                          <Shuffle
+                            className="size-3.5 cursor-pointer"
+                            onClick={handleGenerate}
+                          />
+                        }
+                      />
                    </FormControl>
                  </div>
                </div>
--- a/web/src/components/slider-input-form-field.tsx
+++ b/web/src/components/slider-input-form-field.tsx
@ -54,8 +54,7 @@ export function SliderInputFormField({
          <FormLabel
            tooltip={tooltip}
            className={cn({
-              'text-sm text-muted-foreground whitespace-break-spaces w-1/4':
-                isHorizontal,
+              'text-sm whitespace-break-spaces w-1/4': isHorizontal,
            })}
          >
            {label}