Feature: Added data source functionality #10703 (#11046)

### What problem does this PR solve? Feature: Added data source functionality ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2025-12-26 08:56:47 +08:00 · 2025-11-06 11:53:46 +08:00
parent 15c75bbf15
commit f581a1c4e5
31 changed files with 2526 additions and 16 deletions
--- a/web/src/pages/dataset/dataset-setting/components/added-source-card.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/added-source-card.tsx
@ -0,0 +1,97 @@
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
+import { cn } from '@/lib/utils';
+import {
+  IDataSorceInfo,
+  IDataSourceBase,
+} from '@/pages/user-setting/data-source/interface';
+import { Check } from 'lucide-react';
+import { useMemo } from 'react';
+
+export type IAddedSourceCardProps = IDataSorceInfo & {
+  filterString: string;
+  list: IDataSourceBase[];
+  selectedList: IDataSourceBase[];
+  setSelectedList: (list: IDataSourceBase[]) => void;
+};
+export const AddedSourceCard = (props: IAddedSourceCardProps) => {
+  const {
+    list: originList,
+    name,
+    icon,
+    filterString,
+    selectedList,
+    setSelectedList,
+  } = props;
+
+  const list = useMemo(() => {
+    return originList.map((item) => {
+      const checked = selectedList?.some((i) => i.id === item.id) || false;
+      return {
+        ...item,
+        checked: checked,
+      };
+    });
+  }, [originList, selectedList]);
+
+  const filterList = useMemo(
+    () => list.filter((item) => item.name.indexOf(filterString) > -1),
+    [filterString, list],
+  );
+
+  // const { navigateToDataSourceDetail } = useNavigatePage();
+  // const toDetail = (id: string) => {
+  //   navigateToDataSourceDetail(id);
+  // };
+
+  const onCheck = (item: IDataSourceBase & { checked: boolean }) => {
+    if (item.checked) {
+      setSelectedList(selectedList.filter((i) => i.id !== item.id));
+    } else {
+      setSelectedList([...(selectedList || []), item]);
+    }
+  };
+  return (
+    <>
+      {filterList.length > 0 && (
+        <Card className="bg-transparent border border-border-button px-5 pt-[10px] pb-5 rounded-md">
+          <CardHeader className="flex flex-row items-center justify-between space-y-0 p-0 pb-3">
+            {/* <Users className="mr-2 h-5 w-5 text-[#1677ff]" /> */}
+            <CardTitle className="text-base flex gap-1 font-normal">
+              {icon}
+              {name}
+            </CardTitle>
+          </CardHeader>
+          <CardContent className="p-2 flex flex-col gap-2">
+            {filterList.map((item) => (
+              <div
+                key={item.id}
+                className={cn(
+                  'flex flex-row items-center justify-between rounded-md bg-bg-input px-2 py-1 cursor-pointer',
+                  // { hidden: item.name.indexOf(filterString) <= -1 },
+                )}
+                onClick={() => {
+                  console.log('item--->', item);
+                  // toDetail(item.id);
+                  onCheck(item);
+                }}
+              >
+                <div className="text-sm text-text-secondary ">{item.name}</div>
+                <div className="text-sm text-text-secondary  flex gap-2">
+                  {item.checked && (
+                    <Check
+                      className="cursor-pointer"
+                      size={14}
+                      // onClick={() => {
+                      //   toDetail(item.id);
+                      // }}
+                    />
+                  )}
+                </div>
+              </div>
+            ))}
+          </CardContent>
+        </Card>
+      )}
+    </>
+  );
+};
--- a/web/src/pages/dataset/dataset-setting/components/link-data-source-modal.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/link-data-source-modal.tsx
@ -0,0 +1,86 @@
+import { Button } from '@/components/ui/button';
+import { SearchInput } from '@/components/ui/input';
+import { Modal } from '@/components/ui/modal/modal';
+import { IConnector } from '@/interfaces/database/knowledge';
+import { useListDataSource } from '@/pages/user-setting/data-source/hooks';
+import { IDataSourceBase } from '@/pages/user-setting/data-source/interface';
+import { t } from 'i18next';
+import { useEffect, useState } from 'react';
+import { AddedSourceCard } from './added-source-card';
+
+const LinkDataSourceModal = ({
+  selectedList,
+  open,
+  setOpen,
+  onSubmit,
+}: {
+  selectedList: IConnector[];
+  open: boolean;
+  setOpen: (open: boolean) => void;
+  onSubmit?: (list: IDataSourceBase[] | undefined) => void;
+}) => {
+  const [list, setList] = useState<IDataSourceBase[]>();
+  const [fileterString, setFileterString] = useState('');
+
+  useEffect(() => {
+    setList(selectedList);
+  }, [selectedList]);
+
+  const { categorizedList } = useListDataSource();
+  const handleFormSubmit = (values: any) => {
+    console.log(values, selectedList);
+    onSubmit?.(list);
+  };
+  return (
+    <Modal
+      className="!w-[560px]"
+      title={t('knowledgeConfiguration.linkDataSource')}
+      open={open}
+      onCancel={() => {
+        setList(selectedList);
+      }}
+      onOpenChange={setOpen}
+      showfooter={false}
+    >
+      <div className="flex flex-col gap-4 ">
+        {/* {JSON.stringify(selectedList)} */}
+        <SearchInput
+          value={fileterString}
+          onChange={(e) => setFileterString(e.target.value)}
+        />
+        <div className="flex flex-col gap-3">
+          {categorizedList.map((item, index) => (
+            <AddedSourceCard
+              key={index}
+              selectedList={list as IDataSourceBase[]}
+              setSelectedList={(list) => setList(list)}
+              filterString={fileterString}
+              {...item}
+            />
+          ))}
+        </div>
+        <div className="flex justify-end gap-1">
+          <Button
+            type="button"
+            variant={'outline'}
+            className="btn-primary"
+            onClick={() => {
+              setOpen(false);
+            }}
+          >
+            {t('modal.cancelText')}
+          </Button>
+          <Button
+            type="button"
+            variant={'default'}
+            className="btn-primary"
+            onClick={handleFormSubmit}
+          >
+            {t('modal.okText')}
+          </Button>
+        </div>
+      </div>
+    </Modal>
+  );
+};
+export default LinkDataSourceModal;
--- a/web/src/pages/dataset/dataset-setting/components/link-data-source.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/link-data-source.tsx
@ -0,0 +1,193 @@
+import { Button } from '@/components/ui/button';
+import { Modal } from '@/components/ui/modal/modal';
+import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
+import { IConnector } from '@/interfaces/database/knowledge';
+import { DataSourceInfo } from '@/pages/user-setting/data-source/contant';
+import { IDataSourceBase } from '@/pages/user-setting/data-source/interface';
+import { Link, Settings, Unlink } from 'lucide-react';
+import { useMemo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import LinkDataSourceModal from './link-data-source-modal';
+
+export type IDataSourceNodeProps = IConnector & {
+  icon: React.ReactNode;
+};
+
+export interface ILinkDataSourceProps {
+  data?: IConnector[];
+  handleLinkOrEditSubmit?: (data: IDataSourceBase[] | undefined) => void;
+  unbindFunc?: (item: DataSourceItemProps) => void;
+}
+
+interface DataSourceItemProps extends IDataSourceNodeProps {
+  openLinkModalFunc?: (open: boolean, data?: IDataSourceNodeProps) => void;
+  unbindFunc?: (item: DataSourceItemProps) => void;
+}
+
+const DataSourceItem = (props: DataSourceItemProps) => {
+  const { t } = useTranslation();
+  const { id, name, icon, openLinkModalFunc, unbindFunc } = props;
+
+  const { navigateToDataSourceDetail } = useNavigatePage();
+  const toDetail = (id: string) => {
+    navigateToDataSourceDetail(id);
+  };
+  const openUnlinkModal = () => {
+    Modal.show({
+      visible: true,
+      className: '!w-[560px]',
+      title: t('dataflowParser.unlinkSourceModalTitle'),
+      children: (
+        <div
+          className="text-sm text-text-secondary"
+          dangerouslySetInnerHTML={{
+            __html: t('dataflowParser.unlinkSourceModalContent'),
+          }}
+        ></div>
+      ),
+      onVisibleChange: () => {
+        Modal.hide();
+      },
+      footer: (
+        <div className="flex justify-end gap-2">
+          <Button variant={'outline'} onClick={() => Modal.hide()}>
+            {t('dataflowParser.changeStepModalCancelText')}
+          </Button>
+          <Button
+            variant={'secondary'}
+            className="!bg-state-error text-bg-base"
+            onClick={() => {
+              unbindFunc?.(props);
+              Modal.hide();
+            }}
+          >
+            {t('dataflowParser.unlinkSourceModalConfirmText')}
+          </Button>
+        </div>
+      ),
+    });
+  };
+
+  return (
+    <div className="flex items-center justify-between gap-1 px-2 rounded-md border ">
+      <div className="flex items-center gap-1">
+        {icon}
+        <div>{name}</div>
+      </div>
+      <div className="flex gap-1 items-center">
+        <Button
+          variant={'transparent'}
+          className="border-none"
+          type="button"
+          onClick={() => {
+            toDetail(id);
+          }}
+          // onClick={() =>
+          //   openLinkModalFunc?.(true, { ...omit(props, ['openLinkModalFunc']) })
+          // }
+        >
+          <Settings />
+        </Button>
+        <>
+          <Button
+            type="button"
+            variant={'transparent'}
+            className="border-none"
+            onClick={() => {
+              openUnlinkModal();
+            }}
+          >
+            <Unlink />
+          </Button>
+        </>
+      </div>
+    </div>
+  );
+};
+
+const LinkDataSource = (props: ILinkDataSourceProps) => {
+  const { data, handleLinkOrEditSubmit: submit, unbindFunc } = props;
+  const { t } = useTranslation();
+  const [openLinkModal, setOpenLinkModal] = useState(false);
+
+  const pipelineNode: IDataSourceNodeProps[] = useMemo(() => {
+    if (data && data.length > 0) {
+      return data.map((item) => {
+        return {
+          ...item,
+          id: item?.id,
+          name: item?.name,
+          icon:
+            DataSourceInfo[item?.source as keyof typeof DataSourceInfo]?.icon ||
+            '',
+        } as IDataSourceNodeProps;
+      });
+    }
+    return [];
+  }, [data]);
+
+  const openLinkModalFunc = (open: boolean, data?: IDataSourceNodeProps) => {
+    console.log('open', open, data);
+    setOpenLinkModal(open);
+    // if (data) {
+    //   setCurrentDataSource(data);
+    // } else {
+    //   setCurrentDataSource(undefined);
+    // }
+  };
+
+  const handleLinkOrEditSubmit = (data: IDataSourceBase[] | undefined) => {
+    console.log('handleLinkOrEditSubmit', data);
+    submit?.(data);
+    setOpenLinkModal(false);
+  };
+
+  return (
+    <div className="flex flex-col gap-2">
+      <section className="flex flex-col">
+        <div className="flex items-center gap-1 text-text-primary text-sm">
+          {t('knowledgeConfiguration.dataSource')}
+        </div>
+        <div className="flex justify-between items-center">
+          <div className="text-center text-xs text-text-secondary">
+            {t('knowledgeConfiguration.linkSourceSetTip')}
+          </div>
+          <Button
+            type="button"
+            variant={'transparent'}
+            onClick={() => {
+              openLinkModalFunc?.(true);
+            }}
+          >
+            <Link />
+            <span className="text-xs text-text-primary">
+              {t('knowledgeConfiguration.linkDataSource')}
+            </span>
+          </Button>
+        </div>
+      </section>
+      <section className="flex flex-col gap-2">
+        {pipelineNode.map(
+          (item) =>
+            item.id && (
+              <DataSourceItem
+                key={item.id}
+                openLinkModalFunc={openLinkModalFunc}
+                unbindFunc={unbindFunc}
+                {...item}
+              />
+            ),
+        )}
+      </section>
+      <LinkDataSourceModal
+        selectedList={data as IConnector[]}
+        open={openLinkModal}
+        setOpen={(open: boolean) => {
+          openLinkModalFunc(open);
+        }}
+        onSubmit={handleLinkOrEditSubmit}
+      />
+    </div>
+  );
+};
+export default LinkDataSource;
--- a/web/src/pages/dataset/dataset-setting/form-schema.ts
+++ b/web/src/pages/dataset/dataset-setting/form-schema.ts
@ -76,6 +76,16 @@ export const formSchema = z
      })
      .optional(),
    pagerank: z.number(),
+    connectors: z
+      .array(
+        z.object({
+          id: z.string().optional(),
+          name: z.string().optional(),
+          source: z.string().optional(),
+          ststus: z.string().optional(),
+        }),
+      )
+      .optional(),
    // icon: z.array(z.instanceof(File)),
  })
  .superRefine((data, ctx) => {
--- a/web/src/pages/dataset/dataset-setting/index.tsx
+++ b/web/src/pages/dataset/dataset-setting/index.tsx
@ -7,6 +7,8 @@ import { Form } from '@/components/ui/form';
 import { FormLayout } from '@/constants/form';
 import { DocumentParserType } from '@/constants/knowledge';
 import { PermissionRole } from '@/constants/permission';
+import { DataSourceInfo } from '@/pages/user-setting/data-source/contant';
+import { IDataSourceBase } from '@/pages/user-setting/data-source/interface';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { useEffect, useState } from 'react';
 import { useForm, useWatch } from 'react-hook-form';
@ -19,6 +21,9 @@ import {
 } from '../dataset/generate-button/generate';
 import { ChunkMethodForm } from './chunk-method-form';
 import ChunkMethodLearnMore from './chunk-method-learn-more';
+import LinkDataSource, {
+  IDataSourceNodeProps,
+} from './components/link-data-source';
 import { MainContainer } from './configuration-form-container';
 import { ChunkMethodItem, ParseTypeItem } from './configuration/common-item';
 import { formSchema } from './form-schema';
@ -78,10 +83,12 @@ export default function DatasetSettings() {
      pipeline_id: '',
      parseType: 1,
      pagerank: 0,
+      connectors: [],
    },
  });
  const knowledgeDetails = useFetchKnowledgeConfigurationOnMount(form);
  // const [pipelineData, setPipelineData] = useState<IDataPipelineNodeProps>();
+  const [sourceData, setSourceData] = useState<IDataSourceNodeProps[]>();
  const [graphRagGenerateData, setGraphRagGenerateData] =
    useState<IGenerateLogButtonProps>();
  const [raptorGenerateData, setRaptorGenerateData] =
@ -97,6 +104,19 @@ export default function DatasetSettings() {
      //   linked: true,
      // };
      // setPipelineData(data);
+
+      const source_data: IDataSourceNodeProps[] =
+        knowledgeDetails?.connectors?.map((connector) => {
+          return {
+            ...connector,
+            icon:
+              DataSourceInfo[connector.source as keyof typeof DataSourceInfo]
+                ?.icon || '',
+          };
+        });
+
+      setSourceData(source_data);
+
      setGraphRagGenerateData({
        finish_at: knowledgeDetails.graphrag_task_finish_at,
        task_id: knowledgeDetails.graphrag_task_id,
@ -129,6 +149,23 @@ export default function DatasetSettings() {
  //   }
  // };

+  const handleLinkOrEditSubmit = (data: IDataSourceBase[] | undefined) => {
+    if (data) {
+      const connectors = data.map((connector) => {
+        return {
+          ...connector,
+          icon:
+            DataSourceInfo[connector.source as keyof typeof DataSourceInfo]
+              ?.icon || '',
+        };
+      });
+      setSourceData(connectors as IDataSourceNodeProps[]);
+      form.setValue('connectors', connectors || []);
+      // form.setValue('pipeline_name', data.name || '');
+      // form.setValue('pipeline_avatar', data.avatar || '');
+    }
+  };
+
  const handleDeletePipelineTask = (type: GenerateType) => {
    if (type === GenerateType.KnowledgeGraph) {
      setGraphRagGenerateData({
@ -158,6 +195,19 @@ export default function DatasetSettings() {
    }
    console.log('parseType', parseType);
  }, [parseType, form]);
+
+  const unbindFunc = (data: IDataSourceBase) => {
+    if (data) {
+      const connectors = sourceData?.filter((connector) => {
+        return connector.id !== data.id;
+      });
+      console.log('🚀 ~ DatasetSettings ~ connectors:', connectors);
+      setSourceData(connectors as IDataSourceNodeProps[]);
+      form.setValue('connectors', connectors || []);
+      // form.setValue('pipeline_name', data.name || '');
+      // form.setValue('pipeline_avatar', data.avatar || '');
+    }
+  };
  return (
    <section className="p-5 h-full flex flex-col">
      <TopTitle
@ -205,6 +255,13 @@ export default function DatasetSettings() {
                  data={pipelineData}
                  handleLinkOrEditSubmit={handleLinkOrEditSubmit}
                /> */}
+
+                <Divider />
+                <LinkDataSource
+                  data={sourceData}
+                  handleLinkOrEditSubmit={handleLinkOrEditSubmit}
+                  unbindFunc={unbindFunc}
+                />
              </MainContainer>
            </div>
            <div className="text-right items-center flex justify-end gap-3 w-[768px]">