Fix: Added some prompts and polling functionality to retrieve data source logs. #10703 (#11103)

### What problem does this PR solve?

Fix: Added some prompts and polling functionality to retrieve data
source logs.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

web/src/components/ui/input.tsx

@@ -30,32 +30,47 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
       }
     };
     return (
-      <div className="relative w-full">
+      <>
-        <input
+        {type !== 'password' && (
-          type={type === 'password' && showPassword ? 'text' : type}
+          <input
-          className={cn(
+            type={type === 'password' && showPassword ? 'text' : type}
-            'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
+            className={cn(
-            className,
+              'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
-          )}
+              className,
-          ref={ref}
-          value={inputValue ?? ''}
-          onChange={handleChange}
-          {...restProps}
-        />
-        {type === 'password' && (
-          <button
-            type="button"
-            className="absolute inset-y-0 right-0 pr-3 flex items-center"
-            onClick={() => setShowPassword(!showPassword)}
-          >
-            {showPassword ? (
-              <EyeOff className="h-4 w-4 text-text-secondary" />
-            ) : (
-              <Eye className="h-4 w-4 text-text-secondary" />
             )}
-          </button>
+            ref={ref}
+            value={inputValue ?? ''}
+            onChange={handleChange}
+            {...restProps}
+          />
         )}
-      </div>
+        {type === 'password' && (
+          <div className="relative w-full">
+            <input
+              type={type === 'password' && showPassword ? 'text' : type}
+              className={cn(
+                'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
+                className,
+              )}
+              ref={ref}
+              value={inputValue ?? ''}
+              onChange={handleChange}
+              {...restProps}
+            />
+            <button
+              type="button"
+              className="absolute inset-y-0 right-0 pr-3 flex items-center"
+              onClick={() => setShowPassword(!showPassword)}
+            >
+              {showPassword ? (
+                <EyeOff className="h-4 w-4 text-text-secondary" />
+              ) : (
+                <Eye className="h-4 w-4 text-text-secondary" />
+              )}
+            </button>
+          </div>
+        )}
+      </>
     );
   },
 );

web/src/locales/en.ts

@@ -274,6 +274,10 @@ export default {
       reRankModelWaring: 'Re-rank model is very time consuming.',
     },
     knowledgeConfiguration: {
+      rebuildTip:
+        'Re-downloads files from the linked data source and parses them again.',
+      baseInfo: 'Basic Info',
+      gobalIndex: 'Global Index',
       dataSource: 'Data Source',
       linkSourceSetTip: 'Manage data source linkage with this dataset',
       linkDataSource: 'Link Data Source',
@@ -304,7 +308,7 @@ export default {
       dataFlowPlaceholder: 'Please select a pipeline.',
       buildItFromScratch: 'Build it from scratch',
       dataFlow: 'Pipeline',
-      parseType: 'Ingestion pipeline',
+      parseType: 'Parse Type',
       manualSetup: 'Choose pipeline',
       builtIn: 'Built-in',
       titleDescription:
@@ -692,6 +696,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
       newDocs: 'New Docs',
       timeStarted: 'Time started',
       log: 'Log',
+      confluenceDescription:
+        'Integrate your Confluence workspace to search documentation.',
       s3Description:
         'Connect to your AWS S3 bucket to import and sync stored files.',
       discordDescription:

web/src/locales/zh.ts

@@ -260,6 +260,9 @@ export default {
       theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除',
     },
     knowledgeConfiguration: {
+      rebuildTip: '从所有已关联的数据源重新下载文件并再次解析。',
+      baseInfo: '基础信息',
+      gobalIndex: '全局索引',
       dataSource: '数据源',
       linkSourceSetTip: '管理与此数据集的数据源链接',
       linkDataSource: '链接数据源',
@@ -282,14 +285,14 @@ export default {
       eidtLinkDataPipeline: '编辑pipeline',
       linkPipelineSetTip: '管理与此数据集的数据管道链接',
       default: '默认',
-      dataPipeline: 'pipeline',
+      dataPipeline: 'Ingestion pipeline',
       linkDataPipeline: '关联pipeline',
       enableAutoGenerate: '是否启用自动生成',
       teamPlaceholder: '请选择团队',
       dataFlowPlaceholder: '请选择pipeline',
       buildItFromScratch: '去Scratch构建',
       dataFlow: 'pipeline',
-      parseType: 'Ingestion pipeline',
+      parseType: '解析方法',
       manualSetup: '选择pipeline',
       builtIn: '内置',
       titleDescription: '在这里更新您的知识库详细信息，尤其是切片方法。',
@@ -683,6 +686,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       newDocs: '新文档',
       timeStarted: '开始时间',
       log: '日志',
+      confluenceDescription: '连接你的 Confluence 工作区以搜索文档内容。',
       s3Description: ' 连接你的 AWS S3 存储桶以导入和同步文件。',
       discordDescription: ' 连接你的 Discord 服务器以访问和分析聊天数据。',
       notionDescription: ' 同步 Notion 页面与数据库，用于知识检索。',

web/src/pages/dataset/dataset-overview/interface.ts

@@ -49,7 +49,8 @@ export interface IFileLogItem {
   process_duration: number;
   progress: number;
   progress_msg: string;
-  source_type: string;
+  source_type?: string;
+  source_from?: string;
   status: string;
   task_type: string;
   tenant_id: string;

web/src/pages/dataset/dataset-overview/overview-table.tsx

@@ -107,9 +107,9 @@ export const getFileLogsTableColumns = (
       meta: { cellClassName: 'max-w-[10vw]' },
       cell: ({ row }) => (
         <div className="text-text-primary">
-          {row.original.source_type
+          {row.original.source_from
             ? DataSourceInfo[
-                row.original.source_type as keyof typeof DataSourceInfo
+                row.original.source_from as keyof typeof DataSourceInfo
               ].icon
             : t('localUpload')}
         </div>

web/src/pages/dataset/dataset-setting/components/link-data-source.tsx

@@ -1,9 +1,15 @@
+import { IconFontFill } from '@/components/icon-font';
 import { Button } from '@/components/ui/button';
-import { Modal } from '@/components/ui/modal/modal';
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from '@/components/ui/tooltip';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { IConnector } from '@/interfaces/database/knowledge';
 import { delSourceModal } from '@/pages/user-setting/data-source/component/delete-source-modal';
 import { DataSourceInfo } from '@/pages/user-setting/data-source/contant';
+import { useDataSourceRebuild } from '@/pages/user-setting/data-source/hooks';
 import { IDataSourceBase } from '@/pages/user-setting/data-source/interface';
 import { Link, Settings, Unlink } from 'lucide-react';
 import { useMemo, useState } from 'react';
@@ -27,58 +33,45 @@ interface DataSourceItemProps extends IDataSourceNodeProps {
 
 const DataSourceItem = (props: DataSourceItemProps) => {
   const { t } = useTranslation();
-  const { id, name, icon, unbindFunc } = props;
+  const { id, name, icon, source, unbindFunc } = props;
 
   const { navigateToDataSourceDetail } = useNavigatePage();
+  const { handleRebuild } = useDataSourceRebuild();
   const toDetail = (id: string) => {
     navigateToDataSourceDetail(id);
   };
-  const openUnlinkModal = () => {
-    Modal.show({
-      visible: true,
-      className: '!w-[560px]',
-      title: t('dataflowParser.unlinkSourceModalTitle'),
-      children: (
-        <div
-          className="text-sm text-text-secondary"
-          dangerouslySetInnerHTML={{
-            __html: t('dataflowParser.unlinkSourceModalContent'),
-          }}
-        ></div>
-      ),
-      onVisibleChange: () => {
-        Modal.hide();
-      },
-      footer: (
-        <div className="flex justify-end gap-2">
-          <Button variant={'outline'} onClick={() => Modal.hide()}>
-            {t('dataflowParser.changeStepModalCancelText')}
-          </Button>
-          <Button
-            variant={'secondary'}
-            className="!bg-state-error text-bg-base"
-            onClick={() => {
-              unbindFunc?.(props);
-              Modal.hide();
-            }}
-          >
-            {t('dataflowParser.unlinkSourceModalConfirmText')}
-          </Button>
-        </div>
-      ),
-    });
-  };
 
   return (
-    <div className="flex items-center justify-between gap-1 px-2 rounded-md border ">
+    <div className="flex items-center justify-between gap-1 px-2 h-10 rounded-md border group hover:bg-bg-card">
       <div className="flex items-center gap-1">
         <div className="w-6 h-6 flex-shrink-0">{icon}</div>
+        <div className="text-base text-text-primary">
+          {DataSourceInfo[source].name}
+        </div>
         <div>{name}</div>
       </div>
-      <div className="flex gap-1 items-center">
+      <div className="flex items-center">
+        <Tooltip>
+          <TooltipTrigger>
+            <Button
+              variant={'transparent'}
+              className="border-none hidden group-hover:block"
+              type="button"
+              onClick={() => {
+                handleRebuild({ source_id: id });
+              }}
+            >
+              {/* <Settings /> */}
+              <IconFontFill name="reparse" className="text-text-primary" />
+            </Button>
+          </TooltipTrigger>
+          <TooltipContent>
+            {t('knowledgeConfiguration.rebuildTip')}
+          </TooltipContent>
+        </Tooltip>
         <Button
           variant={'transparent'}
-          className="border-none"
+          className="border-none hidden group-hover:block"
           type="button"
           onClick={() => {
             toDetail(id);
@@ -93,7 +86,7 @@ const DataSourceItem = (props: DataSourceItemProps) => {
           <Button
             type="button"
             variant={'transparent'}
-            className="border-none"
+            className="border-none hidden group-hover:block"
             onClick={() => {
               // openUnlinkModal();
               delSourceModal({
@@ -151,9 +144,12 @@ const LinkDataSource = (props: ILinkDataSourceProps) => {
   return (
     <div className="flex flex-col gap-2">
       <section className="flex flex-col">
-        <div className="flex items-center gap-1 text-text-primary text-sm">
+        <div className="text-base font-medium text-text-primary">
           {t('knowledgeConfiguration.dataSource')}
         </div>
+        {/* <div className="flex items-center gap-1 text-text-primary text-sm">
+          {t('knowledgeConfiguration.dataSource')}
+        </div> */}
         <div className="flex justify-between items-center">
           <div className="text-center text-xs text-text-secondary">
             {t('knowledgeConfiguration.linkSourceSetTip')}

web/src/pages/dataset/dataset-setting/index.tsx

@@ -219,9 +219,14 @@ export default function DatasetSettings() {
           <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6 ">
             <div className="w-[768px] h-[calc(100vh-240px)] pr-1 overflow-y-auto scrollbar-auto">
               <MainContainer className="text-text-secondary">
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.baseInfo')}
+                </div>
                 <GeneralForm></GeneralForm>
                 <Divider />
-
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.gobalIndex')}
+                </div>
                 <GraphRagItems
                   className="border-none p-0"
                   data={graphRagGenerateData as IGenerateLogButtonProps}
@@ -235,6 +240,9 @@ export default function DatasetSettings() {
                   onDelete={() => handleDeletePipelineTask(GenerateType.Raptor)}
                 ></RaptorFormFields>
                 <Divider />
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.dataPipeline')}
+                </div>
                 <ParseTypeItem line={1} />
                 {parseType === 1 && (
                   <ChunkMethodItem line={1}></ChunkMethodItem>

web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx

@@ -66,7 +66,7 @@ const SourceDetailPage = () => {
         render: (fieldProps: FormFieldConfig) => (
           <div className="flex items-center  gap-1 w-full relative">
             <Input {...fieldProps} type={FormFieldType.Number} />
-            <span className="absolute right-0 -translate-x-12 text-text-secondary italic ">
+            <span className="absolute right-0 -translate-x-[58px] text-text-secondary italic ">
               minutes
             </span>
             <button
@@ -96,7 +96,7 @@ const SourceDetailPage = () => {
           return (
             <div className="flex items-center  gap-1 w-full relative">
               <Input {...fieldProps} type={FormFieldType.Number} />
-              <span className="absolute right-0 -translate-x-3 text-text-secondary italic ">
+              <span className="absolute right-0 -translate-x-6 text-text-secondary italic ">
                 hours
               </span>
             </div>
@@ -111,7 +111,7 @@ const SourceDetailPage = () => {
         render: (fieldProps: FormFieldConfig) => (
           <div className="flex items-center  gap-1 w-full relative">
             <Input {...fieldProps} type={FormFieldType.Number} />
-            <span className="absolute right-0 -translate-x-3 text-text-secondary italic ">
+            <span className="absolute right-0 -translate-x-6 text-text-secondary italic ">
               seconds
             </span>
           </div>
@@ -183,7 +183,7 @@ const SourceDetailPage = () => {
           </div>
           <section className="flex flex-col gap-2 mt-6">
             <div className="text-2xl text-text-primary">{t('setting.log')}</div>
-            <DataSourceLogsTable />
+            <DataSourceLogsTable refresh_freq={detail?.refresh_freq || false} />
           </section>
         </CardContent>
       </Card>

web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx

@@ -130,9 +130,14 @@ const columns = ({
 //   pageSize: 10,
 //   total: 0,
 // };
-export const DataSourceLogsTable = () => {
+export const DataSourceLogsTable = ({
+  refresh_freq,
+}: {
+  refresh_freq: number | false;
+}) => {
   // const [pagination, setPagination] = useState(paginationInit);
-  const { data, pagination, setPagination } = useLogListDataSource();
+  const { data, pagination, setPagination } =
+    useLogListDataSource(refresh_freq);
   const navigate = useNavigate();
   const currentPagination = useMemo(
     () => ({

web/src/pages/user-setting/data-source/hooks.ts

@@ -2,6 +2,7 @@ import message from '@/components/ui/message';
 import { useSetModalState } from '@/hooks/common-hooks';
 import { useGetPaginationWithRouter } from '@/hooks/logic-hooks';
 import dataSourceService, {
+  dataSourceRebuild,
   dataSourceResume,
   deleteDataSource,
   featchDataSourceDetail,
@@ -10,7 +11,7 @@ import dataSourceService, {
 import { useQuery, useQueryClient } from '@tanstack/react-query';
 import { t } from 'i18next';
 import { useCallback, useMemo, useState } from 'react';
-import { useSearchParams } from 'umi';
+import { useParams, useSearchParams } from 'umi';
 import { DataSourceInfo, DataSourceKey } from './contant';
 import { IDataSorceInfo, IDataSource, IDataSourceBase } from './interface';
 
@@ -109,14 +110,15 @@ export const useAddDataSource = () => {
   };
 };
 
-export const useLogListDataSource = () => {
+export const useLogListDataSource = (refresh_freq: number | false) => {
   const { pagination, setPagination } = useGetPaginationWithRouter();
   const [currentQueryParameters] = useSearchParams();
   const id = currentQueryParameters.get('id');
 
   const { data, isFetching } = useQuery<{ logs: IDataSource[]; total: number }>(
     {
-      queryKey: ['data-source-logs', id, pagination],
+      queryKey: ['data-source-logs', id, pagination, refresh_freq],
+      refetchInterval: refresh_freq ? refresh_freq * 60 * 1000 : false,
       queryFn: async () => {
         const { data } = await getDataSourceLogs(id as string, {
           page_size: pagination.pageSize,
@@ -186,3 +188,22 @@ export const useDataSourceResume = () => {
   );
   return { handleResume };
 };
+
+export const useDataSourceRebuild = () => {
+  const { id } = useParams();
+  // const [currentQueryParameters] = useSearchParams();
+  // const id = currentQueryParameters.get('id');
+  const handleRebuild = useCallback(
+    async (param: { source_id: string }) => {
+      const { data } = await dataSourceRebuild(param.source_id as string, {
+        kb_id: id as string,
+      });
+      if (data.code === 0) {
+        // queryClient.invalidateQueries({ queryKey: ['data-source-detail', id] });
+        message.success(t(`message.operated`));
+      }
+    },
+    [id],
+  );
+  return { handleRebuild };
+};

web/src/pages/user-setting/data-source/index.tsx

@@ -58,7 +58,17 @@ const DataSource = () => {
     icon,
   }: IDataSorceInfo) => {
     return (
-      <div className="p-[10px] border border-border-button rounded-lg relative group hover:bg-bg-card">
+      <div
+        className="p-[10px] border border-border-button rounded-lg relative group hover:bg-bg-card"
+        onClick={() =>
+          showAddingModal({
+            id,
+            name,
+            description,
+            icon,
+          })
+        }
+      >
         <div className="flex gap-2">
           <div className="w-6 h-6">{icon}</div>
           <div className="flex flex-1 flex-col items-start gap-2">
@@ -67,17 +77,7 @@ const DataSource = () => {
           </div>
         </div>
         <div className=" absolute top-2 right-2">
-          <Button
+          <Button className=" rounded-md px-1 text-bg-base gap-1 bg-text-primary text-xs py-0 h-6 items-center hidden group-hover:flex">
-            onClick={() =>
-              showAddingModal({
-                id,
-                name,
-                description,
-                icon,
-              })
-            }
-            className=" rounded-md px-1 text-bg-base gap-1 bg-text-primary text-xs py-0 h-6 items-center hidden group-hover:flex"
-          >
             <Plus size={12} />
             {t('setting.add')}
           </Button>

web/src/services/data-source-service.ts

@@ -21,10 +21,13 @@ const dataSourceService = registerServer<keyof typeof methods>(
 export const deleteDataSource = (id: string) =>
   request.post(api.dataSourceDel(id));
 export const dataSourceResume = (id: string, data: { resume: boolean }) => {
-  console.log('api.dataSourceResume(id)', data);
   return request.put(api.dataSourceResume(id), { data });
 };
 
+export const dataSourceRebuild = (id: string, data: { kb_id: string }) => {
+  return request.put(api.dataSourceRebuild(id), { data });
+};
+
 export const getDataSourceLogs = (id: string, params?: any) =>
   request.get(api.dataSourceLogs(id), { params });
 export const featchDataSourceDetail = (id: string) =>

web/src/utils/api.ts

@@ -39,6 +39,7 @@ export default {
   dataSourceList: `${api_host}/connector/list`,
   dataSourceDel: (id: string) => `${api_host}/connector/${id}/rm`,
   dataSourceResume: (id: string) => `${api_host}/connector/${id}/resume`,
+  dataSourceRebuild: (id: string) => `${api_host}/connector/${id}/rebuild`,
   dataSourceLogs: (id: string) => `${api_host}/connector/${id}/logs`,
   dataSourceDetail: (id: string) => `${api_host}/connector/${id}`,