Fix: Added some prompts and polling functionality to retrieve data source logs. #10703 (#11103)

### What problem does this PR solve? Fix: Added some prompts and polling functionality to retrieve data source logs. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
2025-12-08 12:32:30 +08:00 · 2025-11-07 14:28:45 +08:00
parent b137de1def
commit edbc396bc6
13 changed files with 155 additions and 95 deletions
--- a/web/src/components/ui/input.tsx
+++ b/web/src/components/ui/input.tsx
@ -30,32 +30,47 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
      }
    };
    return (
-      <div className="relative w-full">
-        <input
-          type={type === 'password' && showPassword ? 'text' : type}
-          className={cn(
-            'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
-            className,
-          )}
-          ref={ref}
-          value={inputValue ?? ''}
-          onChange={handleChange}
-          {...restProps}
-        />
-        {type === 'password' && (
-          <button
-            type="button"
-            className="absolute inset-y-0 right-0 pr-3 flex items-center"
-            onClick={() => setShowPassword(!showPassword)}
-          >
-            {showPassword ? (
-              <EyeOff className="h-4 w-4 text-text-secondary" />
-            ) : (
-              <Eye className="h-4 w-4 text-text-secondary" />
+      <>
+        {type !== 'password' && (
+          <input
+            type={type === 'password' && showPassword ? 'text' : type}
+            className={cn(
+              'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
+              className,
            )}
-          </button>
+            ref={ref}
+            value={inputValue ?? ''}
+            onChange={handleChange}
+            {...restProps}
+          />
        )}
-      </div>
+        {type === 'password' && (
+          <div className="relative w-full">
+            <input
+              type={type === 'password' && showPassword ? 'text' : type}
+              className={cn(
+                'flex h-8 w-full rounded-md border border-input bg-bg-input px-2 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-foreground placeholder:text-text-disabled focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 text-text-primary',
+                className,
+              )}
+              ref={ref}
+              value={inputValue ?? ''}
+              onChange={handleChange}
+              {...restProps}
+            />
+            <button
+              type="button"
+              className="absolute inset-y-0 right-0 pr-3 flex items-center"
+              onClick={() => setShowPassword(!showPassword)}
+            >
+              {showPassword ? (
+                <EyeOff className="h-4 w-4 text-text-secondary" />
+              ) : (
+                <Eye className="h-4 w-4 text-text-secondary" />
+              )}
+            </button>
+          </div>
+        )}
+      </>
    );
  },
 );
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@ -274,6 +274,10 @@ export default {
      reRankModelWaring: 'Re-rank model is very time consuming.',
    },
    knowledgeConfiguration: {
+      rebuildTip:
+        'Re-downloads files from the linked data source and parses them again.',
+      baseInfo: 'Basic Info',
+      gobalIndex: 'Global Index',
      dataSource: 'Data Source',
      linkSourceSetTip: 'Manage data source linkage with this dataset',
      linkDataSource: 'Link Data Source',
@ -304,7 +308,7 @@ export default {
      dataFlowPlaceholder: 'Please select a pipeline.',
      buildItFromScratch: 'Build it from scratch',
      dataFlow: 'Pipeline',
-      parseType: 'Ingestion pipeline',
+      parseType: 'Parse Type',
      manualSetup: 'Choose pipeline',
      builtIn: 'Built-in',
      titleDescription:
@ -692,6 +696,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
      newDocs: 'New Docs',
      timeStarted: 'Time started',
      log: 'Log',
+      confluenceDescription:
+        'Integrate your Confluence workspace to search documentation.',
      s3Description:
        'Connect to your AWS S3 bucket to import and sync stored files.',
      discordDescription:
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@ -260,6 +260,9 @@ export default {
      theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除',
    },
    knowledgeConfiguration: {
+      rebuildTip: '从所有已关联的数据源重新下载文件并再次解析。',
+      baseInfo: '基础信息',
+      gobalIndex: '全局索引',
      dataSource: '数据源',
      linkSourceSetTip: '管理与此数据集的数据源链接',
      linkDataSource: '链接数据源',
@ -282,14 +285,14 @@ export default {
      eidtLinkDataPipeline: '编辑pipeline',
      linkPipelineSetTip: '管理与此数据集的数据管道链接',
      default: '默认',
-      dataPipeline: 'pipeline',
+      dataPipeline: 'Ingestion pipeline',
      linkDataPipeline: '关联pipeline',
      enableAutoGenerate: '是否启用自动生成',
      teamPlaceholder: '请选择团队',
      dataFlowPlaceholder: '请选择pipeline',
      buildItFromScratch: '去Scratch构建',
      dataFlow: 'pipeline',
-      parseType: 'Ingestion pipeline',
+      parseType: '解析方法',
      manualSetup: '选择pipeline',
      builtIn: '内置',
      titleDescription: '在这里更新您的知识库详细信息，尤其是切片方法。',
@ -683,6 +686,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
      newDocs: '新文档',
      timeStarted: '开始时间',
      log: '日志',
+      confluenceDescription: '连接你的 Confluence 工作区以搜索文档内容。',
      s3Description: ' 连接你的 AWS S3 存储桶以导入和同步文件。',
      discordDescription: ' 连接你的 Discord 服务器以访问和分析聊天数据。',
      notionDescription: ' 同步 Notion 页面与数据库，用于知识检索。',
--- a/web/src/pages/dataset/dataset-overview/interface.ts
+++ b/web/src/pages/dataset/dataset-overview/interface.ts
@ -49,7 +49,8 @@ export interface IFileLogItem {
  process_duration: number;
  progress: number;
  progress_msg: string;
-  source_type: string;
+  source_type?: string;
+  source_from?: string;
  status: string;
  task_type: string;
  tenant_id: string;
--- a/web/src/pages/dataset/dataset-overview/overview-table.tsx
+++ b/web/src/pages/dataset/dataset-overview/overview-table.tsx
@ -107,9 +107,9 @@ export const getFileLogsTableColumns = (
      meta: { cellClassName: 'max-w-[10vw]' },
      cell: ({ row }) => (
        <div className="text-text-primary">
-          {row.original.source_type
+          {row.original.source_from
            ? DataSourceInfo[
-                row.original.source_type as keyof typeof DataSourceInfo
+                row.original.source_from as keyof typeof DataSourceInfo
              ].icon
            : t('localUpload')}
        </div>
--- a/web/src/pages/dataset/dataset-setting/components/link-data-source.tsx
+++ b/web/src/pages/dataset/dataset-setting/components/link-data-source.tsx
@ -1,9 +1,15 @@
+import { IconFontFill } from '@/components/icon-font';
 import { Button } from '@/components/ui/button';
-import { Modal } from '@/components/ui/modal/modal';
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from '@/components/ui/tooltip';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { IConnector } from '@/interfaces/database/knowledge';
 import { delSourceModal } from '@/pages/user-setting/data-source/component/delete-source-modal';
 import { DataSourceInfo } from '@/pages/user-setting/data-source/contant';
+import { useDataSourceRebuild } from '@/pages/user-setting/data-source/hooks';
 import { IDataSourceBase } from '@/pages/user-setting/data-source/interface';
 import { Link, Settings, Unlink } from 'lucide-react';
 import { useMemo, useState } from 'react';
@ -27,58 +33,45 @@ interface DataSourceItemProps extends IDataSourceNodeProps {

 const DataSourceItem = (props: DataSourceItemProps) => {
  const { t } = useTranslation();
-  const { id, name, icon, unbindFunc } = props;
+  const { id, name, icon, source, unbindFunc } = props;

  const { navigateToDataSourceDetail } = useNavigatePage();
+  const { handleRebuild } = useDataSourceRebuild();
  const toDetail = (id: string) => {
    navigateToDataSourceDetail(id);
  };
-  const openUnlinkModal = () => {
-    Modal.show({
-      visible: true,
-      className: '!w-[560px]',
-      title: t('dataflowParser.unlinkSourceModalTitle'),
-      children: (
-        <div
-          className="text-sm text-text-secondary"
-          dangerouslySetInnerHTML={{
-            __html: t('dataflowParser.unlinkSourceModalContent'),
-          }}
-        ></div>
-      ),
-      onVisibleChange: () => {
-        Modal.hide();
-      },
-      footer: (
-        <div className="flex justify-end gap-2">
-          <Button variant={'outline'} onClick={() => Modal.hide()}>
-            {t('dataflowParser.changeStepModalCancelText')}
-          </Button>
-          <Button
-            variant={'secondary'}
-            className="!bg-state-error text-bg-base"
-            onClick={() => {
-              unbindFunc?.(props);
-              Modal.hide();
-            }}
-          >
-            {t('dataflowParser.unlinkSourceModalConfirmText')}
-          </Button>
-        </div>
-      ),
-    });
-  };

  return (
-    <div className="flex items-center justify-between gap-1 px-2 rounded-md border ">
+    <div className="flex items-center justify-between gap-1 px-2 h-10 rounded-md border group hover:bg-bg-card">
      <div className="flex items-center gap-1">
        <div className="w-6 h-6 flex-shrink-0">{icon}</div>
+        <div className="text-base text-text-primary">
+          {DataSourceInfo[source].name}
+        </div>
        <div>{name}</div>
      </div>
-      <div className="flex gap-1 items-center">
+      <div className="flex items-center">
+        <Tooltip>
+          <TooltipTrigger>
+            <Button
+              variant={'transparent'}
+              className="border-none hidden group-hover:block"
+              type="button"
+              onClick={() => {
+                handleRebuild({ source_id: id });
+              }}
+            >
+              {/* <Settings /> */}
+              <IconFontFill name="reparse" className="text-text-primary" />
+            </Button>
+          </TooltipTrigger>
+          <TooltipContent>
+            {t('knowledgeConfiguration.rebuildTip')}
+          </TooltipContent>
+        </Tooltip>
        <Button
          variant={'transparent'}
-          className="border-none"
+          className="border-none hidden group-hover:block"
          type="button"
          onClick={() => {
            toDetail(id);
@ -93,7 +86,7 @@ const DataSourceItem = (props: DataSourceItemProps) => {
          <Button
            type="button"
            variant={'transparent'}
-            className="border-none"
+            className="border-none hidden group-hover:block"
            onClick={() => {
              // openUnlinkModal();
              delSourceModal({
@ -151,9 +144,12 @@ const LinkDataSource = (props: ILinkDataSourceProps) => {
  return (
    <div className="flex flex-col gap-2">
      <section className="flex flex-col">
-        <div className="flex items-center gap-1 text-text-primary text-sm">
+        <div className="text-base font-medium text-text-primary">
          {t('knowledgeConfiguration.dataSource')}
        </div>
+        {/* <div className="flex items-center gap-1 text-text-primary text-sm">
+          {t('knowledgeConfiguration.dataSource')}
+        </div> */}
        <div className="flex justify-between items-center">
          <div className="text-center text-xs text-text-secondary">
            {t('knowledgeConfiguration.linkSourceSetTip')}
--- a/web/src/pages/dataset/dataset-setting/index.tsx
+++ b/web/src/pages/dataset/dataset-setting/index.tsx
@ -219,9 +219,14 @@ export default function DatasetSettings() {
          <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6 ">
            <div className="w-[768px] h-[calc(100vh-240px)] pr-1 overflow-y-auto scrollbar-auto">
              <MainContainer className="text-text-secondary">
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.baseInfo')}
+                </div>
                <GeneralForm></GeneralForm>
                <Divider />
-
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.gobalIndex')}
+                </div>
                <GraphRagItems
                  className="border-none p-0"
                  data={graphRagGenerateData as IGenerateLogButtonProps}
@ -235,6 +240,9 @@ export default function DatasetSettings() {
                  onDelete={() => handleDeletePipelineTask(GenerateType.Raptor)}
                ></RaptorFormFields>
                <Divider />
+                <div className="text-base font-medium text-text-primary">
+                  {t('knowledgeConfiguration.dataPipeline')}
+                </div>
                <ParseTypeItem line={1} />
                {parseType === 1 && (
                  <ChunkMethodItem line={1}></ChunkMethodItem>
--- a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx
@ -66,7 +66,7 @@ const SourceDetailPage = () => {
        render: (fieldProps: FormFieldConfig) => (
          <div className="flex items-center  gap-1 w-full relative">
            <Input {...fieldProps} type={FormFieldType.Number} />
-            <span className="absolute right-0 -translate-x-12 text-text-secondary italic ">
+            <span className="absolute right-0 -translate-x-[58px] text-text-secondary italic ">
              minutes
            </span>
            <button
@ -96,7 +96,7 @@ const SourceDetailPage = () => {
          return (
            <div className="flex items-center  gap-1 w-full relative">
              <Input {...fieldProps} type={FormFieldType.Number} />
-              <span className="absolute right-0 -translate-x-3 text-text-secondary italic ">
+              <span className="absolute right-0 -translate-x-6 text-text-secondary italic ">
                hours
              </span>
            </div>
@ -111,7 +111,7 @@ const SourceDetailPage = () => {
        render: (fieldProps: FormFieldConfig) => (
          <div className="flex items-center  gap-1 w-full relative">
            <Input {...fieldProps} type={FormFieldType.Number} />
-            <span className="absolute right-0 -translate-x-3 text-text-secondary italic ">
+            <span className="absolute right-0 -translate-x-6 text-text-secondary italic ">
              seconds
            </span>
          </div>
@ -183,7 +183,7 @@ const SourceDetailPage = () => {
          </div>
          <section className="flex flex-col gap-2 mt-6">
            <div className="text-2xl text-text-primary">{t('setting.log')}</div>
-            <DataSourceLogsTable />
+            <DataSourceLogsTable refresh_freq={detail?.refresh_freq || false} />
          </section>
        </CardContent>
      </Card>
--- a/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx
+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/log-table.tsx
@ -130,9 +130,14 @@ const columns = ({
 //   pageSize: 10,
 //   total: 0,
 // };
-export const DataSourceLogsTable = () => {
+export const DataSourceLogsTable = ({
+  refresh_freq,
+}: {
+  refresh_freq: number | false;
+}) => {
  // const [pagination, setPagination] = useState(paginationInit);
-  const { data, pagination, setPagination } = useLogListDataSource();
+  const { data, pagination, setPagination } =
+    useLogListDataSource(refresh_freq);
  const navigate = useNavigate();
  const currentPagination = useMemo(
    () => ({
--- a/web/src/pages/user-setting/data-source/hooks.ts
+++ b/web/src/pages/user-setting/data-source/hooks.ts
@ -2,6 +2,7 @@ import message from '@/components/ui/message';
 import { useSetModalState } from '@/hooks/common-hooks';
 import { useGetPaginationWithRouter } from '@/hooks/logic-hooks';
 import dataSourceService, {
+  dataSourceRebuild,
  dataSourceResume,
  deleteDataSource,
  featchDataSourceDetail,
@ -10,7 +11,7 @@ import dataSourceService, {
 import { useQuery, useQueryClient } from '@tanstack/react-query';
 import { t } from 'i18next';
 import { useCallback, useMemo, useState } from 'react';
-import { useSearchParams } from 'umi';
+import { useParams, useSearchParams } from 'umi';
 import { DataSourceInfo, DataSourceKey } from './contant';
 import { IDataSorceInfo, IDataSource, IDataSourceBase } from './interface';

@ -109,14 +110,15 @@ export const useAddDataSource = () => {
  };
 };

-export const useLogListDataSource = () => {
+export const useLogListDataSource = (refresh_freq: number | false) => {
  const { pagination, setPagination } = useGetPaginationWithRouter();
  const [currentQueryParameters] = useSearchParams();
  const id = currentQueryParameters.get('id');

  const { data, isFetching } = useQuery<{ logs: IDataSource[]; total: number }>(
    {
-      queryKey: ['data-source-logs', id, pagination],
+      queryKey: ['data-source-logs', id, pagination, refresh_freq],
+      refetchInterval: refresh_freq ? refresh_freq * 60 * 1000 : false,
      queryFn: async () => {
        const { data } = await getDataSourceLogs(id as string, {
          page_size: pagination.pageSize,
@ -186,3 +188,22 @@ export const useDataSourceResume = () => {
  );
  return { handleResume };
 };
+
+export const useDataSourceRebuild = () => {
+  const { id } = useParams();
+  // const [currentQueryParameters] = useSearchParams();
+  // const id = currentQueryParameters.get('id');
+  const handleRebuild = useCallback(
+    async (param: { source_id: string }) => {
+      const { data } = await dataSourceRebuild(param.source_id as string, {
+        kb_id: id as string,
+      });
+      if (data.code === 0) {
+        // queryClient.invalidateQueries({ queryKey: ['data-source-detail', id] });
+        message.success(t(`message.operated`));
+      }
+    },
+    [id],
+  );
+  return { handleRebuild };
+};
--- a/web/src/pages/user-setting/data-source/index.tsx
+++ b/web/src/pages/user-setting/data-source/index.tsx
@ -58,7 +58,17 @@ const DataSource = () => {
    icon,
  }: IDataSorceInfo) => {
    return (
-      <div className="p-[10px] border border-border-button rounded-lg relative group hover:bg-bg-card">
+      <div
+        className="p-[10px] border border-border-button rounded-lg relative group hover:bg-bg-card"
+        onClick={() =>
+          showAddingModal({
+            id,
+            name,
+            description,
+            icon,
+          })
+        }
+      >
        <div className="flex gap-2">
          <div className="w-6 h-6">{icon}</div>
          <div className="flex flex-1 flex-col items-start gap-2">
@ -67,17 +77,7 @@ const DataSource = () => {
          </div>
        </div>
        <div className=" absolute top-2 right-2">
-          <Button
-            onClick={() =>
-              showAddingModal({
-                id,
-                name,
-                description,
-                icon,
-              })
-            }
-            className=" rounded-md px-1 text-bg-base gap-1 bg-text-primary text-xs py-0 h-6 items-center hidden group-hover:flex"
-          >
+          <Button className=" rounded-md px-1 text-bg-base gap-1 bg-text-primary text-xs py-0 h-6 items-center hidden group-hover:flex">
            <Plus size={12} />
            {t('setting.add')}
          </Button>
--- a/web/src/services/data-source-service.ts
+++ b/web/src/services/data-source-service.ts
@ -21,10 +21,13 @@ const dataSourceService = registerServer<keyof typeof methods>(
 export const deleteDataSource = (id: string) =>
  request.post(api.dataSourceDel(id));
 export const dataSourceResume = (id: string, data: { resume: boolean }) => {
-  console.log('api.dataSourceResume(id)', data);
  return request.put(api.dataSourceResume(id), { data });
 };

+export const dataSourceRebuild = (id: string, data: { kb_id: string }) => {
+  return request.put(api.dataSourceRebuild(id), { data });
+};
+
 export const getDataSourceLogs = (id: string, params?: any) =>
  request.get(api.dataSourceLogs(id), { params });
 export const featchDataSourceDetail = (id: string) =>
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@ -39,6 +39,7 @@ export default {
  dataSourceList: `${api_host}/connector/list`,
  dataSourceDel: (id: string) => `${api_host}/connector/${id}/rm`,
  dataSourceResume: (id: string) => `${api_host}/connector/${id}/resume`,
+  dataSourceRebuild: (id: string) => `${api_host}/connector/${id}/rebuild`,
  dataSourceLogs: (id: string) => `${api_host}/connector/${id}/logs`,
  dataSourceDetail: (id: string) => `${api_host}/connector/${id}`,