Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/dataset/dataset-overview/index.tsx

@@ -1,13 +1,16 @@
-import {
-  CircleQuestionMark,
-  Cpu,
-  FileChartLine,
-  HardDriveDownload,
-} from 'lucide-react';
-import { FC, useState } from 'react';
+import { FilterCollection } from '@/components/list-filter-bar/interface';
+import SvgIcon from '@/components/svg-icon';
+import { useIsDarkTheme } from '@/components/theme-provider';
+import { AntToolTip } from '@/components/ui/tooltip';
+import { useFetchDocumentList } from '@/hooks/use-document-request';
+import { t } from 'i18next';
+import { CircleQuestionMark } from 'lucide-react';
+import { FC, useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
+import { RunningStatus, RunningStatusMap } from '../dataset/constant';
 import { LogTabs } from './dataset-common';
 import { DatasetFilter } from './dataset-filter';
+import { useFetchFileLogList, useFetchOverviewTital } from './hook';
 import FileLogsTable from './overview-table';
 
 interface StatCardProps {
@@ -15,15 +18,30 @@ interface StatCardProps {
   value: number;
   icon: JSX.Element;
   children?: JSX.Element;
+  tooltip?: string;
+}
+interface CardFooterProcessProps {
+  success: number;
+  failed: number;
 }
 
-const StatCard: FC<StatCardProps> = ({ title, value, children, icon }) => {
+const StatCard: FC<StatCardProps> = ({
+  title,
+  value,
+  children,
+  icon,
+  tooltip,
+}) => {
   return (
     <div className="bg-bg-card  p-4 rounded-lg border border-border flex flex-col gap-2">
       <div className="flex items-center justify-between">
         <h3 className="flex items-center gap-1 text-sm font-medium text-text-secondary">
           {title}
-          <CircleQuestionMark size={12} />
+          {tooltip && (
+            <AntToolTip title={tooltip} trigger="hover">
+              <CircleQuestionMark size={12} />
+            </AntToolTip>
+          )}
         </h3>
         {icon}
       </div>
@@ -35,115 +53,228 @@ const StatCard: FC<StatCardProps> = ({ title, value, children, icon }) => {
   );
 };
 
-interface CardFooterProcessProps {
-  total: number;
-  completed: number;
-  success: number;
-  failed: number;
-}
 const CardFooterProcess: FC<CardFooterProcessProps> = ({
-  total,
-  completed,
-  success,
-  failed,
+  success = 0,
+  failed = 0,
 }) => {
   const { t } = useTranslation();
-  const successPrecentage = (success / total) * 100;
-  const failedPrecentage = (failed / total) * 100;
   return (
     <div className="flex items-center flex-col gap-2">
-      <div className="flex justify-between w-full text-sm text-text-secondary">
-        <div className="flex items-center gap-2">
-          <div className="flex items-center gap-1">
-            {success}
-            <span>{t('knowledgeDetails.success')}</span>
+      <div className="w-full flex justify-between gap-4 rounded-lg text-sm font-bold text-text-primary">
+        <div className="flex items-center justify-between  rounded-md w-1/2 p-2 bg-state-success-5">
+          <div className="flex items-center rounded-lg gap-1">
+            <div className="w-2 h-2 rounded-full bg-state-success "></div>
+            <div className="font-normal text-text-secondary text-xs">
+              {t('knowledgeDetails.success')}
+            </div>
           </div>
-          <div className="flex items-center gap-1">
-            {failed}
-            <span>{t('knowledgeDetails.failed')}</span>
+          <div>{success || 0}</div>
+        </div>
+        <div className="flex items-center justify-between rounded-md w-1/2 bg-state-error-5 p-2">
+          <div className="flex items-center rounded-lg gap-1">
+            <div className="w-2 h-2 rounded-full bg-state-error"></div>
+            <div className="font-normal text-text-secondary text-xs">
+              {t('knowledgeDetails.failed')}
+            </div>
           </div>
+          <div>{failed || 0}</div>
         </div>
-        <div className="flex items-center gap-1">
-          {completed}
-          <span>{t('knowledgeDetails.completed')}</span>
-        </div>
-      </div>
-      <div className="w-full flex rounded-full h-3 bg-bg-card text-sm font-bold text-text-primary">
-        <div
-          className=" rounded-full h-3 bg-accent-primary"
-          style={{ width: successPrecentage + '%' }}
-        ></div>
-        <div
-          className=" rounded-full h-3 bg-state-error"
-          style={{ width: failedPrecentage + '%' }}
-        ></div>
       </div>
     </div>
   );
 };
+
+const filters = [
+  {
+    field: 'operation_status',
+    label: t('knowledgeDetails.status'),
+    list: Object.values(RunningStatus).map((value) => {
+      // const value = key as RunningStatus;
+      console.log(value);
+      return {
+        id: value,
+        label: RunningStatusMap[value].label,
+      };
+    }),
+  },
+  {
+    field: 'types',
+    label: t('knowledgeDetails.task'),
+    list: [
+      {
+        id: 'Parse',
+        label: 'Parse',
+      },
+      {
+        id: 'Download',
+        label: 'Download',
+      },
+    ],
+  },
+];
 const FileLogsPage: FC = () => {
   const { t } = useTranslation();
-  const [active, setActive] = useState<(typeof LogTabs)[keyof typeof LogTabs]>(
-    LogTabs.FILE_LOGS,
-  );
-  const mockData = Array(30)
-    .fill(0)
-    .map((_, i) => ({
-      id: i === 0 ? '#952734' : `14`,
-      fileName: 'PRD for DealBees 1.2 (1).txt',
-      source: 'GitHub',
-      pipeline: i === 0 ? 'data demo for...' : i === 1 ? 'test' : 'kiki’s demo',
-      startDate: '14/03/2025 14:53:39',
-      task: i === 0 ? 'Parse' : 'Parser',
-      status:
-        i === 0
-          ? 'Success'
-          : i === 1
-            ? 'Failed'
-            : i === 2
-              ? 'Running'
-              : 'Pending',
-    }));
 
-  const pagination = {
-    current: 1,
-    pageSize: 30,
-    total: 100,
-  };
+  const [topAllData, setTopAllData] = useState({
+    totalFiles: {
+      value: 0,
+      precent: 0,
+    },
+    downloads: {
+      value: 0,
+      success: 0,
+      failed: 0,
+    },
+    processing: {
+      value: 0,
+      success: 0,
+      failed: 0,
+    },
+  });
+  const { data: topData } = useFetchOverviewTital();
+  const {
+    pagination: { total: fileTotal },
+  } = useFetchDocumentList();
+
+  useEffect(() => {
+    setTopAllData((prev) => {
+      return {
+        ...prev,
+        processing: {
+          value: topData?.processing || 0,
+          success: topData?.finished || 0,
+          failed: topData?.failed || 0,
+        },
+      };
+    });
+  }, [topData]);
+
+  useEffect(() => {
+    setTopAllData((prev) => {
+      return {
+        ...prev,
+        totalFiles: {
+          value: fileTotal || 0,
+          precent: 0,
+        },
+      };
+    });
+  }, [fileTotal]);
+
+  const {
+    data: tableOriginData,
+    searchString,
+    handleInputChange,
+    pagination,
+    setPagination,
+    active,
+    filterValue,
+    handleFilterSubmit,
+    setActive,
+  } = useFetchFileLogList();
+
+  const tableList = useMemo(() => {
+    console.log('tableList', tableOriginData);
+    if (tableOriginData && tableOriginData.logs?.length) {
+      return tableOriginData.logs.map((item) => {
+        return {
+          ...item,
+          fileName: item.document_name,
+          statusName: item.operation_status,
+        };
+      });
+    }
+  }, [tableOriginData]);
 
   const changeActiveLogs = (active: (typeof LogTabs)[keyof typeof LogTabs]) => {
     setActive(active);
   };
   const handlePaginationChange = (page: number, pageSize: number) => {
     console.log('Pagination changed:', { page, pageSize });
+    setPagination({
+      ...pagination,
+      page,
+      pageSize: pageSize,
+    });
   };
 
+  const isDark = useIsDarkTheme();
+
   return (
     <div className="p-5 min-w-[880px] border-border border rounded-lg mr-5">
       {/* Stats Cards */}
       <div className="grid grid-cols-3 md:grid-cols-3 gap-4 mb-6">
-        <StatCard title="Total Files" value={2827} icon={<FileChartLine />}>
-          <div>+7% from last week</div>
+        <StatCard
+          title={t('datasetOverview.totalFiles')}
+          value={topAllData.totalFiles.value}
+          icon={
+            isDark ? (
+              <SvgIcon name="data-flow/total-files-icon" width={40} />
+            ) : (
+              <SvgIcon name="data-flow/total-files-icon-bri" width={40} />
+            )
+          }
+        >
+          <div>
+            <span className="text-accent-primary">
+              {topAllData.totalFiles.precent > 0 ? '+' : ''}
+              {topAllData.totalFiles.precent}%{' '}
+            </span>
+            <span className="font-normal text-text-secondary text-xs">
+              from last week
+            </span>
+          </div>
         </StatCard>
-        <StatCard title="Downloading" value={28} icon={<HardDriveDownload />}>
+        <StatCard
+          title={t('datasetOverview.downloading')}
+          value={topAllData.downloads.value}
+          icon={
+            isDark ? (
+              <SvgIcon name="data-flow/data-icon" width={40} />
+            ) : (
+              <SvgIcon name="data-flow/data-icon-bri" width={40} />
+            )
+          }
+          tooltip={t('datasetOverview.downloadTip')}
+        >
           <CardFooterProcess
-            total={100}
-            success={8}
-            failed={2}
-            completed={15}
+            success={topAllData.downloads.success}
+            failed={topAllData.downloads.failed}
           />
         </StatCard>
-        <StatCard title="Processing" value={156} icon={<Cpu />}>
-          <CardFooterProcess total={20} success={8} failed={2} completed={15} />
+        <StatCard
+          title={t('datasetOverview.processing')}
+          value={topAllData.processing.value}
+          icon={
+            isDark ? (
+              <SvgIcon name="data-flow/processing-icon" width={40} />
+            ) : (
+              <SvgIcon name="data-flow/processing-icon-bri" width={40} />
+            )
+          }
+          tooltip={t('datasetOverview.processingTip')}
+        >
+          <CardFooterProcess
+            success={topAllData.processing.success}
+            failed={topAllData.processing.failed}
+          />
         </StatCard>
       </div>
 
       {/* Tabs & Search */}
-      <DatasetFilter active={active} setActive={changeActiveLogs} />
+      <DatasetFilter
+        filters={filters as FilterCollection[]}
+        value={filterValue}
+        active={active}
+        setActive={changeActiveLogs}
+        searchString={searchString}
+        onSearchChange={handleInputChange}
+        onChange={handleFilterSubmit}
+      />
 
       {/* Table */}
       <FileLogsTable
-        data={mockData}
+        data={tableList}
         pagination={pagination}
         setPagination={handlePaginationChange}
         pageCount={10}