Feat: Bind data to datasets page #3221 (#4938)

### What problem does this PR solve? Feat: Bind data to datasets page #3221 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2025-12-19 12:06:42 +08:00 · 2025-02-14 09:38:48 +08:00
parent 17fa2e9e8e
commit c1583a3e1d
10 changed files with 571 additions and 256 deletions
--- a/web/src/pages/dataset/dataset/dataset-table.tsx
+++ b/web/src/pages/dataset/dataset/dataset-table.tsx
@ -1,7 +1,6 @@
 'use client';

 import {
-  ColumnDef,
  ColumnFiltersState,
  SortingState,
  VisibilityState,
@ -12,20 +11,10 @@ import {
  getSortedRowModel,
  useReactTable,
 } from '@tanstack/react-table';
-import { ArrowUpDown, MoreHorizontal, Pencil } from 'lucide-react';
 import * as React from 'react';

+import { ChunkMethodDialog } from '@/components/chunk-method-dialog';
 import { Button } from '@/components/ui/button';
-import { Checkbox } from '@/components/ui/checkbox';
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuLabel,
-  DropdownMenuSeparator,
-  DropdownMenuTrigger,
-} from '@/components/ui/dropdown-menu';
-import { Switch } from '@/components/ui/switch';
 import {
  Table,
  TableBody,
@ -34,43 +23,22 @@ import {
  TableHeader,
  TableRow,
 } from '@/components/ui/table';
-import { RunningStatus } from '@/constants/knowledge';
+import { useFetchNextDocumentList } from '@/hooks/document-hooks';
+import { useSetSelectedRecord } from '@/hooks/logic-hooks';
 import { IDocumentInfo } from '@/interfaces/database/document';
-import { useTranslation } from 'react-i18next';
-
-const data: IDocumentInfo[] = [
-  {
-    chunk_num: 1,
-    create_date: 'Thu, 28 Nov 2024 17:10:22 GMT',
-    create_time: 1732785022792,
-    created_by: 'b0975cb4bc3111ee9b830aef05f5e94f',
-    id: '990cb30ead6811efb9b9fa163e197198',
-    kb_id: '25a8cfbe9cd411efbc12fa163e197198',
-    location: 'mian.jpg',
-    name: 'mian.jpg',
-    parser_config: {
-      pages: [[1, 1000000]],
-    },
-    parser_id: 'picture',
-    process_begin_at: 'Thu, 28 Nov 2024 17:10:25 GMT',
-    process_duation: 8.46185,
-    progress: 1,
-    progress_msg:
-      '\nTask has been received.\nPage(1~100000001): Finish OCR: (用小麦粉\n金\nONGXI ...)\nPage(1~100000001): OCR results is too long to use CV LLM.\nPage(1~100000001): Finished slicing files (1 chunks in 0.34s). Start to embedding the content.\nPage(1~100000001): Finished embedding (in 0.35s)! Start to build index!\nPage(1~100000001): Indexing elapsed in 0.02s.\nPage(1~100000001): Done!',
-    run: RunningStatus.RUNNING,
-    size: 19692,
-    source_type: 'local',
-    status: '1',
-    thumbnail:
-      '/v1/document/image/25a8cfbe9cd411efbc12fa163e197198-thumbnail_990cb30ead6811efb9b9fa163e197198.png',
-    token_num: 115,
-    type: 'visual',
-    update_date: 'Thu, 28 Nov 2024 17:10:33 GMT',
-    update_time: 1732785033462,
-  },
-];
+import { getExtension } from '@/utils/document-util';
+import { useMemo } from 'react';
+import { useChangeDocumentParser } from './hooks';
+import { useDatasetTableColumns } from './use-dataset-table-columns';

 export function DatasetTable() {
+  const {
+    // searchString,
+    documents,
+    pagination,
+    // handleInputChange,
+    setPagination,
+  } = useFetchNextDocumentList();
  const [sorting, setSorting] = React.useState<SortingState>([]);
  const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>(
    [],
@ -78,122 +46,31 @@ export function DatasetTable() {
  const [columnVisibility, setColumnVisibility] =
    React.useState<VisibilityState>({});
  const [rowSelection, setRowSelection] = React.useState({});
-  const { t } = useTranslation('translation', {
-    keyPrefix: 'knowledgeDetails',
+
+  const { currentRecord, setRecord } = useSetSelectedRecord<IDocumentInfo>();
+
+  const {
+    changeParserLoading,
+    onChangeParserOk,
+    changeParserVisible,
+    hideChangeParserModal,
+    showChangeParserModal,
+  } = useChangeDocumentParser(currentRecord.id);
+
+  const columns = useDatasetTableColumns({
+    showChangeParserModal,
+    setCurrentRecord: setRecord,
  });

-  const columns: ColumnDef<IDocumentInfo>[] = [
-    {
-      id: 'select',
-      header: ({ table }) => (
-        <Checkbox
-          checked={
-            table.getIsAllPageRowsSelected() ||
-            (table.getIsSomePageRowsSelected() && 'indeterminate')
-          }
-          onCheckedChange={(value) => table.toggleAllPageRowsSelected(!!value)}
-          aria-label="Select all"
-        />
-      ),
-      cell: ({ row }) => (
-        <Checkbox
-          checked={row.getIsSelected()}
-          onCheckedChange={(value) => row.toggleSelected(!!value)}
-          aria-label="Select row"
-        />
-      ),
-      enableSorting: false,
-      enableHiding: false,
-    },
-    {
-      accessorKey: 'name',
-      header: ({ column }) => {
-        return (
-          <Button
-            variant="ghost"
-            onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
-          >
-            {t('name')}
-            <ArrowUpDown />
-          </Button>
-        );
-      },
-      cell: ({ row }) => (
-        <div className="capitalize">{row.getValue('name')}</div>
-      ),
-    },
-    {
-      accessorKey: 'create_time',
-      header: ({ column }) => {
-        return (
-          <Button
-            variant="ghost"
-            onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
-          >
-            {t('uploadDate')}
-            <ArrowUpDown />
-          </Button>
-        );
-      },
-      cell: ({ row }) => (
-        <div className="lowercase">{row.getValue('create_time')}</div>
-      ),
-    },
-    {
-      accessorKey: 'parser_id',
-      header: t('chunkMethod'),
-      cell: ({ row }) => (
-        <div className="capitalize">{row.getValue('parser_id')}</div>
-      ),
-    },
-    {
-      accessorKey: 'run',
-      header: t('parsingStatus'),
-      cell: ({ row }) => (
-        <Button variant="destructive" size={'sm'}>
-          {row.getValue('run')}
-        </Button>
-      ),
-    },
-    {
-      id: 'actions',
-      header: t('action'),
-      enableHiding: false,
-      cell: ({ row }) => {
-        const payment = row.original;
-
-        return (
-          <section className="flex gap-4 items-center">
-            <Switch id="airplane-mode" />
-            <Button variant="secondary" size={'icon'}>
-              <Pencil />
-            </Button>
-            <DropdownMenu>
-              <DropdownMenuTrigger asChild>
-                <Button variant="secondary" size={'icon'}>
-                  <MoreHorizontal />
-                </Button>
-              </DropdownMenuTrigger>
-              <DropdownMenuContent align="end">
-                <DropdownMenuLabel>Actions</DropdownMenuLabel>
-                <DropdownMenuItem
-                  onClick={() => navigator.clipboard.writeText(payment.id)}
-                >
-                  Copy payment ID
-                </DropdownMenuItem>
-                <DropdownMenuSeparator />
-                <DropdownMenuItem>View customer</DropdownMenuItem>
-                <DropdownMenuItem>View payment details</DropdownMenuItem>
-              </DropdownMenuContent>
-            </DropdownMenu>
-          </section>
-        );
-      },
-    },
-  ];
+  const currentPagination = useMemo(() => {
+    return {
+      pageIndex: (pagination.current || 1) - 1,
+      pageSize: pagination.pageSize || 10,
+    };
+  }, [pagination]);

  const table = useReactTable({
-    data,
+    data: documents,
    columns,
    onSortingChange: setSorting,
    onColumnFiltersChange: setColumnFilters,
@ -203,12 +80,29 @@ export function DatasetTable() {
    getFilteredRowModel: getFilteredRowModel(),
    onColumnVisibilityChange: setColumnVisibility,
    onRowSelectionChange: setRowSelection,
+    onPaginationChange: (updaterOrValue: any) => {
+      if (typeof updaterOrValue === 'function') {
+        const nextPagination = updaterOrValue(currentPagination);
+        setPagination({
+          page: nextPagination.pageIndex + 1,
+          pageSize: nextPagination.pageSize,
+        });
+      } else {
+        setPagination({
+          page: updaterOrValue.pageIndex,
+          pageSize: updaterOrValue.pageSize,
+        });
+      }
+    },
+    manualPagination: true, //we're doing manual "server-side" pagination
    state: {
      sorting,
      columnFilters,
      columnVisibility,
      rowSelection,
+      pagination: currentPagination,
    },
+    rowCount: pagination.total ?? 0,
  });

  return (
@ -241,7 +135,10 @@ export function DatasetTable() {
                  data-state={row.getIsSelected() && 'selected'}
                >
                  {row.getVisibleCells().map((cell) => (
-                    <TableCell key={cell.id}>
+                    <TableCell
+                      key={cell.id}
+                      className={cell.column.columnDef.meta?.cellClassName}
+                    >
                      {flexRender(
                        cell.column.columnDef.cell,
                        cell.getContext(),
@ -266,7 +163,7 @@ export function DatasetTable() {
      <div className="flex items-center justify-end space-x-2 py-4">
        <div className="flex-1 text-sm text-muted-foreground">
          {table.getFilteredSelectedRowModel().rows.length} of{' '}
-          {table.getFilteredRowModel().rows.length} row(s) selected.
+          {pagination?.total} row(s) selected.
        </div>
        <div className="space-x-2">
          <Button
@ -287,6 +184,18 @@ export function DatasetTable() {
          </Button>
        </div>
      </div>
+      {changeParserVisible && (
+        <ChunkMethodDialog
+          documentId={currentRecord.id}
+          parserId={currentRecord.parser_id}
+          parserConfig={currentRecord.parser_config}
+          documentExtension={getExtension(currentRecord.name)}
+          onOk={onChangeParserOk}
+          visible={changeParserVisible}
+          hideModal={hideChangeParserModal}
+          loading={changeParserLoading}
+        ></ChunkMethodDialog>
+      )}
    </div>
  );
 }
--- a/web/src/pages/dataset/dataset/hooks.ts
+++ b/web/src/pages/dataset/dataset/hooks.ts
@ -93,7 +93,13 @@ export const useChangeDocumentParser = (documentId: string) => {
  } = useSetModalState();

  const onChangeParserOk = useCallback(
-    async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => {
+    async ({
+      parserId,
+      parserConfig,
+    }: {
+      parserId: string;
+      parserConfig: IChangeParserConfigRequestBody;
+    }) => {
      const ret = await setDocumentParser({
        parserId,
        documentId,
--- a/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx
+++ b/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx
@ -0,0 +1,194 @@
+import SvgIcon from '@/components/svg-icon';
+import { Button } from '@/components/ui/button';
+import { Checkbox } from '@/components/ui/checkbox';
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+  DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
+import { Switch } from '@/components/ui/switch';
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipTrigger,
+} from '@/components/ui/tooltip';
+import { IDocumentInfo } from '@/interfaces/database/document';
+import { cn } from '@/lib/utils';
+import { getExtension } from '@/utils/document-util';
+import { ColumnDef } from '@tanstack/table-core';
+import { ArrowUpDown, MoreHorizontal, Pencil, Wrench } from 'lucide-react';
+import { useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { useChangeDocumentParser } from './hooks';
+
+type UseDatasetTableColumnsType = Pick<
+  ReturnType<typeof useChangeDocumentParser>,
+  'showChangeParserModal'
+> & { setCurrentRecord: (record: IDocumentInfo) => void };
+
+export function useDatasetTableColumns({
+  showChangeParserModal,
+  setCurrentRecord,
+}: UseDatasetTableColumnsType) {
+  const { t } = useTranslation('translation', {
+    keyPrefix: 'knowledgeDetails',
+  });
+
+  // const onShowRenameModal = (record: IDocumentInfo) => {
+  //   setCurrentRecord(record);
+  //   showRenameModal();
+  // };
+  const onShowChangeParserModal = useCallback(
+    (record: IDocumentInfo) => () => {
+      setCurrentRecord(record);
+      showChangeParserModal();
+    },
+    [setCurrentRecord, showChangeParserModal],
+  );
+
+  // const onShowSetMetaModal = useCallback(() => {
+  //   setRecord();
+  //   showSetMetaModal();
+  // }, [setRecord, showSetMetaModal]);
+
+  const columns: ColumnDef<IDocumentInfo>[] = [
+    {
+      id: 'select',
+      header: ({ table }) => (
+        <Checkbox
+          checked={
+            table.getIsAllPageRowsSelected() ||
+            (table.getIsSomePageRowsSelected() && 'indeterminate')
+          }
+          onCheckedChange={(value) => table.toggleAllPageRowsSelected(!!value)}
+          aria-label="Select all"
+        />
+      ),
+      cell: ({ row }) => (
+        <Checkbox
+          checked={row.getIsSelected()}
+          onCheckedChange={(value) => row.toggleSelected(!!value)}
+          aria-label="Select row"
+        />
+      ),
+      enableSorting: false,
+      enableHiding: false,
+    },
+    {
+      accessorKey: 'name',
+      header: ({ column }) => {
+        return (
+          <Button
+            variant="ghost"
+            onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
+          >
+            {t('name')}
+            <ArrowUpDown />
+          </Button>
+        );
+      },
+      meta: { cellClassName: 'max-w-[20vw]' },
+      cell: ({ row }) => {
+        const name: string = row.getValue('name');
+        // return <div className="capitalize">{row.getValue('name')}</div>;
+
+        return (
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <div className="flex gap-2">
+                <SvgIcon
+                  name={`file-icon/${getExtension(name)}`}
+                  width={24}
+                ></SvgIcon>
+                <span className={cn('truncate')}>{name}</span>
+              </div>
+            </TooltipTrigger>
+            <TooltipContent>
+              <p>{name}</p>
+            </TooltipContent>
+          </Tooltip>
+        );
+      },
+    },
+    {
+      accessorKey: 'create_time',
+      header: ({ column }) => {
+        return (
+          <Button
+            variant="ghost"
+            onClick={() => column.toggleSorting(column.getIsSorted() === 'asc')}
+          >
+            {t('uploadDate')}
+            <ArrowUpDown />
+          </Button>
+        );
+      },
+      cell: ({ row }) => (
+        <div className="lowercase">{row.getValue('create_time')}</div>
+      ),
+    },
+    {
+      accessorKey: 'parser_id',
+      header: t('chunkMethod'),
+      cell: ({ row }) => (
+        <div className="capitalize">{row.getValue('parser_id')}</div>
+      ),
+    },
+    {
+      accessorKey: 'run',
+      header: t('parsingStatus'),
+      cell: ({ row }) => (
+        <Button variant="destructive" size={'sm'}>
+          {row.getValue('run')}
+        </Button>
+      ),
+    },
+    {
+      id: 'actions',
+      header: t('action'),
+      enableHiding: false,
+      cell: ({ row }) => {
+        const record = row.original;
+
+        return (
+          <section className="flex gap-4 items-center">
+            <Switch id="airplane-mode" />
+            <Button
+              variant="icon"
+              size={'icon'}
+              onClick={onShowChangeParserModal(record)}
+            >
+              <Wrench />
+            </Button>
+            <Button variant="icon" size={'icon'}>
+              <Pencil />
+            </Button>
+            <DropdownMenu>
+              <DropdownMenuTrigger asChild>
+                <Button variant="icon" size={'icon'}>
+                  <MoreHorizontal />
+                </Button>
+              </DropdownMenuTrigger>
+              <DropdownMenuContent align="end">
+                <DropdownMenuLabel>Actions</DropdownMenuLabel>
+                <DropdownMenuItem
+                  onClick={() => navigator.clipboard.writeText(record.id)}
+                >
+                  Copy payment ID
+                </DropdownMenuItem>
+                <DropdownMenuSeparator />
+                <DropdownMenuItem>View customer</DropdownMenuItem>
+                <DropdownMenuItem>View payment details</DropdownMenuItem>
+              </DropdownMenuContent>
+            </DropdownMenu>
+          </section>
+        );
+      },
+    },
+  ];
+
+  return columns;
+}
--- a/web/src/pages/datasets/index.tsx
+++ b/web/src/pages/datasets/index.tsx
@ -1,87 +1,17 @@
 import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog';
 import ListFilterBar from '@/components/list-filter-bar';
+import { Avatar, AvatarFallback, AvatarImage } from '@/components/ui/avatar';
 import { Button } from '@/components/ui/button';
 import { Card, CardContent } from '@/components/ui/card';
+import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
+import { IKnowledge } from '@/interfaces/database/knowledge';
+import { formatDate } from '@/utils/date';
 import { ChevronRight, Plus, Trash2 } from 'lucide-react';
+import { useMemo } from 'react';
 import { DatasetCreatingDialog } from './dataset-creating-dialog';
 import { useSaveKnowledge } from './hooks';

-const datasets = [
-  {
-    id: 1,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 2,
-    title: 'HR knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 3,
-    title: 'IT knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 4,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 5,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 6,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 7,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 8,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-  {
-    id: 9,
-    title: 'Legal knowledge base',
-    files: '1,242 files',
-    size: '152 MB',
-    created: '12.02.2024',
-    image: 'https://github.com/shadcn.png',
-  },
-];
-
 export default function Datasets() {
  const {
    visible,
@ -92,6 +22,25 @@ export default function Datasets() {
  } = useSaveKnowledge();
  const { navigateToDataset } = useNavigatePage();

+  const {
+    fetchNextPage,
+    data,
+    hasNextPage,
+    searchString,
+    handleInputChange,
+    loading,
+  } = useInfiniteFetchKnowledgeList();
+
+  const nextList: IKnowledge[] = useMemo(() => {
+    const list =
+      data?.pages?.flatMap((x) => (Array.isArray(x.kbs) ? x.kbs : [])) ?? [];
+    return list;
+  }, [data?.pages]);
+
+  const total = useMemo(() => {
+    return data?.pages.at(-1).total ?? 0;
+  }, [data?.pages]);
+
  return (
    <section className="p-8 text-foreground">
      <ListFilterBar title="Datasets" showDialog={showModal}>
@ -99,17 +48,17 @@ export default function Datasets() {
        Create dataset
      </ListFilterBar>
      <div className="grid gap-6 sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-4 xl:grid-cols-6 2xl:grid-cols-8">
-        {datasets.map((dataset) => (
+        {nextList.map((dataset) => (
          <Card
            key={dataset.id}
            className="bg-colors-background-inverse-weak flex-1"
          >
            <CardContent className="p-4">
              <div className="flex justify-between mb-4">
-                <div
-                  className="w-[70px] h-[70px] rounded-xl bg-cover"
-                  style={{ backgroundImage: `url(${dataset.image})` }}
-                />
+                <Avatar className="w-[70px] h-[70px] rounded-lg">
+                  <AvatarImage src={dataset.avatar} />
+                  <AvatarFallback className="rounded-lg">CN</AvatarFallback>
+                </Avatar>
                <ConfirmDeleteDialog>
                  <Button variant="ghost" size="icon">
                    <Trash2 />
@ -118,20 +67,16 @@ export default function Datasets() {
              </div>
              <div className="flex justify-between items-end">
                <div>
-                  <h3 className="text-lg font-semibold mb-2">
-                    {dataset.title}
-                  </h3>
+                  <h3 className="text-lg font-semibold mb-2">{dataset.name}</h3>
+                  <p className="text-sm opacity-80">{dataset.doc_num} files</p>
                  <p className="text-sm opacity-80">
-                    {dataset.files} | {dataset.size}
-                  </p>
-                  <p className="text-sm opacity-80">
-                    Created {dataset.created}
+                    Created {formatDate(dataset.update_time)}
                  </p>
                </div>
                <Button
-                  variant="secondary"
+                  variant="icon"
                  size="icon"
-                  onClick={navigateToDataset}
+                  onClick={navigateToDataset(dataset.id)}
                >
                  <ChevronRight className="h-6 w-6" />
                </Button>
--- a/web/src/pages/home/datasets.tsx
+++ b/web/src/pages/home/datasets.tsx
@ -58,7 +58,7 @@ export function Datasets() {
                    <Button
                      variant="icon"
                      size="icon"
-                      onClick={navigateToDataset}
+                      onClick={navigateToDataset(dataset.id)}
                    >
                      <ChevronRight className="h-6 w-6" />
                    </Button>