Fix: Metadata bugs. (#12111)

### What problem does this PR solve?

Fix: Metadata bugs.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

---------

Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>

web/src/components/list-filter-bar/filter-field.tsx

@@ -118,7 +118,7 @@ export const FilterField = memo(
                     setShowAll(!showAll);
                   }}
                 >
-                  <FormLabel className="text-text-primary">
+                  <FormLabel className="text-text-secondary text-sm">
                     {item.label}
                   </FormLabel>
                   {showAll ? (

web/src/components/list-filter-bar/filter-popover.tsx

@@ -33,6 +33,7 @@ export type CheckboxFormMultipleProps = {
   onChange?: FilterChange;
   onOpenChange?: (open: boolean) => void;
   setOpen(open: boolean): void;
+  filterGroup?: Record<string, string[]>;
 };
 
 function CheckboxFormMultiple({
@@ -40,6 +41,7 @@ function CheckboxFormMultiple({
   value,
   onChange,
   setOpen,
+  filterGroup,
 }: CheckboxFormMultipleProps) {
   const [resolvedFilters, setResolvedFilters] =
     useState<FilterCollection[]>(filters);
@@ -120,6 +122,20 @@ function CheckboxFormMultiple({
     }
   }, [form, value, resolvedFilters, fieldsDict]);
 
+  const filterList = useMemo(() => {
+    const filterSet = filterGroup
+      ? Object.values(filterGroup).reduce<Set<string>>((pre, cur) => {
+          cur.forEach((item) => pre.add(item));
+          return pre;
+        }, new Set())
+      : new Set();
+    return [...filterSet];
+  }, [filterGroup]);
+
+  const notInfilterGroup = useMemo(() => {
+    return filters.filter((x) => !filterList.includes(x.field));
+  }, [filterList, filters]);
+
   return (
     <Form {...form}>
       <form
@@ -127,34 +143,70 @@ function CheckboxFormMultiple({
         className="space-y-8 px-5 py-2.5"
         onReset={() => form.reset()}
       >
-        {filters.map((x) => (
+        <div className="space-y-4">
-          <FormField
+          {filterGroup &&
-            key={x.field}
+            Object.keys(filterGroup).map((key) => {
-            control={form.control}
+              const filterKeys = filterGroup[key];
-            name={x.field}
+              const thisFilters = filters.filter((x) =>
-            render={() => (
+                filterKeys.includes(x.field),
-              <FormItem className="space-y-4">
+              );
-                <div>
+              return (
-                  <FormLabel className="text-text-primary">{x.label}</FormLabel>
+                <div
+                  key={key}
+                  className="flex flex-col space-y-4 border-b border-border-button pb-4"
+                >
+                  <div className="text-text-primary text-sm">{key}</div>
+                  <div className="flex flex-col space-y-4">
+                    {thisFilters.map((x) => (
+                      <FilterField
+                        key={x.field}
+                        item={{ ...x, id: x.field }}
+                        parent={{
+                          ...x,
+                          id: x.field,
+                          field: ``,
+                        }}
+                      />
+                    ))}
+                  </div>
                 </div>
-                {x.list.map((item) => {
+              );
-                  return (
+            })}
-                    <FilterField
+          {notInfilterGroup &&
-                      key={item.id}
+            notInfilterGroup.map((x) => {
-                      item={{ ...item }}
+              return (
-                      parent={{
+                <FormField
-                        ...x,
+                  key={x.field}
-                        id: x.field,
+                  control={form.control}
-                        // field: `${x.field}${item.field ? '.' + item.field : ''}`,
+                  name={x.field}
-                      }}
+                  render={() => (
-                    />
+                    <FormItem className="space-y-4">
-                  );
+                      <div>
-                })}
+                        <FormLabel className="text-text-primary text-sm">
-                <FormMessage />
+                          {x.label}
-              </FormItem>
+                        </FormLabel>
-            )}
+                      </div>
-          />
+                      {x.list.map((item) => {
-        ))}
+                        return (
+                          <FilterField
+                            key={item.id}
+                            item={{ ...item }}
+                            parent={{
+                              ...x,
+                              id: x.field,
+                              // field: `${x.field}${item.field ? '.' + item.field : ''}`,
+                            }}
+                          />
+                        );
+                      })}
+                      <FormMessage />
+                    </FormItem>
+                  )}
+                />
+              );
+            })}
+        </div>
+
         <div className="flex justify-end gap-5">
           <Button
             type="button"
@@ -185,6 +237,7 @@ export function FilterPopover({
   onChange,
   onOpenChange,
   filters,
+  filterGroup,
 }: PropsWithChildren & Omit<CheckboxFormMultipleProps, 'setOpen'>) {
   const [open, setOpen] = useState(false);
   const onOpenChangeFun = useCallback(
@@ -203,6 +256,7 @@ export function FilterPopover({
           value={value}
           filters={filters}
           setOpen={setOpen}
+          filterGroup={filterGroup}
         ></CheckboxFormMultiple>
       </PopoverContent>
     </Popover>

web/src/components/list-filter-bar/index.tsx

@@ -58,9 +58,11 @@ export default function ListFilterBar({
   filters,
   className,
   icon,
+  filterGroup,
 }: PropsWithChildren<IProps & Omit<CheckboxFormMultipleProps, 'setOpen'>> & {
   className?: string;
   icon?: ReactNode;
+  filterGroup?: Record<string, string[]>;
 }) {
   const filterCount = useMemo(() => {
     return typeof value === 'object' && value !== null
@@ -99,6 +101,7 @@ export default function ListFilterBar({
             value={value}
             onChange={onChange}
             filters={filters}
+            filterGroup={filterGroup}
             onOpenChange={onOpenChange}
           >
             <FilterButton count={filterCount}></FilterButton>

web/src/hooks/use-document-request.ts

@@ -122,9 +122,9 @@ export const useFetchDocumentList = () => {
           page: pagination.current,
         },
         {
-          suffix: filterValue.type,
+          suffix: filterValue.type as string[],
-          run_status: filterValue.run,
+          run_status: filterValue.run as string[],
-          metadata: filterValue.metadata,
+          metadata: filterValue.metadata as Record<string, string[]>,
         },
       );
       if (ret.data.code === 0) {

web/src/interfaces/request/knowledge.ts

@@ -33,4 +33,5 @@ export interface IFetchKnowledgeListRequestParams {
 export interface IFetchDocumentListRequestBody {
   suffix?: string[];
   run_status?: string[];
+  metadata?: Record<string, string[]>;
 }

web/src/locales/en.ts

@@ -191,6 +191,8 @@ Procedural Memory: Learned skills, habits, and automated procedures.`,
         fieldName: 'Field name',
         editMetadata: 'Edit metadata',
       },
+      metadataField: 'Metadata field',
+      systemAttribute: 'System attribute',
       localUpload: 'Local upload',
       fileSize: 'File size',
       fileType: 'File type',

web/src/pages/dataset/components/metedata/hook.ts

@@ -198,20 +198,24 @@ export const useManageMetaDataModal = (
   const { setDocumentMeta } = useSetDocumentMeta();
 
   useEffect(() => {
-    if (data) {
+    if (type === MetadataType.Manage) {
-      setTableData(data);
+      if (data) {
-    } else {
+        setTableData(data);
-      setTableData([]);
+      } else {
+        setTableData([]);
+      }
     }
-  }, [data]);
+  }, [data, type]);
 
   useEffect(() => {
-    if (metaData) {
+    if (type !== MetadataType.Manage) {
-      setTableData(metaData);
+      if (metaData) {
-    } else {
+        setTableData(metaData);
-      setTableData([]);
+      } else {
+        setTableData([]);
+      }
     }
-  }, [metaData]);
+  }, [metaData, type]);
 
   const handleDeleteSingleValue = useCallback(
     (field: string, value: string) => {

web/src/pages/dataset/components/metedata/manage-modal.tsx

@@ -54,7 +54,6 @@ export const ManageMetadataModal = (props: IManageModalProps) => {
     values: [],
   });
 
-  const [currentValueIndex, setCurrentValueIndex] = useState<number>(0);
   const [deleteDialogContent, setDeleteDialogContent] = useState({
     visible: false,
     title: '',
@@ -95,12 +94,12 @@ export const ManageMetadataModal = (props: IManageModalProps) => {
       description: '',
       values: [],
     });
-    setCurrentValueIndex(tableData.length || 0);
+    // setCurrentValueIndex(tableData.length || 0);
     showManageValuesModal();
   };
   const handleEditValueRow = useCallback(
-    (data: IMetaDataTableData, index: number) => {
+    (data: IMetaDataTableData) => {
-      setCurrentValueIndex(index);
+      // setCurrentValueIndex(index);
       setValueData(data);
       showManageValuesModal();
     },
@@ -186,7 +185,7 @@ export const ManageMetadataModal = (props: IManageModalProps) => {
               variant={'ghost'}
               className="bg-transparent px-1 py-0"
               onClick={() => {
-                handleEditValueRow(row.original, row.index);
+                handleEditValueRow(row.original);
               }}
             >
               <Settings />
@@ -244,16 +243,32 @@ export const ManageMetadataModal = (props: IManageModalProps) => {
 
   const handleSaveValues = (data: IMetaDataTableData) => {
     setTableData((prev) => {
-      if (currentValueIndex >= prev.length) {
+      //If the keys are the same, they need to be merged.
-        return [...prev, data];
+      const fieldMap = new Map<string, any>();
+
+      prev.forEach((item) => {
+        if (fieldMap.has(item.field)) {
+          const existingItem = fieldMap.get(item.field);
+          const mergedValues = [
+            ...new Set([...existingItem.values, ...item.values]),
+          ];
+          fieldMap.set(item.field, { ...existingItem, values: mergedValues });
+        } else {
+          fieldMap.set(item.field, item);
+        }
+      });
+
+      if (fieldMap.has(data.field)) {
+        const existingItem = fieldMap.get(data.field);
+        const mergedValues = [
+          ...new Set([...existingItem.values, ...data.values]),
+        ];
+        fieldMap.set(data.field, { ...existingItem, values: mergedValues });
       } else {
-        return prev.map((item, index) => {
+        fieldMap.set(data.field, data);
-          if (index === currentValueIndex) {
-            return data;
-          }
-          return item;
-        });
       }
+
+      return Array.from(fieldMap.values());
     });
   };
 

web/src/pages/dataset/components/metedata/manage-values-modal.tsx

@@ -110,8 +110,8 @@ export const ManageValuesModal = (props: IManageValuesProps) => {
 
   // Handle blur event, synchronize to main state
   const handleValueBlur = useCallback(() => {
-    addUpdateValue(metaData.field, [...tempValues]);
+    addUpdateValue(metaData.field, [...new Set([...tempValues])]);
-    handleChange('values', [...tempValues]);
+    handleChange('values', [...new Set([...tempValues])]);
   }, [handleChange, tempValues, metaData, addUpdateValue]);
 
   // Handle delete operation
@@ -139,12 +139,12 @@ export const ManageValuesModal = (props: IManageValuesProps) => {
 
   // Handle adding new value
   const handleAddValue = useCallback(() => {
-    setTempValues((prev) => [...prev, '']);
+    setTempValues((prev) => [...new Set([...prev, ''])]);
 
     // Synchronize to main state
     setMetaData((prev) => ({
       ...prev,
-      values: [...prev.values, ''],
+      values: [...new Set([...prev.values, ''])],
     }));
   }, []);
 

web/src/pages/dataset/dataset-setting/configuration/audio.tsx

@@ -3,11 +3,13 @@ import {
   AutoQuestionsFormField,
 } from '@/components/auto-keywords-form-field';
 import { ConfigurationFormContainer } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function AudioConfiguration() {
   return (
     <ConfigurationFormContainer>
       <>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </>

web/src/pages/dataset/dataset-setting/configuration/book.tsx

@@ -7,6 +7,7 @@ import {
   ConfigurationFormContainer,
   MainContainer,
 } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function BookConfiguration() {
   return (
@@ -16,6 +17,7 @@ export function BookConfiguration() {
       </ConfigurationFormContainer>
 
       <ConfigurationFormContainer>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </ConfigurationFormContainer>

web/src/pages/dataset/dataset-setting/configuration/email.tsx

@@ -3,11 +3,13 @@ import {
   AutoQuestionsFormField,
 } from '@/components/auto-keywords-form-field';
 import { ConfigurationFormContainer } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function EmailConfiguration() {
   return (
     <ConfigurationFormContainer>
       <>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </>

web/src/pages/dataset/dataset-setting/configuration/laws.tsx

@@ -7,6 +7,7 @@ import {
   ConfigurationFormContainer,
   MainContainer,
 } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function LawsConfiguration() {
   return (
@@ -16,6 +17,7 @@ export function LawsConfiguration() {
       </ConfigurationFormContainer>
 
       <ConfigurationFormContainer>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </ConfigurationFormContainer>

web/src/pages/dataset/dataset-setting/configuration/manual.tsx

@@ -7,6 +7,7 @@ import {
   ConfigurationFormContainer,
   MainContainer,
 } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function ManualConfiguration() {
   return (
@@ -16,6 +17,7 @@ export function ManualConfiguration() {
       </ConfigurationFormContainer>
 
       <ConfigurationFormContainer>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </ConfigurationFormContainer>

web/src/pages/dataset/dataset-setting/configuration/one.tsx

@@ -4,12 +4,14 @@ import {
 } from '@/components/auto-keywords-form-field';
 import { LayoutRecognizeFormField } from '@/components/layout-recognize-form-field';
 import { ConfigurationFormContainer } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function OneConfiguration() {
   return (
     <ConfigurationFormContainer>
       <LayoutRecognizeFormField></LayoutRecognizeFormField>
       <>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </>

web/src/pages/dataset/dataset-setting/configuration/paper.tsx

@@ -7,6 +7,7 @@ import {
   ConfigurationFormContainer,
   MainContainer,
 } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function PaperConfiguration() {
   return (
@@ -16,6 +17,7 @@ export function PaperConfiguration() {
       </ConfigurationFormContainer>
 
       <ConfigurationFormContainer>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </ConfigurationFormContainer>

web/src/pages/dataset/dataset-setting/configuration/picture.tsx

@@ -3,11 +3,13 @@ import {
   AutoQuestionsFormField,
 } from '@/components/auto-keywords-form-field';
 import { ConfigurationFormContainer } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function PictureConfiguration() {
   return (
     <ConfigurationFormContainer>
       <>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </>

web/src/pages/dataset/dataset-setting/configuration/presentation.tsx

@@ -7,6 +7,7 @@ import {
   ConfigurationFormContainer,
   MainContainer,
 } from '../configuration-form-container';
+import { AutoMetadata } from './common-item';
 
 export function PresentationConfiguration() {
   return (
@@ -16,6 +17,7 @@ export function PresentationConfiguration() {
       </ConfigurationFormContainer>
 
       <ConfigurationFormContainer>
+        <AutoMetadata />
         <AutoKeywordsFormField></AutoKeywordsFormField>
         <AutoQuestionsFormField></AutoQuestionsFormField>
       </ConfigurationFormContainer>

web/src/pages/dataset/dataset/index.tsx

@@ -16,7 +16,7 @@ import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-reques
 import { Pen, Upload } from 'lucide-react';
 import { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { useManageMetadata } from '../components/metedata/hook';
+import { MetadataType, useManageMetadata } from '../components/metedata/hook';
 import { ManageMetadataModal } from '../components/metedata/manage-modal';
 import { DatasetTable } from './dataset-table';
 import Generate from './generate-button/generate';
@@ -53,7 +53,7 @@ export default function Dataset() {
   const { data: dataSetData } = useFetchKnowledgeBaseConfiguration({
     refreshCount,
   });
-  const { filters, onOpenChange } = useSelectDatasetFilters();
+  const { filters, onOpenChange, filterGroup } = useSelectDatasetFilters();
 
   const {
     createLoading,
@@ -90,6 +90,7 @@ export default function Dataset() {
           onSearchChange={handleInputChange}
           searchString={searchString}
           value={filterValue}
+          filterGroup={filterGroup}
           onChange={handleFilterSubmit}
           onOpenChange={onOpenChange}
           filters={filters}
@@ -105,7 +106,25 @@ export default function Dataset() {
             <Button
               variant={'ghost'}
               className="border border-border-button"
-              onClick={() => showManageMetadataModal()}
+              onClick={() =>
+                showManageMetadataModal({
+                  type: MetadataType.Manage,
+                  isCanAdd: false,
+                  isEditField: true,
+                  title: (
+                    <div className="flex flex-col gap-2">
+                      <div className="text-base font-normal">
+                        {t('knowledgeDetails.metadata.manageMetadata')}
+                      </div>
+                      <div className="text-sm text-text-secondary">
+                        {t(
+                          'knowledgeDetails.metadata.manageMetadataForDataset',
+                        )}
+                      </div>
+                    </div>
+                  ),
+                })
+              }
             >
               <div className="flex gap-1 items-center">
                 <Pen size={14} />
@@ -179,6 +198,7 @@ export default function Dataset() {
             // selectedRowKeys={selectedRowKeys}
             tableData={tableData}
             isCanAdd={metadataConfig.isCanAdd}
+            isEditField={metadataConfig.isEditField}
             isDeleteSingleValue={metadataConfig.isDeleteSingleValue}
             type={metadataConfig.type}
             otherData={metadataConfig.record}

web/src/pages/dataset/dataset/use-select-filters.ts

@@ -49,9 +49,13 @@ export function useSelectDatasetFilters() {
     return [
       { field: 'type', label: 'File Type', list: fileTypes },
       { field: 'run', label: 'Status', list: fileStatus },
-      { field: 'metadata', label: 'metadata', list: metaDataList },
+      { field: 'metadata', label: 'Metadata field', list: metaDataList },
     ] as FilterCollection[];
   }, [fileStatus, fileTypes, metaDataList]);
 
-  return { filters, onOpenChange };
+  const filterGroup = {
+    [t('systemAttribute')]: ['type', 'run'],
+    // [t('metadataField')]: ['metadata'],
+  };
+  return { filters, onOpenChange, filterGroup };
 }