Feat: Remove unnecessary data from the dsl #9869 (#10177)

### What problem does this PR solve?
Feat: Remove unnecessary data from the dsl #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/components/chunk-method-dialog/use-default-parser-values.ts

@@ -1,7 +1,7 @@
 import { IParserConfig } from '@/interfaces/database/document';
 import { useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
-import { DocumentType } from '../layout-recognize-form-field';
+import { ParseDocumentType } from '../layout-recognize-form-field';
 
 export function useDefaultParserValues() {
   const { t } = useTranslation();
@@ -9,7 +9,7 @@ export function useDefaultParserValues() {
   const defaultParserValues = useMemo(() => {
     const defaultParserValues = {
       task_page_size: 12,
-      layout_recognize: DocumentType.DeepDOC,
+      layout_recognize: ParseDocumentType.DeepDOC,
       chunk_token_num: 512,
       delimiter: '\n',
       auto_keywords: 0,

web/src/components/cross-language-form-field.tsx

@@ -22,7 +22,7 @@ const Languages = [
   'Vietnamese',
 ];
 
-const options = Languages.map((x) => ({
+export const crossLanguageOptions = Languages.map((x) => ({
   label: t('language.' + toLower(x)),
   value: x,
 }));
@@ -59,7 +59,7 @@ export const CrossLanguageFormField = ({
           </FormLabel>
           <FormControl>
             <MultiSelect
-              options={options}
+              options={crossLanguageOptions}
               placeholder={t('fileManager.pleaseSelect')}
               maxCount={100}
               {...field}

web/src/components/delimiter-form-field.tsx

@@ -16,7 +16,7 @@ interface IProps {
 }
 
 export const DelimiterInput = forwardRef<HTMLInputElement, InputProps & IProps>(
-  ({ value, onChange, maxLength, defaultValue }, ref) => {
+  ({ value, onChange, maxLength, defaultValue, ...props }, ref) => {
     const nextValue = value?.replaceAll('\n', '\\n');
     const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
       const val = e.target.value;
@@ -30,6 +30,7 @@ export const DelimiterInput = forwardRef<HTMLInputElement, InputProps & IProps>(
         maxLength={maxLength}
         defaultValue={defaultValue}
         ref={ref}
+        {...props}
       ></Input>
     );
   },

web/src/components/layout-recognize-form-field.tsx

@@ -5,6 +5,7 @@ import { cn } from '@/lib/utils';
 import { camelCase } from 'lodash';
 import { useMemo } from 'react';
 import { useFormContext } from 'react-hook-form';
+import { SelectWithSearch } from './originui/select-with-search';
 import {
   FormControl,
   FormField,
@@ -12,9 +13,8 @@ import {
   FormLabel,
   FormMessage,
 } from './ui/form';
-import { RAGFlowSelect } from './ui/select';
 
-export const enum DocumentType {
+export const enum ParseDocumentType {
   DeepDOC = 'DeepDOC',
   PlainText = 'Plain Text',
 }
@@ -22,9 +22,11 @@ export const enum DocumentType {
 export function LayoutRecognizeFormField({
   name = 'parser_config.layout_recognize',
   horizontal = true,
+  optionsWithoutLLM,
 }: {
   name?: string;
   horizontal?: boolean;
+  optionsWithoutLLM?: { value: string; label: string }[];
 }) {
   const form = useFormContext();
 
@@ -32,10 +34,13 @@ export function LayoutRecognizeFormField({
   const allOptions = useSelectLlmOptionsByModelType();
 
   const options = useMemo(() => {
-    const list = [DocumentType.DeepDOC, DocumentType.PlainText].map((x) => ({
+    const list = optionsWithoutLLM
-      label: x === DocumentType.PlainText ? t(camelCase(x)) : 'DeepDoc',
+      ? optionsWithoutLLM
-      value: x,
+      : [ParseDocumentType.DeepDOC, ParseDocumentType.PlainText].map((x) => ({
-    }));
+          label:
+            x === ParseDocumentType.PlainText ? t(camelCase(x)) : 'DeepDoc',
+          value: x,
+        }));
 
     const image2TextList = allOptions[LlmModelType.Image2text].map((x) => {
       return {
@@ -55,7 +60,7 @@ export function LayoutRecognizeFormField({
     });
 
     return [...list, ...image2TextList];
-  }, [allOptions, t]);
+  }, [allOptions, optionsWithoutLLM, t]);
 
   return (
     <FormField
@@ -80,7 +85,10 @@ export function LayoutRecognizeFormField({
               </FormLabel>
               <div className={horizontal ? 'w-3/4' : 'w-full'}>
                 <FormControl>
-                  <RAGFlowSelect {...field} options={options}></RAGFlowSelect>
+                  <SelectWithSearch
+                    {...field}
+                    options={options}
+                  ></SelectWithSearch>
                 </FormControl>
               </div>
             </div>

web/src/locales/en.ts

@@ -1668,6 +1668,7 @@ This delimiter is used to split the input text into several text pieces echo of
       overlappedPercent: 'Overlapped percent',
       searchMethod: 'Search method',
       filenameEmbdWeight: 'Filename embd weight',
+      begin: 'File',
     },
   },
 };

web/src/locales/zh.ts

@@ -1578,6 +1578,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       overlappedPercent: '重叠百分比',
       searchMethod: '搜索方法',
       filenameEmbdWeight: '文件名嵌入权重',
+      begin: '文件',
     },
   },
 };

web/src/pages/agents/hooks/use-create-agent.ts

@@ -2,10 +2,47 @@ import { useSetModalState } from '@/hooks/common-hooks';
 import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request';
 import { DSL } from '@/interfaces/database/agent';
 import { AgentCategory } from '@/pages/agent/constant';
+import { BeginId, Operator } from '@/pages/data-flow/constant';
 import { useCallback } from 'react';
 import { FlowType } from '../constant';
 import { FormSchemaType } from '../create-agent-form';
 
+export const DataflowEmptyDsl = {
+  graph: {
+    nodes: [
+      {
+        id: BeginId,
+        type: 'beginNode',
+        position: {
+          x: 50,
+          y: 200,
+        },
+        data: {
+          label: Operator.Begin,
+          name: Operator.Begin,
+        },
+        sourcePosition: 'left',
+        targetPosition: 'right',
+      },
+    ],
+    edges: [],
+  },
+  components: {
+    [Operator.Begin]: {
+      obj: {
+        component_name: Operator.Begin,
+        params: {},
+      },
+      downstream: [], // other edge target is downstream, edge source is current node id
+      upstream: [], // edge source is upstream, edge target is current node id
+    },
+  },
+  retrieval: [], // reference
+  history: [],
+  path: [],
+  globals: {},
+};
+
 export function useCreateAgentOrPipeline() {
   const { loading, setAgent } = useSetAgent();
   const {
@@ -16,13 +53,13 @@ export function useCreateAgentOrPipeline() {
 
   const handleCreateAgentOrPipeline = useCallback(
     async (data: FormSchemaType) => {
+      const isAgent = data.type === FlowType.Agent;
       const ret = await setAgent({
         title: data.name,
-        dsl: EmptyDsl as DSL,
+        dsl: isAgent ? (EmptyDsl as DSL) : (DataflowEmptyDsl as DSL),
-        canvas_category:
+        canvas_category: isAgent
-          data.type === FlowType.Agent
+          ? AgentCategory.AgentCanvas
-            ? AgentCategory.AgentCanvas
+          : AgentCategory.DataflowCanvas,
-            : AgentCategory.DataflowCanvas,
       });
 
       if (ret.code === 0) {

web/src/pages/data-flow/canvas/index.tsx

@@ -34,6 +34,7 @@ import {
   useHideFormSheetOnNodeDeletion,
   useShowDrawer,
 } from '../hooks/use-show-drawer';
+import { LogSheet } from '../log-sheet';
 import RunSheet from '../run-sheet';
 import { ButtonEdge } from './edge';
 import styles from './index.less';
@@ -93,7 +94,6 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
     chatVisible,
     runVisible,
     hideRunOrChatDrawer,
-    showChatModal,
     showFormDrawer,
   } = useShowDrawer({
     drawerVisible,
@@ -146,6 +146,12 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
     clearActiveDropdown,
   ]);
 
+  const {
+    visible: logSheetVisible,
+    showModal: showLogSheet,
+    hideModal: hideLogSheet,
+  } = useSetModalState();
+
   const onConnect = (connection: Connection) => {
     originalOnConnect(connection);
     isConnectedRef.current = true;
@@ -294,9 +300,10 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) {
       {runVisible && (
         <RunSheet
           hideModal={hideRunOrChatDrawer}
-          showModal={showChatModal}
+          showModal={showLogSheet}
         ></RunSheet>
       )}
+      {logSheetVisible && <LogSheet hideModal={hideLogSheet}></LogSheet>}
     </div>
   );
 }

web/src/pages/data-flow/canvas/node/begin-node.tsx

@@ -36,7 +36,7 @@ function InnerBeginNode({ data, id, selected }: NodeProps<IBeginNode>) {
       <section className="flex items-center  gap-2">
         <OperatorIcon name={data.label as Operator}></OperatorIcon>
         <div className="truncate text-center font-semibold text-sm">
-          {t(`flow.begin`)}
+          {t(`dataflow.begin`)}
         </div>
       </section>
       <section className={cn(styles.generateParameters, 'flex gap-2 flex-col')}>

web/src/pages/data-flow/constant.tsx

@@ -38,10 +38,10 @@ export enum AgentDialogueMode {
   Task = 'task',
 }
 
-export const BeginId = 'begin';
+export const BeginId = 'File';
 
 export enum Operator {
-  Begin = 'Begin',
+  Begin = 'File',
   Note = 'Note',
   Parser = 'Parser',
   Tokenizer = 'Tokenizer',
@@ -80,6 +80,15 @@ export const SwitchOperatorOptions = [
 
 export const SwitchElseTo = 'end_cpn_ids';
 
+export enum TokenizerSearchMethod {
+  Embedding = 'embedding',
+  FullText = 'full_text',
+}
+
+export enum ImageParseMethod {
+  OCR = 'ocr',
+}
+
 const initialQueryBaseValues = {
   query: [],
 };
@@ -287,8 +296,12 @@ export const initialWaitingDialogueValues = {};
 export const initialChunkerValues = { outputs: {} };
 
 export const initialTokenizerValues = {
-  search_method: [],
+  search_method: [
+    TokenizerSearchMethod.Embedding,
+    TokenizerSearchMethod.FullText,
+  ],
   filename_embd_weight: 0.1,
+  fields: ['text'],
   outputs: {},
 };
 
@@ -359,9 +372,14 @@ export const initialStringTransformValues = {
   },
 };
 
-export const initialParserValues = { outputs: {}, parser: [] };
+export const initialParserValues = { outputs: {}, setups: [] };
 
-export const initialSplitterValues = { outputs: {}, chunk_token_size: 512 };
+export const initialSplitterValues = {
+  outputs: {},
+  chunk_token_size: 512,
+  overlapped_percent: 0,
+  delimiters: [{ value: '\n' }],
+};
 
 export const initialHierarchicalMergerValues = { outputs: {} };
 
@@ -450,8 +468,3 @@ export enum FileType {
   Video = 'video',
   Audio = 'audio',
 }
-
-export enum TokenizerSearchMethod {
-  Embedding = 'embedding',
-  FullText = 'full_text',
-}

web/src/pages/data-flow/form/parser-form/common-form-fields.tsx

@@ -42,22 +42,17 @@ export function OutputFormatFormField({
   );
 }
 
-export function ParserMethodFormField({ prefix }: CommonProps) {
+export function ParserMethodFormField({
+  prefix,
+  optionsWithoutLLM,
+}: CommonProps & { optionsWithoutLLM?: { value: string; label: string }[] }) {
   return (
     <LayoutRecognizeFormField
       name={buildFieldNameWithPrefix(`parse_method`, prefix)}
       horizontal={false}
+      optionsWithoutLLM={optionsWithoutLLM}
     ></LayoutRecognizeFormField>
   );
-
-  return (
-    <RAGFlowFormItem
-      name={buildFieldNameWithPrefix(`parse_method`, prefix)}
-      label="parse_method"
-    >
-      <SelectWithSearch options={[]}></SelectWithSearch>
-    </RAGFlowFormItem>
-  );
 }
 
 export function LargeModelFormField({ prefix }: CommonProps) {

web/src/pages/data-flow/form/parser-form/constant.ts

@@ -25,6 +25,7 @@ export enum TextMarkdownOutputFormat {
 
 export enum DocxOutputFormat {
   Markdown = 'markdown',
+  Json = 'json',
 }
 
 export enum PptOutputFormat {
@@ -50,3 +51,15 @@ export const OutputFormatMap = {
   [FileType.Video]: VideoOutputFormat,
   [FileType.Audio]: AudioOutputFormat,
 };
+
+export const InitialOutputFormatMap = {
+  [FileType.PDF]: PdfOutputFormat.Json,
+  [FileType.Spreadsheet]: SpreadsheetOutputFormat.Html,
+  [FileType.Image]: ImageOutputFormat.Text,
+  [FileType.Email]: EmailOutputFormat.Text,
+  [FileType.TextMarkdown]: TextMarkdownOutputFormat.Text,
+  [FileType.Docx]: DocxOutputFormat.Json,
+  [FileType.PowerPoint]: PptOutputFormat.Json,
+  [FileType.Video]: VideoOutputFormat.Json,
+  [FileType.Audio]: AudioOutputFormat.Text,
+};

web/src/pages/data-flow/form/parser-form/email-form-fields.tsx

@@ -2,8 +2,6 @@ import { SelectWithSearch } from '@/components/originui/select-with-search';
 import { RAGFlowFormItem } from '@/components/ragflow-form';
 import { buildOptions } from '@/utils/form';
 import { useTranslation } from 'react-i18next';
-import { FileType } from '../../constant';
-import { OutputFormatFormField } from './common-form-fields';
 import { CommonProps } from './interface';
 import { buildFieldNameWithPrefix } from './utils';
 
@@ -28,10 +26,6 @@ export function EmailFormFields({ prefix }: CommonProps) {
       >
         <SelectWithSearch options={options}></SelectWithSearch>
       </RAGFlowFormItem>
-      <OutputFormatFormField
-        prefix={prefix}
-        fileType={FileType.Email}
-      ></OutputFormatFormField>
     </>
   );
 }

web/src/pages/data-flow/form/parser-form/image-form-fields.tsx

@@ -1,21 +1,33 @@
-import { FileType } from '../../constant';
+import { buildOptions } from '@/utils/form';
-import {
+import { isEmpty } from 'lodash';
-  LargeModelFormField,
+import { useEffect } from 'react';
-  OutputFormatFormField,
+import { useFormContext } from 'react-hook-form';
-  ParserMethodFormField,
+import { ImageParseMethod } from '../../constant';
-} from './common-form-fields';
+import { ParserMethodFormField } from './common-form-fields';
 import { CommonProps } from './interface';
+import { buildFieldNameWithPrefix } from './utils';
+
+const options = buildOptions(ImageParseMethod);
 
 export function ImageFormFields({ prefix }: CommonProps) {
+  const form = useFormContext();
+  const parseMethodName = buildFieldNameWithPrefix('parse_method', prefix);
+
+  useEffect(() => {
+    if (isEmpty(form.getValues(parseMethodName))) {
+      form.setValue(parseMethodName, ImageParseMethod.OCR, {
+        shouldValidate: true,
+        shouldDirty: true,
+      });
+    }
+  }, [form, parseMethodName]);
+
   return (
     <>
-      <ParserMethodFormField prefix={prefix}></ParserMethodFormField>
+      <ParserMethodFormField
-      {/* Multimodal Model */}
-      <LargeModelFormField prefix={prefix}></LargeModelFormField>
-      <OutputFormatFormField
         prefix={prefix}
-        fileType={FileType.Image}
+        optionsWithoutLLM={options}
-      ></OutputFormatFormField>
+      ></ParserMethodFormField>
     </>
   );
 }

web/src/pages/data-flow/form/parser-form/index.tsx

@@ -24,6 +24,7 @@ import { INextOperatorForm } from '../../interface';
 import { buildOutputList } from '../../utils/build-output-list';
 import { Output } from '../components/output';
 import { OutputFormatFormField } from './common-form-fields';
+import { InitialOutputFormatMap } from './constant';
 import { EmailFormFields } from './email-form-fields';
 import { ImageFormFields } from './image-form-fields';
 import { PdfFormFields } from './pdf-form-fields';
@@ -50,14 +51,14 @@ type ParserItemProps = {
 };
 
 export const FormSchema = z.object({
-  parser: z.array(
+  setups: z.array(
     z.object({
       fileFormat: z.string().nullish(),
       output_format: z.string().optional(),
       parse_method: z.string().optional(),
-      llm_id: z.string().optional(),
       lang: z.string().optional(),
       fields: z.array(z.string()).optional(),
+      llm_id: z.string().optional(),
     }),
   ),
 });
@@ -71,10 +72,10 @@ function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) {
   const isHovering = useHover(ref);
 
   const prefix = `${name}.${index}`;
-  const fileFormat = form.getValues(`parser.${index}.fileFormat`);
+  const fileFormat = form.getValues(`setups.${index}.fileFormat`);
 
   const values = form.getValues();
-  const parserList = values.parser.slice(); // Adding, deleting, or modifying the parser array will not change the reference.
+  const parserList = values.setups.slice(); // Adding, deleting, or modifying the parser array will not change the reference.
 
   const filteredFileFormatOptions = useMemo(() => {
     const otherFileFormatList = parserList
@@ -89,7 +90,19 @@ function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) {
   const Widget =
     typeof fileFormat === 'string' && fileFormat in FileFormatWidgetMap
       ? FileFormatWidgetMap[fileFormat as keyof typeof FileFormatWidgetMap]
-      : OutputFormatFormField;
+      : () => <></>;
+
+  const handleFileTypeChange = useCallback(
+    (value: FileType) => {
+      form.setValue(
+        `setups.${index}.output_format`,
+        InitialOutputFormatMap[value],
+        { shouldDirty: true, shouldValidate: true, shouldTouch: true },
+      );
+    },
+    [form, index],
+  );
+
   return (
     <section
       className={cn('space-y-5 px-5 py-2.5 rounded-md', {
@@ -110,11 +123,22 @@ function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) {
         name={buildFieldNameWithPrefix(`fileFormat`, prefix)}
         label={t('dataflow.fileFormats')}
       >
-        <SelectWithSearch
+        {(field) => (
-          options={filteredFileFormatOptions}
+          <SelectWithSearch
-        ></SelectWithSearch>
+            value={field.value}
+            onChange={(val) => {
+              field.onChange(val);
+              handleFileTypeChange(val as FileType);
+            }}
+            options={filteredFileFormatOptions}
+          ></SelectWithSearch>
+        )}
       </RAGFlowFormItem>
       <Widget prefix={prefix} fileType={fileFormat as FileType}></Widget>
+      <OutputFormatFormField
+        prefix={prefix}
+        fileType={fileFormat as FileType}
+      />
       {index < fieldLength - 1 && <Separator />}
     </section>
   );
@@ -130,7 +154,7 @@ const ParserForm = ({ node }: INextOperatorForm) => {
     shouldUnregister: true,
   });
 
-  const name = 'parser';
+  const name = 'setups';
   const { fields, remove, append } = useFieldArray({
     name,
     control: form.control,
@@ -141,9 +165,9 @@ const ParserForm = ({ node }: INextOperatorForm) => {
       fileFormat: null,
       output_format: '',
       parse_method: '',
-      llm_id: '',
       lang: '',
       fields: [],
+      llm_id: '',
     });
   }, [append]);
 

web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx

@@ -1,29 +1,73 @@
-import { CrossLanguageFormField } from '@/components/cross-language-form-field';
+import { crossLanguageOptions } from '@/components/cross-language-form-field';
+import { ParseDocumentType } from '@/components/layout-recognize-form-field';
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { isEmpty } from 'lodash';
+import { useEffect, useMemo } from 'react';
+import { useFormContext, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
-import { FileType } from '../../constant';
+import { ParserMethodFormField } from './common-form-fields';
-import {
-  LargeModelFormField,
-  OutputFormatFormField,
-  ParserMethodFormField,
-} from './common-form-fields';
 import { CommonProps } from './interface';
 import { buildFieldNameWithPrefix } from './utils';
 
 export function PdfFormFields({ prefix }: CommonProps) {
   const { t } = useTranslation();
+  const form = useFormContext();
+
+  const parseMethodName = buildFieldNameWithPrefix('parse_method', prefix);
+
+  const parseMethod = useWatch({
+    name: parseMethodName,
+  });
+  const lang = form.getValues(buildFieldNameWithPrefix('lang', prefix));
+
+  const languageShown = useMemo(() => {
+    return (
+      !isEmpty(parseMethod) &&
+      parseMethod !== ParseDocumentType.DeepDOC &&
+      parseMethod !== ParseDocumentType.PlainText
+    );
+  }, [parseMethod]);
+
+  useEffect(() => {
+    if (languageShown && isEmpty(lang)) {
+      form.setValue(
+        buildFieldNameWithPrefix('lang', prefix),
+        crossLanguageOptions[0].value,
+        {
+          shouldValidate: true,
+          shouldDirty: true,
+        },
+      );
+    }
+  }, [form, lang, languageShown, prefix]);
+
+  useEffect(() => {
+    if (isEmpty(form.getValues(parseMethodName))) {
+      form.setValue(parseMethodName, ParseDocumentType.DeepDOC, {
+        shouldValidate: true,
+        shouldDirty: true,
+      });
+    }
+  }, [form, parseMethodName]);
+
   return (
     <>
       <ParserMethodFormField prefix={prefix}></ParserMethodFormField>
-      {/* Multimodal Model */}
+      {languageShown && (
-      <LargeModelFormField prefix={prefix}></LargeModelFormField>
+        <RAGFlowFormItem
-      <CrossLanguageFormField
+          name={buildFieldNameWithPrefix(`lang`, prefix)}
-        name={buildFieldNameWithPrefix(`lang`, prefix)}
+          label={t('dataflow.lang')}
-        label={t('dataflow.lang')}
+        >
-      ></CrossLanguageFormField>
+          {(field) => (
-      <OutputFormatFormField
+            <SelectWithSearch
-        prefix={prefix}
+              options={crossLanguageOptions}
-        fileType={FileType.Image}
+              value={field.value}
-      ></OutputFormatFormField>
+              onChange={field.onChange}
+            ></SelectWithSearch>
+          )}
+        </RAGFlowFormItem>
+      )}
     </>
   );
 }

web/src/pages/data-flow/form/parser-form/video-form-fields.tsx

@@ -1,21 +1,13 @@
 import {
   LargeModelFormField,
-  OutputFormatFormField,
   OutputFormatFormFieldProps,
 } from './common-form-fields';
 
-export function VideoFormFields({
+export function VideoFormFields({ prefix }: OutputFormatFormFieldProps) {
-  prefix,
-  fileType,
-}: OutputFormatFormFieldProps) {
   return (
     <>
       {/* Multimodal Model */}
       <LargeModelFormField prefix={prefix}></LargeModelFormField>
-      <OutputFormatFormField
-        prefix={prefix}
-        fileType={fileType}
-      ></OutputFormatFormField>
     </>
   );
 }

web/src/pages/data-flow/form/splitter-form/index.tsx

@@ -1,15 +1,15 @@
+import { DelimiterInput } from '@/components/delimiter-form-field';
 import { RAGFlowFormItem } from '@/components/ragflow-form';
 import { SliderInputFormField } from '@/components/slider-input-form-field';
 import { BlockButton, Button } from '@/components/ui/button';
 import { Form } from '@/components/ui/form';
-import { Input } from '@/components/ui/input';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { Trash2 } from 'lucide-react';
 import { memo } from 'react';
 import { useFieldArray, useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
-import { initialChunkerValues, initialSplitterValues } from '../../constant';
+import { initialSplitterValues } from '../../constant';
 import { useFormValues } from '../../hooks/use-form-values';
 import { useWatchFormChange } from '../../hooks/use-watch-form-change';
 import { INextOperatorForm } from '../../interface';
@@ -32,7 +32,7 @@ export const FormSchema = z.object({
 export type SplitterFormSchemaType = z.infer<typeof FormSchema>;
 
 const SplitterForm = ({ node }: INextOperatorForm) => {
-  const defaultValues = useFormValues(initialChunkerValues, node);
+  const defaultValues = useFormValues(initialSplitterValues, node);
   const { t } = useTranslation();
 
   const form = useForm<SplitterFormSchemaType>({
@@ -59,32 +59,34 @@ const SplitterForm = ({ node }: INextOperatorForm) => {
         <SliderInputFormField
           name="overlapped_percent"
           max={0.3}
-          min={0.1}
+          min={0}
           step={0.01}
           label={t('dataflow.overlappedPercent')}
         ></SliderInputFormField>
-        <span>{t('flow.delimiters')}</span>
+        <section>
-        {fields.map((field, index) => (
+          <span className="mb-2 inline-block">{t('flow.delimiters')}</span>
-          <div key={field.id} className="flex items-center gap-2">
+          {fields.map((field, index) => (
-            <div className="space-y-2 flex-1">
+            <div key={field.id} className="flex items-center gap-2">
-              <RAGFlowFormItem
+              <div className="space-y-2 flex-1">
-                name={`${name}.${index}.value`}
+                <RAGFlowFormItem
-                label="delimiter"
+                  name={`${name}.${index}.value`}
-                labelClassName="!hidden"
+                  label="delimiter"
+                  labelClassName="!hidden"
+                >
+                  <DelimiterInput className="!m-0"></DelimiterInput>
+                </RAGFlowFormItem>
+              </div>
+              <Button
+                type="button"
+                variant={'ghost'}
+                onClick={() => remove(index)}
               >
-                <Input className="!m-0"></Input>
+                <Trash2 />
-              </RAGFlowFormItem>
+              </Button>
             </div>
-            <Button
+          ))}
-              type="button"
+        </section>
-              variant={'ghost'}
+        <BlockButton onClick={() => append({ value: '\n' })}>
-              onClick={() => remove(index)}
-            >
-              <Trash2 />
-            </Button>
-          </div>
-        ))}
-        <BlockButton onClick={() => append({ value: '' })}>
           {t('common.add')}
         </BlockButton>
       </FormWrapper>

web/src/pages/data-flow/form/tokenizer-form/index.tsx

@@ -25,6 +25,8 @@ export const FormSchema = z.object({
 
 const SearchMethodOptions = buildOptions(TokenizerSearchMethod);
 
+const FieldsOptions = [{ label: 'text', value: 'text' }];
+
 const TokenizerForm = ({ node }: INextOperatorForm) => {
   const { t } = useTranslation();
   const defaultValues = useFormValues(initialTokenizerValues, node);
@@ -59,6 +61,16 @@ const TokenizerForm = ({ node }: INextOperatorForm) => {
           max={0.5}
           step={0.01}
         ></SliderInputFormField>
+        <RAGFlowFormItem name="fields" label={t('dataflow.fields')}>
+          {(field) => (
+            <MultiSelect
+              options={FieldsOptions}
+              onValueChange={field.onChange}
+              defaultValue={field.value}
+              variant="inverted"
+            />
+          )}
+        </RAGFlowFormItem>
       </FormWrapper>
       <div className="p-5">
         <Output list={outputList}></Output>

web/src/pages/data-flow/hooks/use-run-dataflow.ts

@@ -19,7 +19,7 @@ export function useRunDataflow(showLogSheet: () => void) {
         id,
         query: '',
         session_id: null,
-        inputs: fileResponseData,
+        files: [fileResponseData.file],
       });
       console.log('🚀 ~ useRunDataflow ~ res:', res);
 

web/src/pages/data-flow/hooks/use-watch-form-change.ts

@@ -4,15 +4,16 @@ import useGraphStore from '../store';
 
 export function useWatchFormChange(id?: string, form?: UseFormReturn<any>) {
   let values = useWatch({ control: form?.control });
+  console.log(
+    '🚀 ~ useWatchFormChange ~ values:',
+    JSON.stringify(values, null, 2),
+  );
+
   const updateNodeForm = useGraphStore((state) => state.updateNodeForm);
 
   useEffect(() => {
-    // Manually triggered form updates are synchronized to the canvas
     if (id) {
-      values = form?.getValues() || {};
+      updateNodeForm(id, values);
-      let nextValues: any = values;
-
-      updateNodeForm(id, nextValues);
     }
-  }, [form?.formState.isDirty, id, updateNodeForm, values]);
+  }, [id, updateNodeForm, values]);
 }

web/src/pages/data-flow/log-sheet/dataflow-timeline.tsx

@@ -0,0 +1,3 @@
+export function DataflowTimeline() {
+  return <div>xx</div>;
+}

web/src/pages/data-flow/log-sheet/index.tsx

@@ -0,0 +1,28 @@
+import {
+  Sheet,
+  SheetContent,
+  SheetHeader,
+  SheetTitle,
+} from '@/components/ui/sheet';
+import { IModalProps } from '@/interfaces/common';
+import { cn } from '@/lib/utils';
+import { NotebookText } from 'lucide-react';
+import 'react18-json-view/src/style.css';
+
+type LogSheetProps = IModalProps<any>;
+
+export function LogSheet({ hideModal }: LogSheetProps) {
+  return (
+    <Sheet open onOpenChange={hideModal} modal={false}>
+      <SheetContent className={cn('top-20 right-[620px]')}>
+        <SheetHeader>
+          <SheetTitle className="flex items-center gap-1">
+            <NotebookText className="size-4" />
+            Log
+          </SheetTitle>
+        </SheetHeader>
+        <section className="max-h-[82vh] overflow-auto mt-6"></section>
+      </SheetContent>
+    </Sheet>
+  );
+}

web/src/pages/data-flow/run-sheet/index.tsx

@@ -13,7 +13,7 @@ import { UploaderForm } from './uploader';
 const RunSheet = ({ hideModal, showModal: showLogSheet }: IModalProps<any>) => {
   const { t } = useTranslation();
 
-  const { run, loading } = useRunDataflow(() => {});
+  const { run, loading } = useRunDataflow(showLogSheet!);
 
   return (
     <Sheet onOpenChange={hideModal} open modal={false}>

web/src/pages/data-flow/utils.ts

@@ -1,6 +1,5 @@
 import {
   IAgentForm,
-  ICategorizeForm,
   ICategorizeItem,
   ICategorizeItemResult,
 } from '@/interfaces/database/agent';
@@ -22,6 +21,7 @@ import pipe from 'lodash/fp/pipe';
 import isObject from 'lodash/isObject';
 import {
   CategorizeAnchorPointPositions,
+  FileType,
   NoDebugOperatorsList,
   NodeHandleId,
   Operator,
@@ -31,15 +31,6 @@ import { ParserFormSchemaType } from './form/parser-form';
 import { SplitterFormSchemaType } from './form/splitter-form';
 import { BeginQuery, IPosition } from './interface';
 
-function buildAgentExceptionGoto(edges: Edge[], nodeId: string) {
-  const exceptionEdges = edges.filter(
-    (x) =>
-      x.source === nodeId && x.sourceHandle === NodeHandleId.AgentException,
-  );
-
-  return exceptionEdges.map((x) => x.target);
-}
-
 const buildComponentDownstreamOrUpstream = (
   edges: Edge[],
   nodeId: string,
@@ -80,70 +71,6 @@ const removeUselessDataInTheOperator = curry(
     return params;
   },
 );
-// initialize data for operators without parameters
-// const initializeOperatorParams = curry((operatorName: string, values: any) => {
-//   if (isEmpty(values)) {
-//     return initialFormValuesMap[operatorName as Operator];
-//   }
-//   return values;
-// });
-
-function buildAgentTools(edges: Edge[], nodes: Node[], nodeId: string) {
-  const node = nodes.find((x) => x.id === nodeId);
-  const params = { ...(node?.data.form ?? {}) };
-  if (node && node.data.label === Operator.Agent) {
-    const bottomSubAgentEdges = edges.filter(
-      (x) => x.source === nodeId && x.sourceHandle === NodeHandleId.AgentBottom,
-    );
-
-    (params as IAgentForm).tools = (params as IAgentForm).tools.concat(
-      bottomSubAgentEdges.map((x) => {
-        const {
-          params: formData,
-          id,
-          name,
-        } = buildAgentTools(edges, nodes, x.target);
-
-        return {
-          component_name: Operator.Agent,
-          id,
-          name: name as string, // Cast name to string and provide fallback
-          params: { ...formData },
-        };
-      }),
-    );
-  }
-  return { params, name: node?.data.name, id: node?.id };
-}
-
-function filterTargetsBySourceHandleId(edges: Edge[], handleId: string) {
-  return edges.filter((x) => x.sourceHandle === handleId).map((x) => x.target);
-}
-
-function buildCategorize(edges: Edge[], nodes: Node[], nodeId: string) {
-  const node = nodes.find((x) => x.id === nodeId);
-  const params = { ...(node?.data.form ?? {}) } as ICategorizeForm;
-  if (node && node.data.label === Operator.Categorize) {
-    const subEdges = edges.filter((x) => x.source === nodeId);
-
-    const items = params.items || [];
-
-    const nextCategoryDescription = items.reduce<
-      ICategorizeForm['category_description']
-    >((pre, val) => {
-      const key = val.name;
-      pre[key] = {
-        ...omit(val, 'name', 'uuid'),
-        examples: val.examples?.map((x) => x.value) || [],
-        to: filterTargetsBySourceHandleId(subEdges, val.uuid),
-      };
-      return pre;
-    }, {});
-
-    params.category_description = nextCategoryDescription;
-  }
-  return omit(params, 'items');
-}
 
 const buildOperatorParams = (operatorName: string) =>
   pipe(
@@ -151,7 +78,7 @@ const buildOperatorParams = (operatorName: string) =>
     // initializeOperatorParams(operatorName), // Final processing, for guarantee
   );
 
-const ExcludeOperators = [Operator.Note, Operator.Tool];
+const ExcludeOperators = [Operator.Note];
 
 export function isBottomSubAgent(edges: Edge[], nodeId?: string) {
   const edge = edges.find(
@@ -171,14 +98,51 @@ function transformObjectArrayToPureArray(
 }
 
 function transformParserParams(params: ParserFormSchemaType) {
-  return params.parser.reduce<
+  const setups = params.setups.reduce<
-    Record<string, ParserFormSchemaType['parser'][0]>
+    Record<string, ParserFormSchemaType['setups'][0]>
   >((pre, cur) => {
     if (cur.fileFormat) {
-      pre[cur.fileFormat] = omit(cur, 'fileFormat');
+      let filteredSetup: Partial<ParserFormSchemaType['setups'][0]> = {
+        output_format: cur.output_format,
+      };
+
+      switch (cur.fileFormat) {
+        case FileType.PDF:
+          filteredSetup = {
+            ...filteredSetup,
+            parse_method: cur.parse_method,
+            lang: cur.lang,
+          };
+          break;
+        case FileType.Image:
+          filteredSetup = {
+            ...filteredSetup,
+            parse_method: cur.parse_method,
+          };
+          break;
+        case FileType.Email:
+          filteredSetup = {
+            ...filteredSetup,
+            fields: cur.fields,
+          };
+          break;
+        case FileType.Video:
+        case FileType.Audio:
+          filteredSetup = {
+            ...filteredSetup,
+            llm_id: cur.llm_id,
+          };
+          break;
+        default:
+          break;
+      }
+
+      pre[cur.fileFormat] = filteredSetup;
     }
     return pre;
   }, {});
+
+  return { ...params, setups };
 }
 
 function transformSplitterParams(params: SplitterFormSchemaType) {
@@ -218,18 +182,6 @@ export const buildDslComponentsByGraph = (
       let params = x?.data.form ?? {};
 
       switch (operatorName) {
-        case Operator.Agent: {
-          const { params: formData } = buildAgentTools(edges, nodes, id);
-          params = {
-            ...formData,
-            exception_goto: buildAgentExceptionGoto(edges, id),
-          };
-          break;
-        }
-        case Operator.Categorize:
-          params = buildCategorize(edges, nodes, id);
-          break;
-
         case Operator.Parser:
           params = transformParserParams(params);
           break;