Feat: Add ParserForm to the data pipeline #9869 (#9986)

### What problem does this PR solve?

Feat: Add ParserForm to the data pipeline  #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx

@@ -17,9 +17,6 @@ import {
   TooltipTrigger,
 } from '@/components/ui/tooltip';
 import { IModalProps } from '@/interfaces/common';
-import { Operator } from '@/pages/agent/constant';
-import { AgentInstanceContext, HandleContext } from '@/pages/agent/context';
-import OperatorIcon from '@/pages/agent/operator-icon';
 import { Position } from '@xyflow/react';
 import { t } from 'i18next';
 import { lowerFirst } from 'lodash';
@@ -32,6 +29,9 @@ import {
   useRef,
 } from 'react';
 import { useTranslation } from 'react-i18next';
+import { Operator } from '../../../constant';
+import { AgentInstanceContext, HandleContext } from '../../../context';
+import OperatorIcon from '../../../operator-icon';
 
 type OperatorItemProps = {
   operators: Operator[];
@@ -134,7 +134,13 @@ function AccordionOperators({
         </AccordionTrigger>
         <AccordionContent className="flex flex-col gap-4 text-balance">
           <OperatorItemList
-            operators={[Operator.Agent, Operator.Retrieval]}
+            operators={[
+              Operator.Agent,
+              Operator.Retrieval,
+              Operator.Parser,
+              Operator.Chunker,
+              Operator.Tokenizer,
+            ]}
             isCustomDropdown={isCustomDropdown}
             mousePosition={mousePosition}
           ></OperatorItemList>
@@ -186,23 +192,7 @@ function AccordionOperators({
         </AccordionTrigger>
         <AccordionContent className="flex flex-col gap-4 text-balance">
           <OperatorItemList
-            operators={[
+            operators={[Operator.ExeSQL, Operator.Email, Operator.Invoke]}
-              Operator.TavilySearch,
-              Operator.TavilyExtract,
-              Operator.ExeSQL,
-              Operator.Google,
-              Operator.YahooFinance,
-              Operator.Email,
-              Operator.DuckDuckGo,
-              Operator.Wikipedia,
-              Operator.GoogleScholar,
-              Operator.ArXiv,
-              Operator.PubMed,
-              Operator.GitHub,
-              Operator.Invoke,
-              Operator.WenCai,
-              Operator.SearXNG,
-            ]}
             isCustomDropdown={isCustomDropdown}
             mousePosition={mousePosition}
           ></OperatorItemList>

web/src/pages/data-flow/constant.tsx

@@ -317,7 +317,7 @@ export const initialCodeValues = {
 
 export const initialWaitingDialogueValues = {};
 
-export const initialChunkerValues = {};
+export const initialChunkerValues = { outputs: {} };
 
 export const initialTokenizerValues = {};
 
@@ -388,7 +388,7 @@ export const initialStringTransformValues = {
   },
 };
 
-export const initialParserValues = {};
+export const initialParserValues = { outputs: {} };
 
 export const CategorizeAnchorPointPositions = [
   { top: 1, right: 34 },

web/src/pages/data-flow/form-sheet/form-config-map.tsx

@@ -2,6 +2,7 @@ import { Operator } from '../constant';
 import AgentForm from '../form/agent-form';
 import BeginForm from '../form/begin-form';
 import CategorizeForm from '../form/categorize-form';
+import ChunkerForm from '../form/chunker-form';
 import CodeForm from '../form/code-form';
 import CrawlerForm from '../form/crawler-form';
 import EmailForm from '../form/email-form';
@@ -11,11 +12,13 @@ import IterationForm from '../form/iteration-form';
 import IterationStartForm from '../form/iteration-start-from';
 import KeywordExtractForm from '../form/keyword-extract-form';
 import MessageForm from '../form/message-form';
+import ParserForm from '../form/parser-form';
 import RelevantForm from '../form/relevant-form';
 import RetrievalForm from '../form/retrieval-form/next';
 import RewriteQuestionForm from '../form/rewrite-question-form';
 import StringTransformForm from '../form/string-transform-form';
 import SwitchForm from '../form/switch-form';
+import TokenizerForm from '../form/tokenizer-form';
 import UserFillUpForm from '../form/user-fill-up-form';
 
 export const FormConfigMap = {
@@ -82,4 +85,13 @@ export const FormConfigMap = {
   [Operator.StringTransform]: {
     component: StringTransformForm,
   },
+  [Operator.Parser]: {
+    component: ParserForm,
+  },
+  [Operator.Chunker]: {
+    component: ChunkerForm,
+  },
+  [Operator.Tokenizer]: {
+    component: TokenizerForm,
+  },
 };

web/src/pages/data-flow/form/chunker-form/index.tsx

@@ -0,0 +1,140 @@
+import { FormContainer } from '@/components/form-container';
+import NumberInput from '@/components/originui/number-input';
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { useTranslate } from '@/hooks/common-hooks';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { memo } from 'react';
+import { useForm, useFormContext } from 'react-hook-form';
+import { z } from 'zod';
+import { initialChunkerValues } from '../../constant';
+import { useFormValues } from '../../hooks/use-form-values';
+import { useWatchFormChange } from '../../hooks/use-watch-form-change';
+import { INextOperatorForm } from '../../interface';
+import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options';
+import { buildOutputList } from '../../utils/build-output-list';
+import { ApiKeyField } from '../components/api-key-field';
+import { FormWrapper } from '../components/form-wrapper';
+import { Output } from '../components/output';
+import { QueryVariable } from '../components/query-variable';
+
+const outputList = buildOutputList(initialChunkerValues.outputs);
+
+export const GoogleFormPartialSchema = {
+  api_key: z.string(),
+  country: z.string(),
+  language: z.string(),
+};
+
+export const FormSchema = z.object({
+  ...GoogleFormPartialSchema,
+  q: z.string(),
+  start: z.number(),
+  num: z.number(),
+});
+
+export function GoogleFormWidgets() {
+  const form = useFormContext();
+  const { t } = useTranslate('flow');
+
+  return (
+    <>
+      <FormField
+        control={form.control}
+        name={`country`}
+        render={({ field }) => (
+          <FormItem className="flex-1">
+            <FormLabel>{t('country')}</FormLabel>
+            <FormControl>
+              <SelectWithSearch
+                {...field}
+                options={GoogleCountryOptions}
+              ></SelectWithSearch>
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name={`language`}
+        render={({ field }) => (
+          <FormItem className="flex-1">
+            <FormLabel>{t('language')}</FormLabel>
+            <FormControl>
+              <SelectWithSearch
+                {...field}
+                options={GoogleLanguageOptions}
+              ></SelectWithSearch>
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+    </>
+  );
+}
+
+const ChunkerForm = ({ node }: INextOperatorForm) => {
+  const { t } = useTranslate('flow');
+  const defaultValues = useFormValues(initialChunkerValues, node);
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    defaultValues,
+    resolver: zodResolver(FormSchema),
+  });
+
+  useWatchFormChange(node?.id, form);
+
+  return (
+    <Form {...form}>
+      <FormWrapper>
+        <FormContainer>
+          <QueryVariable name="q"></QueryVariable>
+        </FormContainer>
+        <FormContainer>
+          <ApiKeyField placeholder={t('apiKeyPlaceholder')}></ApiKeyField>
+          <FormField
+            control={form.control}
+            name={`start`}
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('flowStart')}</FormLabel>
+                <FormControl>
+                  <NumberInput {...field} className="w-full"></NumberInput>
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name={`num`}
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('flowNum')}</FormLabel>
+                <FormControl>
+                  <NumberInput {...field} className="w-full"></NumberInput>
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <GoogleFormWidgets></GoogleFormWidgets>
+        </FormContainer>
+      </FormWrapper>
+      <div className="p-5">
+        <Output list={outputList}></Output>
+      </div>
+    </Form>
+  );
+};
+
+export default memo(ChunkerForm);

web/src/pages/data-flow/form/parser-form/index.tsx

@@ -14,7 +14,7 @@ import { zodResolver } from '@hookform/resolvers/zod';
 import { memo } from 'react';
 import { useForm, useFormContext } from 'react-hook-form';
 import { z } from 'zod';
-import { initialGoogleValues } from '../../constant';
+import { initialParserValues } from '../../constant';
 import { useFormValues } from '../../hooks/use-form-values';
 import { useWatchFormChange } from '../../hooks/use-watch-form-change';
 import { INextOperatorForm } from '../../interface';
@@ -25,7 +25,7 @@ import { FormWrapper } from '../components/form-wrapper';
 import { Output } from '../components/output';
 import { QueryVariable } from '../components/query-variable';
 
-const outputList = buildOutputList(initialGoogleValues.outputs);
+const outputList = buildOutputList(initialParserValues.outputs);
 
 export const GoogleFormPartialSchema = {
   api_key: z.string(),
@@ -84,7 +84,7 @@ export function GoogleFormWidgets() {
 
 const ParserForm = ({ node }: INextOperatorForm) => {
   const { t } = useTranslate('flow');
-  const defaultValues = useFormValues(initialGoogleValues, node);
+  const defaultValues = useFormValues(initialParserValues, node);
 
   const form = useForm<z.infer<typeof FormSchema>>({
     defaultValues,

web/src/pages/data-flow/form/tokenizer-form/index.tsx

@@ -0,0 +1,140 @@
+import { FormContainer } from '@/components/form-container';
+import NumberInput from '@/components/originui/number-input';
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { useTranslate } from '@/hooks/common-hooks';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { memo } from 'react';
+import { useForm, useFormContext } from 'react-hook-form';
+import { z } from 'zod';
+import { initialChunkerValues } from '../../constant';
+import { useFormValues } from '../../hooks/use-form-values';
+import { useWatchFormChange } from '../../hooks/use-watch-form-change';
+import { INextOperatorForm } from '../../interface';
+import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options';
+import { buildOutputList } from '../../utils/build-output-list';
+import { ApiKeyField } from '../components/api-key-field';
+import { FormWrapper } from '../components/form-wrapper';
+import { Output } from '../components/output';
+import { QueryVariable } from '../components/query-variable';
+
+const outputList = buildOutputList(initialChunkerValues.outputs);
+
+export const GoogleFormPartialSchema = {
+  api_key: z.string(),
+  country: z.string(),
+  language: z.string(),
+};
+
+export const FormSchema = z.object({
+  ...GoogleFormPartialSchema,
+  q: z.string(),
+  start: z.number(),
+  num: z.number(),
+});
+
+export function GoogleFormWidgets() {
+  const form = useFormContext();
+  const { t } = useTranslate('flow');
+
+  return (
+    <>
+      <FormField
+        control={form.control}
+        name={`country`}
+        render={({ field }) => (
+          <FormItem className="flex-1">
+            <FormLabel>{t('country')}</FormLabel>
+            <FormControl>
+              <SelectWithSearch
+                {...field}
+                options={GoogleCountryOptions}
+              ></SelectWithSearch>
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+      <FormField
+        control={form.control}
+        name={`language`}
+        render={({ field }) => (
+          <FormItem className="flex-1">
+            <FormLabel>{t('language')}</FormLabel>
+            <FormControl>
+              <SelectWithSearch
+                {...field}
+                options={GoogleLanguageOptions}
+              ></SelectWithSearch>
+            </FormControl>
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+    </>
+  );
+}
+
+const TokenizerForm = ({ node }: INextOperatorForm) => {
+  const { t } = useTranslate('flow');
+  const defaultValues = useFormValues(initialChunkerValues, node);
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    defaultValues,
+    resolver: zodResolver(FormSchema),
+  });
+
+  useWatchFormChange(node?.id, form);
+
+  return (
+    <Form {...form}>
+      <FormWrapper>
+        <FormContainer>
+          <QueryVariable name="q"></QueryVariable>
+        </FormContainer>
+        <FormContainer>
+          <ApiKeyField placeholder={t('apiKeyPlaceholder')}></ApiKeyField>
+          <FormField
+            control={form.control}
+            name={`start`}
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('flowStart')}</FormLabel>
+                <FormControl>
+                  <NumberInput {...field} className="w-full"></NumberInput>
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name={`num`}
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('flowNum')}</FormLabel>
+                <FormControl>
+                  <NumberInput {...field} className="w-full"></NumberInput>
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <GoogleFormWidgets></GoogleFormWidgets>
+        </FormContainer>
+      </FormWrapper>
+      <div className="p-5">
+        <Output list={outputList}></Output>
+      </div>
+    </Form>
+  );
+};
+
+export default memo(TokenizerForm);

web/src/pages/data-flow/operator-icon.tsx

@@ -1,6 +1,11 @@
 import { IconFont } from '@/components/icon-font';
 import { cn } from '@/lib/utils';
-import { HousePlus } from 'lucide-react';
+import {
+  FileChartColumnIncreasing,
+  Grid3x3,
+  HousePlus,
+  ListMinus,
+} from 'lucide-react';
 import { Operator } from './constant';
 
 interface IProps {
@@ -25,13 +30,19 @@ export const OperatorIconMap = {
   [Operator.Email]: 'sendemail-0',
 };
 
+export const SVGIconMap = {
+  [Operator.Parser]: FileChartColumnIncreasing,
+  [Operator.Chunker]: Grid3x3,
+  [Operator.Tokenizer]: ListMinus,
+};
+
 const Empty = () => {
   return <div className="hidden"></div>;
 };
 
 const OperatorIcon = ({ name, className }: IProps) => {
   const Icon = OperatorIconMap[name as keyof typeof OperatorIconMap] || Empty;
-  const SvgIcon = Empty;
+  const SvgIcon = SVGIconMap[name as keyof typeof SVGIconMap] || Empty;
 
   if (name === Operator.Begin) {
     return (
@@ -49,7 +60,7 @@ const OperatorIcon = ({ name, className }: IProps) => {
   return typeof Icon === 'string' ? (
     <IconFont name={Icon} className={cn('size-5 ', className)}></IconFont>
   ) : (
-    <SvgIcon></SvgIcon>
+    <SvgIcon className="size-5"></SvgIcon>
   );
 };