From a255c78b5987c290332d91039e80472677ca3785 Mon Sep 17 00:00:00 2001 From: balibabu Date: Tue, 9 Sep 2025 09:50:46 +0800 Subject: [PATCH] Feat: Add ParserForm to the data pipeline #9869 (#9986) ### What problem does this PR solve? Feat: Add ParserForm to the data pipeline #9869 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- .../node/dropdown/next-step-dropdown.tsx | 32 ++-- web/src/pages/data-flow/constant.tsx | 4 +- .../data-flow/form-sheet/form-config-map.tsx | 12 ++ .../data-flow/form/chunker-form/index.tsx | 140 ++++++++++++++++++ .../data-flow/form/parser-form/index.tsx | 6 +- .../data-flow/form/tokenizer-form/index.tsx | 140 ++++++++++++++++++ web/src/pages/data-flow/operator-icon.tsx | 17 ++- 7 files changed, 322 insertions(+), 29 deletions(-) create mode 100644 web/src/pages/data-flow/form/chunker-form/index.tsx create mode 100644 web/src/pages/data-flow/form/tokenizer-form/index.tsx diff --git a/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx b/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx index 6d9f32453..88c35dae6 100644 --- a/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx +++ b/web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx @@ -17,9 +17,6 @@ import { TooltipTrigger, } from '@/components/ui/tooltip'; import { IModalProps } from '@/interfaces/common'; -import { Operator } from '@/pages/agent/constant'; -import { AgentInstanceContext, HandleContext } from '@/pages/agent/context'; -import OperatorIcon from '@/pages/agent/operator-icon'; import { Position } from '@xyflow/react'; import { t } from 'i18next'; import { lowerFirst } from 'lodash'; @@ -32,6 +29,9 @@ import { useRef, } from 'react'; import { useTranslation } from 'react-i18next'; +import { Operator } from '../../../constant'; +import { AgentInstanceContext, HandleContext } from '../../../context'; +import OperatorIcon from '../../../operator-icon'; type OperatorItemProps = { operators: Operator[]; @@ -134,7 +134,13 @@ function AccordionOperators({ @@ -186,23 +192,7 @@ function AccordionOperators({ diff --git a/web/src/pages/data-flow/constant.tsx b/web/src/pages/data-flow/constant.tsx index ea158722f..d1d124f48 100644 --- a/web/src/pages/data-flow/constant.tsx +++ b/web/src/pages/data-flow/constant.tsx @@ -317,7 +317,7 @@ export const initialCodeValues = { export const initialWaitingDialogueValues = {}; -export const initialChunkerValues = {}; +export const initialChunkerValues = { outputs: {} }; export const initialTokenizerValues = {}; @@ -388,7 +388,7 @@ export const initialStringTransformValues = { }, }; -export const initialParserValues = {}; +export const initialParserValues = { outputs: {} }; export const CategorizeAnchorPointPositions = [ { top: 1, right: 34 }, diff --git a/web/src/pages/data-flow/form-sheet/form-config-map.tsx b/web/src/pages/data-flow/form-sheet/form-config-map.tsx index 44dc2ca2e..f00d93009 100644 --- a/web/src/pages/data-flow/form-sheet/form-config-map.tsx +++ b/web/src/pages/data-flow/form-sheet/form-config-map.tsx @@ -2,6 +2,7 @@ import { Operator } from '../constant'; import AgentForm from '../form/agent-form'; import BeginForm from '../form/begin-form'; import CategorizeForm from '../form/categorize-form'; +import ChunkerForm from '../form/chunker-form'; import CodeForm from '../form/code-form'; import CrawlerForm from '../form/crawler-form'; import EmailForm from '../form/email-form'; @@ -11,11 +12,13 @@ import IterationForm from '../form/iteration-form'; import IterationStartForm from '../form/iteration-start-from'; import KeywordExtractForm from '../form/keyword-extract-form'; import MessageForm from '../form/message-form'; +import ParserForm from '../form/parser-form'; import RelevantForm from '../form/relevant-form'; import RetrievalForm from '../form/retrieval-form/next'; import RewriteQuestionForm from '../form/rewrite-question-form'; import StringTransformForm from '../form/string-transform-form'; import SwitchForm from '../form/switch-form'; +import TokenizerForm from '../form/tokenizer-form'; import UserFillUpForm from '../form/user-fill-up-form'; export const FormConfigMap = { @@ -82,4 +85,13 @@ export const FormConfigMap = { [Operator.StringTransform]: { component: StringTransformForm, }, + [Operator.Parser]: { + component: ParserForm, + }, + [Operator.Chunker]: { + component: ChunkerForm, + }, + [Operator.Tokenizer]: { + component: TokenizerForm, + }, }; diff --git a/web/src/pages/data-flow/form/chunker-form/index.tsx b/web/src/pages/data-flow/form/chunker-form/index.tsx new file mode 100644 index 000000000..cfc32e82f --- /dev/null +++ b/web/src/pages/data-flow/form/chunker-form/index.tsx @@ -0,0 +1,140 @@ +import { FormContainer } from '@/components/form-container'; +import NumberInput from '@/components/originui/number-input'; +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { useTranslate } from '@/hooks/common-hooks'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { memo } from 'react'; +import { useForm, useFormContext } from 'react-hook-form'; +import { z } from 'zod'; +import { initialChunkerValues } from '../../constant'; +import { useFormValues } from '../../hooks/use-form-values'; +import { useWatchFormChange } from '../../hooks/use-watch-form-change'; +import { INextOperatorForm } from '../../interface'; +import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options'; +import { buildOutputList } from '../../utils/build-output-list'; +import { ApiKeyField } from '../components/api-key-field'; +import { FormWrapper } from '../components/form-wrapper'; +import { Output } from '../components/output'; +import { QueryVariable } from '../components/query-variable'; + +const outputList = buildOutputList(initialChunkerValues.outputs); + +export const GoogleFormPartialSchema = { + api_key: z.string(), + country: z.string(), + language: z.string(), +}; + +export const FormSchema = z.object({ + ...GoogleFormPartialSchema, + q: z.string(), + start: z.number(), + num: z.number(), +}); + +export function GoogleFormWidgets() { + const form = useFormContext(); + const { t } = useTranslate('flow'); + + return ( + <> + ( + + {t('country')} + + + + + + )} + /> + ( + + {t('language')} + + + + + + )} + /> + + ); +} + +const ChunkerForm = ({ node }: INextOperatorForm) => { + const { t } = useTranslate('flow'); + const defaultValues = useFormValues(initialChunkerValues, node); + + const form = useForm>({ + defaultValues, + resolver: zodResolver(FormSchema), + }); + + useWatchFormChange(node?.id, form); + + return ( +
+ + + + + + + ( + + {t('flowStart')} + + + + + + )} + /> + ( + + {t('flowNum')} + + + + + + )} + /> + + + +
+ +
+
+ ); +}; + +export default memo(ChunkerForm); diff --git a/web/src/pages/data-flow/form/parser-form/index.tsx b/web/src/pages/data-flow/form/parser-form/index.tsx index 3418d83a9..1a84c304a 100644 --- a/web/src/pages/data-flow/form/parser-form/index.tsx +++ b/web/src/pages/data-flow/form/parser-form/index.tsx @@ -14,7 +14,7 @@ import { zodResolver } from '@hookform/resolvers/zod'; import { memo } from 'react'; import { useForm, useFormContext } from 'react-hook-form'; import { z } from 'zod'; -import { initialGoogleValues } from '../../constant'; +import { initialParserValues } from '../../constant'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; @@ -25,7 +25,7 @@ import { FormWrapper } from '../components/form-wrapper'; import { Output } from '../components/output'; import { QueryVariable } from '../components/query-variable'; -const outputList = buildOutputList(initialGoogleValues.outputs); +const outputList = buildOutputList(initialParserValues.outputs); export const GoogleFormPartialSchema = { api_key: z.string(), @@ -84,7 +84,7 @@ export function GoogleFormWidgets() { const ParserForm = ({ node }: INextOperatorForm) => { const { t } = useTranslate('flow'); - const defaultValues = useFormValues(initialGoogleValues, node); + const defaultValues = useFormValues(initialParserValues, node); const form = useForm>({ defaultValues, diff --git a/web/src/pages/data-flow/form/tokenizer-form/index.tsx b/web/src/pages/data-flow/form/tokenizer-form/index.tsx new file mode 100644 index 000000000..b48581390 --- /dev/null +++ b/web/src/pages/data-flow/form/tokenizer-form/index.tsx @@ -0,0 +1,140 @@ +import { FormContainer } from '@/components/form-container'; +import NumberInput from '@/components/originui/number-input'; +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { useTranslate } from '@/hooks/common-hooks'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { memo } from 'react'; +import { useForm, useFormContext } from 'react-hook-form'; +import { z } from 'zod'; +import { initialChunkerValues } from '../../constant'; +import { useFormValues } from '../../hooks/use-form-values'; +import { useWatchFormChange } from '../../hooks/use-watch-form-change'; +import { INextOperatorForm } from '../../interface'; +import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options'; +import { buildOutputList } from '../../utils/build-output-list'; +import { ApiKeyField } from '../components/api-key-field'; +import { FormWrapper } from '../components/form-wrapper'; +import { Output } from '../components/output'; +import { QueryVariable } from '../components/query-variable'; + +const outputList = buildOutputList(initialChunkerValues.outputs); + +export const GoogleFormPartialSchema = { + api_key: z.string(), + country: z.string(), + language: z.string(), +}; + +export const FormSchema = z.object({ + ...GoogleFormPartialSchema, + q: z.string(), + start: z.number(), + num: z.number(), +}); + +export function GoogleFormWidgets() { + const form = useFormContext(); + const { t } = useTranslate('flow'); + + return ( + <> + ( + + {t('country')} + + + + + + )} + /> + ( + + {t('language')} + + + + + + )} + /> + + ); +} + +const TokenizerForm = ({ node }: INextOperatorForm) => { + const { t } = useTranslate('flow'); + const defaultValues = useFormValues(initialChunkerValues, node); + + const form = useForm>({ + defaultValues, + resolver: zodResolver(FormSchema), + }); + + useWatchFormChange(node?.id, form); + + return ( +
+ + + + + + + ( + + {t('flowStart')} + + + + + + )} + /> + ( + + {t('flowNum')} + + + + + + )} + /> + + + +
+ +
+
+ ); +}; + +export default memo(TokenizerForm); diff --git a/web/src/pages/data-flow/operator-icon.tsx b/web/src/pages/data-flow/operator-icon.tsx index a11145ace..2184569d6 100644 --- a/web/src/pages/data-flow/operator-icon.tsx +++ b/web/src/pages/data-flow/operator-icon.tsx @@ -1,6 +1,11 @@ import { IconFont } from '@/components/icon-font'; import { cn } from '@/lib/utils'; -import { HousePlus } from 'lucide-react'; +import { + FileChartColumnIncreasing, + Grid3x3, + HousePlus, + ListMinus, +} from 'lucide-react'; import { Operator } from './constant'; interface IProps { @@ -25,13 +30,19 @@ export const OperatorIconMap = { [Operator.Email]: 'sendemail-0', }; +export const SVGIconMap = { + [Operator.Parser]: FileChartColumnIncreasing, + [Operator.Chunker]: Grid3x3, + [Operator.Tokenizer]: ListMinus, +}; + const Empty = () => { return
; }; const OperatorIcon = ({ name, className }: IProps) => { const Icon = OperatorIconMap[name as keyof typeof OperatorIconMap] || Empty; - const SvgIcon = Empty; + const SvgIcon = SVGIconMap[name as keyof typeof SVGIconMap] || Empty; if (name === Operator.Begin) { return ( @@ -49,7 +60,7 @@ const OperatorIcon = ({ name, className }: IProps) => { return typeof Icon === 'string' ? ( ) : ( - + ); };