diff --git a/web/src/components/chunk-method-dialog/use-default-parser-values.ts b/web/src/components/chunk-method-dialog/use-default-parser-values.ts index 6b52b50fd..fce389608 100644 --- a/web/src/components/chunk-method-dialog/use-default-parser-values.ts +++ b/web/src/components/chunk-method-dialog/use-default-parser-values.ts @@ -1,7 +1,7 @@ import { IParserConfig } from '@/interfaces/database/document'; import { useCallback, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; -import { DocumentType } from '../layout-recognize-form-field'; +import { ParseDocumentType } from '../layout-recognize-form-field'; export function useDefaultParserValues() { const { t } = useTranslation(); @@ -9,7 +9,7 @@ export function useDefaultParserValues() { const defaultParserValues = useMemo(() => { const defaultParserValues = { task_page_size: 12, - layout_recognize: DocumentType.DeepDOC, + layout_recognize: ParseDocumentType.DeepDOC, chunk_token_num: 512, delimiter: '\n', auto_keywords: 0, diff --git a/web/src/components/cross-language-form-field.tsx b/web/src/components/cross-language-form-field.tsx index 3322324e1..8855112e5 100644 --- a/web/src/components/cross-language-form-field.tsx +++ b/web/src/components/cross-language-form-field.tsx @@ -22,7 +22,7 @@ const Languages = [ 'Vietnamese', ]; -const options = Languages.map((x) => ({ +export const crossLanguageOptions = Languages.map((x) => ({ label: t('language.' + toLower(x)), value: x, })); @@ -59,7 +59,7 @@ export const CrossLanguageFormField = ({ ( - ({ value, onChange, maxLength, defaultValue }, ref) => { + ({ value, onChange, maxLength, defaultValue, ...props }, ref) => { const nextValue = value?.replaceAll('\n', '\\n'); const handleInputChange = (e: React.ChangeEvent) => { const val = e.target.value; @@ -30,6 +30,7 @@ export const DelimiterInput = forwardRef( maxLength={maxLength} defaultValue={defaultValue} ref={ref} + {...props} > ); }, diff --git a/web/src/components/layout-recognize-form-field.tsx b/web/src/components/layout-recognize-form-field.tsx index 2bfdbe014..c1a91796e 100644 --- a/web/src/components/layout-recognize-form-field.tsx +++ b/web/src/components/layout-recognize-form-field.tsx @@ -5,6 +5,7 @@ import { cn } from '@/lib/utils'; import { camelCase } from 'lodash'; import { useMemo } from 'react'; import { useFormContext } from 'react-hook-form'; +import { SelectWithSearch } from './originui/select-with-search'; import { FormControl, FormField, @@ -12,9 +13,8 @@ import { FormLabel, FormMessage, } from './ui/form'; -import { RAGFlowSelect } from './ui/select'; -export const enum DocumentType { +export const enum ParseDocumentType { DeepDOC = 'DeepDOC', PlainText = 'Plain Text', } @@ -22,9 +22,11 @@ export const enum DocumentType { export function LayoutRecognizeFormField({ name = 'parser_config.layout_recognize', horizontal = true, + optionsWithoutLLM, }: { name?: string; horizontal?: boolean; + optionsWithoutLLM?: { value: string; label: string }[]; }) { const form = useFormContext(); @@ -32,10 +34,13 @@ export function LayoutRecognizeFormField({ const allOptions = useSelectLlmOptionsByModelType(); const options = useMemo(() => { - const list = [DocumentType.DeepDOC, DocumentType.PlainText].map((x) => ({ - label: x === DocumentType.PlainText ? t(camelCase(x)) : 'DeepDoc', - value: x, - })); + const list = optionsWithoutLLM + ? optionsWithoutLLM + : [ParseDocumentType.DeepDOC, ParseDocumentType.PlainText].map((x) => ({ + label: + x === ParseDocumentType.PlainText ? t(camelCase(x)) : 'DeepDoc', + value: x, + })); const image2TextList = allOptions[LlmModelType.Image2text].map((x) => { return { @@ -55,7 +60,7 @@ export function LayoutRecognizeFormField({ }); return [...list, ...image2TextList]; - }, [allOptions, t]); + }, [allOptions, optionsWithoutLLM, t]); return (
- +
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index cafa7c753..eca4c1d35 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1668,6 +1668,7 @@ This delimiter is used to split the input text into several text pieces echo of overlappedPercent: 'Overlapped percent', searchMethod: 'Search method', filenameEmbdWeight: 'Filename embd weight', + begin: 'File', }, }, }; diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 9f93c221a..531135e2d 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1578,6 +1578,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于 overlappedPercent: '重叠百分比', searchMethod: '搜索方法', filenameEmbdWeight: '文件名嵌入权重', + begin: '文件', }, }, }; diff --git a/web/src/pages/agents/hooks/use-create-agent.ts b/web/src/pages/agents/hooks/use-create-agent.ts index 01014335a..853756e7c 100644 --- a/web/src/pages/agents/hooks/use-create-agent.ts +++ b/web/src/pages/agents/hooks/use-create-agent.ts @@ -2,10 +2,47 @@ import { useSetModalState } from '@/hooks/common-hooks'; import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request'; import { DSL } from '@/interfaces/database/agent'; import { AgentCategory } from '@/pages/agent/constant'; +import { BeginId, Operator } from '@/pages/data-flow/constant'; import { useCallback } from 'react'; import { FlowType } from '../constant'; import { FormSchemaType } from '../create-agent-form'; +export const DataflowEmptyDsl = { + graph: { + nodes: [ + { + id: BeginId, + type: 'beginNode', + position: { + x: 50, + y: 200, + }, + data: { + label: Operator.Begin, + name: Operator.Begin, + }, + sourcePosition: 'left', + targetPosition: 'right', + }, + ], + edges: [], + }, + components: { + [Operator.Begin]: { + obj: { + component_name: Operator.Begin, + params: {}, + }, + downstream: [], // other edge target is downstream, edge source is current node id + upstream: [], // edge source is upstream, edge target is current node id + }, + }, + retrieval: [], // reference + history: [], + path: [], + globals: {}, +}; + export function useCreateAgentOrPipeline() { const { loading, setAgent } = useSetAgent(); const { @@ -16,13 +53,13 @@ export function useCreateAgentOrPipeline() { const handleCreateAgentOrPipeline = useCallback( async (data: FormSchemaType) => { + const isAgent = data.type === FlowType.Agent; const ret = await setAgent({ title: data.name, - dsl: EmptyDsl as DSL, - canvas_category: - data.type === FlowType.Agent - ? AgentCategory.AgentCanvas - : AgentCategory.DataflowCanvas, + dsl: isAgent ? (EmptyDsl as DSL) : (DataflowEmptyDsl as DSL), + canvas_category: isAgent + ? AgentCategory.AgentCanvas + : AgentCategory.DataflowCanvas, }); if (ret.code === 0) { diff --git a/web/src/pages/data-flow/canvas/index.tsx b/web/src/pages/data-flow/canvas/index.tsx index 87fc9bef0..bd252406c 100644 --- a/web/src/pages/data-flow/canvas/index.tsx +++ b/web/src/pages/data-flow/canvas/index.tsx @@ -34,6 +34,7 @@ import { useHideFormSheetOnNodeDeletion, useShowDrawer, } from '../hooks/use-show-drawer'; +import { LogSheet } from '../log-sheet'; import RunSheet from '../run-sheet'; import { ButtonEdge } from './edge'; import styles from './index.less'; @@ -93,7 +94,6 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) { chatVisible, runVisible, hideRunOrChatDrawer, - showChatModal, showFormDrawer, } = useShowDrawer({ drawerVisible, @@ -146,6 +146,12 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) { clearActiveDropdown, ]); + const { + visible: logSheetVisible, + showModal: showLogSheet, + hideModal: hideLogSheet, + } = useSetModalState(); + const onConnect = (connection: Connection) => { originalOnConnect(connection); isConnectedRef.current = true; @@ -294,9 +300,10 @@ function DataFlowCanvas({ drawerVisible, hideDrawer }: IProps) { {runVisible && ( )} + {logSheetVisible && } ); } diff --git a/web/src/pages/data-flow/canvas/node/begin-node.tsx b/web/src/pages/data-flow/canvas/node/begin-node.tsx index be80d56ae..2d3db3cba 100644 --- a/web/src/pages/data-flow/canvas/node/begin-node.tsx +++ b/web/src/pages/data-flow/canvas/node/begin-node.tsx @@ -36,7 +36,7 @@ function InnerBeginNode({ data, id, selected }: NodeProps) {
- {t(`flow.begin`)} + {t(`dataflow.begin`)}
diff --git a/web/src/pages/data-flow/constant.tsx b/web/src/pages/data-flow/constant.tsx index 9d1bc0f90..8c0d3459d 100644 --- a/web/src/pages/data-flow/constant.tsx +++ b/web/src/pages/data-flow/constant.tsx @@ -38,10 +38,10 @@ export enum AgentDialogueMode { Task = 'task', } -export const BeginId = 'begin'; +export const BeginId = 'File'; export enum Operator { - Begin = 'Begin', + Begin = 'File', Note = 'Note', Parser = 'Parser', Tokenizer = 'Tokenizer', @@ -80,6 +80,15 @@ export const SwitchOperatorOptions = [ export const SwitchElseTo = 'end_cpn_ids'; +export enum TokenizerSearchMethod { + Embedding = 'embedding', + FullText = 'full_text', +} + +export enum ImageParseMethod { + OCR = 'ocr', +} + const initialQueryBaseValues = { query: [], }; @@ -287,8 +296,12 @@ export const initialWaitingDialogueValues = {}; export const initialChunkerValues = { outputs: {} }; export const initialTokenizerValues = { - search_method: [], + search_method: [ + TokenizerSearchMethod.Embedding, + TokenizerSearchMethod.FullText, + ], filename_embd_weight: 0.1, + fields: ['text'], outputs: {}, }; @@ -359,9 +372,14 @@ export const initialStringTransformValues = { }, }; -export const initialParserValues = { outputs: {}, parser: [] }; +export const initialParserValues = { outputs: {}, setups: [] }; -export const initialSplitterValues = { outputs: {}, chunk_token_size: 512 }; +export const initialSplitterValues = { + outputs: {}, + chunk_token_size: 512, + overlapped_percent: 0, + delimiters: [{ value: '\n' }], +}; export const initialHierarchicalMergerValues = { outputs: {} }; @@ -450,8 +468,3 @@ export enum FileType { Video = 'video', Audio = 'audio', } - -export enum TokenizerSearchMethod { - Embedding = 'embedding', - FullText = 'full_text', -} diff --git a/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx index 82742b5ad..c63ad6e92 100644 --- a/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx @@ -42,22 +42,17 @@ export function OutputFormatFormField({ ); } -export function ParserMethodFormField({ prefix }: CommonProps) { +export function ParserMethodFormField({ + prefix, + optionsWithoutLLM, +}: CommonProps & { optionsWithoutLLM?: { value: string; label: string }[] }) { return ( ); - - return ( - - - - ); } export function LargeModelFormField({ prefix }: CommonProps) { diff --git a/web/src/pages/data-flow/form/parser-form/constant.ts b/web/src/pages/data-flow/form/parser-form/constant.ts index c4cede934..a029009db 100644 --- a/web/src/pages/data-flow/form/parser-form/constant.ts +++ b/web/src/pages/data-flow/form/parser-form/constant.ts @@ -25,6 +25,7 @@ export enum TextMarkdownOutputFormat { export enum DocxOutputFormat { Markdown = 'markdown', + Json = 'json', } export enum PptOutputFormat { @@ -50,3 +51,15 @@ export const OutputFormatMap = { [FileType.Video]: VideoOutputFormat, [FileType.Audio]: AudioOutputFormat, }; + +export const InitialOutputFormatMap = { + [FileType.PDF]: PdfOutputFormat.Json, + [FileType.Spreadsheet]: SpreadsheetOutputFormat.Html, + [FileType.Image]: ImageOutputFormat.Text, + [FileType.Email]: EmailOutputFormat.Text, + [FileType.TextMarkdown]: TextMarkdownOutputFormat.Text, + [FileType.Docx]: DocxOutputFormat.Json, + [FileType.PowerPoint]: PptOutputFormat.Json, + [FileType.Video]: VideoOutputFormat.Json, + [FileType.Audio]: AudioOutputFormat.Text, +}; diff --git a/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx index 8014e2801..fc0b32b35 100644 --- a/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx @@ -2,8 +2,6 @@ import { SelectWithSearch } from '@/components/originui/select-with-search'; import { RAGFlowFormItem } from '@/components/ragflow-form'; import { buildOptions } from '@/utils/form'; import { useTranslation } from 'react-i18next'; -import { FileType } from '../../constant'; -import { OutputFormatFormField } from './common-form-fields'; import { CommonProps } from './interface'; import { buildFieldNameWithPrefix } from './utils'; @@ -28,10 +26,6 @@ export function EmailFormFields({ prefix }: CommonProps) { > - ); } diff --git a/web/src/pages/data-flow/form/parser-form/image-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/image-form-fields.tsx index 987b4f0be..8ee2b09f2 100644 --- a/web/src/pages/data-flow/form/parser-form/image-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/image-form-fields.tsx @@ -1,21 +1,33 @@ -import { FileType } from '../../constant'; -import { - LargeModelFormField, - OutputFormatFormField, - ParserMethodFormField, -} from './common-form-fields'; +import { buildOptions } from '@/utils/form'; +import { isEmpty } from 'lodash'; +import { useEffect } from 'react'; +import { useFormContext } from 'react-hook-form'; +import { ImageParseMethod } from '../../constant'; +import { ParserMethodFormField } from './common-form-fields'; import { CommonProps } from './interface'; +import { buildFieldNameWithPrefix } from './utils'; + +const options = buildOptions(ImageParseMethod); export function ImageFormFields({ prefix }: CommonProps) { + const form = useFormContext(); + const parseMethodName = buildFieldNameWithPrefix('parse_method', prefix); + + useEffect(() => { + if (isEmpty(form.getValues(parseMethodName))) { + form.setValue(parseMethodName, ImageParseMethod.OCR, { + shouldValidate: true, + shouldDirty: true, + }); + } + }, [form, parseMethodName]); + return ( <> - - {/* Multimodal Model */} - - + optionsWithoutLLM={options} + > ); } diff --git a/web/src/pages/data-flow/form/parser-form/index.tsx b/web/src/pages/data-flow/form/parser-form/index.tsx index dafb5f0eb..baf7a88a4 100644 --- a/web/src/pages/data-flow/form/parser-form/index.tsx +++ b/web/src/pages/data-flow/form/parser-form/index.tsx @@ -24,6 +24,7 @@ import { INextOperatorForm } from '../../interface'; import { buildOutputList } from '../../utils/build-output-list'; import { Output } from '../components/output'; import { OutputFormatFormField } from './common-form-fields'; +import { InitialOutputFormatMap } from './constant'; import { EmailFormFields } from './email-form-fields'; import { ImageFormFields } from './image-form-fields'; import { PdfFormFields } from './pdf-form-fields'; @@ -50,14 +51,14 @@ type ParserItemProps = { }; export const FormSchema = z.object({ - parser: z.array( + setups: z.array( z.object({ fileFormat: z.string().nullish(), output_format: z.string().optional(), parse_method: z.string().optional(), - llm_id: z.string().optional(), lang: z.string().optional(), fields: z.array(z.string()).optional(), + llm_id: z.string().optional(), }), ), }); @@ -71,10 +72,10 @@ function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) { const isHovering = useHover(ref); const prefix = `${name}.${index}`; - const fileFormat = form.getValues(`parser.${index}.fileFormat`); + const fileFormat = form.getValues(`setups.${index}.fileFormat`); const values = form.getValues(); - const parserList = values.parser.slice(); // Adding, deleting, or modifying the parser array will not change the reference. + const parserList = values.setups.slice(); // Adding, deleting, or modifying the parser array will not change the reference. const filteredFileFormatOptions = useMemo(() => { const otherFileFormatList = parserList @@ -89,7 +90,19 @@ function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) { const Widget = typeof fileFormat === 'string' && fileFormat in FileFormatWidgetMap ? FileFormatWidgetMap[fileFormat as keyof typeof FileFormatWidgetMap] - : OutputFormatFormField; + : () => <>; + + const handleFileTypeChange = useCallback( + (value: FileType) => { + form.setValue( + `setups.${index}.output_format`, + InitialOutputFormatMap[value], + { shouldDirty: true, shouldValidate: true, shouldTouch: true }, + ); + }, + [form, index], + ); + return (
- + {(field) => ( + { + field.onChange(val); + handleFileTypeChange(val as FileType); + }} + options={filteredFileFormatOptions} + > + )} + {index < fieldLength - 1 && }
); @@ -130,7 +154,7 @@ const ParserForm = ({ node }: INextOperatorForm) => { shouldUnregister: true, }); - const name = 'parser'; + const name = 'setups'; const { fields, remove, append } = useFieldArray({ name, control: form.control, @@ -141,9 +165,9 @@ const ParserForm = ({ node }: INextOperatorForm) => { fileFormat: null, output_format: '', parse_method: '', - llm_id: '', lang: '', fields: [], + llm_id: '', }); }, [append]); diff --git a/web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx index 8b7ee0afc..b38669019 100644 --- a/web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx @@ -1,29 +1,73 @@ -import { CrossLanguageFormField } from '@/components/cross-language-form-field'; +import { crossLanguageOptions } from '@/components/cross-language-form-field'; +import { ParseDocumentType } from '@/components/layout-recognize-form-field'; +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { isEmpty } from 'lodash'; +import { useEffect, useMemo } from 'react'; +import { useFormContext, useWatch } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; -import { FileType } from '../../constant'; -import { - LargeModelFormField, - OutputFormatFormField, - ParserMethodFormField, -} from './common-form-fields'; +import { ParserMethodFormField } from './common-form-fields'; import { CommonProps } from './interface'; import { buildFieldNameWithPrefix } from './utils'; export function PdfFormFields({ prefix }: CommonProps) { const { t } = useTranslation(); + const form = useFormContext(); + + const parseMethodName = buildFieldNameWithPrefix('parse_method', prefix); + + const parseMethod = useWatch({ + name: parseMethodName, + }); + const lang = form.getValues(buildFieldNameWithPrefix('lang', prefix)); + + const languageShown = useMemo(() => { + return ( + !isEmpty(parseMethod) && + parseMethod !== ParseDocumentType.DeepDOC && + parseMethod !== ParseDocumentType.PlainText + ); + }, [parseMethod]); + + useEffect(() => { + if (languageShown && isEmpty(lang)) { + form.setValue( + buildFieldNameWithPrefix('lang', prefix), + crossLanguageOptions[0].value, + { + shouldValidate: true, + shouldDirty: true, + }, + ); + } + }, [form, lang, languageShown, prefix]); + + useEffect(() => { + if (isEmpty(form.getValues(parseMethodName))) { + form.setValue(parseMethodName, ParseDocumentType.DeepDOC, { + shouldValidate: true, + shouldDirty: true, + }); + } + }, [form, parseMethodName]); + return ( <> - {/* Multimodal Model */} - - - + {languageShown && ( + + {(field) => ( + + )} + + )} ); } diff --git a/web/src/pages/data-flow/form/parser-form/video-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/video-form-fields.tsx index 88e07cacb..79f64d6fd 100644 --- a/web/src/pages/data-flow/form/parser-form/video-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/video-form-fields.tsx @@ -1,21 +1,13 @@ import { LargeModelFormField, - OutputFormatFormField, OutputFormatFormFieldProps, } from './common-form-fields'; -export function VideoFormFields({ - prefix, - fileType, -}: OutputFormatFormFieldProps) { +export function VideoFormFields({ prefix }: OutputFormatFormFieldProps) { return ( <> {/* Multimodal Model */} - ); } diff --git a/web/src/pages/data-flow/form/splitter-form/index.tsx b/web/src/pages/data-flow/form/splitter-form/index.tsx index 3807a8674..26a0198c5 100644 --- a/web/src/pages/data-flow/form/splitter-form/index.tsx +++ b/web/src/pages/data-flow/form/splitter-form/index.tsx @@ -1,15 +1,15 @@ +import { DelimiterInput } from '@/components/delimiter-form-field'; import { RAGFlowFormItem } from '@/components/ragflow-form'; import { SliderInputFormField } from '@/components/slider-input-form-field'; import { BlockButton, Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; -import { Input } from '@/components/ui/input'; import { zodResolver } from '@hookform/resolvers/zod'; import { Trash2 } from 'lucide-react'; import { memo } from 'react'; import { useFieldArray, useForm } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; -import { initialChunkerValues, initialSplitterValues } from '../../constant'; +import { initialSplitterValues } from '../../constant'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; @@ -32,7 +32,7 @@ export const FormSchema = z.object({ export type SplitterFormSchemaType = z.infer; const SplitterForm = ({ node }: INextOperatorForm) => { - const defaultValues = useFormValues(initialChunkerValues, node); + const defaultValues = useFormValues(initialSplitterValues, node); const { t } = useTranslation(); const form = useForm({ @@ -59,32 +59,34 @@ const SplitterForm = ({ node }: INextOperatorForm) => { - {t('flow.delimiters')} - {fields.map((field, index) => ( -
-
- + {t('flow.delimiters')} + {fields.map((field, index) => ( +
+
+ + + +
+
- -
- ))} - append({ value: '' })}> + ))} +
+ append({ value: '\n' })}> {t('common.add')} diff --git a/web/src/pages/data-flow/form/tokenizer-form/index.tsx b/web/src/pages/data-flow/form/tokenizer-form/index.tsx index ac215e290..c9b272d46 100644 --- a/web/src/pages/data-flow/form/tokenizer-form/index.tsx +++ b/web/src/pages/data-flow/form/tokenizer-form/index.tsx @@ -25,6 +25,8 @@ export const FormSchema = z.object({ const SearchMethodOptions = buildOptions(TokenizerSearchMethod); +const FieldsOptions = [{ label: 'text', value: 'text' }]; + const TokenizerForm = ({ node }: INextOperatorForm) => { const { t } = useTranslation(); const defaultValues = useFormValues(initialTokenizerValues, node); @@ -59,6 +61,16 @@ const TokenizerForm = ({ node }: INextOperatorForm) => { max={0.5} step={0.01} > + + {(field) => ( + + )} +
diff --git a/web/src/pages/data-flow/hooks/use-run-dataflow.ts b/web/src/pages/data-flow/hooks/use-run-dataflow.ts index da0858f6d..9de8a7066 100644 --- a/web/src/pages/data-flow/hooks/use-run-dataflow.ts +++ b/web/src/pages/data-flow/hooks/use-run-dataflow.ts @@ -19,7 +19,7 @@ export function useRunDataflow(showLogSheet: () => void) { id, query: '', session_id: null, - inputs: fileResponseData, + files: [fileResponseData.file], }); console.log('🚀 ~ useRunDataflow ~ res:', res); diff --git a/web/src/pages/data-flow/hooks/use-watch-form-change.ts b/web/src/pages/data-flow/hooks/use-watch-form-change.ts index 534c2e2a9..8c23dfb7f 100644 --- a/web/src/pages/data-flow/hooks/use-watch-form-change.ts +++ b/web/src/pages/data-flow/hooks/use-watch-form-change.ts @@ -4,15 +4,16 @@ import useGraphStore from '../store'; export function useWatchFormChange(id?: string, form?: UseFormReturn) { let values = useWatch({ control: form?.control }); + console.log( + '🚀 ~ useWatchFormChange ~ values:', + JSON.stringify(values, null, 2), + ); + const updateNodeForm = useGraphStore((state) => state.updateNodeForm); useEffect(() => { - // Manually triggered form updates are synchronized to the canvas if (id) { - values = form?.getValues() || {}; - let nextValues: any = values; - - updateNodeForm(id, nextValues); + updateNodeForm(id, values); } - }, [form?.formState.isDirty, id, updateNodeForm, values]); + }, [id, updateNodeForm, values]); } diff --git a/web/src/pages/data-flow/log-sheet/dataflow-timeline.tsx b/web/src/pages/data-flow/log-sheet/dataflow-timeline.tsx new file mode 100644 index 000000000..7876750fa --- /dev/null +++ b/web/src/pages/data-flow/log-sheet/dataflow-timeline.tsx @@ -0,0 +1,3 @@ +export function DataflowTimeline() { + return
xx
; +} diff --git a/web/src/pages/data-flow/log-sheet/index.tsx b/web/src/pages/data-flow/log-sheet/index.tsx new file mode 100644 index 000000000..d8db50907 --- /dev/null +++ b/web/src/pages/data-flow/log-sheet/index.tsx @@ -0,0 +1,28 @@ +import { + Sheet, + SheetContent, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; +import { IModalProps } from '@/interfaces/common'; +import { cn } from '@/lib/utils'; +import { NotebookText } from 'lucide-react'; +import 'react18-json-view/src/style.css'; + +type LogSheetProps = IModalProps; + +export function LogSheet({ hideModal }: LogSheetProps) { + return ( + + + + + + Log + + +
+ + + ); +} diff --git a/web/src/pages/data-flow/run-sheet/index.tsx b/web/src/pages/data-flow/run-sheet/index.tsx index 264a0883a..4a2c70681 100644 --- a/web/src/pages/data-flow/run-sheet/index.tsx +++ b/web/src/pages/data-flow/run-sheet/index.tsx @@ -13,7 +13,7 @@ import { UploaderForm } from './uploader'; const RunSheet = ({ hideModal, showModal: showLogSheet }: IModalProps) => { const { t } = useTranslation(); - const { run, loading } = useRunDataflow(() => {}); + const { run, loading } = useRunDataflow(showLogSheet!); return ( diff --git a/web/src/pages/data-flow/utils.ts b/web/src/pages/data-flow/utils.ts index 49bd18e2e..a13a3bc7b 100644 --- a/web/src/pages/data-flow/utils.ts +++ b/web/src/pages/data-flow/utils.ts @@ -1,6 +1,5 @@ import { IAgentForm, - ICategorizeForm, ICategorizeItem, ICategorizeItemResult, } from '@/interfaces/database/agent'; @@ -22,6 +21,7 @@ import pipe from 'lodash/fp/pipe'; import isObject from 'lodash/isObject'; import { CategorizeAnchorPointPositions, + FileType, NoDebugOperatorsList, NodeHandleId, Operator, @@ -31,15 +31,6 @@ import { ParserFormSchemaType } from './form/parser-form'; import { SplitterFormSchemaType } from './form/splitter-form'; import { BeginQuery, IPosition } from './interface'; -function buildAgentExceptionGoto(edges: Edge[], nodeId: string) { - const exceptionEdges = edges.filter( - (x) => - x.source === nodeId && x.sourceHandle === NodeHandleId.AgentException, - ); - - return exceptionEdges.map((x) => x.target); -} - const buildComponentDownstreamOrUpstream = ( edges: Edge[], nodeId: string, @@ -80,70 +71,6 @@ const removeUselessDataInTheOperator = curry( return params; }, ); -// initialize data for operators without parameters -// const initializeOperatorParams = curry((operatorName: string, values: any) => { -// if (isEmpty(values)) { -// return initialFormValuesMap[operatorName as Operator]; -// } -// return values; -// }); - -function buildAgentTools(edges: Edge[], nodes: Node[], nodeId: string) { - const node = nodes.find((x) => x.id === nodeId); - const params = { ...(node?.data.form ?? {}) }; - if (node && node.data.label === Operator.Agent) { - const bottomSubAgentEdges = edges.filter( - (x) => x.source === nodeId && x.sourceHandle === NodeHandleId.AgentBottom, - ); - - (params as IAgentForm).tools = (params as IAgentForm).tools.concat( - bottomSubAgentEdges.map((x) => { - const { - params: formData, - id, - name, - } = buildAgentTools(edges, nodes, x.target); - - return { - component_name: Operator.Agent, - id, - name: name as string, // Cast name to string and provide fallback - params: { ...formData }, - }; - }), - ); - } - return { params, name: node?.data.name, id: node?.id }; -} - -function filterTargetsBySourceHandleId(edges: Edge[], handleId: string) { - return edges.filter((x) => x.sourceHandle === handleId).map((x) => x.target); -} - -function buildCategorize(edges: Edge[], nodes: Node[], nodeId: string) { - const node = nodes.find((x) => x.id === nodeId); - const params = { ...(node?.data.form ?? {}) } as ICategorizeForm; - if (node && node.data.label === Operator.Categorize) { - const subEdges = edges.filter((x) => x.source === nodeId); - - const items = params.items || []; - - const nextCategoryDescription = items.reduce< - ICategorizeForm['category_description'] - >((pre, val) => { - const key = val.name; - pre[key] = { - ...omit(val, 'name', 'uuid'), - examples: val.examples?.map((x) => x.value) || [], - to: filterTargetsBySourceHandleId(subEdges, val.uuid), - }; - return pre; - }, {}); - - params.category_description = nextCategoryDescription; - } - return omit(params, 'items'); -} const buildOperatorParams = (operatorName: string) => pipe( @@ -151,7 +78,7 @@ const buildOperatorParams = (operatorName: string) => // initializeOperatorParams(operatorName), // Final processing, for guarantee ); -const ExcludeOperators = [Operator.Note, Operator.Tool]; +const ExcludeOperators = [Operator.Note]; export function isBottomSubAgent(edges: Edge[], nodeId?: string) { const edge = edges.find( @@ -171,14 +98,51 @@ function transformObjectArrayToPureArray( } function transformParserParams(params: ParserFormSchemaType) { - return params.parser.reduce< - Record + const setups = params.setups.reduce< + Record >((pre, cur) => { if (cur.fileFormat) { - pre[cur.fileFormat] = omit(cur, 'fileFormat'); + let filteredSetup: Partial = { + output_format: cur.output_format, + }; + + switch (cur.fileFormat) { + case FileType.PDF: + filteredSetup = { + ...filteredSetup, + parse_method: cur.parse_method, + lang: cur.lang, + }; + break; + case FileType.Image: + filteredSetup = { + ...filteredSetup, + parse_method: cur.parse_method, + }; + break; + case FileType.Email: + filteredSetup = { + ...filteredSetup, + fields: cur.fields, + }; + break; + case FileType.Video: + case FileType.Audio: + filteredSetup = { + ...filteredSetup, + llm_id: cur.llm_id, + }; + break; + default: + break; + } + + pre[cur.fileFormat] = filteredSetup; } return pre; }, {}); + + return { ...params, setups }; } function transformSplitterParams(params: SplitterFormSchemaType) { @@ -218,18 +182,6 @@ export const buildDslComponentsByGraph = ( let params = x?.data.form ?? {}; switch (operatorName) { - case Operator.Agent: { - const { params: formData } = buildAgentTools(edges, nodes, id); - params = { - ...formData, - exception_goto: buildAgentExceptionGoto(edges, id), - }; - break; - } - case Operator.Categorize: - params = buildCategorize(edges, nodes, id); - break; - case Operator.Parser: params = transformParserParams(params); break;