diff --git a/web/src/constants/agent.ts b/web/src/constants/agent.ts index 35ac2a6f9..6cb0cc92a 100644 --- a/web/src/constants/agent.ts +++ b/web/src/constants/agent.ts @@ -1,3 +1,6 @@ +import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat'; +import { ChatVariableEnabledField, variableEnabledFieldMap } from './chat'; + export enum ProgrammingLanguage { Python = 'python', Javascript = 'javascript', @@ -26,3 +29,21 @@ export enum AgentGlobals { } export const AgentGlobalsSysQueryWithBrace = `{${AgentGlobals.SysQuery}}`; + +export const variableCheckBoxFieldMap = Object.keys( + variableEnabledFieldMap, +).reduce>((pre, cur) => { + pre[cur] = setInitialChatVariableEnabledFieldValue( + cur as ChatVariableEnabledField, + ); + return pre; +}, {}); + +export const initialLlmBaseValues = { + ...variableCheckBoxFieldMap, + temperature: 0.1, + top_p: 0.3, + frequency_penalty: 0.7, + presence_penalty: 0.4, + max_tokens: 256, +}; diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 3aa04a5c2..a40749102 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1705,6 +1705,13 @@ This delimiter is used to split the input text into several text pieces echo of exportJson: 'Export JSON', viewResult: 'View Result', running: 'Running', + context: 'Context Generator', + contextDescription: 'Context Generator', + summary: 'Summary', + keywords: 'Keywords', + questions: 'Questions', + metadata: 'Metadata', + fieldName: 'Result Destination', }, }, }; diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 045ca20fc..639ae678e 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1623,6 +1623,13 @@ General:实体和关系提取提示来自 GitHub - microsoft/graphrag:基于 exportJson: '导出 JSON', viewResult: '查看结果', running: '运行中', + context: '上下文生成器', + contextDescription: '上下文生成器', + summary: '摘要', + keywords: '关键词', + questions: '问题', + metadata: '元数据', + fieldName: '结果目的地', }, }, }; diff --git a/web/src/pages/agent/constant.tsx b/web/src/pages/agent/constant.tsx index e37f327c1..a0ca13b00 100644 --- a/web/src/pages/agent/constant.tsx +++ b/web/src/pages/agent/constant.tsx @@ -7,6 +7,7 @@ import { AgentGlobalsSysQueryWithBrace, CodeTemplateStrMap, ProgrammingLanguage, + initialLlmBaseValues, } from '@/constants/agent'; export enum AgentDialogueMode { @@ -14,13 +15,8 @@ export enum AgentDialogueMode { Task = 'task', } -import { - ChatVariableEnabledField, - variableEnabledFieldMap, -} from '@/constants/chat'; import { ModelVariableType } from '@/constants/knowledge'; import i18n from '@/locales/config'; -import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat'; import { t } from 'i18next'; // DuckDuckGo's channel options @@ -271,24 +267,6 @@ export const initialBeginValues = { prologue: `Hi! I'm your assistant. What can I do for you?`, }; -export const variableCheckBoxFieldMap = Object.keys( - variableEnabledFieldMap, -).reduce>((pre, cur) => { - pre[cur] = setInitialChatVariableEnabledFieldValue( - cur as ChatVariableEnabledField, - ); - return pre; -}, {}); - -const initialLlmBaseValues = { - ...variableCheckBoxFieldMap, - temperature: 0.1, - top_p: 0.3, - frequency_penalty: 0.7, - presence_penalty: 0.4, - max_tokens: 256, -}; - export const initialGenerateValues = { ...initialLlmBaseValues, prompt: i18n.t('flow.promptText'), diff --git a/web/src/pages/data-flow/canvas/node/index.tsx b/web/src/pages/data-flow/canvas/node/index.tsx index a1d48955b..c7f2b2933 100644 --- a/web/src/pages/data-flow/canvas/node/index.tsx +++ b/web/src/pages/data-flow/canvas/node/index.tsx @@ -2,7 +2,6 @@ import { IRagNode } from '@/interfaces/database/flow'; import { NodeProps, Position } from '@xyflow/react'; import { memo } from 'react'; import { NodeHandleId } from '../../constant'; -import { needsSingleStepDebugging } from '../../utils'; import { CommonHandle } from './handle'; import { LeftHandleStyle, RightHandleStyle } from './handle-icon'; import NodeHeader from './node-header'; @@ -16,12 +15,7 @@ function InnerRagNode({ selected, }: NodeProps) { return ( - + store.deleteNodeById); diff --git a/web/src/pages/data-flow/constant.tsx b/web/src/pages/data-flow/constant.tsx index 6572a515e..f3f61bc0b 100644 --- a/web/src/pages/data-flow/constant.tsx +++ b/web/src/pages/data-flow/constant.tsx @@ -1,3 +1,5 @@ +import { ParseDocumentType } from '@/components/layout-recognize-form-field'; +import { initialLlmBaseValues } from '@/constants/agent'; import { ChatVariableEnabledField, variableEnabledFieldMap, @@ -15,6 +17,89 @@ import { WrapText, } from 'lucide-react'; +export enum FileType { + PDF = 'pdf', + Spreadsheet = 'spreadsheet', + Image = 'image', + Email = 'email', + TextMarkdown = 'text&markdown', + Docx = 'word', + PowerPoint = 'slides', + Video = 'video', + Audio = 'audio', +} + +export enum PdfOutputFormat { + Json = 'json', + Markdown = 'markdown', +} + +export enum SpreadsheetOutputFormat { + Json = 'json', + Html = 'html', +} + +export enum ImageOutputFormat { + Text = 'text', +} + +export enum EmailOutputFormat { + Json = 'json', + Text = 'text', +} + +export enum TextMarkdownOutputFormat { + Text = 'text', +} + +export enum DocxOutputFormat { + Markdown = 'markdown', + Json = 'json', +} + +export enum PptOutputFormat { + Json = 'json', +} + +export enum VideoOutputFormat { + Json = 'json', +} + +export enum AudioOutputFormat { + Text = 'text', +} + +export const OutputFormatMap = { + [FileType.PDF]: PdfOutputFormat, + [FileType.Spreadsheet]: SpreadsheetOutputFormat, + [FileType.Image]: ImageOutputFormat, + [FileType.Email]: EmailOutputFormat, + [FileType.TextMarkdown]: TextMarkdownOutputFormat, + [FileType.Docx]: DocxOutputFormat, + [FileType.PowerPoint]: PptOutputFormat, + [FileType.Video]: VideoOutputFormat, + [FileType.Audio]: AudioOutputFormat, +}; + +export const InitialOutputFormatMap = { + [FileType.PDF]: PdfOutputFormat.Json, + [FileType.Spreadsheet]: SpreadsheetOutputFormat.Html, + [FileType.Image]: ImageOutputFormat.Text, + [FileType.Email]: EmailOutputFormat.Text, + [FileType.TextMarkdown]: TextMarkdownOutputFormat.Text, + [FileType.Docx]: DocxOutputFormat.Json, + [FileType.PowerPoint]: PptOutputFormat.Json, + [FileType.Video]: VideoOutputFormat.Json, + [FileType.Audio]: AudioOutputFormat.Text, +}; + +export enum ContextGeneratorFieldName { + Summary = 'summary', + Keywords = 'keywords', + Questions = 'questions', + Metadata = 'metadata', +} + export enum PromptRole { User = 'user', Assistant = 'assistant', @@ -83,9 +168,28 @@ export enum TokenizerFields { Summary = 'summary', } +export enum ParserFields { + From = 'from', + To = 'to', + Cc = 'cc', + Bcc = 'bcc', + Date = 'date', + Subject = 'subject', + Body = 'body', + Attachments = 'attachments', +} + export const initialBeginValues = { - mode: AgentDialogueMode.Conversational, - prologue: `Hi! I'm your assistant. What can I do for you?`, + outputs: { + name: { + type: 'string', + value: '', + }, + file: { + type: 'Object', + value: {}, + }, + }, }; export const variableCheckBoxFieldMap = Object.keys( @@ -125,10 +229,40 @@ export enum StringTransformDelimiter { Space = ' ', } -export const initialParserValues = { outputs: {}, setups: [] }; +export const initialParserValues = { + outputs: { + markdown: { type: 'string', value: '' }, + text: { type: 'string', value: '' }, + html: { type: 'string', value: '' }, + json: { type: 'Array', value: [] }, + }, + setups: [ + { + fileFormat: FileType.PDF, + output_format: PdfOutputFormat.Json, + parse_method: ParseDocumentType.DeepDOC, + }, + { + fileFormat: FileType.Spreadsheet, + output_format: SpreadsheetOutputFormat.Html, + }, + { + fileFormat: FileType.Image, + output_format: ImageOutputFormat.Text, + parse_method: ImageParseMethod.OCR, + }, + { + fileFormat: FileType.Email, + fields: Object.values(ParserFields), + output_format: EmailOutputFormat.Text, + }, + ], +}; export const initialSplitterValues = { - outputs: {}, + outputs: { + chunks: { type: 'Array', value: [] }, + }, chunk_token_size: 512, overlapped_percent: 0, delimiters: [{ value: '\n' }], @@ -143,7 +277,9 @@ export enum Hierarchy { } export const initialHierarchicalMergerValues = { - outputs: {}, + outputs: { + chunks: { type: 'Array', value: [] }, + }, hierarchy: Hierarchy.H3, levels: [ { expressions: [{ expression: '^#[^#]' }] }, @@ -154,6 +290,8 @@ export const initialHierarchicalMergerValues = { }; export const initialContextValues = { + ...initialLlmBaseValues, + field_name: [ContextGeneratorFieldName.Summary], outputs: {}, }; @@ -232,18 +370,6 @@ export enum AgentExceptionMethod { Goto = 'goto', } -export enum FileType { - PDF = 'pdf', - Spreadsheet = 'spreadsheet', - Image = 'image', - Email = 'email', - TextMarkdown = 'text&markdown', - Docx = 'word', - PowerPoint = 'slides', - Video = 'video', - Audio = 'audio', -} - export const FileTypeSuffixMap = { [FileType.PDF]: ['pdf'], [FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'], diff --git a/web/src/pages/data-flow/form/context-form/index.tsx b/web/src/pages/data-flow/form/context-form/index.tsx index 337965c57..d1986bf10 100644 --- a/web/src/pages/data-flow/form/context-form/index.tsx +++ b/web/src/pages/data-flow/form/context-form/index.tsx @@ -1,18 +1,24 @@ -import { DelimiterInput } from '@/components/delimiter-form-field'; +import { LargeModelFormField } from '@/components/large-model-form-field'; +import { LlmSettingSchema } from '@/components/llm-setting-items/next'; import { RAGFlowFormItem } from '@/components/ragflow-form'; -import { SliderInputFormField } from '@/components/slider-input-form-field'; -import { BlockButton, Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; +import { MultiSelect } from '@/components/ui/multi-select'; +import { useBuildPromptExtraPromptOptions } from '@/pages/agent/form/agent-form/use-build-prompt-options'; +import { PromptEditor } from '@/pages/agent/form/components/prompt-editor'; +import { buildOptions } from '@/utils/form'; import { zodResolver } from '@hookform/resolvers/zod'; -import { Trash2 } from 'lucide-react'; import { memo } from 'react'; -import { useFieldArray, useForm } from 'react-hook-form'; +import { useForm } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; -import { initialContextValues } from '../../constant'; +import { + ContextGeneratorFieldName, + initialContextValues, +} from '../../constant'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; +import useGraphStore from '../../store'; import { buildOutputList } from '../../utils/build-output-list'; import { FormWrapper } from '../components/form-wrapper'; import { Output } from '../components/output'; @@ -20,13 +26,10 @@ import { Output } from '../components/output'; const outputList = buildOutputList(initialContextValues.outputs); export const FormSchema = z.object({ - chunk_token_size: z.number(), - delimiters: z.array( - z.object({ - value: z.string().optional(), - }), - ), - overlapped_percent: z.number(), // 0.0 - 0.3 + sys_prompt: z.string(), + prompts: z.string().optional(), + ...LlmSettingSchema, + field_name: z.array(z.string()), }); export type ContextFormSchemaType = z.infer; @@ -39,58 +42,39 @@ const ContextForm = ({ node }: INextOperatorForm) => { defaultValues, resolver: zodResolver(FormSchema), }); - const name = 'delimiters'; - const { fields, append, remove } = useFieldArray({ - name: name, - control: form.control, - }); + const { edges } = useGraphStore((state) => state); + + const { extraOptions } = useBuildPromptExtraPromptOptions(edges, node?.id); + + const options = buildOptions(ContextGeneratorFieldName, t, 'dataflow'); useWatchFormChange(node?.id, form); return (
- - -
- {t('flow.delimiters')} -
- {fields.map((field, index) => ( -
-
- - - -
- -
- ))} -
-
- append({ value: '\n' })}> - {t('common.add')} - + + + + + + + + + {(field) => ( + + )} +
diff --git a/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx index d9b51bd7a..46809ad2c 100644 --- a/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/common-form-fields.tsx @@ -8,8 +8,7 @@ import { import { RAGFlowFormItem } from '@/components/ragflow-form'; import { buildOptions } from '@/utils/form'; import { useTranslation } from 'react-i18next'; -import { FileType } from '../../constant'; -import { OutputFormatMap } from './constant'; +import { FileType, OutputFormatMap } from '../../constant'; import { CommonProps } from './interface'; import { buildFieldNameWithPrefix } from './utils'; diff --git a/web/src/pages/data-flow/form/parser-form/constant.ts b/web/src/pages/data-flow/form/parser-form/constant.ts deleted file mode 100644 index a029009db..000000000 --- a/web/src/pages/data-flow/form/parser-form/constant.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { FileType } from '../../constant'; - -export enum PdfOutputFormat { - Json = 'json', - Markdown = 'markdown', -} - -export enum SpreadsheetOutputFormat { - Json = 'json', - Html = 'html', -} - -export enum ImageOutputFormat { - Text = 'text', -} - -export enum EmailOutputFormat { - Json = 'json', - Text = 'text', -} - -export enum TextMarkdownOutputFormat { - Text = 'text', -} - -export enum DocxOutputFormat { - Markdown = 'markdown', - Json = 'json', -} - -export enum PptOutputFormat { - Json = 'json', -} - -export enum VideoOutputFormat { - Json = 'json', -} - -export enum AudioOutputFormat { - Text = 'text', -} - -export const OutputFormatMap = { - [FileType.PDF]: PdfOutputFormat, - [FileType.Spreadsheet]: SpreadsheetOutputFormat, - [FileType.Image]: ImageOutputFormat, - [FileType.Email]: EmailOutputFormat, - [FileType.TextMarkdown]: TextMarkdownOutputFormat, - [FileType.Docx]: DocxOutputFormat, - [FileType.PowerPoint]: PptOutputFormat, - [FileType.Video]: VideoOutputFormat, - [FileType.Audio]: AudioOutputFormat, -}; - -export const InitialOutputFormatMap = { - [FileType.PDF]: PdfOutputFormat.Json, - [FileType.Spreadsheet]: SpreadsheetOutputFormat.Html, - [FileType.Image]: ImageOutputFormat.Text, - [FileType.Email]: EmailOutputFormat.Text, - [FileType.TextMarkdown]: TextMarkdownOutputFormat.Text, - [FileType.Docx]: DocxOutputFormat.Json, - [FileType.PowerPoint]: PptOutputFormat.Json, - [FileType.Video]: VideoOutputFormat.Json, - [FileType.Audio]: AudioOutputFormat.Text, -}; diff --git a/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx b/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx index fc0b32b35..640c271f9 100644 --- a/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx +++ b/web/src/pages/data-flow/form/parser-form/email-form-fields.tsx @@ -1,20 +1,12 @@ -import { SelectWithSearch } from '@/components/originui/select-with-search'; import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { MultiSelect } from '@/components/ui/multi-select'; import { buildOptions } from '@/utils/form'; import { useTranslation } from 'react-i18next'; +import { ParserFields } from '../../constant'; import { CommonProps } from './interface'; import { buildFieldNameWithPrefix } from './utils'; -const options = buildOptions([ - 'from', - 'to', - 'cc', - 'bcc', - 'date', - 'subject', - 'body', - 'attachments', -]); +const options = buildOptions(ParserFields); export function EmailFormFields({ prefix }: CommonProps) { const { t } = useTranslation(); @@ -24,7 +16,14 @@ export function EmailFormFields({ prefix }: CommonProps) { name={buildFieldNameWithPrefix(`fields`, prefix)} label={t('dataflow.fields')} > - + {(field) => ( + + )} ); diff --git a/web/src/pages/data-flow/form/parser-form/index.tsx b/web/src/pages/data-flow/form/parser-form/index.tsx index 3e08301f1..7bd05766b 100644 --- a/web/src/pages/data-flow/form/parser-form/index.tsx +++ b/web/src/pages/data-flow/form/parser-form/index.tsx @@ -17,14 +17,17 @@ import { } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; -import { FileType, initialParserValues } from '../../constant'; +import { + FileType, + InitialOutputFormatMap, + initialParserValues, +} from '../../constant'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; import { buildOutputList } from '../../utils/build-output-list'; import { Output } from '../components/output'; import { OutputFormatFormField } from './common-form-fields'; -import { InitialOutputFormatMap } from './constant'; import { EmailFormFields } from './email-form-fields'; import { ImageFormFields } from './image-form-fields'; import { PdfFormFields } from './pdf-form-fields'; diff --git a/web/src/pages/data-flow/hooks/use-add-node.ts b/web/src/pages/data-flow/hooks/use-add-node.ts index 111f81e75..df3a14592 100644 --- a/web/src/pages/data-flow/hooks/use-add-node.ts +++ b/web/src/pages/data-flow/hooks/use-add-node.ts @@ -1,3 +1,4 @@ +import { useFetchModelId } from '@/hooks/logic-hooks'; import { Connection, Node, Position, ReactFlowInstance } from '@xyflow/react'; import humanId from 'human-id'; import { lowerFirst } from 'lodash'; @@ -22,6 +23,8 @@ import { } from '../utils'; export const useInitializeOperatorParams = () => { + const llmId = useFetchModelId(); + const initialFormValuesMap = useMemo(() => { return { [Operator.Begin]: initialBeginValues, @@ -30,9 +33,9 @@ export const useInitializeOperatorParams = () => { [Operator.Tokenizer]: initialTokenizerValues, [Operator.Splitter]: initialSplitterValues, [Operator.HierarchicalMerger]: initialHierarchicalMergerValues, - [Operator.Context]: initialContextValues, + [Operator.Context]: { ...initialContextValues, llm_id: llmId }, }; - }, []); + }, [llmId]); const initializeOperatorParams = useCallback( (operatorName: Operator) => { diff --git a/web/src/pages/data-flow/operator-icon.tsx b/web/src/pages/data-flow/operator-icon.tsx index 7af1719e8..fd0bbf6bf 100644 --- a/web/src/pages/data-flow/operator-icon.tsx +++ b/web/src/pages/data-flow/operator-icon.tsx @@ -4,6 +4,7 @@ import { Blocks, File, FileChartColumnIncreasing, + FileStack, Heading, ListMinus, } from 'lucide-react'; @@ -24,6 +25,7 @@ export const SVGIconMap = { [Operator.Tokenizer]: ListMinus, [Operator.Splitter]: Blocks, [Operator.HierarchicalMerger]: Heading, + [Operator.Context]: FileStack, }; const Empty = () => { diff --git a/web/src/pages/data-flow/utils.ts b/web/src/pages/data-flow/utils.ts index 32cbb8772..4ca27fb39 100644 --- a/web/src/pages/data-flow/utils.ts +++ b/web/src/pages/data-flow/utils.ts @@ -1,7 +1,6 @@ import { IAgentForm } from '@/interfaces/database/agent'; import { DSLComponents, RAGFlowNodeType } from '@/interfaces/database/flow'; -import { removeUselessFieldsFromValues } from '@/utils/form'; -import { Edge, Node, XYPosition } from '@xyflow/react'; +import { Edge, XYPosition } from '@xyflow/react'; import { FormInstance, FormListFieldData } from 'antd'; import { humanId } from 'human-id'; import { curry, get, intersectionWith, isEmpty, isEqual, sample } from 'lodash'; @@ -24,24 +23,13 @@ const buildComponentDownstreamOrUpstream = ( edges: Edge[], nodeId: string, isBuildDownstream = true, - nodes: Node[], ) => { return edges .filter((y) => { - const node = nodes.find((x) => x.id === nodeId); let isNotUpstreamTool = true; let isNotUpstreamAgent = true; let isNotExceptionGoto = true; - if (isBuildDownstream && node?.data.label === Operator.Agent) { - isNotExceptionGoto = y.sourceHandle !== NodeHandleId.AgentException; - // Exclude the tool operator downstream of the agent operator - isNotUpstreamTool = !y.target.startsWith(Operator.Tool); - // Exclude the agent operator downstream of the agent operator - isNotUpstreamAgent = !( - y.target.startsWith(Operator.Agent) && - y.targetHandle === NodeHandleId.AgentTop - ); - } + return ( y[isBuildDownstream ? 'source' : 'target'] === nodeId && isNotUpstreamTool && @@ -54,9 +42,9 @@ const buildComponentDownstreamOrUpstream = ( const removeUselessDataInTheOperator = curry( (operatorName: string, params: Record) => { - if (operatorName === Operator.Categorize) { - return removeUselessFieldsFromValues(params, ''); - } + // if (operatorName === Operator.Categorize) { + // return removeUselessFieldsFromValues(params, ''); + // } return params; }, ); @@ -197,8 +185,8 @@ export const buildDslComponentsByGraph = ( component_name: operatorName, params: buildOperatorParams(operatorName)(params) ?? {}, }, - downstream: buildComponentDownstreamOrUpstream(edges, id, true, nodes), - upstream: buildComponentDownstreamOrUpstream(edges, id, false, nodes), + downstream: buildComponentDownstreamOrUpstream(edges, id, true), + upstream: buildComponentDownstreamOrUpstream(edges, id, false), parent_id: x?.parentId, }; }); @@ -352,25 +340,6 @@ export const generateNodeNamesWithIncreasingIndex = ( export const duplicateNodeForm = (nodeData?: RAGFlowNodeType['data']) => { const form: Record = { ...(nodeData?.form ?? {}) }; - // Delete the downstream node corresponding to the to field of the Categorize operator - if (nodeData?.label === Operator.Categorize) { - form.category_description = Object.keys(form.category_description).reduce< - Record> - >((pre, cur) => { - pre[cur] = { - ...form.category_description[cur], - to: undefined, - }; - return pre; - }, {}); - } - - // Delete the downstream nodes corresponding to the yes and no fields of the Relevant operator - if (nodeData?.label === Operator.Relevant) { - form.yes = undefined; - form.no = undefined; - } - return { ...(nodeData ?? { label: '' }), form,