Feat: Added the context operator form for data flow #9869 (#10270)

### What problem does this PR solve?
Feat: Added the context operator form for data flow #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu
2025-09-24 19:58:16 +08:00
committed by GitHub
parent afb8a84f7b
commit 8be7380b79
15 changed files with 254 additions and 227 deletions

View File

@ -1,3 +1,6 @@
import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat';
import { ChatVariableEnabledField, variableEnabledFieldMap } from './chat';
export enum ProgrammingLanguage {
Python = 'python',
Javascript = 'javascript',
@ -26,3 +29,21 @@ export enum AgentGlobals {
}
export const AgentGlobalsSysQueryWithBrace = `{${AgentGlobals.SysQuery}}`;
export const variableCheckBoxFieldMap = Object.keys(
variableEnabledFieldMap,
).reduce<Record<string, boolean>>((pre, cur) => {
pre[cur] = setInitialChatVariableEnabledFieldValue(
cur as ChatVariableEnabledField,
);
return pre;
}, {});
export const initialLlmBaseValues = {
...variableCheckBoxFieldMap,
temperature: 0.1,
top_p: 0.3,
frequency_penalty: 0.7,
presence_penalty: 0.4,
max_tokens: 256,
};

View File

@ -1705,6 +1705,13 @@ This delimiter is used to split the input text into several text pieces echo of
exportJson: 'Export JSON',
viewResult: 'View Result',
running: 'Running',
context: 'Context Generator',
contextDescription: 'Context Generator',
summary: 'Summary',
keywords: 'Keywords',
questions: 'Questions',
metadata: 'Metadata',
fieldName: 'Result Destination',
},
},
};

View File

@ -1623,6 +1623,13 @@ General实体和关系提取提示来自 GitHub - microsoft/graphrag基于
exportJson: '导出 JSON',
viewResult: '查看结果',
running: '运行中',
context: '上下文生成器',
contextDescription: '上下文生成器',
summary: '摘要',
keywords: '关键词',
questions: '问题',
metadata: '元数据',
fieldName: '结果目的地',
},
},
};

View File

@ -7,6 +7,7 @@ import {
AgentGlobalsSysQueryWithBrace,
CodeTemplateStrMap,
ProgrammingLanguage,
initialLlmBaseValues,
} from '@/constants/agent';
export enum AgentDialogueMode {
@ -14,13 +15,8 @@ export enum AgentDialogueMode {
Task = 'task',
}
import {
ChatVariableEnabledField,
variableEnabledFieldMap,
} from '@/constants/chat';
import { ModelVariableType } from '@/constants/knowledge';
import i18n from '@/locales/config';
import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat';
import { t } from 'i18next';
// DuckDuckGo's channel options
@ -271,24 +267,6 @@ export const initialBeginValues = {
prologue: `Hi! I'm your assistant. What can I do for you?`,
};
export const variableCheckBoxFieldMap = Object.keys(
variableEnabledFieldMap,
).reduce<Record<string, boolean>>((pre, cur) => {
pre[cur] = setInitialChatVariableEnabledFieldValue(
cur as ChatVariableEnabledField,
);
return pre;
}, {});
const initialLlmBaseValues = {
...variableCheckBoxFieldMap,
temperature: 0.1,
top_p: 0.3,
frequency_penalty: 0.7,
presence_penalty: 0.4,
max_tokens: 256,
};
export const initialGenerateValues = {
...initialLlmBaseValues,
prompt: i18n.t('flow.promptText'),

View File

@ -2,7 +2,6 @@ import { IRagNode } from '@/interfaces/database/flow';
import { NodeProps, Position } from '@xyflow/react';
import { memo } from 'react';
import { NodeHandleId } from '../../constant';
import { needsSingleStepDebugging } from '../../utils';
import { CommonHandle } from './handle';
import { LeftHandleStyle, RightHandleStyle } from './handle-icon';
import NodeHeader from './node-header';
@ -16,12 +15,7 @@ function InnerRagNode({
selected,
}: NodeProps<IRagNode>) {
return (
<ToolBar
selected={selected}
id={id}
label={data.label}
showRun={needsSingleStepDebugging(data.label)}
>
<ToolBar selected={selected} id={id} label={data.label}>
<NodeWrapper selected={selected}>
<CommonHandle
id={NodeHandleId.End}

View File

@ -34,7 +34,7 @@ export function ToolBar({
children,
label,
id,
showRun = true,
showRun = false,
}: ToolBarProps) {
const deleteNodeById = useGraphStore((store) => store.deleteNodeById);

View File

@ -1,3 +1,5 @@
import { ParseDocumentType } from '@/components/layout-recognize-form-field';
import { initialLlmBaseValues } from '@/constants/agent';
import {
ChatVariableEnabledField,
variableEnabledFieldMap,
@ -15,6 +17,89 @@ import {
WrapText,
} from 'lucide-react';
export enum FileType {
PDF = 'pdf',
Spreadsheet = 'spreadsheet',
Image = 'image',
Email = 'email',
TextMarkdown = 'text&markdown',
Docx = 'word',
PowerPoint = 'slides',
Video = 'video',
Audio = 'audio',
}
export enum PdfOutputFormat {
Json = 'json',
Markdown = 'markdown',
}
export enum SpreadsheetOutputFormat {
Json = 'json',
Html = 'html',
}
export enum ImageOutputFormat {
Text = 'text',
}
export enum EmailOutputFormat {
Json = 'json',
Text = 'text',
}
export enum TextMarkdownOutputFormat {
Text = 'text',
}
export enum DocxOutputFormat {
Markdown = 'markdown',
Json = 'json',
}
export enum PptOutputFormat {
Json = 'json',
}
export enum VideoOutputFormat {
Json = 'json',
}
export enum AudioOutputFormat {
Text = 'text',
}
export const OutputFormatMap = {
[FileType.PDF]: PdfOutputFormat,
[FileType.Spreadsheet]: SpreadsheetOutputFormat,
[FileType.Image]: ImageOutputFormat,
[FileType.Email]: EmailOutputFormat,
[FileType.TextMarkdown]: TextMarkdownOutputFormat,
[FileType.Docx]: DocxOutputFormat,
[FileType.PowerPoint]: PptOutputFormat,
[FileType.Video]: VideoOutputFormat,
[FileType.Audio]: AudioOutputFormat,
};
export const InitialOutputFormatMap = {
[FileType.PDF]: PdfOutputFormat.Json,
[FileType.Spreadsheet]: SpreadsheetOutputFormat.Html,
[FileType.Image]: ImageOutputFormat.Text,
[FileType.Email]: EmailOutputFormat.Text,
[FileType.TextMarkdown]: TextMarkdownOutputFormat.Text,
[FileType.Docx]: DocxOutputFormat.Json,
[FileType.PowerPoint]: PptOutputFormat.Json,
[FileType.Video]: VideoOutputFormat.Json,
[FileType.Audio]: AudioOutputFormat.Text,
};
export enum ContextGeneratorFieldName {
Summary = 'summary',
Keywords = 'keywords',
Questions = 'questions',
Metadata = 'metadata',
}
export enum PromptRole {
User = 'user',
Assistant = 'assistant',
@ -83,9 +168,28 @@ export enum TokenizerFields {
Summary = 'summary',
}
export enum ParserFields {
From = 'from',
To = 'to',
Cc = 'cc',
Bcc = 'bcc',
Date = 'date',
Subject = 'subject',
Body = 'body',
Attachments = 'attachments',
}
export const initialBeginValues = {
mode: AgentDialogueMode.Conversational,
prologue: `Hi! I'm your assistant. What can I do for you?`,
outputs: {
name: {
type: 'string',
value: '',
},
file: {
type: 'Object',
value: {},
},
},
};
export const variableCheckBoxFieldMap = Object.keys(
@ -125,10 +229,40 @@ export enum StringTransformDelimiter {
Space = ' ',
}
export const initialParserValues = { outputs: {}, setups: [] };
export const initialParserValues = {
outputs: {
markdown: { type: 'string', value: '' },
text: { type: 'string', value: '' },
html: { type: 'string', value: '' },
json: { type: 'Array<object>', value: [] },
},
setups: [
{
fileFormat: FileType.PDF,
output_format: PdfOutputFormat.Json,
parse_method: ParseDocumentType.DeepDOC,
},
{
fileFormat: FileType.Spreadsheet,
output_format: SpreadsheetOutputFormat.Html,
},
{
fileFormat: FileType.Image,
output_format: ImageOutputFormat.Text,
parse_method: ImageParseMethod.OCR,
},
{
fileFormat: FileType.Email,
fields: Object.values(ParserFields),
output_format: EmailOutputFormat.Text,
},
],
};
export const initialSplitterValues = {
outputs: {},
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},
chunk_token_size: 512,
overlapped_percent: 0,
delimiters: [{ value: '\n' }],
@ -143,7 +277,9 @@ export enum Hierarchy {
}
export const initialHierarchicalMergerValues = {
outputs: {},
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},
hierarchy: Hierarchy.H3,
levels: [
{ expressions: [{ expression: '^#[^#]' }] },
@ -154,6 +290,8 @@ export const initialHierarchicalMergerValues = {
};
export const initialContextValues = {
...initialLlmBaseValues,
field_name: [ContextGeneratorFieldName.Summary],
outputs: {},
};
@ -232,18 +370,6 @@ export enum AgentExceptionMethod {
Goto = 'goto',
}
export enum FileType {
PDF = 'pdf',
Spreadsheet = 'spreadsheet',
Image = 'image',
Email = 'email',
TextMarkdown = 'text&markdown',
Docx = 'word',
PowerPoint = 'slides',
Video = 'video',
Audio = 'audio',
}
export const FileTypeSuffixMap = {
[FileType.PDF]: ['pdf'],
[FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'],

View File

@ -1,18 +1,24 @@
import { DelimiterInput } from '@/components/delimiter-form-field';
import { LargeModelFormField } from '@/components/large-model-form-field';
import { LlmSettingSchema } from '@/components/llm-setting-items/next';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import { BlockButton, Button } from '@/components/ui/button';
import { Form } from '@/components/ui/form';
import { MultiSelect } from '@/components/ui/multi-select';
import { useBuildPromptExtraPromptOptions } from '@/pages/agent/form/agent-form/use-build-prompt-options';
import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';
import { buildOptions } from '@/utils/form';
import { zodResolver } from '@hookform/resolvers/zod';
import { Trash2 } from 'lucide-react';
import { memo } from 'react';
import { useFieldArray, useForm } from 'react-hook-form';
import { useForm } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import { initialContextValues } from '../../constant';
import {
ContextGeneratorFieldName,
initialContextValues,
} from '../../constant';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
import useGraphStore from '../../store';
import { buildOutputList } from '../../utils/build-output-list';
import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
@ -20,13 +26,10 @@ import { Output } from '../components/output';
const outputList = buildOutputList(initialContextValues.outputs);
export const FormSchema = z.object({
chunk_token_size: z.number(),
delimiters: z.array(
z.object({
value: z.string().optional(),
}),
),
overlapped_percent: z.number(), // 0.0 - 0.3
sys_prompt: z.string(),
prompts: z.string().optional(),
...LlmSettingSchema,
field_name: z.array(z.string()),
});
export type ContextFormSchemaType = z.infer<typeof FormSchema>;
@ -39,58 +42,39 @@ const ContextForm = ({ node }: INextOperatorForm) => {
defaultValues,
resolver: zodResolver(FormSchema),
});
const name = 'delimiters';
const { fields, append, remove } = useFieldArray({
name: name,
control: form.control,
});
const { edges } = useGraphStore((state) => state);
const { extraOptions } = useBuildPromptExtraPromptOptions(edges, node?.id);
const options = buildOptions(ContextGeneratorFieldName, t, 'dataflow');
useWatchFormChange(node?.id, form);
return (
<Form {...form}>
<FormWrapper>
<SliderInputFormField
name="chunk_token_size"
max={2048}
label={t('knowledgeConfiguration.chunkTokenNumber')}
></SliderInputFormField>
<SliderInputFormField
name="overlapped_percent"
max={0.3}
min={0}
step={0.01}
label={t('dataflow.overlappedPercent')}
></SliderInputFormField>
<section>
<span className="mb-2 inline-block">{t('flow.delimiters')}</span>
<div className="space-y-4">
{fields.map((field, index) => (
<div key={field.id} className="flex items-center gap-2">
<div className="space-y-2 flex-1">
<RAGFlowFormItem
name={`${name}.${index}.value`}
label="delimiter"
labelClassName="!hidden"
>
<DelimiterInput className="!m-0"></DelimiterInput>
</RAGFlowFormItem>
</div>
<Button
type="button"
variant={'ghost'}
onClick={() => remove(index)}
>
<Trash2 />
</Button>
</div>
))}
</div>
</section>
<BlockButton onClick={() => append({ value: '\n' })}>
{t('common.add')}
</BlockButton>
<LargeModelFormField></LargeModelFormField>
<RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt">
<PromptEditor
placeholder={t('flow.messagePlaceholder')}
showToolbar={true}
extraOptions={extraOptions}
></PromptEditor>
</RAGFlowFormItem>
<RAGFlowFormItem label={t('flow.userPrompt')} name="prompts">
<PromptEditor showToolbar={true}></PromptEditor>
</RAGFlowFormItem>
<RAGFlowFormItem label={t('dataflow.fieldName')} name="field_name">
{(field) => (
<MultiSelect
onValueChange={field.onChange}
placeholder={t('dataFlowPlaceholder')}
defaultValue={field.value}
options={options}
></MultiSelect>
)}
</RAGFlowFormItem>
</FormWrapper>
<div className="p-5">
<Output list={outputList}></Output>

View File

@ -8,8 +8,7 @@ import {
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { buildOptions } from '@/utils/form';
import { useTranslation } from 'react-i18next';
import { FileType } from '../../constant';
import { OutputFormatMap } from './constant';
import { FileType, OutputFormatMap } from '../../constant';
import { CommonProps } from './interface';
import { buildFieldNameWithPrefix } from './utils';

View File

@ -1,65 +0,0 @@
import { FileType } from '../../constant';
export enum PdfOutputFormat {
Json = 'json',
Markdown = 'markdown',
}
export enum SpreadsheetOutputFormat {
Json = 'json',
Html = 'html',
}
export enum ImageOutputFormat {
Text = 'text',
}
export enum EmailOutputFormat {
Json = 'json',
Text = 'text',
}
export enum TextMarkdownOutputFormat {
Text = 'text',
}
export enum DocxOutputFormat {
Markdown = 'markdown',
Json = 'json',
}
export enum PptOutputFormat {
Json = 'json',
}
export enum VideoOutputFormat {
Json = 'json',
}
export enum AudioOutputFormat {
Text = 'text',
}
export const OutputFormatMap = {
[FileType.PDF]: PdfOutputFormat,
[FileType.Spreadsheet]: SpreadsheetOutputFormat,
[FileType.Image]: ImageOutputFormat,
[FileType.Email]: EmailOutputFormat,
[FileType.TextMarkdown]: TextMarkdownOutputFormat,
[FileType.Docx]: DocxOutputFormat,
[FileType.PowerPoint]: PptOutputFormat,
[FileType.Video]: VideoOutputFormat,
[FileType.Audio]: AudioOutputFormat,
};
export const InitialOutputFormatMap = {
[FileType.PDF]: PdfOutputFormat.Json,
[FileType.Spreadsheet]: SpreadsheetOutputFormat.Html,
[FileType.Image]: ImageOutputFormat.Text,
[FileType.Email]: EmailOutputFormat.Text,
[FileType.TextMarkdown]: TextMarkdownOutputFormat.Text,
[FileType.Docx]: DocxOutputFormat.Json,
[FileType.PowerPoint]: PptOutputFormat.Json,
[FileType.Video]: VideoOutputFormat.Json,
[FileType.Audio]: AudioOutputFormat.Text,
};

View File

@ -1,20 +1,12 @@
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { MultiSelect } from '@/components/ui/multi-select';
import { buildOptions } from '@/utils/form';
import { useTranslation } from 'react-i18next';
import { ParserFields } from '../../constant';
import { CommonProps } from './interface';
import { buildFieldNameWithPrefix } from './utils';
const options = buildOptions([
'from',
'to',
'cc',
'bcc',
'date',
'subject',
'body',
'attachments',
]);
const options = buildOptions(ParserFields);
export function EmailFormFields({ prefix }: CommonProps) {
const { t } = useTranslation();
@ -24,7 +16,14 @@ export function EmailFormFields({ prefix }: CommonProps) {
name={buildFieldNameWithPrefix(`fields`, prefix)}
label={t('dataflow.fields')}
>
<SelectWithSearch options={options}></SelectWithSearch>
{(field) => (
<MultiSelect
options={options}
onValueChange={field.onChange}
defaultValue={field.value}
variant="inverted"
></MultiSelect>
)}
</RAGFlowFormItem>
</>
);

View File

@ -17,14 +17,17 @@ import {
} from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import { FileType, initialParserValues } from '../../constant';
import {
FileType,
InitialOutputFormatMap,
initialParserValues,
} from '../../constant';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
import { buildOutputList } from '../../utils/build-output-list';
import { Output } from '../components/output';
import { OutputFormatFormField } from './common-form-fields';
import { InitialOutputFormatMap } from './constant';
import { EmailFormFields } from './email-form-fields';
import { ImageFormFields } from './image-form-fields';
import { PdfFormFields } from './pdf-form-fields';

View File

@ -1,3 +1,4 @@
import { useFetchModelId } from '@/hooks/logic-hooks';
import { Connection, Node, Position, ReactFlowInstance } from '@xyflow/react';
import humanId from 'human-id';
import { lowerFirst } from 'lodash';
@ -22,6 +23,8 @@ import {
} from '../utils';
export const useInitializeOperatorParams = () => {
const llmId = useFetchModelId();
const initialFormValuesMap = useMemo(() => {
return {
[Operator.Begin]: initialBeginValues,
@ -30,9 +33,9 @@ export const useInitializeOperatorParams = () => {
[Operator.Tokenizer]: initialTokenizerValues,
[Operator.Splitter]: initialSplitterValues,
[Operator.HierarchicalMerger]: initialHierarchicalMergerValues,
[Operator.Context]: initialContextValues,
[Operator.Context]: { ...initialContextValues, llm_id: llmId },
};
}, []);
}, [llmId]);
const initializeOperatorParams = useCallback(
(operatorName: Operator) => {

View File

@ -4,6 +4,7 @@ import {
Blocks,
File,
FileChartColumnIncreasing,
FileStack,
Heading,
ListMinus,
} from 'lucide-react';
@ -24,6 +25,7 @@ export const SVGIconMap = {
[Operator.Tokenizer]: ListMinus,
[Operator.Splitter]: Blocks,
[Operator.HierarchicalMerger]: Heading,
[Operator.Context]: FileStack,
};
const Empty = () => {

View File

@ -1,7 +1,6 @@
import { IAgentForm } from '@/interfaces/database/agent';
import { DSLComponents, RAGFlowNodeType } from '@/interfaces/database/flow';
import { removeUselessFieldsFromValues } from '@/utils/form';
import { Edge, Node, XYPosition } from '@xyflow/react';
import { Edge, XYPosition } from '@xyflow/react';
import { FormInstance, FormListFieldData } from 'antd';
import { humanId } from 'human-id';
import { curry, get, intersectionWith, isEmpty, isEqual, sample } from 'lodash';
@ -24,24 +23,13 @@ const buildComponentDownstreamOrUpstream = (
edges: Edge[],
nodeId: string,
isBuildDownstream = true,
nodes: Node[],
) => {
return edges
.filter((y) => {
const node = nodes.find((x) => x.id === nodeId);
let isNotUpstreamTool = true;
let isNotUpstreamAgent = true;
let isNotExceptionGoto = true;
if (isBuildDownstream && node?.data.label === Operator.Agent) {
isNotExceptionGoto = y.sourceHandle !== NodeHandleId.AgentException;
// Exclude the tool operator downstream of the agent operator
isNotUpstreamTool = !y.target.startsWith(Operator.Tool);
// Exclude the agent operator downstream of the agent operator
isNotUpstreamAgent = !(
y.target.startsWith(Operator.Agent) &&
y.targetHandle === NodeHandleId.AgentTop
);
}
return (
y[isBuildDownstream ? 'source' : 'target'] === nodeId &&
isNotUpstreamTool &&
@ -54,9 +42,9 @@ const buildComponentDownstreamOrUpstream = (
const removeUselessDataInTheOperator = curry(
(operatorName: string, params: Record<string, unknown>) => {
if (operatorName === Operator.Categorize) {
return removeUselessFieldsFromValues(params, '');
}
// if (operatorName === Operator.Categorize) {
// return removeUselessFieldsFromValues(params, '');
// }
return params;
},
);
@ -197,8 +185,8 @@ export const buildDslComponentsByGraph = (
component_name: operatorName,
params: buildOperatorParams(operatorName)(params) ?? {},
},
downstream: buildComponentDownstreamOrUpstream(edges, id, true, nodes),
upstream: buildComponentDownstreamOrUpstream(edges, id, false, nodes),
downstream: buildComponentDownstreamOrUpstream(edges, id, true),
upstream: buildComponentDownstreamOrUpstream(edges, id, false),
parent_id: x?.parentId,
};
});
@ -352,25 +340,6 @@ export const generateNodeNamesWithIncreasingIndex = (
export const duplicateNodeForm = (nodeData?: RAGFlowNodeType['data']) => {
const form: Record<string, any> = { ...(nodeData?.form ?? {}) };
// Delete the downstream node corresponding to the to field of the Categorize operator
if (nodeData?.label === Operator.Categorize) {
form.category_description = Object.keys(form.category_description).reduce<
Record<string, Record<string, any>>
>((pre, cur) => {
pre[cur] = {
...form.category_description[cur],
to: undefined,
};
return pre;
}, {});
}
// Delete the downstream nodes corresponding to the yes and no fields of the Relevant operator
if (nodeData?.label === Operator.Relevant) {
form.yes = undefined;
form.no = undefined;
}
return {
...(nodeData ?? { label: '' }),
form,