pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-22 05:50:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: Create a data flow #9869 (#10131)

Browse Source 
### What problem does this PR solve?

Feat: Create a data flow #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
balibabu
2025-09-17 17:54:21 +08:00
committed by GitHub
parent ccb255919a
commit cf1f523d03
17 changed files with 189 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx
View File

@ -1,9 +1,3 @@
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from '@/components/ui/accordion';
import {
DropdownMenu,
DropdownMenuContent,
@ -124,84 +118,16 @@ function AccordionOperators({
mousePosition?: { x: number; y: number };
}) {
return (
<Accordion
type="multiple"
className="px-2 text-text-title max-h-[45vh] overflow-auto"
defaultValue={['item-1', 'item-2', 'item-3', 'item-4', 'item-5']}
>
<AccordionItem value="item-1">
<AccordionTrigger className="text-xl">
{t('flow.foundation')}
</AccordionTrigger>
<AccordionContent className="flex flex-col gap-4 text-balance">
<OperatorItemList
operators={[
Operator.Agent,
Operator.Retrieval,
Operator.Parser,
Operator.Chunker,
Operator.Tokenizer,
Operator.Splitter,
Operator.HierarchicalMerger,
]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
</AccordionContent>
</AccordionItem>
<AccordionItem value="item-2">
<AccordionTrigger className="text-xl">
{t('flow.dialog')}
</AccordionTrigger>
<AccordionContent className="flex flex-col gap-4 text-balance">
<OperatorItemList
operators={[Operator.Message, Operator.UserFillUp]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
</AccordionContent>
</AccordionItem>
<AccordionItem value="item-3">
<AccordionTrigger className="text-xl">
{t('flow.flow')}
</AccordionTrigger>
<AccordionContent className="flex flex-col gap-4 text-balance">
<OperatorItemList
operators={[
Operator.Switch,
Operator.Iteration,
Operator.Categorize,
]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
</AccordionContent>
</AccordionItem>
<AccordionItem value="item-4">
<AccordionTrigger className="text-xl">
{t('flow.dataManipulation')}
</AccordionTrigger>
<AccordionContent className="flex flex-col gap-4 text-balance">
<OperatorItemList
operators={[Operator.Code, Operator.StringTransform]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
</AccordionContent>
</AccordionItem>
<AccordionItem value="item-5">
<AccordionTrigger className="text-xl">
{t('flow.tools')}
</AccordionTrigger>
<AccordionContent className="flex flex-col gap-4 text-balance">
<OperatorItemList
operators={[Operator.ExeSQL, Operator.Email, Operator.Invoke]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
</AccordionContent>
</AccordionItem>
</Accordion>
<OperatorItemList
operators={[
Operator.Parser,
Operator.Tokenizer,
Operator.Splitter,
Operator.HierarchicalMerger,
]}
isCustomDropdown={isCustomDropdown}
mousePosition={mousePosition}
></OperatorItemList>
);
}

51
web/src/pages/data-flow/constant.tsx
View File

@ -307,7 +307,11 @@ export const initialWaitingDialogueValues = {};
export const initialChunkerValues = { outputs: {} };
export const initialTokenizerValues = {};
export const initialTokenizerValues = {
search_method: [],
filename_embd_weight: 0.1,
outputs: {},
};
export const initialAgentValues = {
...initialLlmBaseValues,
@ -401,42 +405,10 @@ export const CategorizeAnchorPointPositions = [
// no connection lines are allowed between key and value
export const RestrictedUpstreamMap = {
[Operator.Begin]: [Operator.Relevant],
[Operator.Categorize]: [Operator.Begin, Operator.Categorize],
[Operator.Retrieval]: [Operator.Begin, Operator.Retrieval],
[Operator.Message]: [
Operator.Begin,
Operator.Message,
Operator.Retrieval,
Operator.RewriteQuestion,
Operator.Categorize,
],
[Operator.Relevant]: [Operator.Begin],
[Operator.RewriteQuestion]: [
Operator.Begin,
Operator.Message,
Operator.RewriteQuestion,
Operator.Relevant,
],
[Operator.KeywordExtract]: [
Operator.Begin,
Operator.Message,
Operator.Relevant,
],
[Operator.ExeSQL]: [Operator.Begin],
[Operator.Switch]: [Operator.Begin],
[Operator.Concentrator]: [Operator.Begin],
[Operator.Crawler]: [Operator.Begin],
[Operator.Note]: [],
[Operator.Invoke]: [Operator.Begin],
[Operator.Email]: [Operator.Begin],
[Operator.Iteration]: [Operator.Begin],
[Operator.IterationStart]: [Operator.Begin],
[Operator.Code]: [Operator.Begin],
[Operator.WaitingDialogue]: [Operator.Begin],
[Operator.Agent]: [Operator.Begin],
[Operator.StringTransform]: [Operator.Begin],
[Operator.UserFillUp]: [Operator.Begin],
[Operator.Tool]: [Operator.Begin],
[Operator.Parser]: [Operator.Begin],
[Operator.Splitter]: [Operator.Begin],
[Operator.HierarchicalMerger]: [Operator.Begin],
[Operator.Tokenizer]: [Operator.Begin],
};
export const NodeMap = {
@ -529,3 +501,8 @@ export enum FileType {
Video = 'video',
Audio = 'audio',
}
export enum TokenizerSearchMethod {
Embedding = 'embedding',
FullText = 'full_text',
}

4
web/src/pages/data-flow/form/hierarchical-merger-form/index.tsx
View File

@ -44,6 +44,8 @@ export const FormSchema = z.object({
),
});
export type HierarchicalMergerFormSchemaType = z.infer<typeof FormSchema>;
type RegularExpressionsProps = {
index: number;
parentName: string;
@ -113,7 +115,7 @@ export function RegularExpressions({
const HierarchicalMergerForm = ({ node }: INextOperatorForm) => {
const defaultValues = useFormValues(initialHierarchicalMergerValues, node);
const form = useForm<z.infer<typeof FormSchema>>({
const form = useForm<HierarchicalMergerFormSchemaType>({
defaultValues,
resolver: zodResolver(FormSchema),
});

4
web/src/pages/data-flow/form/parser-form/index.tsx
View File

@ -62,11 +62,11 @@ export const FormSchema = z.object({
),
});
export type FormSchemaType = z.infer<typeof FormSchema>;
export type ParserFormSchemaType = z.infer<typeof FormSchema>;
function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) {
const { t } = useTranslation();
const form = useFormContext<FormSchemaType>();
const form = useFormContext<ParserFormSchemaType>();
const ref = useRef(null);
const isHovering = useHover(ref);

4
web/src/pages/data-flow/form/splitter-form/index.tsx
View File

@ -29,10 +29,12 @@ export const FormSchema = z.object({
overlapped_percent: z.number(), // 0.0 - 0.3
});
export type SplitterFormSchemaType = z.infer<typeof FormSchema>;
const SplitterForm = ({ node }: INextOperatorForm) => {
const defaultValues = useFormValues(initialChunkerValues, node);
const form = useForm<z.infer<typeof FormSchema>>({
const form = useForm<SplitterFormSchemaType>({
defaultValues,
resolver: zodResolver(FormSchema),
});

135
web/src/pages/data-flow/form/tokenizer-form/index.tsx
View File

@ -1,94 +1,38 @@
import { FormContainer } from '@/components/form-container';
import NumberInput from '@/components/originui/number-input';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from '@/components/ui/form';
import { useTranslate } from '@/hooks/common-hooks';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import { Form } from '@/components/ui/form';
import { MultiSelect } from '@/components/ui/multi-select';
import { buildOptions } from '@/utils/form';
import { zodResolver } from '@hookform/resolvers/zod';
import { memo } from 'react';
import { useForm, useFormContext } from 'react-hook-form';
import { useForm } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import { initialChunkerValues } from '../../constant';
import { initialTokenizerValues, TokenizerSearchMethod } from '../../constant';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options';
import { buildOutputList } from '../../utils/build-output-list';
import { ApiKeyField } from '../components/api-key-field';
import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
import { QueryVariable } from '../components/query-variable';
const outputList = buildOutputList(initialChunkerValues.outputs);
export const GoogleFormPartialSchema = {
api_key: z.string(),
country: z.string(),
language: z.string(),
};
const outputList = buildOutputList(initialTokenizerValues.outputs);
export const FormSchema = z.object({
...GoogleFormPartialSchema,
q: z.string(),
start: z.number(),
num: z.number(),
search_method: z.array(z.string()).min(1),
filename_embd_weight: z.number(),
});
export function GoogleFormWidgets() {
const form = useFormContext();
const { t } = useTranslate('flow');
return (
<>
<FormField
control={form.control}
name={`country`}
render={({ field }) => (
<FormItem className="flex-1">
<FormLabel>{t('country')}</FormLabel>
<FormControl>
<SelectWithSearch
{...field}
options={GoogleCountryOptions}
></SelectWithSearch>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name={`language`}
render={({ field }) => (
<FormItem className="flex-1">
<FormLabel>{t('language')}</FormLabel>
<FormControl>
<SelectWithSearch
{...field}
options={GoogleLanguageOptions}
></SelectWithSearch>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
</>
);
}
const SearchMethodOptions = buildOptions(TokenizerSearchMethod);
const TokenizerForm = ({ node }: INextOperatorForm) => {
const { t } = useTranslate('flow');
const defaultValues = useFormValues(initialChunkerValues, node);
const { t } = useTranslation();
const defaultValues = useFormValues(initialTokenizerValues, node);
const form = useForm<z.infer<typeof FormSchema>>({
defaultValues,
resolver: zodResolver(FormSchema),
mode: 'onChange',
});
useWatchFormChange(node?.id, form);
@ -96,39 +40,22 @@ const TokenizerForm = ({ node }: INextOperatorForm) => {
return (
<Form {...form}>
<FormWrapper>
<FormContainer>
<QueryVariable name="q"></QueryVariable>
</FormContainer>
<FormContainer>
<ApiKeyField placeholder={t('apiKeyPlaceholder')}></ApiKeyField>
<FormField
control={form.control}
name={`start`}
render={({ field }) => (
<FormItem>
<FormLabel>{t('flowStart')}</FormLabel>
<FormControl>
<NumberInput {...field} className="w-full"></NumberInput>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name={`num`}
render={({ field }) => (
<FormItem>
<FormLabel>{t('flowNum')}</FormLabel>
<FormControl>
<NumberInput {...field} className="w-full"></NumberInput>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<GoogleFormWidgets></GoogleFormWidgets>
</FormContainer>
<RAGFlowFormItem name="search_method" label={t('search_method')}>
{(field) => (
<MultiSelect
options={SearchMethodOptions}
onValueChange={field.onChange}
defaultValue={field.value}
variant="inverted"
/>
)}
</RAGFlowFormItem>
<SliderInputFormField
name="filename_embd_weight"
label="filename_embd_weight"
max={0.5}
step={0.01}
></SliderInputFormField>
</FormWrapper>
<div className="p-5">
<Output list={outputList}></Output>

1
web/src/pages/data-flow/hooks/use-save-graph.ts
View File

@ -22,6 +22,7 @@ export const useSaveGraph = (showMessage: boolean = true) => {
return setAgent({
id,
title: data.title,
canvas_category: data.canvas_category,
dsl: buildDslData(currentNodes),
});
},

68
web/src/pages/data-flow/utils.ts
View File

@ -9,7 +9,15 @@ import { removeUselessFieldsFromValues } from '@/utils/form';
import { Edge, Node, XYPosition } from '@xyflow/react';
import { FormInstance, FormListFieldData } from 'antd';
import { humanId } from 'human-id';
import { curry, get, intersectionWith, isEqual, omit, sample } from 'lodash';
import {
curry,
get,
intersectionWith,
isEmpty,
isEqual,
omit,
sample,
} from 'lodash';
import pipe from 'lodash/fp/pipe';
import isObject from 'lodash/isObject';
import {
@ -18,6 +26,9 @@ import {
NodeHandleId,
Operator,
} from './constant';
import { HierarchicalMergerFormSchemaType } from './form/hierarchical-merger-form';
import { ParserFormSchemaType } from './form/parser-form';
import { SplitterFormSchemaType } from './form/splitter-form';
import { BeginQuery, IPosition } from './interface';
function buildAgentExceptionGoto(edges: Edge[], nodeId: string) {
@ -63,10 +74,7 @@ const buildComponentDownstreamOrUpstream = (
const removeUselessDataInTheOperator = curry(
(operatorName: string, params: Record<string, unknown>) => {
if (
operatorName === Operator.Generate ||
operatorName === Operator.Categorize
) {
if (operatorName === Operator.Categorize) {
return removeUselessFieldsFromValues(params, '');
}
return params;
@ -151,6 +159,44 @@ export function isBottomSubAgent(edges: Edge[], nodeId?: string) {
);
return !!edge;
}
// Because the array of react-hook-form must be object data,
// it needs to be converted into a simple data type array required by the backend
function transformObjectArrayToPureArray(
list: Array<Record<string, any>>,
field: string,
) {
return Array.isArray(list)
? list.filter((x) => !isEmpty(x[field])).map((y) => y[field])
: [];
}
function transformParserParams(params: ParserFormSchemaType) {
return params.parser.reduce<
Record<string, ParserFormSchemaType['parser'][0]>
>((pre, cur) => {
if (cur.fileFormat) {
pre[cur.fileFormat] = omit(cur, 'fileFormat');
}
return pre;
}, {});
}
function transformSplitterParams(params: SplitterFormSchemaType) {
return {
...params,
delimiters: transformObjectArrayToPureArray(params.delimiters, 'value'),
};
}
function transformHierarchicalMergerParams(
params: HierarchicalMergerFormSchemaType,
) {
const levels = params.levels.map((x) =>
transformObjectArrayToPureArray(x.expressions, 'expression'),
);
return { ...params, hierarchy: Number(params.hierarchy), levels };
}
// construct a dsl based on the node information of the graph
export const buildDslComponentsByGraph = (
@ -184,6 +230,18 @@ export const buildDslComponentsByGraph = (
params = buildCategorize(edges, nodes, id);
break;
case Operator.Parser:
params = transformParserParams(params);
break;
case Operator.Splitter:
params = transformSplitterParams(params);
break;
case Operator.HierarchicalMerger:
params = transformHierarchicalMergerParams(params);
break;
default:
break;
}