diff --git a/agent/component/list_operations.py b/agent/component/list_operations.py new file mode 100644 index 000000000..c29d79ea6 --- /dev/null +++ b/agent/component/list_operations.py @@ -0,0 +1,149 @@ +from abc import ABC +import os +from agent.component.base import ComponentBase, ComponentParamBase +from api.utils.api_utils import timeout + +class ListOperationsParam(ComponentParamBase): + """ + Define the List Operations component parameters. + """ + def __init__(self): + super().__init__() + self.query = "" + self.operations = "topN" + self.n=0 + self.sort_method = "asc" + self.filter = { + "operator": "=", + "value": "" + } + self.outputs = { + "result": { + "value": [], + "type": "Array of ?" + }, + "first": { + "value": "", + "type": "?" + }, + "last": { + "value": "", + "type": "?" + } + } + + def check(self): + self.check_empty(self.query, "query") + self.check_valid_value(self.operations, "Support operations", ["topN","head","tail","filter","sort","drop_duplicates"]) + + def get_input_form(self) -> dict[str, dict]: + return {} + + +class ListOperations(ComponentBase,ABC): + component_name = "ListOperations" + + @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))) + def _invoke(self, **kwargs): + self.input_objects=[] + inputs = getattr(self._param, "query", None) + self.inputs=self._canvas.get_variable_value(inputs) + self.set_input_value(inputs, self.inputs) + if self._param.operations == "topN": + self._topN() + elif self._param.operations == "head": + self._head() + elif self._param.operations == "tail": + self._tail() + elif self._param.operations == "filter": + self._filter() + elif self._param.operations == "sort": + self._sort() + elif self._param.operations == "drop_duplicates": + self._drop_duplicates() + + + def _coerce_n(self): + try: + return int(getattr(self._param, "n", 0)) + except Exception: + return 0 + + def _set_outputs(self, outputs): + self._param.outputs["result"]["value"] = outputs + self._param.outputs["first"]["value"] = outputs[0] if outputs else None + self._param.outputs["last"]["value"] = outputs[-1] if outputs else None + + def _topN(self): + n = self._coerce_n() + if n < 1: + outputs = [] + else: + n = min(n, len(self.inputs)) + outputs = self.inputs[:n] + self._set_outputs(outputs) + + def _head(self): + n = self._coerce_n() + if 1 <= n <= len(self.inputs): + outputs = [self.inputs[n - 1]] + else: + outputs = [] + self._set_outputs(outputs) + + def _tail(self): + n = self._coerce_n() + if 1 <= n <= len(self.inputs): + outputs = [self.inputs[-n]] + else: + outputs = [] + self._set_outputs(outputs) + + def _filter(self): + self._set_outputs([i for i in self.inputs if self._eval(self._norm(i),self._param.filter["operator"],self._param.filter["value"])]) + + def _norm(self,v): + s = "" if v is None else str(v) + return s + + def _eval(self, v, operator, value): + if operator == "=": + return v == value + elif operator == "≠": + return v != value + elif operator == "contains": + return value in v + elif operator == "start with": + return v.startswith(value) + elif operator == "end with": + return v.endswith(value) + else: + return False + + def _sort(self): + if self._param.sort_method == "asc": + self._set_outputs(sorted(self.inputs)) + elif self._param.sort_method == "desc": + self._set_outputs(sorted(self.inputs, reverse=True)) + + def _drop_duplicates(self): + seen = set() + outs = [] + for item in self.inputs: + k = self._hashable(item) + if k in seen: + continue + seen.add(k) + outs.append(item) + self._set_outputs(outs) + + def _hashable(self,x): + if isinstance(x, dict): + return tuple(sorted((k, self._hashable(v)) for k, v in x.items())) + if isinstance(x, (list, tuple)): + return tuple(self._hashable(v) for v in x) + if isinstance(x, set): + return tuple(sorted(self._hashable(v) for v in x)) + return x + def thoughts(self) -> str: + return "ListOperation in progress" diff --git a/web/src/constants/agent.tsx b/web/src/constants/agent.tsx index 6ee8ab516..3a8411ce3 100644 --- a/web/src/constants/agent.tsx +++ b/web/src/constants/agent.tsx @@ -109,6 +109,7 @@ export enum Operator { SearXNG = 'SearXNG', Placeholder = 'Placeholder', DataOperations = 'DataOperations', + ListOperations = 'ListOperations', VariableAssigner = 'VariableAssigner', VariableAggregator = 'VariableAggregator', File = 'File', // pipeline diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 9a0569ab5..b9f374f7c 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1591,6 +1591,8 @@ This delimiter is used to split the input text into several text pieces echo of codeDescription: 'It allows developers to write custom Python logic.', dataOperations: 'Data operations', dataOperationsDescription: 'Perform various operations on a Data object.', + listOperations: 'List operations', + listOperationsDescription: 'Perform operations on a list.', variableAssigner: 'Variable assigner', variableAssignerDescription: 'This component performs operations on Data objects, including extracting, filtering, and editing keys and values in the Data.', @@ -1806,6 +1808,19 @@ Important structured information may include: names, dates, locations, events, k removeKeys: 'Remove keys', renameKeys: 'Rename keys', }, + ListOperationsOptions: { + topN: 'Top N', + head: 'Head', + tail: 'Tail', + sort: 'Sort', + filter: 'Filter', + dropDuplicates: 'Drop duplicates', + }, + sortMethod: 'Sort method', + SortMethodOptions: { + asc: 'Ascending', + desc: 'Descending', + }, }, llmTools: { bad_calculator: { diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index c065986f2..ce21c5a30 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1508,6 +1508,8 @@ General:实体和关系提取提示来自 GitHub - microsoft/graphrag:基于 codeDescription: '它允许开发人员编写自定义 Python 逻辑。', dataOperations: '数据操作', dataOperationsDescription: '对数据对象执行各种操作。', + listOperations: '列表操作', + listOperationsDescription: '对列表对象执行各种操作。', variableAssigner: '变量赋值器', variableAssignerDescription: '此组件对数据对象执行操作,包括提取、筛选和编辑数据中的键和值。', @@ -1679,6 +1681,19 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`, removeKeys: '删除键', renameKeys: '重命名键', }, + ListOperationsOptions: { + topN: '取前N项', + head: '取前第N项', + tail: '取后第N项', + sort: '排序', + filter: '筛选', + dropDuplicates: '去重', + }, + sortMethod: '排序方式', + SortMethodOptions: { + asc: '升序', + desc: '降序', + }, }, footer: { profile: 'All rights reserved @ React', diff --git a/web/src/pages/agent/canvas/index.tsx b/web/src/pages/agent/canvas/index.tsx index 5f78e8185..f2fc983e2 100644 --- a/web/src/pages/agent/canvas/index.tsx +++ b/web/src/pages/agent/canvas/index.tsx @@ -61,6 +61,7 @@ import { FileNode } from './node/file-node'; import { InvokeNode } from './node/invoke-node'; import { IterationNode, IterationStartNode } from './node/iteration-node'; import { KeywordNode } from './node/keyword-node'; +import { ListOperationsNode } from './node/list-operations-node'; import { MessageNode } from './node/message-node'; import NoteNode from './node/note-node'; import ParserNode from './node/parser-node'; @@ -101,6 +102,7 @@ export const nodeTypes: NodeTypes = { splitterNode: SplitterNode, contextNode: ExtractorNode, dataOperationsNode: DataOperationsNode, + listOperationsNode: ListOperationsNode, variableAssignerNode: VariableAssignerNode, variableAggregatorNode: VariableAggregatorNode, }; diff --git a/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx b/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx index 232ab78ff..8fd96f55f 100644 --- a/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx +++ b/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx @@ -79,6 +79,7 @@ export function AccordionOperators({ Operator.Code, Operator.StringTransform, Operator.DataOperations, + Operator.ListOperations, // Operator.VariableAssigner, Operator.VariableAggregator, ]} diff --git a/web/src/pages/agent/canvas/node/list-operations-node.tsx b/web/src/pages/agent/canvas/node/list-operations-node.tsx new file mode 100644 index 000000000..5b2778c92 --- /dev/null +++ b/web/src/pages/agent/canvas/node/list-operations-node.tsx @@ -0,0 +1,22 @@ +import { BaseNode } from '@/interfaces/database/agent'; +import { NodeProps } from '@xyflow/react'; +import { camelCase } from 'lodash'; +import { useTranslation } from 'react-i18next'; +import { RagNode } from '.'; +import { ListOperationsFormSchemaType } from '../../form/list-operations-form'; +import { LabelCard } from './card'; + +export function ListOperationsNode({ + ...props +}: NodeProps>) { + const { data } = props; + const { t } = useTranslation(); + + return ( + + + {t(`flow.ListOperationsOptions.${camelCase(data.form?.operations)}`)} + + + ); +} diff --git a/web/src/pages/agent/constant/index.tsx b/web/src/pages/agent/constant/index.tsx index 45341abf4..7aad5e4a3 100644 --- a/web/src/pages/agent/constant/index.tsx +++ b/web/src/pages/agent/constant/index.tsx @@ -595,6 +595,35 @@ export const initialDataOperationsValues = { }, }, }; +export enum SortMethod { + Asc = 'asc', + Desc = 'desc', +} + +export enum ListOperations { + TopN = 'topN', + Head = 'head', + Tail = 'tail', + Filter = 'filter', + Sort = 'sort', + DropDuplicates = 'drop_duplicates', +} + +export const initialListOperationsValues = { + query: '', + operations: ListOperations.TopN, + outputs: { + result: { + type: 'Array', + }, + first: { + type: '?', + }, + last: { + type: '?', + }, + }, +}; export const initialVariableAssignerValues = {}; @@ -673,6 +702,7 @@ export const RestrictedUpstreamMap = { [Operator.Tool]: [Operator.Begin], [Operator.Placeholder]: [Operator.Begin], [Operator.DataOperations]: [Operator.Begin], + [Operator.ListOperations]: [Operator.Begin], [Operator.Parser]: [Operator.Begin], // pipeline [Operator.Splitter]: [Operator.Begin], [Operator.HierarchicalMerger]: [Operator.Begin], @@ -729,6 +759,7 @@ export const NodeMap = { [Operator.HierarchicalMerger]: 'splitterNode', [Operator.Extractor]: 'contextNode', [Operator.DataOperations]: 'dataOperationsNode', + [Operator.ListOperations]: 'listOperationsNode', [Operator.VariableAssigner]: 'variableAssignerNode', [Operator.VariableAggregator]: 'variableAggregatorNode', }; diff --git a/web/src/pages/agent/form-sheet/form-config-map.tsx b/web/src/pages/agent/form-sheet/form-config-map.tsx index c291e4e05..37ab4cf2f 100644 --- a/web/src/pages/agent/form-sheet/form-config-map.tsx +++ b/web/src/pages/agent/form-sheet/form-config-map.tsx @@ -21,6 +21,7 @@ import IterationForm from '../form/iteration-form'; import IterationStartForm from '../form/iteration-start-from'; import Jin10Form from '../form/jin10-form'; import KeywordExtractForm from '../form/keyword-extract-form'; +import ListOperationsForm from '../form/list-operations-form'; import MessageForm from '../form/message-form'; import ParserForm from '../form/parser-form'; import PubMedForm from '../form/pubmed-form'; @@ -184,6 +185,9 @@ export const FormConfigMap = { [Operator.DataOperations]: { component: DataOperationsForm, }, + [Operator.ListOperations]: { + component: ListOperationsForm, + }, [Operator.VariableAssigner]: { component: VariableAssignerForm, }, diff --git a/web/src/pages/agent/form/list-operations-form/index.tsx b/web/src/pages/agent/form/list-operations-form/index.tsx new file mode 100644 index 000000000..5803fe055 --- /dev/null +++ b/web/src/pages/agent/form/list-operations-form/index.tsx @@ -0,0 +1,140 @@ +import NumberInput from '@/components/originui/number-input'; +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Separator } from '@/components/ui/separator'; +import { useBuildSwitchOperatorOptions } from '@/hooks/logic-hooks/use-build-operator-options'; +import { buildOptions } from '@/utils/form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { memo } from 'react'; +import { useForm, useWatch } from 'react-hook-form'; +import { useTranslation } from 'react-i18next'; +import { z } from 'zod'; +import { + DataOperationsOperatorOptions, + JsonSchemaDataType, + ListOperations, + SortMethod, + initialListOperationsValues, +} from '../../constant'; +import { useFormValues } from '../../hooks/use-form-values'; +import { useWatchFormChange } from '../../hooks/use-watch-form-change'; +import { INextOperatorForm } from '../../interface'; +import { buildOutputList } from '../../utils/build-output-list'; +import { FormWrapper } from '../components/form-wrapper'; +import { Output, OutputSchema } from '../components/output'; +import { PromptEditor } from '../components/prompt-editor'; +import { QueryVariable } from '../components/query-variable'; + +export const RetrievalPartialSchema = { + query: z.string(), + operations: z.string(), + n: z.number().int().min(0).optional(), + sort_method: z.string().optional(), + filter: z + .object({ + value: z.string().optional(), + operator: z.string().optional(), + }) + .optional(), + ...OutputSchema, +}; + +export const FormSchema = z.object(RetrievalPartialSchema); + +export type ListOperationsFormSchemaType = z.infer; + +const outputList = buildOutputList(initialListOperationsValues.outputs); + +function ListOperationsForm({ node }: INextOperatorForm) { + const { t } = useTranslation(); + + const defaultValues = useFormValues(initialListOperationsValues, node); + + const form = useForm({ + defaultValues: defaultValues, + mode: 'onChange', + resolver: zodResolver(FormSchema), + shouldUnregister: true, + }); + + const operations = useWatch({ control: form.control, name: 'operations' }); + + const ListOperationsOptions = buildOptions( + ListOperations, + t, + `flow.ListOperationsOptions`, + true, + ); + const SortMethodOptions = buildOptions( + SortMethod, + t, + `flow.SortMethodOptions`, + true, + ); + const operatorOptions = useBuildSwitchOperatorOptions( + DataOperationsOperatorOptions, + ); + useWatchFormChange(node?.id, form, true); + + return ( +
+ + + + + + + {[ + ListOperations.TopN, + ListOperations.Head, + ListOperations.Tail, + ].includes(operations as ListOperations) && ( + ( + + {t('flowNum')} + + + + + + )} + /> + )} + {[ListOperations.Sort].includes(operations as ListOperations) && ( + + + + )} + {[ListOperations.Filter].includes(operations as ListOperations) && ( +
+ + + + + + + +
+ )} + +
+
+ ); +} + +export default memo(ListOperationsForm); diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts index ed092a01b..44091f1b1 100644 --- a/web/src/pages/agent/hooks/use-add-node.ts +++ b/web/src/pages/agent/hooks/use-add-node.ts @@ -31,6 +31,7 @@ import { initialIterationValues, initialJin10Values, initialKeywordExtractValues, + initialListOperationsValues, initialMessageValues, initialNoteValues, initialParserValues, @@ -129,6 +130,7 @@ export const useInitializeOperatorParams = () => { prompts: t('flow.prompts.user.summary'), }, [Operator.DataOperations]: initialDataOperationsValues, + [Operator.ListOperations]: initialListOperationsValues, [Operator.VariableAssigner]: initialVariableAssignerValues, [Operator.VariableAggregator]: initialVariableAggregatorValues, }; diff --git a/web/src/pages/agent/operator-icon.tsx b/web/src/pages/agent/operator-icon.tsx index a7ece8ead..44fe9d01a 100644 --- a/web/src/pages/agent/operator-icon.tsx +++ b/web/src/pages/agent/operator-icon.tsx @@ -14,7 +14,7 @@ import { ReactComponent as YahooFinanceIcon } from '@/assets/svg/yahoo-finance.s import { IconFont } from '@/components/icon-font'; import { cn } from '@/lib/utils'; -import { Equal, FileCode, HousePlus, Variable } from 'lucide-react'; +import { Columns3, Equal, FileCode, HousePlus, Variable } from 'lucide-react'; import { Operator } from './constant'; interface IProps { @@ -57,6 +57,7 @@ export const SVGIconMap = { }; export const LucideIconMap = { [Operator.DataOperations]: FileCode, + [Operator.ListOperations]: Columns3, [Operator.VariableAssigner]: Equal, [Operator.VariableAggregator]: Variable, }; diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts index 3312b7236..a7d4248ff 100644 --- a/web/src/pages/agent/utils.ts +++ b/web/src/pages/agent/utils.ts @@ -328,7 +328,6 @@ export const buildDslComponentsByGraph = ( case Operator.DataOperations: params = transformDataOperationsParams(params); break; - default: break; }