Feat: Allow other operators to reference the structured output defined by the agent operator. #10866 (#10886)

### What problem does this PR solve?

Feat: Allow other operators to reference the structured output defined
by the agent operator. #10866
### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/pages/agent/constant/index.tsx

@@ -584,6 +584,8 @@ export const initialCodeValues = {
 
 export const initialWaitingDialogueValues = {};
 
+export const AgentStructuredOutputField = 'structured';
+
 export const initialAgentValues = {
   ...initialLlmBaseValues,
   description: '',
@@ -615,7 +617,7 @@ export const initialAgentValues = {
       type: 'string',
       value: '',
     },
-    structured: {},
+    [AgentStructuredOutputField]: {},
   },
 };
 

web/src/pages/agent/form/agent-form/structured-output-dialog.tsx

@@ -34,7 +34,7 @@ export function StructuredOutputDialog({
         <DialogHeader>
           <DialogTitle> {t('flow.structuredOutput.configuration')}</DialogTitle>
         </DialogHeader>
-        <section className="flex">
+        <section className="flex overflow-auto">
           <div className="flex-1">
             <SchemaVisualEditor schema={schema} onChange={setSchema} />
           </div>

web/src/pages/agent/form/components/prompt-editor/index.css

@@ -28,10 +28,10 @@
   cursor: pointer;
   line-height: 16px;
   font-size: 15px;
-  display: flex;
+  /* display: flex;
   align-content: center;
   flex-direction: row;
-  flex-shrink: 0;
+  flex-shrink: 0; */
   border: 0;
 }
 

web/src/pages/agent/form/components/prompt-editor/utils.ts

@@ -0,0 +1,44 @@
+import { JSONSchema } from '@/components/jsonjoy-builder';
+import { Operator } from '@/constants/agent';
+import { isPlainObject } from 'lodash';
+
+export function filterAgentStructuredOutput(
+  structuredOutput: JSONSchema,
+  operator?: string,
+) {
+  if (typeof structuredOutput === 'boolean') {
+    return structuredOutput;
+  }
+  if (
+    structuredOutput.properties &&
+    isPlainObject(structuredOutput.properties)
+  ) {
+    const filterByPredicate = (predicate: (value: JSONSchema) => boolean) => {
+      const properties = Object.entries({
+        ...structuredOutput.properties,
+      }).reduce(
+        (pre, [key, value]) => {
+          if (predicate(value)) {
+            pre[key] = value;
+          }
+          return pre;
+        },
+        {} as Record<string, JSONSchema>,
+      );
+
+      return { ...structuredOutput, properties };
+    };
+
+    if (operator === Operator.Iteration) {
+      return filterByPredicate(
+        (value) => typeof value !== 'boolean' && value.type === 'array',
+      );
+    } else {
+      return filterByPredicate(
+        (value) => typeof value !== 'boolean' && value.type !== 'array',
+      );
+    }
+  }
+
+  return structuredOutput;
+}

web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx

@@ -30,10 +30,21 @@ import * as ReactDOM from 'react-dom';
 
 import { $createVariableNode } from './variable-node';
 
+import {
+  HoverCard,
+  HoverCardContent,
+  HoverCardTrigger,
+} from '@/components/ui/hover-card';
+import { Operator } from '@/constants/agent';
+import { cn } from '@/lib/utils';
+import { AgentStructuredOutputField } from '@/pages/agent/constant';
 import { useBuildQueryVariableOptions } from '@/pages/agent/hooks/use-get-begin-query';
+import useGraphStore from '@/pages/agent/store';
+import { get, isPlainObject } from 'lodash';
 import { PromptIdentity } from '../../agent-form/use-build-prompt-options';
 import { ProgrammaticTag } from './constant';
 import './index.css';
+import { filterAgentStructuredOutput } from './utils';
 class VariableInnerOption extends MenuOption {
   label: string;
   value: string;
@@ -71,6 +82,10 @@ class VariableOption extends MenuOption {
   }
 }
 
+function getNodeId(value: string) {
+  return value.split('@').at(0);
+}
+
 function VariablePickerMenuItem({
   index,
   option,
@@ -82,6 +97,59 @@ function VariablePickerMenuItem({
     option: VariableOption | VariableInnerOption,
   ) => void;
 }) {
+  const { getOperatorTypeFromId, getNode, clickedNodeId } = useGraphStore(
+    (state) => state,
+  );
+
+  const showSecondaryMenu = useCallback(
+    (value: string, outputLabel: string) => {
+      const nodeId = getNodeId(value);
+      return (
+        getOperatorTypeFromId(nodeId) === Operator.Agent &&
+        outputLabel === AgentStructuredOutputField
+      );
+    },
+    [getOperatorTypeFromId],
+  );
+
+  const renderAgentStructuredOutput = useCallback(
+    (values: any, option: VariableInnerOption) => {
+      if (isPlainObject(values) && 'properties' in values) {
+        return (
+          <ul className="border-l">
+            {Object.entries(values.properties).map(([key, value]) => {
+              const nextOption = new VariableInnerOption(
+                option.label + `.${key}`,
+                option.value + `.${key}`,
+                option.parentLabel,
+                option.icon,
+              );
+
+              const dataType = get(value, 'type');
+
+              return (
+                <li key={key} className="pl-1">
+                  <div
+                    onClick={() => selectOptionAndCleanUp(nextOption)}
+                    className="hover:bg-bg-card p-1 text-text-primary rounded-sm flex justify-between"
+                  >
+                    {key}
+                    <span className="text-text-secondary">{dataType}</span>
+                  </div>
+                  {dataType === 'object' &&
+                    renderAgentStructuredOutput(value, nextOption)}
+                </li>
+              );
+            })}
+          </ul>
+        );
+      }
+
+      return <div></div>;
+    },
+    [selectOptionAndCleanUp],
+  );
+
   return (
     <li
       key={option.key}
@@ -93,15 +161,56 @@ function VariablePickerMenuItem({
       <div>
         <span className="text text-text-secondary">{option.title}</span>
         <ul className="pl-2 py-1">
-          {option.options.map((x) => (
+          {option.options.map((x) => {
-            <li
+            const shouldShowSecondary = showSecondaryMenu(x.value, x.label);
-              key={x.value}
+
-              onClick={() => selectOptionAndCleanUp(x)}
+            if (shouldShowSecondary) {
-              className="hover:bg-bg-card p-1 text-text-primary rounded-sm"
+              const node = getNode(getNodeId(x.value));
-            >
+              const structuredOutput = get(
-              {x.label}
+                node,
-            </li>
+                `data.form.outputs.${AgentStructuredOutputField}`,
-          ))}
+              );
+
+              const filteredStructuredOutput = filterAgentStructuredOutput(
+                structuredOutput,
+                getOperatorTypeFromId(clickedNodeId),
+              );
+
+              return (
+                <HoverCard key={x.value} openDelay={100} closeDelay={100}>
+                  <HoverCardTrigger asChild>
+                    <li className="hover:bg-bg-card p-1 text-text-primary rounded-sm">
+                      {x.label}
+                    </li>
+                  </HoverCardTrigger>
+                  <HoverCardContent
+                    side="left"
+                    align="start"
+                    className={cn(
+                      'min-w-[140px]  border border-border rounded-md shadow-lg p-0',
+                    )}
+                  >
+                    <section className="p-2">
+                      <div className="p-1">
+                        {x.parentLabel} structured output:
+                      </div>
+                      {renderAgentStructuredOutput(filteredStructuredOutput, x)}
+                    </section>
+                  </HoverCardContent>
+                </HoverCard>
+              );
+            }
+
+            return (
+              <li
+                key={x.value}
+                onClick={() => selectOptionAndCleanUp(x)}
+                className="hover:bg-bg-card p-1 text-text-primary rounded-sm"
+              >
+                {x.label}
+              </li>
+            );
+          })}
         </ul>
       </div>
     </li>
@@ -130,6 +239,9 @@ export default function VariablePickerMenuPlugin({
   baseOptions,
 }: VariablePickerMenuPluginProps): JSX.Element {
   const [editor] = useLexicalComposerContext();
+  const getOperatorTypeFromId = useGraphStore(
+    (state) => state.getOperatorTypeFromId,
+  );
 
   // const checkForTriggerMatch = useBasicTypeaheadTriggerMatch('/', {
   //   minLength: 0,
@@ -200,9 +312,28 @@ export default function VariablePickerMenuPlugin({
         return pre.concat(cur.options);
       }, []);
 
+      // agent structured output
+      const fields = value.split('@');
+      if (
+        getOperatorTypeFromId(fields.at(0)) === Operator.Agent &&
+        fields.at(1)?.startsWith(AgentStructuredOutputField)
+      ) {
+        // is agent structured output
+        const agentOption = children.find((x) => value.includes(x.value));
+        const jsonSchemaFields = fields
+          .at(1)
+          ?.slice(AgentStructuredOutputField.length);
+
+        return {
+          ...agentOption,
+          label: (agentOption?.label ?? '') + jsonSchemaFields,
+          value: value,
+        };
+      }
+
       return children.find((x) => x.value === value);
     },
-    [options],
+    [getOperatorTypeFromId, options],
   );
 
   const onSelectOption = useCallback(