Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/pages/agent/canvas/edge/index.tsx

@@ -43,7 +43,9 @@ function InnerButtonEdge({
     targetPosition,
   });
   const selectedStyle = useMemo(() => {
-    return selected ? { strokeWidth: 1, stroke: 'var(--accent-primary)' } : {};
+    return selected
+      ? { strokeWidth: 1, stroke: 'rgb(var(--accent-primary))' }
+      : {};
   }, [selected]);
 
   const placeholderHighlightStyle = useMemo(() => {
@@ -67,7 +69,7 @@ function InnerButtonEdge({
       let index = idx - 1;
       while (index >= 0) {
         if (path[index] === source) {
-          return { strokeWidth: 1, stroke: 'var(--accent-primary)' };
+          return { strokeWidth: 1, stroke: 'rgb(var(--accent-primary))' };
         }
         index--;
       }

web/src/pages/agent/constant.tsx

@@ -7,6 +7,7 @@ import {
   AgentGlobalsSysQueryWithBrace,
   CodeTemplateStrMap,
   ProgrammingLanguage,
+  initialLlmBaseValues,
 } from '@/constants/agent';
 
 export enum AgentDialogueMode {
@@ -14,13 +15,8 @@ export enum AgentDialogueMode {
   Task = 'task',
 }
 
-import {
-  ChatVariableEnabledField,
-  variableEnabledFieldMap,
-} from '@/constants/chat';
 import { ModelVariableType } from '@/constants/knowledge';
 import i18n from '@/locales/config';
-import { setInitialChatVariableEnabledFieldValue } from '@/utils/chat';
 import { t } from 'i18next';
 
 // DuckDuckGo's channel options
@@ -276,24 +272,6 @@ export const initialBeginValues = {
   prologue: `Hi! I'm your assistant. What can I do for you?`,
 };
 
-export const variableCheckBoxFieldMap = Object.keys(
-  variableEnabledFieldMap,
-).reduce<Record<string, boolean>>((pre, cur) => {
-  pre[cur] = setInitialChatVariableEnabledFieldValue(
-    cur as ChatVariableEnabledField,
-  );
-  return pre;
-}, {});
-
-const initialLlmBaseValues = {
-  ...variableCheckBoxFieldMap,
-  temperature: 0.1,
-  top_p: 0.3,
-  frequency_penalty: 0.7,
-  presence_penalty: 0.4,
-  max_tokens: 256,
-};
-
 export const initialGenerateValues = {
   ...initialLlmBaseValues,
   prompt: i18n.t('flow.promptText'),

web/src/pages/agent/debug-content/uploader.tsx

@@ -86,7 +86,7 @@ export function FileUploadDirectUpload({
           </div>
           <p className="font-medium text-sm">Drag & drop files here</p>
           <p className="text-muted-foreground text-xs">
-            Or click to browse (max 2 files)
+            Or click to browse (max 1 files)
           </p>
         </div>
         <FileUploadTrigger asChild>

web/src/pages/agent/form/components/prompt-editor/index.tsx

@@ -55,7 +55,7 @@ type IProps = {
   onChange?: (value?: string) => void;
   placeholder?: ReactNode;
 } & PromptContentProps &
-  Pick<VariablePickerMenuPluginProps, 'extraOptions'>;
+  Pick<VariablePickerMenuPluginProps, 'extraOptions' | 'baseOptions'>;
 
 function PromptContent({
   showToolbar = true,
@@ -126,6 +126,7 @@ export function PromptEditor({
   showToolbar,
   multiLine = true,
   extraOptions,
+  baseOptions,
 }: IProps) {
   const { t } = useTranslation();
   const initialConfig: InitialConfigType = {
@@ -177,6 +178,7 @@ export function PromptEditor({
         <VariablePickerMenuPlugin
           value={value}
           extraOptions={extraOptions}
+          baseOptions={baseOptions}
         ></VariablePickerMenuPlugin>
         <PasteHandlerPlugin />
         <VariableOnChangePlugin

web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx

@@ -10,7 +10,6 @@ import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext
 import {
   LexicalTypeaheadMenuPlugin,
   MenuOption,
-  useBasicTypeaheadTriggerMatch,
 } from '@lexical/react/LexicalTypeaheadMenuPlugin';
 import {
   $createParagraphNode,
@@ -109,29 +108,56 @@ function VariablePickerMenuItem({
   );
 }
 
+export type VariablePickerMenuOptionType = {
+  label: string;
+  title: string;
+  value?: string;
+  options: Array<{
+    label: string;
+    value: string;
+    icon: ReactNode;
+  }>;
+};
+
 export type VariablePickerMenuPluginProps = {
   value?: string;
-  extraOptions?: Array<{
-    label: string;
-    title: string;
-    options: Array<{ label: string; value: string; icon?: ReactNode }>;
-  }>;
+  extraOptions?: VariablePickerMenuOptionType[];
+  baseOptions?: VariablePickerMenuOptionType[];
 };
 export default function VariablePickerMenuPlugin({
   value,
   extraOptions,
+  baseOptions,
 }: VariablePickerMenuPluginProps): JSX.Element {
   const [editor] = useLexicalComposerContext();
-  const isFirstRender = useRef(true);
 
-  const checkForTriggerMatch = useBasicTypeaheadTriggerMatch('/', {
-    minLength: 0,
-  });
+  // const checkForTriggerMatch = useBasicTypeaheadTriggerMatch('/', {
+  //   minLength: 0,
+  // });
+
+  const testTriggerFn = React.useCallback((text: string) => {
+    const lastChar = text.slice(-1);
+    if (lastChar === '/') {
+      console.log('Found trigger character "/"');
+      return {
+        leadOffset: text.length - 1,
+        matchingString: '',
+        replaceableString: '/',
+      };
+    }
+    return null;
+  }, []);
+
+  const previousValue = useRef<string | undefined>();
 
   const [queryString, setQueryString] = React.useState<string | null>('');
 
   let options = useBuildQueryVariableOptions();
 
+  if (baseOptions) {
+    options = baseOptions as typeof options;
+  }
+
   const buildNextOptions = useCallback(() => {
     let filteredOptions = [...options, ...(extraOptions ?? [])];
     if (queryString) {
@@ -267,8 +293,8 @@ export default function VariablePickerMenuPlugin({
   );
 
   useEffect(() => {
-    if (editor && value && isFirstRender.current) {
-      isFirstRender.current = false;
+    if (editor && value && value !== previousValue.current) {
+      previousValue.current = value;
       editor.update(
         () => {
           parseTextToVariableNodes(value);
@@ -278,6 +304,21 @@ export default function VariablePickerMenuPlugin({
     }
   }, [parseTextToVariableNodes, editor, value]);
 
+  // Fixed the issue where the cursor would go to the end when changing its own data
+  useEffect(() => {
+    return editor.registerUpdateListener(({ editorState, tags }) => {
+      // If we trigger the programmatic update ourselves, we should not write back to avoid an infinite loop.
+      if (tags.has(ProgrammaticTag)) return;
+
+      editorState.read(() => {
+        const text = $getRoot().getTextContent();
+        if (text !== previousValue.current) {
+          previousValue.current = text;
+        }
+      });
+    });
+  }, [editor]);
+
   return (
     <LexicalTypeaheadMenuPlugin<VariableOption | VariableInnerOption>
       onQueryChange={setQueryString}
@@ -288,7 +329,7 @@ export default function VariablePickerMenuPlugin({
           closeMenu,
         )
       }
-      triggerFn={checkForTriggerMatch}
+      triggerFn={testTriggerFn}
       options={buildNextOptions()}
       menuRenderFn={(anchorElementRef, { selectOptionAndCleanUp }) => {
         const nextOptions = buildNextOptions();

web/src/pages/agent/form/iteration-form/use-build-options.ts

@@ -1,7 +1,7 @@
+import { buildOutputOptions } from '@/utils/canvas-util';
 import { isEmpty } from 'lodash';
 import { useMemo } from 'react';
 import { Operator } from '../../constant';
-import { buildOutputOptions } from '../../hooks/use-get-begin-query';
 import useGraphStore from '../../store';
 
 export function useBuildSubNodeOutputOptions(nodeId?: string) {

web/src/pages/agent/hooks/use-get-begin-query.tsx

@@ -1,19 +1,11 @@
 import { AgentGlobals } from '@/constants/agent';
 import { useFetchAgent } from '@/hooks/use-agent-request';
 import { RAGFlowNodeType } from '@/interfaces/database/flow';
-import { Edge } from '@xyflow/react';
+import { buildNodeOutputOptions } from '@/utils/canvas-util';
 import { DefaultOptionType } from 'antd/es/select';
 import { t } from 'i18next';
-import { isEmpty } from 'lodash';
 import get from 'lodash/get';
-import {
-  ReactNode,
-  useCallback,
-  useContext,
-  useEffect,
-  useMemo,
-  useState,
-} from 'react';
+import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import {
   AgentDialogueMode,
   BeginId,
@@ -83,72 +75,18 @@ export const useGetBeginNodeDataQueryIsSafe = () => {
   return isBeginNodeDataQuerySafe;
 };
 
-function filterAllUpstreamNodeIds(edges: Edge[], nodeIds: string[]) {
-  return nodeIds.reduce<string[]>((pre, nodeId) => {
-    const currentEdges = edges.filter((x) => x.target === nodeId);
-
-    const upstreamNodeIds: string[] = currentEdges.map((x) => x.source);
-
-    const ids = upstreamNodeIds.concat(
-      filterAllUpstreamNodeIds(edges, upstreamNodeIds),
-    );
-
-    ids.forEach((x) => {
-      if (pre.every((y) => y !== x)) {
-        pre.push(x);
-      }
-    });
-
-    return pre;
-  }, []);
-}
-
-export function buildOutputOptions(
-  outputs: Record<string, any> = {},
-  nodeId?: string,
-  parentLabel?: string | ReactNode,
-  icon?: ReactNode,
-) {
-  return Object.keys(outputs).map((x) => ({
-    label: x,
-    value: `${nodeId}@${x}`,
-    parentLabel,
-    icon,
-    type: outputs[x]?.type,
-  }));
-}
-
 export function useBuildNodeOutputOptions(nodeId?: string) {
   const nodes = useGraphStore((state) => state.nodes);
   const edges = useGraphStore((state) => state.edges);
 
-  const nodeOutputOptions = useMemo(() => {
-    if (!nodeId) {
-      return [];
-    }
-    const upstreamIds = filterAllUpstreamNodeIds(edges, [nodeId]);
-
-    const nodeWithOutputList = nodes.filter(
-      (x) =>
-        upstreamIds.some((y) => y === x.id) && !isEmpty(x.data?.form?.outputs),
-    );
-
-    return nodeWithOutputList
-      .filter((x) => x.id !== nodeId)
-      .map((x) => ({
-        label: x.data.name,
-        value: x.id,
-        title: x.data.name,
-        options: buildOutputOptions(
-          x.data.form.outputs,
-          x.id,
-          x.data.name,
-          <OperatorIcon name={x.data.label as Operator} />,
-        ),
-      }));
+  return useMemo(() => {
+    return buildNodeOutputOptions({
+      nodes,
+      edges,
+      nodeId,
+      Icon: ({ name }) => <OperatorIcon name={name as Operator}></OperatorIcon>,
+    });
   }, [edges, nodeId, nodes]);
-
-  return nodeOutputOptions;
 }
 
 // exclude nodes with branches

web/src/pages/agent/log-sheet/workflow-timeline.tsx

@@ -23,7 +23,7 @@ import { ITraceData } from '@/interfaces/database/agent';
 import { cn } from '@/lib/utils';
 import { t } from 'i18next';
 import { get, isEmpty, isEqual, uniqWith } from 'lodash';
-import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo } from 'react';
 import JsonView from 'react18-json-view';
 import { Operator } from '../constant';
 import { useCacheChatLog } from '../hooks/use-cache-chat-log';
@@ -116,12 +116,12 @@ export const WorkFlowTimeline = ({
   isShare,
 }: LogFlowTimelineProps) => {
   // const getNode = useGraphStore((state) => state.getNode);
-  const [isStopFetchTrace, setISStopFetchTrace] = useState(false);
 
-  const { data: traceData, setMessageId } = useFetchMessageTrace(
-    isStopFetchTrace,
-    canvasId,
-  );
+  const {
+    data: traceData,
+    setMessageId,
+    setISStopFetchTrace,
+  } = useFetchMessageTrace(canvasId);
 
   useEffect(() => {
     setMessageId(currentMessageId);
@@ -133,7 +133,7 @@ export const WorkFlowTimeline = ({
 
   useEffect(() => {
     setISStopFetchTrace(!sendLoading);
-  }, [sendLoading]);
+  }, [sendLoading, setISStopFetchTrace]);
 
   const startedNodeList = useMemo(() => {
     const finish = currentEventListWithoutMessage?.some(
@@ -151,7 +151,7 @@ export const WorkFlowTimeline = ({
       }
       return pre;
     }, []);
-  }, [currentEventListWithoutMessage, sendLoading]);
+  }, [currentEventListWithoutMessage, sendLoading, setISStopFetchTrace]);
 
   const getElapsedTime = (nodeId: string) => {
     if (nodeId === 'begin') {

web/src/pages/agent/run-sheet/index.tsx

@@ -51,7 +51,7 @@ const RunSheet = ({
   );
 
   return (
-    <Sheet onOpenChange={hideModal} open>
+    <Sheet onOpenChange={hideModal} open modal={false}>
       <SheetContent className={cn('top-20 p-2')}>
         <SheetHeader>
           <SheetTitle>{t('flow.testRun')}</SheetTitle>