Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/utils/api.ts

@@ -45,6 +45,18 @@ export default {
   getKnowledgeGraph: (knowledgeId: string) =>
     `${api_host}/kb/${knowledgeId}/knowledge_graph`,
   getMeta: `${api_host}/kb/get_meta`,
+  getKnowledgeBasicInfo: `${api_host}/kb/basic_info`,
+  // data pipeline log
+  fetchDataPipelineLog: `${api_host}/kb/list_pipeline_logs`,
+  get_pipeline_detail: `${api_host}/kb/pipeline_log_detail`,
+  fetchPipelineDatasetLogs: `${api_host}/kb/list_pipeline_dataset_logs`,
+  runGraphRag: `${api_host}/kb/run_graphrag`,
+  traceGraphRag: `${api_host}/kb/trace_graphrag`,
+  runRaptor: `${api_host}/kb/run_raptor`,
+  traceRaptor: `${api_host}/kb/trace_raptor`,
+  unbindPipelineTask: ({ kb_id, type }: { kb_id: string; type: string }) =>
+    `${api_host}/kb/unbind_task?kb_id=${kb_id}&pipeline_task_type=${type}`,
+  pipelineRerun: `${api_host}/canvas/rerun`,
 
   // tags
   listTag: (knowledgeId: string) => `${api_host}/kb/${knowledgeId}/tags`,
@@ -138,7 +150,6 @@ export default {
   // flow
   listTemplates: `${api_host}/canvas/templates`,
   listCanvas: `${api_host}/canvas/list`,
-  listCanvasTeam: `${api_host}/canvas/listteam`,
   getCanvas: `${api_host}/canvas/get`,
   getCanvasSSE: `${api_host}/canvas/getsse`,
   removeCanvas: `${api_host}/canvas/rm`,
@@ -165,6 +176,8 @@ export default {
   fetchExternalAgentInputs: (canvasId: string) =>
     `${ExternalApi}${api_host}/agentbots/${canvasId}/inputs`,
   prompt: `${api_host}/canvas/prompts`,
+  cancelDataflow: (id: string) => `${api_host}/canvas/cancel/${id}`,
+  downloadFile: `${api_host}/canvas/download`,
 
   // mcp server
   listMcpServer: `${api_host}/mcp_server/list`,
@@ -190,4 +203,11 @@ export default {
   mindmapShare: `${ExternalApi}${api_host}/searchbots/mindmap`,
   getRelatedQuestionsShare: `${ExternalApi}${api_host}/searchbots/related_questions`,
   retrievalTestShare: `${ExternalApi}${api_host}/searchbots/retrieval_test`,
+
+  // data pipeline
+  fetchDataflow: (id: string) => `${api_host}/dataflow/get/${id}`,
+  setDataflow: `${api_host}/dataflow/set`,
+  removeDataflow: `${api_host}/dataflow/rm`,
+  listDataflow: `${api_host}/dataflow/list`,
+  runDataflow: `${api_host}/dataflow/run`,
 };

web/src/utils/canvas-util.tsx

@@ -0,0 +1,75 @@
+import { BaseNode } from '@/interfaces/database/agent';
+import { Edge } from '@xyflow/react';
+import { isEmpty } from 'lodash';
+import { ComponentType, ReactNode } from 'react';
+
+export function filterAllUpstreamNodeIds(edges: Edge[], nodeIds: string[]) {
+  return nodeIds.reduce<string[]>((pre, nodeId) => {
+    const currentEdges = edges.filter((x) => x.target === nodeId);
+
+    const upstreamNodeIds: string[] = currentEdges.map((x) => x.source);
+
+    const ids = upstreamNodeIds.concat(
+      filterAllUpstreamNodeIds(edges, upstreamNodeIds),
+    );
+
+    ids.forEach((x) => {
+      if (pre.every((y) => y !== x)) {
+        pre.push(x);
+      }
+    });
+
+    return pre;
+  }, []);
+}
+
+export function buildOutputOptions(
+  outputs: Record<string, any> = {},
+  nodeId?: string,
+  parentLabel?: string | ReactNode,
+  icon?: ReactNode,
+) {
+  return Object.keys(outputs).map((x) => ({
+    label: x,
+    value: `${nodeId}@${x}`,
+    parentLabel,
+    icon,
+    type: outputs[x]?.type,
+  }));
+}
+
+export function buildNodeOutputOptions({
+  nodes,
+  edges,
+  nodeId,
+  Icon,
+}: {
+  nodes: BaseNode[];
+  edges: Edge[];
+  nodeId?: string;
+  Icon: ComponentType<{ name: string }>;
+}) {
+  if (!nodeId) {
+    return [];
+  }
+  const upstreamIds = filterAllUpstreamNodeIds(edges, [nodeId]);
+
+  const nodeWithOutputList = nodes.filter(
+    (x) =>
+      upstreamIds.some((y) => y === x.id) && !isEmpty(x.data?.form?.outputs),
+  );
+
+  return nodeWithOutputList
+    .filter((x) => x.id !== nodeId)
+    .map((x) => ({
+      label: x.data.name,
+      value: x.id,
+      title: x.data.name,
+      options: buildOutputOptions(
+        x.data.form.outputs,
+        x.id,
+        x.data.name,
+        <Icon name={x.data.name} />,
+      ),
+    }));
+}

web/src/utils/common-util.ts

@@ -154,8 +154,11 @@ function getCSSVariableValue(variableName: string): string {
   return value;
 }
 
-// Parse the color and convert to RGBA
-export function parseColorToRGBA(color: string): [number, number, number] {
+/**Parse the color and convert to RGB,
+ * #fff -> [255, 255, 255]
+ * var(--text-primary) -> [var(--text-primary-r), var(--text-primary-g), var(--text-primary-b)]
+ * */
+export function parseColorToRGB(color: string): [number, number, number] {
   // Handling CSS variables (e.g. var(--accent-primary))
   let colorStr = color;
   if (colorStr.startsWith('var(')) {
@@ -172,6 +175,32 @@ export function parseColorToRGBA(color: string): [number, number, number] {
     colorStr = getCSSVariableValue(varName);
   }
 
+  // Handle rgb(var(--accent-primary)) format
+  if (colorStr.startsWith('rgb(var(')) {
+    const varMatch = colorStr.match(/rgb\(var\(([^)]+)\)\)/);
+    if (!varMatch) {
+      console.error(`Invalid nested CSS variable: ${color}`);
+      return [0, 0, 0];
+    }
+    const varName = varMatch[1];
+    if (!varName) {
+      console.error(`Invalid nested CSS variable: ${colorStr}`);
+      return [0, 0, 0];
+    }
+    // Get the CSS variable value which should be in format "r, g, b"
+    const rgbValues = getCSSVariableValue(varName);
+    const rgbMatch = rgbValues.match(/^(\d+),?\s*(\d+),?\s*(\d+)$/);
+    if (rgbMatch) {
+      return [
+        parseInt(rgbMatch[1]),
+        parseInt(rgbMatch[2]),
+        parseInt(rgbMatch[3]),
+      ];
+    }
+    console.error(`Unsupported RGB CSS variable format: ${rgbValues}`);
+    return [0, 0, 0];
+  }
+
   // Handles hexadecimal colors (e.g. #FF5733)
   if (colorStr.startsWith('#')) {
     const cleanedHex = colorStr.replace(/^#/, '');
@@ -205,3 +234,14 @@ export function parseColorToRGBA(color: string): [number, number, number] {
   console.error(`Unsupported colorStr format: ${colorStr}`);
   return [0, 0, 0];
 }
+
+/**
+ *
+ * @param color eg: #fff, or var(--color-text-primary)
+ * @param opcity 0~1
+ * @return rgba(r,g,b,opcity)
+ */
+export function parseColorToRGBA(color: string, opcity = 1): string {
+  const [r, g, b] = parseColorToRGB(color);
+  return `rgba(${r},${g},${b},${opcity})`;
+}

web/src/utils/component-util.ts

@@ -1,3 +1,13 @@
-export function buildSelectOptions(list: Array<string>) {
+export function buildSelectOptions(
+  list: Array<any>,
+  keyName?: string,
+  valueName?: string,
+) {
+  if (!Array.isArray(list) || !list.length) {
+    return [];
+  }
+  if (keyName && valueName) {
+    return list.map((x) => ({ label: x[valueName], value: x[keyName] }));
+  }
   return list.map((x) => ({ label: x, value: x }));
 }

web/src/utils/dataset-util.ts

@@ -7,27 +7,3 @@ export function isKnowledgeGraphParser(parserId: DocumentParserType) {
 export function isNaiveParser(parserId: DocumentParserType) {
   return parserId === DocumentParserType.Naive;
 }
-
-export type FilterType = {
-  id: string;
-  label: string;
-  count: number;
-};
-
-export function groupListByType<T extends Record<string, any>>(
-  list: T[],
-  idField: string,
-  labelField: string,
-) {
-  const fileTypeList: FilterType[] = [];
-  list.forEach((x) => {
-    const item = fileTypeList.find((y) => y.id === x[idField]);
-    if (!item) {
-      fileTypeList.push({ id: x[idField], label: x[labelField], count: 1 });
-    } else {
-      item.count += 1;
-    }
-  });
-
-  return fileTypeList;
-}

web/src/utils/date.ts

@@ -43,3 +43,21 @@ export function formatStandardDate(date: any) {
   }
   return parsedDate.format('YYYY-MM-DD');
 }
+
+export function formatSecondsToHumanReadable(seconds: number): string {
+  if (isNaN(seconds) || seconds < 0) {
+    return '0s';
+  }
+
+  const h = Math.floor(seconds / 3600);
+  const m = Math.floor((seconds % 3600) / 60);
+  // const s = toFixed(seconds % 60, 3);
+  const s = seconds % 60;
+  const formattedSeconds = s === 0 ? '0' : s.toFixed(3).replace(/\.?0+$/, '');
+  const parts = [];
+  if (h > 0) parts.push(`${h}h `);
+  if (m > 0) parts.push(`${m}m `);
+  if (s || parts.length === 0) parts.push(`${formattedSeconds}s`);
+
+  return parts.join('');
+}

web/src/utils/form.ts

@@ -35,7 +35,9 @@ export function buildOptions(
 ) {
   if (t) {
     return Object.values(data).map((val) => ({
-      label: t(`${prefix ? prefix + '.' : ''}${val.toLowerCase()}`),
+      label: t(
+        `${prefix ? prefix + '.' : ''}${typeof val === 'string' ? val.toLowerCase() : val}`,
+      ),
       value: val,
     }));
   }

web/src/utils/list-filter-util.ts

@@ -0,0 +1,29 @@
+export type FilterType = {
+  id: string;
+  label: string;
+  count: number;
+};
+
+export function groupListByType<T extends Record<string, any>>(
+  list: T[],
+  idField: string,
+  labelField: string,
+) {
+  const fileTypeList: FilterType[] = [];
+  list.forEach((x) => {
+    const item = fileTypeList.find((y) => y.id === x[idField]);
+    if (!item) {
+      fileTypeList.push({ id: x[idField], label: x[labelField], count: 1 });
+    } else {
+      item.count += 1;
+    }
+  });
+
+  return fileTypeList;
+}
+
+export function buildOwnersFilter<T extends Record<string, any>>(list: T[]) {
+  const owners = groupListByType(list, 'tenant_id', 'nickname');
+
+  return { field: 'owner', list: owners, label: 'Owner' };
+}