Feat: Use data pipeline to visualize the parsing configuration of the knowledge base (#10423)

### What problem does this PR solve?

#9869

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: jinhai <haijin.chn@gmail.com>
Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: chanx <1243304602@qq.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
Co-authored-by: Lynn <lynn_inf@hotmail.com>
Co-authored-by: 纷繁下的无奈 <zhileihuang@126.com>
Co-authored-by: huangzl <huangzl@shinemo.com>
Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com>
Co-authored-by: Wilmer <33392318@qq.com>
Co-authored-by: Adrian Weidig <adrianweidig@gmx.net>
Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yongteng Lei <yongtengrey@outlook.com>
Co-authored-by: Liu An <asiro@qq.com>
Co-authored-by: buua436 <66937541+buua436@users.noreply.github.com>
Co-authored-by: BadwomanCraZY <511528396@qq.com>
Co-authored-by: cucusenok <31804608+cucusenok@users.noreply.github.com>
Co-authored-by: Russell Valentine <russ@coldstonelabs.org>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Billy Bao <newyorkupperbay@gmail.com>
Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: TensorNull <129579691+TensorNull@users.noreply.github.com>
Co-authored-by: TensorNull <tensor.null@gmail.com>
Co-authored-by: TeslaZY <TeslaZY@outlook.com>
Co-authored-by: Ajay <160579663+aybanda@users.noreply.github.com>
Co-authored-by: AB <aj@Ajays-MacBook-Air.local>
Co-authored-by: 天海蒼灆 <huangaoqin@tecpie.com>
Co-authored-by: He Wang <wanghechn@qq.com>
Co-authored-by: Atsushi Hatakeyama <atu729@icloud.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Mohamed Mathari <155896313+melmathari@users.noreply.github.com>
Co-authored-by: Mohamed Mathari <nocodeventure@Mac-mini-van-Mohamed.fritz.box>
Co-authored-by: Stephen Hu <stephenhu@seismic.com>
Co-authored-by: Shaun Zhang <zhangwfjh@users.noreply.github.com>
Co-authored-by: zhimeng123 <60221886+zhimeng123@users.noreply.github.com>
Co-authored-by: mxc <mxc@example.com>
Co-authored-by: Dominik Novotný <50611433+SgtMarmite@users.noreply.github.com>
Co-authored-by: EVGENY M <168018528+rjohny55@users.noreply.github.com>
Co-authored-by: mcoder6425 <mcoder64@gmail.com>
Co-authored-by: lemsn <lemsn@msn.com>
Co-authored-by: lemsn <lemsn@126.com>
Co-authored-by: Adrian Gora <47756404+adagora@users.noreply.github.com>
Co-authored-by: Womsxd <45663319+Womsxd@users.noreply.github.com>
Co-authored-by: FatMii <39074672+FatMii@users.noreply.github.com>

web/src/services/agent-service.ts

@@ -1,4 +1,7 @@
-import { IAgentLogsRequest } from '@/interfaces/database/agent';
+import {
+  IAgentLogsRequest,
+  IPipeLineListRequest,
+} from '@/interfaces/database/agent';
 import api from '@/utils/api';
 import { registerNextServer } from '@/utils/register-server';
 import request from '@/utils/request';
@@ -14,7 +17,6 @@ const {
   testDbConnect,
   getInputElements,
   debug,
-  listCanvasTeam,
   settingCanvas,
   uploadCanvasFile,
   trace,
@@ -26,6 +28,7 @@ const {
   fetchAgentLogs,
   fetchExternalAgentInputs,
   prompt,
+  cancelDataflow,
 } = api;
 
 const methods = {
@@ -81,10 +84,6 @@ const methods = {
     url: debug,
     method: 'post',
   },
-  listCanvasTeam: {
-    url: listCanvasTeam,
-    method: 'get',
-  },
   settingCanvas: {
     url: settingCanvas,
     method: 'post',
@@ -117,6 +116,10 @@ const methods = {
     url: prompt,
     method: 'get',
   },
+  cancelDataflow: {
+    url: cancelDataflow,
+    method: 'put',
+  },
 } as const;
 
 const agentService = registerNextServer<keyof typeof methods>(methods);
@@ -131,4 +134,8 @@ export const fetchAgentLogsByCanvasId = (
   return request.get(methods.fetchAgentLogs.url(canvasId), { params: params });
 };
 
+export const fetchPipeLineList = (params: IPipeLineListRequest) => {
+  return request.get(api.listCanvas, { params: params });
+};
+
 export default agentService;

web/src/services/dataflow-service.ts

@@ -0,0 +1,37 @@
+import api from '@/utils/api';
+import { registerNextServer } from '@/utils/register-server';
+
+const {
+  listDataflow,
+  removeDataflow,
+  fetchDataflow,
+  runDataflow,
+  setDataflow,
+} = api;
+
+const methods = {
+  listDataflow: {
+    url: listDataflow,
+    method: 'get',
+  },
+  removeDataflow: {
+    url: removeDataflow,
+    method: 'post',
+  },
+  fetchDataflow: {
+    url: fetchDataflow,
+    method: 'get',
+  },
+  runDataflow: {
+    url: runDataflow,
+    method: 'post',
+  },
+  setDataflow: {
+    url: setDataflow,
+    method: 'post',
+  },
+} as const;
+
+const dataflowService = registerNextServer<keyof typeof methods>(methods);
+
+export default dataflowService;

web/src/services/knowledge-service.ts

@@ -4,6 +4,7 @@ import {
   IFetchKnowledgeListRequestBody,
   IFetchKnowledgeListRequestParams,
 } from '@/interfaces/request/knowledge';
+import { ProcessingType } from '@/pages/dataset/dataset-overview/dataset-common';
 import api from '@/utils/api';
 import registerServer from '@/utils/register-server';
 import request, { post } from '@/utils/request';
@@ -39,6 +40,13 @@ const {
   setMeta,
   getMeta,
   retrievalTestShare,
+  getKnowledgeBasicInfo,
+  fetchDataPipelineLog,
+  fetchPipelineDatasetLogs,
+  runGraphRag,
+  traceGraphRag,
+  runRaptor,
+  traceRaptor,
 } = api;
 
 const methods = {
@@ -169,6 +177,43 @@ const methods = {
     url: retrievalTestShare,
     method: 'post',
   },
+  getKnowledgeBasicInfo: {
+    url: getKnowledgeBasicInfo,
+    method: 'get',
+  },
+  fetchDataPipelineLog: {
+    url: fetchDataPipelineLog,
+    method: 'post',
+  },
+  fetchPipelineDatasetLogs: {
+    url: fetchPipelineDatasetLogs,
+    method: 'post',
+  },
+  get_pipeline_detail: {
+    url: api.get_pipeline_detail,
+    method: 'get',
+  },
+
+  runGraphRag: {
+    url: runGraphRag,
+    method: 'post',
+  },
+  traceGraphRag: {
+    url: traceGraphRag,
+    method: 'get',
+  },
+  runRaptor: {
+    url: runRaptor,
+    method: 'post',
+  },
+  traceRaptor: {
+    url: traceRaptor,
+    method: 'get',
+  },
+  pipelineRerun: {
+    url: api.pipelineRerun,
+    method: 'post',
+  },
 };
 
 const kbService = registerServer<keyof typeof methods>(methods, request);
@@ -205,4 +250,23 @@ export const listDocument = (
 export const documentFilter = (kb_id: string) =>
   request.post(api.get_dataset_filter, { kb_id });
 
+export const listDataPipelineLogDocument = (
+  params?: IFetchKnowledgeListRequestParams,
+  body?: IFetchDocumentListRequestBody,
+) => request.post(api.fetchDataPipelineLog, { data: body || {}, params });
+export const listPipelineDatasetLogs = (
+  params?: IFetchKnowledgeListRequestParams,
+  body?: IFetchDocumentListRequestBody,
+) => request.post(api.fetchPipelineDatasetLogs, { data: body || {}, params });
+
+export function deletePipelineTask({
+  kb_id,
+  type,
+}: {
+  kb_id: string;
+  type: ProcessingType;
+}) {
+  return request.delete(api.unbindPipelineTask({ kb_id, type }));
+}
+
 export default kbService;