Feat: Add HierarchicalMergerForm #9869 (#10122)

### What problem does this PR solve?
Feat:  Add HierarchicalMergerForm #9869

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

deepdoc/parser/excel_parser.py

@@ -22,10 +22,10 @@ from openpyxl import Workbook, load_workbook
 from rag.nlp import find_codec
 
 # copied from `/openpyxl/cell/cell.py`
-ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')
+ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")
+
 
 class RAGFlowExcelParser:
-
     @staticmethod
     def _load_excel_to_workbook(file_like_object):
         if isinstance(file_like_object, bytes):
@@ -36,7 +36,7 @@ class RAGFlowExcelParser:
         file_head = file_like_object.read(4)
         file_like_object.seek(0)
 
-        if not (file_head.startswith(b'PK\x03\x04') or file_head.startswith(b'\xD0\xCF\x11\xE0')):
+        if not (file_head.startswith(b"PK\x03\x04") or file_head.startswith(b"\xd0\xcf\x11\xe0")):
             logging.info("Not an Excel file, converting CSV to Excel Workbook")
 
             try:
@@ -48,7 +48,7 @@ class RAGFlowExcelParser:
                 raise Exception(f"Failed to parse CSV and convert to Excel Workbook: {e_csv}")
 
         try:
-            return load_workbook(file_like_object,data_only= True)
+            return load_workbook(file_like_object, data_only=True)
         except Exception as e:
             logging.info(f"openpyxl load error: {e}, try pandas instead")
             try:
@@ -59,7 +59,7 @@ class RAGFlowExcelParser:
                 except Exception as ex:
                     logging.info(f"pandas with default engine load error: {ex}, try calamine instead")
                     file_like_object.seek(0)
-                    df = pd.read_excel(file_like_object, engine='calamine')
+                    df = pd.read_excel(file_like_object, engine="calamine")
                     return RAGFlowExcelParser._dataframe_to_workbook(df)
             except Exception as e_pandas:
                 raise Exception(f"pandas.read_excel error: {e_pandas}, original openpyxl error: {e}")
@@ -116,9 +116,7 @@ class RAGFlowExcelParser:
                 tb = ""
                 tb += f"<table><caption>{sheetname}</caption>"
                 tb += tb_rows_0
-                for r in list(
-                    rows[1 + chunk_i * chunk_rows: min(1 + (chunk_i + 1) * chunk_rows, len(rows))]
-                ):
+                for r in list(rows[1 + chunk_i * chunk_rows : min(1 + (chunk_i + 1) * chunk_rows, len(rows))]):
                     tb += "<tr>"
                     for i, c in enumerate(r):
                         if c.value is None:
@@ -133,8 +131,16 @@ class RAGFlowExcelParser:
 
     def markdown(self, fnm):
         import pandas as pd
+
         file_like_object = BytesIO(fnm) if not isinstance(fnm, str) else fnm
-        df = pd.read_excel(file_like_object)
+        try:
+            file_like_object.seek(0)
+            df = pd.read_excel(file_like_object)
+        except Exception as e:
+            logging.warning(f"Parse spreadsheet error: {e}, trying to interpret as CSV file")
+            file_like_object.seek(0)
+            df = pd.read_csv(file_like_object)
+        df = df.replace(r"^\s*$", "", regex=True)
         return df.to_markdown(index=False)
 
     def __call__(self, fnm):

docker/service_conf.yaml.template

@@ -29,7 +29,6 @@ redis:
   db: 1
   password: '${REDIS_PASSWORD:-infini_rag_flow}'
   host: '${REDIS_HOST:-redis}:6379'
-
 # postgres:
 #   name: '${POSTGRES_DBNAME:-rag_flow}'
 #   user: '${POSTGRES_USER:-rag_flow}'
@@ -65,15 +64,26 @@ redis:
 #   secret: 'secret'
 #   tenant_id: 'tenant_id'
 #   container_name: 'container_name'
+# The OSS object storage uses the MySQL configuration above by default. If you need to switch to another object storage service, please uncomment and configure the following parameters.
+# opendal:
+#   scheme: 'mysql'  # Storage type, such as s3, oss, azure, etc.
+#   config:
+#     oss_table: 'opendal_storage'
 # user_default_llm:
-#   factory: 'Tongyi-Qianwen'
-#   api_key: 'sk-xxxxxxxxxxxxx'
-#   base_url: ''
+#   factory: 'BAAI'
+#   api_key: 'backup'
+#   base_url: 'backup_base_url'
 #   default_models:
-#     chat_model: 'qwen-plus'
-#     embedding_model: 'BAAI/bge-large-zh-v1.5@BAAI'
-#     rerank_model: ''
-#     asr_model: ''
+#     chat_model:
+#       name: 'qwen2.5-7b-instruct'
+#       factory: 'xxxx'
+#       api_key: 'xxxx'
+#       base_url: 'https://api.xx.com'
+#     embedding_model:
+#       name: 'bge-m3'
+#     rerank_model: 'bge-reranker-v2'
+#     asr_model:
+#       model: 'whisper-large-v3' # alias of name
 #     image2text_model: ''
 # oauth:
 #   oauth2:
@@ -109,3 +119,14 @@ redis:
 #   switch: false
 #   component: false
 #   dataset: false
+# smtp:
+#   mail_server: ""
+#   mail_port: 465
+#   mail_use_ssl: true
+#   mail_use_tls: false
+#   mail_username: ""
+#   mail_password: ""
+#   mail_default_sender:
+#     - "RAGFlow" # display name
+#     - "" # sender email address
+#   mail_frontend_url: "https://your-frontend.example.com"

rag/app/naive.py

@@ -41,37 +41,43 @@ class Docx(DocxParser):
         pass
 
     def get_picture(self, document, paragraph):
-        img = paragraph._element.xpath('.//pic:pic')
-        if not img:
-            return None
-        img = img[0]
-        embed = img.xpath('.//a:blip/@r:embed')
-        if not embed:
-            return None
-        embed = embed[0]
-        try:
-            related_part = document.part.related_parts[embed]
-            image_blob = related_part.image.blob
-        except UnrecognizedImageError:
-            logging.info("Unrecognized image format. Skipping image.")
-            return None
-        except UnexpectedEndOfFileError:
-            logging.info("EOF was unexpectedly encountered while reading an image stream. Skipping image.")
-            return None
-        except InvalidImageStreamError:
-            logging.info("The recognized image stream appears to be corrupted. Skipping image.")
-            return None
-        except UnicodeDecodeError:
-            logging.info("The recognized image stream appears to be corrupted. Skipping image.")
-            return None
-        except Exception:
-            logging.info("The recognized image stream appears to be corrupted. Skipping image.")
-            return None
-        try:
-            image = Image.open(BytesIO(image_blob)).convert('RGB')
-            return image
-        except Exception:
+        imgs = paragraph._element.xpath('.//pic:pic')
+        if not imgs:
             return None
+        res_img = None
+        for img in imgs:
+            embed = img.xpath('.//a:blip/@r:embed')
+            if not embed:
+                continue
+            embed = embed[0]
+            try:
+                related_part = document.part.related_parts[embed]
+                image_blob = related_part.image.blob
+            except UnrecognizedImageError:
+                logging.info("Unrecognized image format. Skipping image.")
+                continue
+            except UnexpectedEndOfFileError:
+                logging.info("EOF was unexpectedly encountered while reading an image stream. Skipping image.")
+                continue
+            except InvalidImageStreamError:
+                logging.info("The recognized image stream appears to be corrupted. Skipping image.")
+                continue
+            except UnicodeDecodeError:
+                logging.info("The recognized image stream appears to be corrupted. Skipping image.")
+                continue
+            except Exception:
+                logging.info("The recognized image stream appears to be corrupted. Skipping image.")
+                continue
+            try:
+                image = Image.open(BytesIO(image_blob)).convert('RGB')
+                if res_img is None:
+                    res_img = image
+                else:
+                    res_img = concat_img(res_img, image)
+            except Exception:
+                continue
+
+        return res_img
 
     def __clean(self, line):
         line = re.sub(r"\u3000", " ", line).strip()

rag/flow/chunker/chunker.py

@@ -73,11 +73,13 @@ class Chunker(ProcessBase):
 
     def _general(self, from_upstream: ChunkerFromUpstream):
         self.callback(random.randint(1, 5) / 100.0, "Start to chunk via `General`.")
-        if from_upstream.output_format in ["markdown", "text"]:
+        if from_upstream.output_format in ["markdown", "text", "html"]:
             if from_upstream.output_format == "markdown":
                 payload = from_upstream.markdown_result
-            else:  # == "text"
+            elif from_upstream.output_format == "text":
                 payload = from_upstream.text_result
+            else:  # == "html"
+                payload = from_upstream.html_result
 
             if not payload:
                 payload = ""
@@ -90,6 +92,7 @@ class Chunker(ProcessBase):
             )
             return [{"text": c} for c in cks]
 
+        # json
         sections, section_images = [], []
         for o in from_upstream.json_result or []:
             sections.append((o.get("text", ""), o.get("position_tag", "")))

rag/flow/chunker/schema.py

@@ -29,7 +29,7 @@ class ChunkerFromUpstream(BaseModel):
     json_result: list[dict[str, Any]] | None = Field(default=None, alias="json")
     markdown_result: str | None = Field(default=None, alias="markdown")
     text_result: str | None = Field(default=None, alias="text")
-    html_result: str | None = Field(default=None, alias="html")
+    html_result: list[str] | None = Field(default=None, alias="html")
 
     model_config = ConfigDict(populate_by_name=True, extra="forbid")
 

rag/flow/parser/parser.py

@@ -12,6 +12,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+import logging
 import random
 
 import trio
@@ -29,8 +30,18 @@ class ParserParam(ProcessParamBase):
     def __init__(self):
         super().__init__()
         self.allowed_output_format = {
-            "pdf": ["json", "markdown"],
-            "excel": ["json", "markdown", "html"],
+            "pdf": [
+                "json",
+                "markdown",
+            ],
+            "spreadsheet": [
+                "json",
+                "markdown",
+                "html",
+            ],
+            "word": [
+                "json",
+            ],
             "ppt": [],
             "image": [],
             "email": [],
@@ -44,12 +55,29 @@ class ParserParam(ProcessParamBase):
                 "parse_method": "deepdoc",  # deepdoc/plain_text/vlm
                 "vlm_name": "",
                 "lang": "Chinese",
-                "suffix": ["pdf"],
+                "suffix": [
+                    "pdf",
+                ],
                 "output_format": "json",
             },
-            "excel": {
+            "spreadsheet": {
                 "output_format": "html",
-                "suffix": ["xls", "xlsx", "csv"],
+                "suffix": [
+                    "xls",
+                    "xlsx",
+                    "csv",
+                ],
+            },
+            "word": {
+                "suffix": [
+                    "doc",
+                    "docx",
+                ],
+                "output_format": "json",
+            },
+            "markdown": {
+                "suffix": ["md", "markdown"],
+                "output_format": "json",
             },
             "ppt": {},
             "image": {
@@ -76,10 +104,15 @@ class ParserParam(ProcessParamBase):
             pdf_output_format = pdf_config.get("output_format", "")
             self.check_valid_value(pdf_output_format, "PDF output format abnormal.", self.allowed_output_format["pdf"])
 
-        excel_config = self.setups.get("excel", "")
-        if excel_config:
-            excel_output_format = excel_config.get("output_format", "")
-            self.check_valid_value(excel_output_format, "Excel output format abnormal.", self.allowed_output_format["excel"])
+        spreadsheet_config = self.setups.get("spreadsheet", "")
+        if spreadsheet_config:
+            spreadsheet_output_format = spreadsheet_config.get("output_format", "")
+            self.check_valid_value(spreadsheet_output_format, "Spreadsheet output format abnormal.", self.allowed_output_format["spreadsheet"])
+
+        doc_config = self.setups.get("doc", "")
+        if doc_config:
+            doc_output_format = doc_config.get("output_format", "")
+            self.check_valid_value(doc_output_format, "Word processer document output format abnormal.", self.allowed_output_format["doc"])
 
         image_config = self.setups.get("image", "")
         if image_config:
@@ -93,10 +126,13 @@ class ParserParam(ProcessParamBase):
 class Parser(ProcessBase):
     component_name = "Parser"
 
-    def _pdf(self, blob):
+    def _pdf(self, from_upstream: ParserFromUpstream):
         self.callback(random.randint(1, 5) / 100.0, "Start to work on a PDF.")
+
+        blob = from_upstream.blob
         conf = self._param.setups["pdf"]
         self.set_output("output_format", conf["output_format"])
+
         if conf.get("parse_method") == "deepdoc":
             bboxes = RAGFlowPdfParser().parse_into_bboxes(blob, callback=self.callback)
         elif conf.get("parse_method") == "plain_text":
@@ -110,6 +146,7 @@ class Parser(ProcessBase):
             for t, poss in lines:
                 pn, x0, x1, top, bott = poss.split(" ")
                 bboxes.append({"page_number": int(pn), "x0": float(x0), "x1": float(x1), "top": float(top), "bottom": float(bott), "text": t})
+
         if conf.get("output_format") == "json":
             self.set_output("json", bboxes)
         if conf.get("output_format") == "markdown":
@@ -123,23 +160,93 @@ class Parser(ProcessBase):
                 mkdn += b.get("text", "") + "\n"
             self.set_output("markdown", mkdn)
 
-    def _excel(self, blob):
-        self.callback(random.randint(1, 5) / 100.0, "Start to work on a Excel.")
-        conf = self._param.setups["excel"]
+    def _spreadsheet(self, from_upstream: ParserFromUpstream):
+        self.callback(random.randint(1, 5) / 100.0, "Start to work on a Spreadsheet.")
+
+        blob = from_upstream.blob
+        conf = self._param.setups["spreadsheet"]
         self.set_output("output_format", conf["output_format"])
-        excel_parser = ExcelParser()
+
+        print("spreadsheet {conf=}", flush=True)
+        spreadsheet_parser = ExcelParser()
         if conf.get("output_format") == "html":
-            html = excel_parser.html(blob, 1000000000)
+            html = spreadsheet_parser.html(blob, 1000000000)
             self.set_output("html", html)
         elif conf.get("output_format") == "json":
-            self.set_output("json", [{"text": txt} for txt in excel_parser(blob) if txt])
+            self.set_output("json", [{"text": txt} for txt in spreadsheet_parser(blob) if txt])
         elif conf.get("output_format") == "markdown":
-            self.set_output("markdown", excel_parser.markdown(blob))
+            self.set_output("markdown", spreadsheet_parser.markdown(blob))
+
+    def _word(self, from_upstream: ParserFromUpstream):
+        from tika import parser as  word_parser
+
+        self.callback(random.randint(1, 5) / 100.0, "Start to work on a Word Processor Document")
+
+        blob = from_upstream.blob
+        name = from_upstream.name
+        conf = self._param.setups["word"]
+        self.set_output("output_format", conf["output_format"])
+
+        print("word {conf=}", flush=True)
+        doc_parsed = word_parser.from_buffer(blob)
+
+        sections = []
+        if doc_parsed.get("content"):
+            sections = doc_parsed["content"].split("\n")
+            sections = [{"text": section} for section in sections if section]
+        else:
+            logging.warning(f"tika.parser got empty content from {name}.")
+
+        # json
+        assert conf.get("output_format") == "json", "have to be json for doc"
+        if conf.get("output_format") == "json":
+            self.set_output("json", sections)
+
+    def _markdown(self, from_upstream: ParserFromUpstream):
+        from functools import reduce
+
+        from rag.app.naive import Markdown as naive_markdown_parser
+        from rag.nlp import concat_img
+
+        self.callback(random.randint(1, 5) / 100.0, "Start to work on a Word Processor Document")
+
+        blob = from_upstream.blob
+        name = from_upstream.name
+        conf = self._param.setups["markdown"]
+        self.set_output("output_format", conf["output_format"])
+
+        print("markdown {conf=}", flush=True)
+
+        markdown_parser = naive_markdown_parser()
+        sections, tables = markdown_parser(name, blob, separate_tables=False)
+
+        # json
+        assert conf.get("output_format") == "json", "have to be json for doc"
+        if conf.get("output_format") == "json":
+            json_results = []
+
+            for section_text, _ in sections:
+                json_result = {
+                    "text": section_text,
+                }
+
+                images = markdown_parser.get_pictures(section_text) if section_text else None
+                if images:
+                    # If multiple images found, combine them using concat_img
+                    combined_image = reduce(concat_img, images) if len(images) > 1 else images[0]
+                    json_result["image"] = combined_image
+
+                json_results.append(json_result)
+
+            self.set_output("json", json_results)
+
 
     async def _invoke(self, **kwargs):
         function_map = {
             "pdf": self._pdf,
-            "excel": self._excel,
+            "markdown": self._markdown,
+            "spreadsheet": self._spreadsheet,
+            "word": self._word
         }
         try:
             from_upstream = ParserFromUpstream.model_validate(kwargs)
@@ -150,5 +257,5 @@ class Parser(ProcessBase):
         for p_type, conf in self._param.setups.items():
             if from_upstream.name.split(".")[-1].lower() not in conf.get("suffix", []):
                 continue
-            await trio.to_thread.run_sync(function_map[p_type], from_upstream.blob)
+            await trio.to_thread.run_sync(function_map[p_type], from_upstream)
             break

rag/flow/tests/dsl_examples/general_pdf_all.json

@@ -23,16 +23,31 @@
                   ],
                   "output_format": "json"
                 },
-                "excel": {
-                  "output_format": "html",
+                "spreadsheet": {
                   "suffix": [
                     "xls",
                     "xlsx",
                     "csv"
-                  ]
+                  ],
+                  "output_format": "html"
+                },
+                "word": {
+                  "suffix": [
+                    "doc",
+                    "docx"
+                  ],
+                  "output_format": "json"
+                },
+                "markdown": {
+                  "suffix": [
+                    "md",
+                    "markdown"
+                  ],
+                  "output_format": "json"
                 }
               }
             }
+          }
         },
         "downstream": ["Chunker:0"],
         "upstream": ["Begin"]

rag/flow/tokenizer/schema.py

@@ -31,7 +31,7 @@ class TokenizerFromUpstream(BaseModel):
     json_result: list[dict[str, Any]] | None = Field(default=None, alias="json")
     markdown_result: str | None = Field(default=None, alias="markdown")
     text_result: str | None = Field(default=None, alias="text")
-    html_result: str | None = Field(default=None, alias="html")
+    html_result: list[str] | None = Field(default=None, alias="html")
 
     model_config = ConfigDict(populate_by_name=True, extra="forbid")
 

rag/flow/tokenizer/tokenizer.py

@@ -117,11 +117,13 @@ class Tokenizer(ProcessBase):
                     ck["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(ck["content_ltks"])
                     if i % 100 == 99:
                         self.callback(i * 1.0 / len(chunks) / parts)
-            elif from_upstream.output_format in ["markdown", "text"]:
+            elif from_upstream.output_format in ["markdown", "text", "html"]:
                 if from_upstream.output_format == "markdown":
                     payload = from_upstream.markdown_result
-                else:  # == "text"
+                elif from_upstream.output_format == "text":
                     payload = from_upstream.text_result
+                else:  # == "html"
+                    payload = from_upstream.html_result
 
                 if not payload:
                     return ""

rag/llm/cv_model.py

@@ -124,17 +124,19 @@ class Base(ABC):
                 mime = "image/jpeg"
             b64 = base64.b64encode(data).decode("utf-8")
             return f"data:{mime};base64,{b64}"
-        buffered = BytesIO()
-        fmt = "JPEG"
-        try:
-            image.save(buffered, format="JPEG")
-        except Exception:
-            buffered = BytesIO()  # reset buffer before saving PNG
-            image.save(buffered, format="PNG")
-            fmt = "PNG"
-        data = buffered.getvalue()
-        b64 = base64.b64encode(data).decode("utf-8")
-        mime = f"image/{fmt.lower()}"
+        with BytesIO() as buffered:
+            fmt = "JPEG"
+            try:
+                image.save(buffered, format="JPEG")
+            except Exception:
+                 # reset buffer before saving PNG
+                buffered.seek(0)
+                buffered.truncate()
+                image.save(buffered, format="PNG")
+                fmt = "PNG"
+            data = buffered.getvalue()
+            b64 = base64.b64encode(data).decode("utf-8")
+            mime = f"image/{fmt.lower()}"
         return f"data:{mime};base64,{b64}"
 
     def prompt(self, b64):
@@ -519,24 +521,24 @@ class GeminiCV(Base):
             else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out."
         )
         b64 = self.image2base64(image)
-        img = open(BytesIO(base64.b64decode(b64)))
-        input = [prompt, img]
-        res = self.model.generate_content(input)
-        img.close()
-        return res.text, res.usage_metadata.total_token_count
+        with BytesIO(base64.b64decode(b64)) as bio:
+            img = open(bio)
+            input = [prompt, img]
+            res = self.model.generate_content(input)
+            img.close()
+            return res.text, res.usage_metadata.total_token_count
 
     def describe_with_prompt(self, image, prompt=None):
         from PIL.Image import open
 
         b64 = self.image2base64(image)
         vision_prompt = prompt if prompt else vision_llm_describe_prompt()
-        img = open(BytesIO(base64.b64decode(b64)))
-        input = [vision_prompt, img]
-        res = self.model.generate_content(
-            input,
-        )
-        img.close()
-        return res.text, res.usage_metadata.total_token_count
+        with BytesIO(base64.b64decode(b64)) as bio:
+            img = open(bio)
+            input = [vision_prompt, img]
+            res = self.model.generate_content(input)
+            img.close()
+            return res.text, res.usage_metadata.total_token_count
 
     def chat(self, system, history, gen_conf, images=[]):
         generation_config = dict(temperature=gen_conf.get("temperature", 0.3), top_p=gen_conf.get("top_p", 0.7))

rag/llm/embedding_model.py

@@ -751,6 +751,8 @@ class SILICONFLOWEmbed(Base):
         token_count = 0
         for i in range(0, len(texts), batch_size):
             texts_batch = texts[i : i + batch_size]
+            texts_batch = [" " if not text.strip() else text for text in texts_batch]
+
             payload = {
                 "model": self.model_name,
                 "input": texts_batch,
@@ -935,7 +937,7 @@ class GiteeEmbed(SILICONFLOWEmbed):
         if not base_url:
             base_url = "https://ai.gitee.com/v1/embeddings"
         super().__init__(key, model_name, base_url)
-        
+
 class DeepInfraEmbed(OpenAIEmbed):
     _FACTORY_NAME = "DeepInfra"
 
@@ -951,4 +953,4 @@ class Ai302Embed(Base):
     def __init__(self, key, model_name, base_url="https://api.302.ai/v1/embeddings"):
         if not base_url:
             base_url = "https://api.302.ai/v1/embeddings"
-        super().__init__(key, model_name, base_url)
+        super().__init__(key, model_name, base_url)

rag/nlp/__init__.py

@@ -518,7 +518,7 @@ def hierarchical_merge(bull, sections, depth):
     return res
 
 
-def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？", overlapped_percent=0):
+def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。；！？", overlapped_percent=0):
     from deepdoc.parser.pdf_parser import RAGFlowPdfParser
     if not sections:
         return []
@@ -534,7 +534,7 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？", overl
             pos = ""
         if tnum < 8:
             pos = ""
-        # Ensure that the length of the merged chunk does not exceed chunk_token_num  
+        # Ensure that the length of the merged chunk does not exceed chunk_token_num
         if cks[-1] == "" or tk_nums[-1] > chunk_token_num * (100 - overlapped_percent)/100.:
             if cks:
                 overlapped = RAGFlowPdfParser.remove_tag(cks[-1])
@@ -638,10 +638,10 @@ def concat_img(img1, img2):
         return img2
     if not img1 and not img2:
         return None
-    
+
     if img1 is img2:
         return img1
-    
+
     if isinstance(img1, Image.Image) and isinstance(img2, Image.Image):
         pixel_data1 = img1.tobytes()
         pixel_data2 = img2.tobytes()

web/package-lock.json

@@ -66,7 +66,7 @@
         "jsencrypt": "^3.3.2",
         "lexical": "^0.23.1",
         "lodash": "^4.17.21",
-        "lucide-react": "^0.508.0",
+        "lucide-react": "^0.542.0",
         "mammoth": "^1.7.2",
         "next-themes": "^0.4.6",
         "openai-speech-stream-player": "^1.0.8",
@@ -25113,9 +25113,9 @@
       }
     },
     "node_modules/lucide-react": {
-      "version": "0.508.0",
-      "resolved": "https://registry.npmmirror.com/lucide-react/-/lucide-react-0.508.0.tgz",
-      "integrity": "sha512-gcP16PnexqtOFrTtv98kVsGzTfnbPekzZiQfByi2S89xfk7E/4uKE1USZqccIp58v42LqkO7MuwpCqshwSrJCg==",
+      "version": "0.542.0",
+      "resolved": "https://registry.npmmirror.com/lucide-react/-/lucide-react-0.542.0.tgz",
+      "integrity": "sha512-w3hD8/SQB7+lzU2r4VdFyzzOzKnUjTZIF/MQJGSSvni7Llewni4vuViRppfRAa2guOsY5k4jZyxw/i9DQHv+dw==",
       "license": "ISC",
       "peerDependencies": {
         "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"

web/package.json

@@ -79,7 +79,7 @@
     "jsencrypt": "^3.3.2",
     "lexical": "^0.23.1",
     "lodash": "^4.17.21",
-    "lucide-react": "^0.508.0",
+    "lucide-react": "^0.542.0",
     "mammoth": "^1.7.2",
     "next-themes": "^0.4.6",
     "openai-speech-stream-player": "^1.0.8",

web/src/assets/svg/data-flow/data-icon-bri.svg

@@ -0,0 +1,15 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M35.3194 10.6367H20.4258C19.4857 10.6367 18.7236 11.3988 18.7236 12.3388V34.892C18.7236 35.8321 19.4857 36.5942 20.4258 36.5942H35.3194C36.2594 36.5942 37.0215 35.8321 37.0215 34.892V12.3388C37.0215 11.3988 36.2594 10.6367 35.3194 10.6367Z" fill="url(#paint0_linear_488_37636)"/>
+<path d="M31.0639 4.25391H5.10642C4.16637 4.25391 3.4043 5.01597 3.4043 5.95603V18.2965C3.4043 19.2365 4.16637 19.9986 5.10642 19.9986H31.0639C32.0039 19.9986 32.766 19.2365 32.766 18.2965V5.95603C32.766 5.01597 32.0039 4.25391 31.0639 4.25391Z" fill="#00BEB4" fill-opacity="0.1"/>
+<path d="M31.0639 4.25391C32.0039 4.25391 32.766 5.01597 32.766 5.95603V18.2965C32.766 19.2365 32.0039 19.9986 31.0639 19.9986H5.10642C4.16637 19.9986 3.4043 19.2365 3.4043 18.2965V5.95603C3.4043 5.01597 4.16637 4.25391 5.10642 4.25391H31.0639ZM31.0639 4.67944H5.10642C4.40138 4.67944 3.82983 5.25099 3.82983 5.95603V18.2965C3.82983 19.0015 4.40138 19.5731 5.10642 19.5731H31.0639C31.7689 19.5731 32.3405 19.0015 32.3405 18.2965V5.95603C32.3405 5.25099 31.7689 4.67944 31.0639 4.67944Z" fill="#00BEB4"/>
+<path d="M31.0639 22.5547H5.10642C4.16637 22.5547 3.4043 23.3168 3.4043 24.2568V34.8951C3.4043 35.8352 4.16637 36.5972 5.10642 36.5972H31.0639C32.0039 36.5972 32.766 35.8352 32.766 34.8951V24.2568C32.766 23.3168 32.0039 22.5547 31.0639 22.5547Z" fill="#00BEB4" fill-opacity="0.1"/>
+<path d="M31.0639 22.5547C32.0039 22.5547 32.766 23.3168 32.766 24.2568V34.8951C32.766 35.8352 32.0039 36.5972 31.0639 36.5972H5.10642C4.16637 36.5972 3.4043 35.8352 3.4043 34.8951V24.2568C3.4043 23.3168 4.16637 22.5547 5.10642 22.5547H31.0639ZM31.0639 22.9802H5.10642C4.40138 22.9802 3.82983 23.5518 3.82983 24.2568V34.8951C3.82983 35.6002 4.40138 36.1717 5.10642 36.1717H31.0639C31.7689 36.1717 32.3405 35.6002 32.3405 34.8951V24.2568C32.3405 23.5518 31.7689 22.9802 31.0639 22.9802Z" fill="#00BEB4"/>
+<path d="M10.6384 14.8949C12.2835 14.8949 13.6171 13.5613 13.6171 11.9162C13.6171 10.2711 12.2835 8.9375 10.6384 8.9375C8.99329 8.9375 7.65967 10.2711 7.65967 11.9162C7.65967 13.5613 8.99329 14.8949 10.6384 14.8949Z" fill="#00BEB4"/>
+<path d="M10.6384 32.766C12.2835 32.766 13.6171 31.4324 13.6171 29.7873C13.6171 28.1422 12.2835 26.8086 10.6384 26.8086C8.99329 26.8086 7.65967 28.1422 7.65967 29.7873C7.65967 31.4324 8.99329 32.766 10.6384 32.766Z" fill="#00BEB4"/>
+<defs>
+<linearGradient id="paint0_linear_488_37636" x1="933.617" y1="10.6367" x2="933.617" y2="2606.38" gradientUnits="userSpaceOnUse">
+<stop stop-color="#C9F1EF"/>
+<stop offset="1" stop-color="#00BEB4"/>
+</linearGradient>
+</defs>
+</svg>

web/src/assets/svg/data-flow/data-icon.svg

@@ -0,0 +1,15 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M35.3194 10.6387H20.4258C19.4857 10.6387 18.7236 11.4007 18.7236 12.3408V34.894C18.7236 35.834 19.4857 36.5961 20.4258 36.5961H35.3194C36.2594 36.5961 37.0215 35.834 37.0215 34.894V12.3408C37.0215 11.4007 36.2594 10.6387 35.3194 10.6387Z" fill="url(#paint0_linear_491_41413)"/>
+<path d="M31.0639 4.25586H5.10642C4.16637 4.25586 3.4043 5.01793 3.4043 5.95799V18.2984C3.4043 19.2385 4.16637 20.0005 5.10642 20.0005H31.0639C32.0039 20.0005 32.766 19.2385 32.766 18.2984V5.95799C32.766 5.01793 32.0039 4.25586 31.0639 4.25586Z" fill="#00BEB4" fill-opacity="0.2"/>
+<path d="M31.0639 4.25586C32.0039 4.25586 32.766 5.01793 32.766 5.95799V18.2984C32.766 19.2385 32.0039 20.0005 31.0639 20.0005H5.10642C4.16637 20.0005 3.4043 19.2385 3.4043 18.2984V5.95799C3.4043 5.01793 4.16637 4.25586 5.10642 4.25586H31.0639ZM31.0639 4.68139H5.10642C4.40138 4.68139 3.82983 5.25294 3.82983 5.95799V18.2984C3.82983 19.0035 4.40138 19.575 5.10642 19.575H31.0639C31.7689 19.575 32.3405 19.0035 32.3405 18.2984V5.95799C32.3405 5.25294 31.7689 4.68139 31.0639 4.68139Z" fill="#226365"/>
+<path d="M31.0639 22.5527H5.10642C4.16637 22.5527 3.4043 23.3148 3.4043 24.2549V34.8932C3.4043 35.8332 4.16637 36.5953 5.10642 36.5953H31.0639C32.0039 36.5953 32.766 35.8332 32.766 34.8932V24.2549C32.766 23.3148 32.0039 22.5527 31.0639 22.5527Z" fill="#3A9093" fill-opacity="0.2"/>
+<path d="M31.0639 22.5527C32.0039 22.5527 32.766 23.3148 32.766 24.2549V34.8932C32.766 35.8332 32.0039 36.5953 31.0639 36.5953H5.10642C4.16637 36.5953 3.4043 35.8332 3.4043 34.8932V24.2549C3.4043 23.3148 4.16637 22.5527 5.10642 22.5527H31.0639ZM31.0639 22.9783H5.10642C4.40138 22.9783 3.82983 23.5498 3.82983 24.2549V34.8932C3.82983 35.5982 4.40138 36.1698 5.10642 36.1698H31.0639C31.7689 36.1698 32.3405 35.5982 32.3405 34.8932V24.2549C32.3405 23.5498 31.7689 22.9783 31.0639 22.9783Z" fill="#226365"/>
+<path d="M10.6384 14.893C12.2835 14.893 13.6171 13.5594 13.6171 11.9143C13.6171 10.2692 12.2835 8.93555 10.6384 8.93555C8.99329 8.93555 7.65967 10.2692 7.65967 11.9143C7.65967 13.5594 8.99329 14.893 10.6384 14.893Z" fill="#3A9093"/>
+<path d="M10.6384 32.766C12.2835 32.766 13.6171 31.4324 13.6171 29.7873C13.6171 28.1422 12.2835 26.8086 10.6384 26.8086C8.99329 26.8086 7.65967 28.1422 7.65967 29.7873C7.65967 31.4324 8.99329 32.766 10.6384 32.766Z" fill="#3A9093"/>
+<defs>
+<linearGradient id="paint0_linear_491_41413" x1="933.617" y1="10.6387" x2="933.617" y2="2606.38" gradientUnits="userSpaceOnUse">
+<stop stop-color="#1B3C3D"/>
+<stop offset="1" stop-color="#164142"/>
+</linearGradient>
+</defs>
+</svg>

web/src/assets/svg/data-flow/processing-icon-bri.svg

@@ -0,0 +1,6 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M21.8074 21.9283L30.4051 33.9033C30.9531 34.667 30.7785 35.7307 30.0148 36.2787C29.7258 36.4865 29.3785 36.5982 29.0223 36.5982H11.8273C10.8871 36.5982 10.125 35.8361 10.125 34.8963C10.125 34.54 10.2367 34.1928 10.4445 33.9033L19.0422 21.9283C19.5902 21.1646 20.6539 20.99 21.4176 21.5385C21.5676 21.6463 21.6996 21.7779 21.8074 21.9283Z" fill="#C6EFED"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M5.94336 3.39844H34.0285C35.9086 3.39844 37.4328 4.92266 37.4328 6.80273V27.2281C37.4328 29.1082 35.9086 30.6324 34.0285 30.6324H5.94336C4.06328 30.6324 2.53906 29.1082 2.53906 27.2281V6.80273C2.53906 4.92266 4.06328 3.39844 5.94336 3.39844Z" fill="#00BEB4" fill-opacity="0.2"/>
+<path d="M34.0422 3.40625C35.9223 3.40625 37.4465 4.93047 37.4465 6.81055V27.2359C37.4465 29.116 35.9223 30.6402 34.0422 30.6402H5.95703C4.07695 30.6402 2.55273 29.116 2.55273 27.2359V6.81055C2.55273 4.93047 4.07695 3.40625 5.95703 3.40625H34.0422ZM34.0422 3.83164H5.95703C4.31211 3.83164 2.97852 5.16523 2.97852 6.81055V27.2359C2.97852 28.8812 4.31211 30.2148 5.95703 30.2148H34.0422C35.6871 30.2148 37.0207 28.8812 37.0207 27.2359V6.81055C37.0207 5.16523 35.6871 3.83164 34.0422 3.83164Z" fill="#00BEB4"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M19.9785 11.6797C20.6836 11.6797 21.2551 12.2512 21.2551 12.9562V21.0414C21.2551 21.7465 20.6836 22.318 19.9785 22.318C19.2734 22.318 18.702 21.7465 18.702 21.0414V12.9562C18.702 12.2512 19.2734 11.6797 19.9785 11.6797ZM11.0422 11.6797C11.7473 11.6797 12.3187 12.2512 12.3187 12.9562V21.0414C12.3187 21.7465 11.7473 22.318 11.0422 22.318C10.3371 22.318 9.76562 21.7465 9.76562 21.0414V12.9562C9.76562 12.2512 10.3371 11.6797 11.0422 11.6797ZM28.9145 11.6797C29.6195 11.6797 30.191 12.2512 30.191 12.9562V21.0414C30.191 21.7465 29.6195 22.318 28.9145 22.318C28.2094 22.318 27.6379 21.7465 27.6379 21.0414V12.9562C27.6379 12.2512 28.2094 11.6797 28.9145 11.6797Z" fill="#00BEB4"/>
+</svg>

web/src/assets/svg/data-flow/processing-icon.svg

@@ -0,0 +1,6 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M21.8074 21.9264L30.4051 33.9014C30.9531 34.665 30.7785 35.7287 30.0148 36.2767C29.7258 36.4846 29.3785 36.5963 29.0223 36.5963H11.8273C10.8871 36.5963 10.125 35.8342 10.125 34.8943C10.125 34.5381 10.2367 34.1908 10.4445 33.9014L19.0422 21.9264C19.5902 21.1627 20.6539 20.9881 21.4176 21.5365C21.5676 21.6443 21.6996 21.776 21.8074 21.9264Z" fill="#1C3C3D"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M5.94336 3.39844H34.0285C35.9086 3.39844 37.4328 4.92266 37.4328 6.80273V27.2281C37.4328 29.1082 35.9086 30.6324 34.0285 30.6324H5.94336C4.06328 30.6324 2.53906 29.1082 2.53906 27.2281V6.80273C2.53906 4.92266 4.06328 3.39844 5.94336 3.39844Z" fill="#00BEB4" fill-opacity="0.2"/>
+<path d="M34.0422 3.4043C35.9223 3.4043 37.4465 4.92852 37.4465 6.80859V27.234C37.4465 29.1141 35.9223 30.6383 34.0422 30.6383H5.95703C4.07695 30.6383 2.55273 29.1141 2.55273 27.234V6.80859C2.55273 4.92852 4.07695 3.4043 5.95703 3.4043H34.0422ZM34.0422 3.82969H5.95703C4.31211 3.82969 2.97852 5.16328 2.97852 6.80859V27.234C2.97852 28.8793 4.31211 30.2129 5.95703 30.2129H34.0422C35.6871 30.2129 37.0207 28.8793 37.0207 27.234V6.80859C37.0207 5.16328 35.6871 3.82969 34.0422 3.82969Z" fill="#1B3B3C"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M19.9785 11.6797C20.6836 11.6797 21.2551 12.2512 21.2551 12.9562V21.0414C21.2551 21.7465 20.6836 22.318 19.9785 22.318C19.2734 22.318 18.702 21.7465 18.702 21.0414V12.9562C18.702 12.2512 19.2734 11.6797 19.9785 11.6797ZM11.0422 11.6797C11.7473 11.6797 12.3187 12.2512 12.3187 12.9562V21.0414C12.3187 21.7465 11.7473 22.318 11.0422 22.318C10.3371 22.318 9.76562 21.7465 9.76562 21.0414V12.9562C9.76562 12.2512 10.3371 11.6797 11.0422 11.6797ZM28.9145 11.6797C29.6195 11.6797 30.191 12.2512 30.191 12.9562V21.0414C30.191 21.7465 29.6195 22.318 28.9145 22.318C28.2094 22.318 27.6379 21.7465 27.6379 21.0414V12.9562C27.6379 12.2512 28.2094 11.6797 28.9145 11.6797Z" fill="#00BEB4"/>
+</svg>

web/src/assets/svg/data-flow/total-files-icon-bri.svg

@@ -0,0 +1,6 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.0291 4.67969C11.8025 4.67969 12.4787 5.20078 12.6752 5.94844L13.3494 8.50937H31.4275C33.1599 8.50937 34.6158 9.81055 34.8103 11.5316L37.0205 31.1062C37.231 32.9746 35.8877 34.6602 34.0193 34.8711C33.8927 34.8852 33.765 34.8926 33.6377 34.8926H6.30289C4.92476 34.8926 3.79547 33.7988 3.75094 32.4215L3.11734 12.7746H3.115L2.90719 6.4375C2.87633 5.49805 3.61304 4.71133 4.5525 4.68047C4.57086 4.68008 4.58961 4.67969 4.60836 4.67969H11.0291Z" fill="#00BEB4" fill-opacity="0.1"/>
+<path d="M11.0291 4.67969C11.8025 4.67969 12.4787 5.20078 12.6752 5.94844L13.349 8.50937H31.4275C33.1599 8.50937 34.6158 9.81055 34.8103 11.5316L37.0205 31.1062C37.231 32.9746 35.8877 34.6602 34.0193 34.8711C33.8927 34.8852 33.765 34.8926 33.6377 34.8926H6.30289C4.92476 34.8926 3.79547 33.7988 3.75094 32.4215L3.11656 12.7742L2.90719 6.4375C2.87633 5.49805 3.61304 4.71133 4.5525 4.68047L4.58023 4.67969H11.0291ZM11.0291 5.10508H4.59078L4.56656 5.10586C3.86187 5.12891 3.30914 5.71914 3.33219 6.42344L3.54195 12.7605L4.17633 32.4078C4.21344 33.5555 5.15445 34.4668 6.30289 34.4668H33.6377C33.749 34.4668 33.8607 34.4605 33.9716 34.448C35.6064 34.2637 36.7822 32.7887 36.5974 31.1539L34.3873 11.5797C34.2173 10.0734 32.9431 8.93516 31.4275 8.93516H13.0209L12.9377 8.61758L12.2638 6.05703C12.1162 5.49609 11.6091 5.10508 11.0291 5.10508Z" fill="#00BEB4"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.72812 12.7656H36.6539C38.0637 12.7656 39.207 13.9086 39.207 15.3188C39.207 15.4328 39.1992 15.5465 39.184 15.6594L36.9922 31.943C36.7648 33.6324 35.323 34.8934 33.6184 34.8934H6.37969C4.96953 34.8934 3.82617 33.75 3.82617 32.3398C3.82617 32.2102 3.83633 32.0801 3.85586 31.952L6.36367 15.6523C6.61914 13.9914 8.04805 12.7656 9.72812 12.7656Z" fill="#CAF2F0"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.98438 14.6172H20.4848C20.899 14.6172 21.2348 14.9529 21.2348 15.3672C21.2348 15.7814 20.899 16.1172 20.4848 16.1172H8.98438C8.57013 16.1172 8.23438 15.7814 8.23438 15.3672C8.23438 14.9529 8.57013 14.6172 8.98438 14.6172Z" fill="#00BEB4"/>
+</svg>

web/src/assets/svg/data-flow/total-files-icon.svg

@@ -0,0 +1,6 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.0291 4.68164C11.8025 4.68164 12.4787 5.20273 12.6752 5.95039L13.3494 8.51133H31.4275C33.1599 8.51133 34.6158 9.8125 34.8103 11.5336L37.0205 31.1082C37.231 32.9766 35.8877 34.6621 34.0193 34.873C33.8927 34.8871 33.765 34.8945 33.6377 34.8945H6.30289C4.92476 34.8945 3.79547 33.8008 3.75094 32.4234L3.11734 12.7766H3.115L2.90719 6.43945C2.87633 5.5 3.61304 4.71328 4.5525 4.68242C4.57086 4.68203 4.58961 4.68164 4.60836 4.68164H11.0291Z" fill="#1F3232"/>
+<path d="M11.0291 4.68164C11.8025 4.68164 12.4787 5.20273 12.6752 5.95039L13.349 8.51133H31.4275C33.1599 8.51133 34.6158 9.8125 34.8103 11.5336L37.0205 31.1082C37.231 32.9766 35.8877 34.6621 34.0193 34.873C33.8927 34.8871 33.765 34.8945 33.6377 34.8945H6.30289C4.92476 34.8945 3.79547 33.8008 3.75094 32.4234L3.11656 12.7762L2.90719 6.43945C2.87633 5.5 3.61304 4.71328 4.5525 4.68242L4.58023 4.68164H11.0291ZM11.0291 5.10703H4.59078L4.56656 5.10781C3.86187 5.13086 3.30914 5.72109 3.33219 6.42539L3.54195 12.7625L4.17633 32.4098C4.21344 33.5574 5.15445 34.4687 6.30289 34.4687H33.6377C33.749 34.4687 33.8607 34.4625 33.9716 34.45C35.6064 34.2656 36.7822 32.7906 36.5974 31.1559L34.3873 11.5816C34.2173 10.0754 32.9431 8.93711 31.4275 8.93711H13.0209L12.9377 8.61953L12.2638 6.05898C12.1162 5.49805 11.6091 5.10703 11.0291 5.10703Z" fill="#1B3B3C"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.72812 12.7656H36.6539C38.0637 12.7656 39.207 13.9086 39.207 15.3188C39.207 15.4328 39.1992 15.5465 39.184 15.6594L36.9922 31.943C36.7648 33.6324 35.323 34.8934 33.6184 34.8934H6.37969C4.96953 34.8934 3.82617 33.75 3.82617 32.3398C3.82617 32.2102 3.83633 32.0801 3.85586 31.952L6.36367 15.6523C6.61914 13.9914 8.04805 12.7656 9.72812 12.7656Z" fill="#1B3B3C"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.98438 14.6172H20.4848C20.899 14.6172 21.2348 14.9529 21.2348 15.3672C21.2348 15.7814 20.899 16.1172 20.4848 16.1172H8.98438C8.57013 16.1172 8.23438 15.7814 8.23438 15.3672C8.23438 14.9529 8.57013 14.6172 8.98438 14.6172Z" fill="#00BEB4"/>
+</svg>

web/src/assets/svg/rerun.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1757483419289" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="22299" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M998.765714 523.629714c13.824 0 25.014857 11.190857 25.014857 25.014857a475.282286 475.282286 0 0 1-875.593142 256.219429l-27.574858 55.149714a25.014857 25.014857 0 1 1-44.763428-22.454857l44.178286-88.283428a24.868571 24.868571 0 0 1 26.550857-25.526858 25.014857 25.014857 0 0 1 8.265143 0.804572l99.474285 24.868571a25.014857 25.014857 0 0 1-12.068571 48.566857l-46.372572-11.556571A425.252571 425.252571 0 0 0 973.750857 548.571429c0-13.897143 11.190857-25.014857 25.014857-25.014858zM430.957714 365.714286l6.729143 0.658285c2.633143 0.438857 285.549714 160.109714 285.549714 160.109715 20.114286 17.846857 7.314286 34.523429-6.582857 45.933714-1.828571 1.462857-194.779429 113.078857-249.929143 144.969143l-10.678857 6.217143-3.876571 2.194285c-16.676571 8.923429-39.497143 8.923429-47.250286-11.995428-0.877714-2.194286-2.267429-250.221714-2.56-303.396572L402.285714 400.457143l0.731429-0.512c0.731429-18.651429 8.265143-38.034286 34.669714-33.645714z m-15.945143-273.408a475.282286 475.282286 0 0 1 533.869715 200.045714l27.501714-55.149714a25.014857 25.014857 0 1 1 44.690286 22.454857l-44.105143 88.283428a24.868571 24.868571 0 0 1-26.624 25.526858 24.868571 24.868571 0 0 1-8.192-0.804572l-99.547429-24.868571a25.014857 25.014857 0 0 1 12.068572-48.566857l46.445714 11.629714A425.252571 425.252571 0 0 0 123.245714 548.571429a25.014857 25.014857 0 0 1-50.029714 0 475.282286 475.282286 0 0 1 341.796571-456.265143z" fill="#3BA05C" p-id="22300"></path></svg>

web/src/components/cross-language-form-field.tsx

@@ -30,11 +30,13 @@ const options = Languages.map((x) => ({
 type CrossLanguageItemProps = {
   name?: string;
   vertical?: boolean;
+  label?: string;
 };
 
 export const CrossLanguageFormField = ({
   name = 'prompt_config.cross_languages',
   vertical = true,
+  label,
 }: CrossLanguageItemProps) => {
   const { t } = useTranslation();
   const form = useFormContext();
@@ -53,7 +55,7 @@ export const CrossLanguageFormField = ({
           })}
         >
           <FormLabel tooltip={t('chat.crossLanguageTip')}>
-            {t('chat.crossLanguage')}
+            {label || t('chat.crossLanguage')}
           </FormLabel>
           <FormControl>
             <MultiSelect

web/src/components/data-pipeline-select/index.tsx

@@ -0,0 +1,76 @@
+import { useTranslate } from '@/hooks/common-hooks';
+import { buildSelectOptions } from '@/utils/component-util';
+import { ArrowUpRight } from 'lucide-react';
+import { useFormContext } from 'react-hook-form';
+import {
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '../ui/form';
+import { RAGFlowSelect } from '../ui/select';
+
+interface IProps {
+  toDataPipeline?: () => void;
+  formFieldName: string;
+}
+
+const data = [
+  { id: '1', name: 'data-pipeline-1' },
+  { id: '2', name: 'data-pipeline-2' },
+  { id: '3', name: 'data-pipeline-3' },
+  { id: '4', name: 'data-pipeline-4' },
+];
+export function DataFlowItem(props: IProps) {
+  const { toDataPipeline, formFieldName } = props;
+  const { t } = useTranslate('knowledgeConfiguration');
+  const form = useFormContext();
+  const toDataPipLine = () => {
+    // window.open('/data-pipeline');
+
+    toDataPipeline?.();
+  };
+  const options = buildSelectOptions(data, 'id', 'name');
+  return (
+    <FormField
+      control={form.control}
+      name={formFieldName}
+      render={({ field }) => (
+        <FormItem className=" items-center space-y-0 ">
+          <div className="flex flex-col gap-1">
+            <div className="flex gap-2 justify-between ">
+              <FormLabel
+                tooltip={t('dataFlowTip')}
+                className="text-sm text-text-primary whitespace-wrap "
+              >
+                {t('dataFlow')}
+              </FormLabel>
+              <div
+                className="text-sm flex text-text-primary cursor-pointer"
+                onClick={toDataPipLine}
+              >
+                {t('buildItFromScratch')}
+                <ArrowUpRight size={14} />
+              </div>
+            </div>
+
+            <div className="text-muted-foreground">
+              <FormControl>
+                <RAGFlowSelect
+                  {...field}
+                  placeholder={t('dataFlowPlaceholder')}
+                  options={options}
+                />
+              </FormControl>
+            </div>
+          </div>
+          <div className="flex pt-1">
+            <div className="w-1/4"></div>
+            <FormMessage />
+          </div>
+        </FormItem>
+      )}
+    />
+  );
+}

web/src/components/file-status-badge.tsx

@@ -0,0 +1,61 @@
+// src/pages/dataset/file-logs/file-status-badge.tsx
+import { FC } from 'react';
+/**
+ * params: status: 0 not run yet 1 running, 2 cancel, 3 success, 4 fail
+ */
+interface StatusBadgeProps {
+  // status: 'Success' | 'Failed' | 'Running' | 'Pending';
+  status: 0 | 1 | 2 | 3 | 4;
+  name?: string;
+}
+
+const FileStatusBadge: FC<StatusBadgeProps> = ({ status, name }) => {
+  const getStatusColor = () => {
+    // #3ba05c  → rgb(59, 160, 92)   // state-success
+    // #d8494b  → rgb(216, 73, 75)   // state-error
+    // #00beb4  → rgb(0, 190, 180)   // accent-primary
+    // #faad14  → rgb(250, 173, 20)  // state-warning
+    switch (status) {
+      case 3:
+        return `bg-[rgba(59,160,92,0.1)] text-state-success`;
+      case 4:
+        return `bg-[rgba(216,73,75,0.1)] text-state-error`;
+      case 1:
+        return `bg-[rgba(0,190,180,0.1)] text-accent-primary`;
+      case 0:
+        return `bg-[rgba(250,173,20,0.1)] text-state-warning`;
+      default:
+        return 'bg-gray-500/10 text-white';
+    }
+  };
+
+  const getBgStatusColor = () => {
+    // #3ba05c  → rgb(59, 160, 92)   // state-success
+    // #d8494b  → rgb(216, 73, 75)   // state-error
+    // #00beb4  → rgb(0, 190, 180)   // accent-primary
+    // #faad14  → rgb(250, 173, 20)  // state-warning
+    switch (status) {
+      case 3:
+        return `bg-[rgba(59,160,92,1)] text-state-success`;
+      case 4:
+        return `bg-[rgba(216,73,75,1)] text-state-error`;
+      case 1:
+        return `bg-[rgba(0,190,180,1)] text-accent-primary`;
+      case 0:
+        return `bg-[rgba(250,173,20,1)] text-state-warning`;
+      default:
+        return 'bg-gray-500/10 text-white';
+    }
+  };
+
+  return (
+    <span
+      className={`inline-flex items-center w-[75px] px-2 py-1 rounded-full text-xs font-medium ${getStatusColor()}`}
+    >
+      <div className={`w-1 h-1 mr-1 rounded-full ${getBgStatusColor()}`}></div>
+      {name || ''}
+    </span>
+  );
+};
+
+export default FileStatusBadge;

web/src/components/icon-font.tsx

@@ -4,6 +4,7 @@ import { getExtension } from '@/utils/document-util';
 
 type IconFontType = {
   name: string;
+
   className?: string;
 };
 
@@ -13,6 +14,23 @@ export const IconFont = ({ name, className }: IconFontType) => (
   </svg>
 );
 
+export function IconFontFill({
+  name,
+  className,
+  isFill = true,
+}: IconFontType & { isFill?: boolean }) {
+  return (
+    <span className={cn('size-4', className)}>
+      <svg
+        className={cn('size-4', className)}
+        style={{ fill: isFill ? 'currentColor' : '' }}
+      >
+        <use xlinkHref={`#icon-${name}`} />
+      </svg>
+    </span>
+  );
+}
+
 export function FileIcon({
   name,
   className,

web/src/components/layout-recognize-form-field.tsx

@@ -1,6 +1,7 @@
 import { LlmModelType } from '@/constants/knowledge';
 import { useTranslate } from '@/hooks/common-hooks';
 import { useSelectLlmOptionsByModelType } from '@/hooks/llm-hooks';
+import { cn } from '@/lib/utils';
 import { camelCase } from 'lodash';
 import { useMemo } from 'react';
 import { useFormContext } from 'react-hook-form';
@@ -18,7 +19,13 @@ export const enum DocumentType {
   PlainText = 'Plain Text',
 }
 
-export function LayoutRecognizeFormField() {
+export function LayoutRecognizeFormField({
+  name = 'parser_config.layout_recognize',
+  horizontal = true,
+}: {
+  name?: string;
+  horizontal?: boolean;
+}) {
   const form = useFormContext();
 
   const { t } = useTranslate('knowledgeDetails');
@@ -53,33 +60,32 @@ export function LayoutRecognizeFormField() {
   return (
     <FormField
       control={form.control}
-      name="parser_config.layout_recognize"
+      name={name}
       render={({ field }) => {
-        if (typeof field.value === 'undefined') {
-          // default value set
-          form.setValue(
-            'parser_config.layout_recognize',
-            form.formState.defaultValues?.parser_config?.layout_recognize ??
-              'DeepDOC',
-          );
-        }
         return (
-          <FormItem className=" items-center space-y-0 ">
-            <div className="flex items-center">
+          <FormItem className={'items-center space-y-0 '}>
+            <div
+              className={cn('flex', {
+                'flex-col ': !horizontal,
+                'items-center': horizontal,
+              })}
+            >
               <FormLabel
                 tooltip={t('layoutRecognizeTip')}
-                className="text-sm text-muted-foreground whitespace-wrap w-1/4"
+                className={cn('text-sm text-muted-foreground whitespace-wrap', {
+                  ['w-1/4']: horizontal,
+                })}
               >
                 {t('layoutRecognize')}
               </FormLabel>
-              <div className="w-3/4">
+              <div className={horizontal ? 'w-3/4' : 'w-full'}>
                 <FormControl>
                   <RAGFlowSelect {...field} options={options}></RAGFlowSelect>
                 </FormControl>
               </div>
             </div>
             <div className="flex pt-1">
-              <div className="w-1/4"></div>
+              <div className={horizontal ? 'w-1/4' : 'w-full'}></div>
               <FormMessage />
             </div>
           </FormItem>

web/src/components/llm-setting-items/llm-form-field.tsx

@@ -0,0 +1,25 @@
+import { LlmModelType } from '@/constants/knowledge';
+import { useComposeLlmOptionsByModelTypes } from '@/hooks/llm-hooks';
+import { useTranslation } from 'react-i18next';
+import { SelectWithSearch } from '../originui/select-with-search';
+import { RAGFlowFormItem } from '../ragflow-form';
+
+type LLMFormFieldProps = {
+  options?: any[];
+  name?: string;
+};
+
+export function LLMFormField({ options, name }: LLMFormFieldProps) {
+  const { t } = useTranslation();
+
+  const modelOptions = useComposeLlmOptionsByModelTypes([
+    LlmModelType.Chat,
+    LlmModelType.Image2text,
+  ]);
+
+  return (
+    <RAGFlowFormItem name={name || 'llm_id'} label={t('chat.model')}>
+      <SelectWithSearch options={options || modelOptions}></SelectWithSearch>
+    </RAGFlowFormItem>
+  );
+}

web/src/components/llm-setting-items/next.tsx

@@ -1,11 +1,9 @@
-import { LlmModelType, ModelVariableType } from '@/constants/knowledge';
+import { ModelVariableType } from '@/constants/knowledge';
 import { useTranslate } from '@/hooks/common-hooks';
-import { useComposeLlmOptionsByModelTypes } from '@/hooks/llm-hooks';
 import { camelCase } from 'lodash';
 import { useCallback } from 'react';
 import { useFormContext } from 'react-hook-form';
 import { z } from 'zod';
-import { SelectWithSearch } from '../originui/select-with-search';
 import {
   FormControl,
   FormField,
@@ -20,6 +18,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from '../ui/select';
+import { LLMFormField } from './llm-form-field';
 import { SliderInputSwitchFormField } from './slider';
 import { useHandleFreedomChange } from './use-watch-change';
 
@@ -61,11 +60,6 @@ export function LlmSettingFieldItems({
   const form = useFormContext();
   const { t } = useTranslate('chat');
 
-  const modelOptions = useComposeLlmOptionsByModelTypes([
-    LlmModelType.Chat,
-    LlmModelType.Image2text,
-  ]);
-
   const getFieldWithPrefix = useCallback(
     (name: string) => {
       return prefix ? `${prefix}.${name}` : name;
@@ -82,22 +76,7 @@ export function LlmSettingFieldItems({
 
   return (
     <div className="space-y-5">
-      <FormField
-        control={form.control}
-        name={'llm_id'}
-        render={({ field }) => (
-          <FormItem>
-            <FormLabel>{t('model')}</FormLabel>
-            <FormControl>
-              <SelectWithSearch
-                options={options || modelOptions}
-                {...field}
-              ></SelectWithSearch>
-            </FormControl>
-            <FormMessage />
-          </FormItem>
-        )}
-      />
+      <LLMFormField options={options}></LLMFormField>
       <FormField
         control={form.control}
         name={'parameter'}

web/src/components/originui/timeline.tsx

@@ -1,6 +1,7 @@
 'use client';
 
 import { cn } from '@/lib/utils';
+import { parseColorToRGBA } from '@/utils/common-util';
 import { Slot } from '@radix-ui/react-slot';
 import * as React from 'react';
 
@@ -197,7 +198,208 @@ function TimelineTitle({
   );
 }
 
+interface TimelineIndicatorNodeProps {
+  nodeSize?: string | number;
+  iconColor?: string;
+  lineColor?: string;
+  textColor?: string;
+  indicatorBgColor?: string;
+  indicatorBorderColor?: string;
+}
+interface TimelineNode
+  extends Omit<
+      React.HTMLAttributes<HTMLDivElement>,
+      'id' | 'title' | 'content'
+    >,
+    TimelineIndicatorNodeProps {
+  id: string | number;
+  title?: React.ReactNode;
+  content?: React.ReactNode;
+  date?: React.ReactNode;
+  icon?: React.ReactNode;
+  completed?: boolean;
+  clickable?: boolean;
+  activeStyle?: TimelineIndicatorNodeProps;
+}
+
+interface CustomTimelineProps extends React.HTMLAttributes<HTMLDivElement> {
+  nodes: TimelineNode[];
+  activeStep?: number;
+  nodeSize?: string | number;
+  onStepChange?: (step: number, id: string | number) => void;
+  orientation?: 'horizontal' | 'vertical';
+  lineStyle?: 'solid' | 'dashed';
+  lineColor?: string;
+  indicatorColor?: string;
+  defaultValue?: number;
+  activeStyle?: TimelineIndicatorNodeProps;
+}
+
+const CustomTimeline = ({
+  nodes,
+  activeStep,
+  nodeSize = 12,
+  onStepChange,
+  orientation = 'horizontal',
+  lineStyle = 'solid',
+  lineColor = 'var(--text-secondary)',
+  indicatorColor = 'var(--accent-primary)',
+  defaultValue = 1,
+  className,
+  activeStyle,
+  ...props
+}: CustomTimelineProps) => {
+  const [internalActiveStep, setInternalActiveStep] =
+    React.useState(defaultValue);
+  const _lineColor = `rgb(${parseColorToRGBA(lineColor)})`;
+  console.log(lineColor, _lineColor);
+  const currentActiveStep = activeStep ?? internalActiveStep;
+
+  const handleStepChange = (step: number, id: string | number) => {
+    if (activeStep === undefined) {
+      setInternalActiveStep(step);
+    }
+    onStepChange?.(step, id);
+  };
+  const [r, g, b] = parseColorToRGBA(indicatorColor);
+  return (
+    <Timeline
+      value={currentActiveStep}
+      onValueChange={(step) => handleStepChange(step, nodes[step - 1]?.id)}
+      orientation={orientation}
+      className={className}
+      {...props}
+    >
+      {nodes.map((node, index) => {
+        const step = index + 1;
+        const isCompleted = node.completed ?? step <= currentActiveStep;
+        const isActive = step === currentActiveStep;
+        const isClickable = node.clickable ?? true;
+        const _activeStyle = node.activeStyle ?? (activeStyle || {});
+        const _nodeSizeTemp =
+          isActive && _activeStyle?.nodeSize
+            ? _activeStyle?.nodeSize
+            : node.nodeSize ?? nodeSize;
+        const _nodeSize =
+          typeof _nodeSizeTemp === 'number'
+            ? `${_nodeSizeTemp}px`
+            : _nodeSizeTemp;
+        console.log('icon-size', nodeSize, node.nodeSize, _nodeSize);
+        // const activeStyle = _activeStyle || {};
+
+        return (
+          <TimelineItem
+            key={node.id}
+            step={step}
+            className={cn(
+              node.className,
+              isClickable &&
+                'cursor-pointer hover:opacity-80 transition-opacity',
+              isCompleted && 'data-[completed]:data-completed/timeline-item',
+              isActive && 'relative z-10',
+            )}
+            onClick={() => isClickable && handleStepChange(step, node.id)}
+          >
+            <TimelineSeparator
+              className={cn(
+                'group-data-[orientation=horizontal]/timeline:-top-6 group-data-[orientation=horizontal]/timeline:h-0.1  group-data-[orientation=horizontal]/timeline:-translate-y-1/2',
+                'group-data-[orientation=vertical]/timeline:-left-6 group-data-[orientation=vertical]/timeline:w-0.1 group-data-[orientation=vertical]/timeline:-translate-x-1/2 ',
+                // `group-data-[orientation=horizontal]/timeline:w-[calc(100%-0.5rem-1rem)] group-data-[orientation=vertical]/timeline:h-[calc(100%-1rem-1rem)] group-data-[orientation=vertical]/timeline:translate-y-7 group-data-[orientation=horizontal]/timeline:translate-x-7`,
+              )}
+              style={{
+                border:
+                  lineStyle === 'dashed'
+                    ? `1px dashed ${isActive ? _activeStyle.lineColor || _lineColor : _lineColor}`
+                    : lineStyle === 'solid'
+                      ? `1px solid ${isActive ? _activeStyle.lineColor || _lineColor : _lineColor}`
+                      : 'none',
+                backgroundColor: 'transparent',
+                width:
+                  orientation === 'horizontal'
+                    ? `calc(100% - ${_nodeSize} - 2px - 0.1rem)`
+                    : '1px',
+                height:
+                  orientation === 'vertical'
+                    ? `calc(100% - ${_nodeSize} - 2px - 0.1rem)`
+                    : '1px',
+                transform: `translate(${
+                  orientation === 'horizontal' ? `${_nodeSize}` : '0'
+                }, ${orientation === 'vertical' ? `${_nodeSize}` : '0'})`,
+              }}
+            />
+
+            <TimelineIndicator
+              className={cn(
+                'flex items-center justify-center p-1',
+                isCompleted && 'bg-primary border-primary',
+                !isCompleted && 'border-text-secondary bg-bg-base',
+              )}
+              style={{
+                width: _nodeSize,
+                height: _nodeSize,
+                borderColor: isActive
+                  ? _activeStyle.indicatorBorderColor || indicatorColor
+                  : isCompleted
+                    ? indicatorColor
+                    : '',
+                // backgroundColor: isActive
+                //   ? _activeStyle.indicatorBgColor || indicatorColor
+                //   : isCompleted
+                //     ? indicatorColor
+                //     : '',
+                backgroundColor: isActive
+                  ? _activeStyle.indicatorBgColor ||
+                    `rgba(${r}, ${g}, ${b}, 0.1)`
+                  : isCompleted
+                    ? `rgba(${r}, ${g}, ${b}, 0.1)`
+                    : '',
+              }}
+            >
+              {node.icon && (
+                <div
+                  className={cn(
+                    'text-current',
+                    `w-[${_nodeSize}] h-[${_nodeSize}]`,
+                    isActive &&
+                      `text-primary w-[${_activeStyle.nodeSize || _nodeSize}] h-[${_activeStyle.nodeSize || _nodeSize}]`,
+                  )}
+                  style={{
+                    color: isActive ? _activeStyle.iconColor : undefined,
+                  }}
+                >
+                  {node.icon}
+                </div>
+              )}
+            </TimelineIndicator>
+
+            <TimelineHeader>
+              {node.date && <TimelineDate>{node.date}</TimelineDate>}
+              <TimelineTitle
+                className={cn(
+                  'text-sm font-medium',
+                  isActive && _activeStyle.textColor
+                    ? `text-${_activeStyle.textColor}`
+                    : '',
+                )}
+                style={{
+                  color: isActive ? _activeStyle.textColor : undefined,
+                }}
+              >
+                {node.title}
+              </TimelineTitle>
+            </TimelineHeader>
+            {node.content && <TimelineContent>{node.content}</TimelineContent>}
+          </TimelineItem>
+        );
+      })}
+    </Timeline>
+  );
+};
+
+CustomTimeline.displayName = 'CustomTimeline';
+
 export {
+  CustomTimeline,
   Timeline,
   TimelineContent,
   TimelineDate,
@@ -206,4 +408,5 @@ export {
   TimelineItem,
   TimelineSeparator,
   TimelineTitle,
+  type TimelineNode,
 };

web/src/components/ragflow-form.tsx

@@ -15,6 +15,8 @@ type RAGFlowFormItemProps = {
   tooltip?: ReactNode;
   children: ReactNode | ((field: ControllerRenderProps) => ReactNode);
   horizontal?: boolean;
+  required?: boolean;
+  labelClassName?: string;
 };
 
 export function RAGFlowFormItem({
@@ -23,6 +25,8 @@ export function RAGFlowFormItem({
   tooltip,
   children,
   horizontal = false,
+  required = false,
+  labelClassName,
 }: RAGFlowFormItemProps) {
   const form = useFormContext();
   return (
@@ -35,7 +39,11 @@ export function RAGFlowFormItem({
             'flex items-center': horizontal,
           })}
         >
-          <FormLabel tooltip={tooltip} className={cn({ 'w-1/4': horizontal })}>
+          <FormLabel
+            required={required}
+            tooltip={tooltip}
+            className={cn({ 'w-1/4': horizontal }, labelClassName)}
+          >
             {label}
           </FormLabel>
           <FormControl>

web/src/components/spotlight.tsx

@@ -3,10 +3,22 @@ import React from 'react';
 
 interface SpotlightProps {
   className?: string;
+  opcity?: number;
+  coverage?: number;
 }
-
-const Spotlight: React.FC<SpotlightProps> = ({ className }) => {
+/**
+ *
+ * @param opcity 0~1 default 0.5
+ * @param coverage 0~100 default 60
+ * @returns
+ */
+const Spotlight: React.FC<SpotlightProps> = ({
+  className,
+  opcity = 0.5,
+  coverage = 60,
+}) => {
   const isDark = useIsDarkTheme();
+  const rgb = isDark ? '255, 255, 255' : '194, 221, 243';
   return (
     <div
       className={`absolute inset-0  opacity-80 ${className} rounded-lg`}
@@ -18,9 +30,7 @@ const Spotlight: React.FC<SpotlightProps> = ({ className }) => {
       <div
         className="absolute inset-0"
         style={{
-          background: isDark
-            ? 'radial-gradient(circle at 50% 190%, #fff4 0%, #fff0 60%)'
-            : 'radial-gradient(circle at 50% 190%, #E4F3FF 0%, #E4F3FF00  60%)',
+          background: `radial-gradient(circle at 50% 190%, rgba(${rgb},${opcity}) 0%, rgba(${rgb},0) ${coverage}%)`,
           pointerEvents: 'none',
         }}
       ></div>

web/src/components/ui/dialog.tsx

@@ -38,7 +38,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        'fixed left-[50%] top-[50%] z-50 grid w-full max-w-xl  translate-x-[-50%] translate-y-[-50%] gap-4 border bg-colors-background-neutral-standard p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg',
+        'fixed left-[50%] top-[50%] z-50 grid w-full max-w-xl  translate-x-[-50%] translate-y-[-50%] gap-4 border bg-bg-base p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg',
         className,
       )}
       {...props}

web/src/components/ui/modal/modal.tsx

@@ -187,7 +187,7 @@ const Modal: ModalType = ({
             )}
 
             {/* content */}
-            <div className="py-2 px-6 overflow-y-auto max-h-[80vh] focus-visible:!outline-none">
+            <div className="py-2 px-6 overflow-y-auto scrollbar-auto max-h-[80vh] focus-visible:!outline-none">
               {destroyOnClose && !open ? null : children}
             </div>
 

web/src/constants/knowledge.ts

@@ -57,6 +57,7 @@ export enum LlmModelType {
 export enum KnowledgeSearchParams {
   DocumentId = 'doc_id',
   KnowledgeId = 'id',
+  Type = 'type',
 }
 
 export enum DocumentType {

web/src/hooks/logic-hooks/navigate-hooks.ts

@@ -87,7 +87,7 @@ export const useNavigatePage = () => {
     (id: string, knowledgeId?: string) => () => {
       navigate(
         // `${Routes.ParsedResult}/${id}?${QueryStringMap.KnowledgeId}=${knowledgeId}`,
-        `${Routes.ParsedResult}/chunks?id=${knowledgeId}&doc_id=${id}`,
+        `${Routes.DataflowResult}?id=${knowledgeId}&doc_id=${id}&type=chunk`,
       );
     },
     [navigate],
@@ -125,6 +125,16 @@ export const useNavigatePage = () => {
     [navigate],
   );
 
+  const navigateToDataflowResult = useCallback(
+    (id: string, knowledgeId?: string) => () => {
+      navigate(
+        // `${Routes.ParsedResult}/${id}?${QueryStringMap.KnowledgeId}=${knowledgeId}`,
+        `${Routes.DataflowResult}?id=${knowledgeId}&doc_id=${id}&type=dataflow`,
+      );
+    },
+    [navigate],
+  );
+
   return {
     navigateToDatasetList,
     navigateToDataset,
@@ -144,5 +154,6 @@ export const useNavigatePage = () => {
     navigateToFiles,
     navigateToAgentList,
     navigateToOldProfile,
+    navigateToDataflowResult,
   };
 };

web/src/hooks/route-hook.ts

@@ -29,6 +29,7 @@ export const useGetKnowledgeSearchParams = () => {
   const [currentQueryParameters] = useSearchParams();
 
   return {
+    type: currentQueryParameters.get(KnowledgeSearchParams.Type) || '',
     documentId:
       currentQueryParameters.get(KnowledgeSearchParams.DocumentId) || '',
     knowledgeId:

web/src/hooks/use-agent-request.ts

@@ -24,9 +24,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { useDebounce } from 'ahooks';
 import { get, set } from 'lodash';
 import { useCallback, useState } from 'react';
-import { useTranslation } from 'react-i18next';
 import { useParams, useSearchParams } from 'umi';
-import { v4 as uuid } from 'uuid';
 import {
   useGetPaginationWithRouter,
   useHandleSearchChange,
@@ -80,7 +78,7 @@ export const EmptyDsl = {
         component_name: 'Begin',
         params: {},
       },
-      downstream: ['Answer:China'], // other edge target is downstream, edge source is current node id
+      downstream: [], // other edge target is downstream, edge source is current node id
       upstream: [], // edge source is upstream, edge target is current node id
     },
   },
@@ -96,21 +94,11 @@ export const EmptyDsl = {
 };
 
 export const useFetchAgentTemplates = () => {
-  const { t } = useTranslation();
-
   const { data } = useQuery<IFlowTemplate[]>({
     queryKey: [AgentApiAction.FetchAgentTemplates],
     initialData: [],
     queryFn: async () => {
       const { data } = await agentService.listTemplates();
-      if (Array.isArray(data?.data)) {
-        data.data.unshift({
-          id: uuid(),
-          title: t('flow.blank'),
-          description: t('flow.createFromNothing'),
-          dsl: EmptyDsl,
-        });
-      }
 
       return data.data;
     },

web/src/hooks/use-dataflow-request.ts

@@ -0,0 +1,127 @@
+import message from '@/components/ui/message';
+import { IFlow } from '@/interfaces/database/agent';
+import { Operator } from '@/pages/data-flow/constant';
+import dataflowService from '@/services/dataflow-service';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { useTranslation } from 'react-i18next';
+import { useParams } from 'umi';
+
+export const enum DataflowApiAction {
+  ListDataflow = 'listDataflow',
+  RemoveDataflow = 'removeDataflow',
+  FetchDataflow = 'fetchDataflow',
+  RunDataflow = 'runDataflow',
+  SetDataflow = 'setDataflow',
+}
+
+export const EmptyDsl = {
+  graph: {
+    nodes: [
+      {
+        id: Operator.Begin,
+        type: 'beginNode',
+        position: {
+          x: 50,
+          y: 200,
+        },
+        data: {
+          label: 'Begin',
+          name: Operator.Begin,
+        },
+        sourcePosition: 'left',
+        targetPosition: 'right',
+      },
+    ],
+    edges: [],
+  },
+  components: {
+    begin: {
+      obj: {
+        component_name: 'Begin',
+        params: {},
+      },
+      downstream: [], // other edge target is downstream, edge source is current node id
+      upstream: [], // edge source is upstream, edge target is current node id
+    },
+  },
+  retrieval: [], // reference
+  history: [],
+  path: [],
+};
+
+export const useRemoveDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.RemoveDataflow],
+    mutationFn: async (ids: string[]) => {
+      const { data } = await dataflowService.removeDataflow({
+        canvas_ids: ids,
+      });
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.ListDataflow],
+        });
+
+        message.success(t('message.deleted'));
+      }
+      return data.code;
+    },
+  });
+
+  return { data, loading, removeDataflow: mutateAsync };
+};
+
+export const useSetDataflow = () => {
+  const queryClient = useQueryClient();
+  const { t } = useTranslation();
+
+  const {
+    data,
+    isPending: loading,
+    mutateAsync,
+  } = useMutation({
+    mutationKey: [DataflowApiAction.SetDataflow],
+    mutationFn: async (params: Partial<IFlow>) => {
+      const { data } = await dataflowService.setDataflow(params);
+      if (data.code === 0) {
+        queryClient.invalidateQueries({
+          queryKey: [DataflowApiAction.FetchDataflow],
+        });
+
+        message.success(t(`message.${params.id ? 'modified' : 'created'}`));
+      }
+      return data?.code;
+    },
+  });
+
+  return { data, loading, setDataflow: mutateAsync };
+};
+
+export const useFetchDataflow = () => {
+  const { id } = useParams();
+
+  const {
+    data,
+    isFetching: loading,
+    refetch,
+  } = useQuery<IFlow>({
+    queryKey: [DataflowApiAction.FetchDataflow, id],
+    gcTime: 0,
+    initialData: {} as IFlow,
+    enabled: !!id,
+    refetchOnWindowFocus: false,
+    queryFn: async () => {
+      const { data } = await dataflowService.fetchDataflow(id);
+
+      return data?.data ?? ({} as IFlow);
+    },
+  });
+
+  return { data, loading, refetch };
+};

web/src/interfaces/database/agent.ts

@@ -41,8 +41,8 @@ export interface DSL {
   path?: string[];
   answer?: any[];
   graph?: IGraph;
-  messages: Message[];
-  reference: IReference[];
+  messages?: Message[];
+  reference?: IReference[];
   globals: Record<string, any>;
   retrieval: IReference[];
 }

web/src/locales/en.ts

@@ -102,6 +102,28 @@ export default {
       noMoreData: `That's all. Nothing more.`,
     },
     knowledgeDetails: {
+      generateKnowledgeGraph:
+        'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.',
+      generateRaptor:
+        'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.',
+      generate: 'Generate',
+      raptor: 'Raptor',
+      knowledgeGraph: 'Knowledge Graph',
+      processingType: 'Processing Type',
+      dataPipeline: 'Data Pipeline',
+      operations: 'Operations',
+      status: 'Status',
+      task: 'Task',
+      startDate: 'Start Date',
+      source: 'Source',
+      fileName: 'File Name',
+      datasetLogs: 'Dataset Logs',
+      fileLogs: 'File Logs',
+      overview: 'Overview',
+      success: 'Success',
+      failed: 'Failed',
+      completed: 'Completed',
+      processLog: 'Process Log',
       created: 'Created',
       learnMore: 'Learn More',
       general: 'General',
@@ -195,6 +217,7 @@ export default {
       chunk: 'Chunk',
       bulk: 'Bulk',
       cancel: 'Cancel',
+      close: 'Close',
       rerankModel: 'Rerank model',
       rerankPlaceholder: 'Please select',
       rerankTip: `Optional. If left empty, RAGFlow will use a combination of weighted keyword similarity and weighted vector cosine similarity; if a rerank model is selected, a weighted reranking score will replace the weighted vector cosine similarity. Please be aware that using a rerank model will significantly increase the system's response time. If you wish to use a rerank model, ensure you use a SaaS reranker; if you prefer a locally deployed rerank model, ensure you start RAGFlow with docker-compose-gpu.yml.`,
@@ -238,6 +261,17 @@ export default {
       reRankModelWaring: 'Re-rank model is very time consuming.',
     },
     knowledgeConfiguration: {
+      default: 'Default',
+      dataPipeline: 'Data Pipeline',
+      linkDataPipeline: 'Link Data Pipeline',
+      enableAutoGenerate: 'Enable Auto Generate',
+      teamPlaceholder: 'Please select a team.',
+      dataFlowPlaceholder: 'Please select a data flow.',
+      buildItFromScratch: 'Build it from scratch',
+      dataFlow: 'Data Flow',
+      parseType: 'Parse Type',
+      manualSetup: 'Manual Setup',
+      builtIn: 'Built-in',
       titleDescription:
         'Update your knowledge base configuration here, particularly the chunking method.',
       name: 'Knowledge base name',
@@ -934,7 +968,7 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
       exceptionMethod: 'Exception method',
       maxRounds: 'Max reflection rounds',
       delayEfterError: 'Delay after error',
-      maxRetries: 'Max retries',
+      maxRetries: 'Max reflection rounds',
       advancedSettings: 'Advanced Settings',
       addTools: 'Add Tools',
       sysPromptDefultValue: `
@@ -1589,5 +1623,24 @@ This delimiter is used to split the input text into several text pieces echo of
       total: 'Total {{total}}',
       page: '{{page}} /Page',
     },
+    dataflowParser: {
+      parseSummary: 'Parse Summary',
+      parseSummaryTip: 'Parser：deepdoc',
+      rerunFromCurrentStep: 'Rerun From Current Step',
+      rerunFromCurrentStepTip: 'Changes detected. Click to re-run.',
+    },
+    dataflow: {
+      parser: 'Parser',
+      parserDescription: 'Parser',
+      chunker: 'Chunker',
+      chunkerDescription: 'Chunker',
+      tokenizer: 'Tokenizer',
+      tokenizerDescription: 'Tokenizer',
+      outputFormat: 'Output format',
+      lang: 'Language',
+      fileFormats: 'File formats',
+      fields: 'Fields',
+      addParser: 'Add Parser',
+    },
   },
 };

web/src/locales/zh.ts

@@ -94,6 +94,24 @@ export default {
       noMoreData: '没有更多数据了',
     },
     knowledgeDetails: {
+      generate: '生成',
+      raptor: 'Raptor',
+      knowledgeGraph: '知识图谱',
+      processingType: '处理类型',
+      dataPipeline: '数据管道',
+      operations: '操作',
+      status: '状态',
+      task: '任务',
+      startDate: '开始时间',
+      source: '来源',
+      fileName: '文件名',
+      datasetLogs: '数据集日志',
+      fileLogs: '文件日志',
+      overview: '概览',
+      success: '成功',
+      failed: '失败',
+      completed: '已完成',
+      processLog: '处理进度日志',
       created: '创建于',
       learnMore: '了解更多',
       general: '通用',
@@ -183,6 +201,7 @@ export default {
       chunk: '解析块',
       bulk: '批量',
       cancel: '取消',
+      close: '关闭',
       rerankModel: 'Rerank模型',
       rerankPlaceholder: '请选择',
       rerankTip: `非必选项：若不选择 rerank 模型，系统将默认采用关键词相似度与向量余弦相似度相结合的混合查询方式；如果设置了 rerank 模型，则混合查询中的向量相似度部分将被 rerank 打分替代。请注意：采用 rerank 模型会非常耗时。如需选用 rerank 模型，建议使用 SaaS 的 rerank 模型服务；如果你倾向使用本地部署的 rerank 模型，请务必确保你使用 docker-compose-gpu.yml 启动 RAGFlow。`,
@@ -227,6 +246,17 @@ export default {
       theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除',
     },
     knowledgeConfiguration: {
+      default: '默认',
+      dataPipeline: '数据流',
+      linkDataPipeline: '关联数据流',
+      enableAutoGenerate: '是否启用自动生成',
+      teamPlaceholder: '请选择团队',
+      dataFlowPlaceholder: '请选择数据流',
+      buildItFromScratch: '去Scratch构建',
+      dataFlow: '数据流',
+      parseType: '切片方法',
+      manualSetup: '手动设置',
+      builtIn: '内置',
       titleDescription: '在这里更新您的知识库详细信息，尤其是切片方法。',
       name: '知识库名称',
       photo: '知识库图片',
@@ -892,7 +922,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       exceptionMethod: '异常处理方法',
       maxRounds: '最大反思轮数',
       delayEfterError: '错误后延迟',
-      maxRetries: '最大重试次数',
+      maxRetries: '最大反思轮数',
       advancedSettings: '高级设置',
       addTools: '添加工具',
       sysPromptDefultValue: `
@@ -1501,5 +1531,24 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       total: '总共 {{total}} 条',
       page: '{{page}}条/页',
     },
+    dataflowParser: {
+      parseSummary: '解析摘要',
+      parseSummaryTip: '解析器: deepdoc',
+      rerunFromCurrentStep: '从当前步骤重新运行',
+      rerunFromCurrentStepTip: '已修改，点击重新运行。',
+    },
+    dataflow: {
+      parser: '解析器',
+      parserDescription: '解析器',
+      chunker: '分块器',
+      chunkerDescription: '分块器',
+      tokenizer: '分词器',
+      tokenizerDescription: '分词器',
+      outputFormat: '输出格式',
+      lang: '语言',
+      fileFormats: '文件格式',
+      fields: '字段',
+      addParser: '增加解析器',
+    },
   },
 };

web/src/pages/agent/canvas/index.tsx

@@ -40,6 +40,7 @@ import { useCacheChatLog } from '../hooks/use-cache-chat-log';
 import { useMoveNote } from '../hooks/use-move-note';
 import { useDropdownManager } from './context';
 
+import Spotlight from '@/components/spotlight';
 import {
   useHideFormSheetOnNodeDeletion,
   useShowDrawer,
@@ -309,6 +310,7 @@ function AgentCanvas({ drawerVisible, hideDrawer }: IProps) {
           onBeforeDelete={handleBeforeDelete}
         >
           <AgentBackground></AgentBackground>
+          <Spotlight className="z-0" opcity={0.7} coverage={70} />
           <Controls position={'bottom-center'} orientation="horizontal">
             <ControlButton>
               <Tooltip>

web/src/pages/agent/form/agent-form/index.tsx

@@ -57,13 +57,6 @@ const FormSchema = z.object({
   //   )
   //   .optional(),
   message_history_window_size: z.coerce.number(),
-  tools: z
-    .array(
-      z.object({
-        component_name: z.string(),
-      }),
-    )
-    .optional(),
   ...LlmSettingSchema,
   max_retries: z.coerce.number(),
   delay_after_error: z.coerce.number().optional(),

web/src/pages/agent/form/agent-form/use-values.ts

@@ -1,15 +1,21 @@
 import { useFetchModelId } from '@/hooks/logic-hooks';
 import { RAGFlowNodeType } from '@/interfaces/database/flow';
-import { get, isEmpty } from 'lodash';
+import { get, isEmpty, omit } from 'lodash';
 import { useMemo } from 'react';
 import { initialAgentValues } from '../../constant';
 
+// You need to exclude the mcp and tools fields that are not in the form,
+// otherwise the form data update will reset the tools or mcp data to an array
+function omitToolsAndMcp(values: Record<string, any>) {
+  return omit(values, ['mcp', 'tools']);
+}
+
 export function useValues(node?: RAGFlowNodeType) {
   const llmId = useFetchModelId();
 
   const defaultValues = useMemo(
     () => ({
-      ...initialAgentValues,
+      ...omitToolsAndMcp(initialAgentValues),
       llm_id: llmId,
       prompts: '',
     }),
@@ -24,7 +30,7 @@ export function useValues(node?: RAGFlowNodeType) {
     }
 
     return {
-      ...formData,
+      ...omitToolsAndMcp(formData),
       prompts: get(formData, 'prompts.0.content', ''),
     };
   }, [defaultValues, node?.data?.form]);

web/src/pages/agent/hooks/use-export-json.ts

@@ -1,71 +1,17 @@
-import { useToast } from '@/components/hooks/use-toast';
-import { FileMimeType, Platform } from '@/constants/common';
-import { useSetModalState } from '@/hooks/common-hooks';
 import { useFetchAgent } from '@/hooks/use-agent-request';
-import { IGraph } from '@/interfaces/database/flow';
 import { downloadJsonFile } from '@/utils/file-util';
-import { message } from 'antd';
-import isEmpty from 'lodash/isEmpty';
 import { useCallback } from 'react';
-import { useTranslation } from 'react-i18next';
 import { useBuildDslData } from './use-build-dsl';
-import { useSetGraphInfo } from './use-set-graph';
 
-export const useHandleExportOrImportJsonFile = () => {
+export const useHandleExportJsonFile = () => {
   const { buildDslData } = useBuildDslData();
-  const {
-    visible: fileUploadVisible,
-    hideModal: hideFileUploadModal,
-    showModal: showFileUploadModal,
-  } = useSetModalState();
-  const setGraphInfo = useSetGraphInfo();
   const { data } = useFetchAgent();
-  const { t } = useTranslation();
-  const { toast } = useToast();
-
-  const onFileUploadOk = useCallback(
-    async ({
-      fileList,
-      platform,
-    }: {
-      fileList: File[];
-      platform: Platform;
-    }) => {
-      console.log('🚀 ~ useHandleExportOrImportJsonFile ~ platform:', platform);
-      if (fileList.length > 0) {
-        const file = fileList[0];
-        if (file.type !== FileMimeType.Json) {
-          toast({ title: t('flow.jsonUploadTypeErrorMessage') });
-          return;
-        }
-
-        const graphStr = await file.text();
-        const errorMessage = t('flow.jsonUploadContentErrorMessage');
-        try {
-          const graph = JSON.parse(graphStr);
-          if (graphStr && !isEmpty(graph) && Array.isArray(graph?.nodes)) {
-            setGraphInfo(graph ?? ({} as IGraph));
-            hideFileUploadModal();
-          } else {
-            message.error(errorMessage);
-          }
-        } catch (error) {
-          message.error(errorMessage);
-        }
-      }
-    },
-    [hideFileUploadModal, setGraphInfo, t, toast],
-  );
 
   const handleExportJson = useCallback(() => {
     downloadJsonFile(buildDslData().graph, `${data.title}.json`);
   }, [buildDslData, data.title]);
 
   return {
-    fileUploadVisible,
     handleExportJson,
-    handleImportJson: showFileUploadModal,
-    hideFileUploadModal,
-    onFileUploadOk,
   };
 };

web/src/pages/agent/index.tsx

@@ -24,7 +24,6 @@ import { ReactFlowProvider } from '@xyflow/react';
 import {
   ChevronDown,
   CirclePlay,
-  Download,
   History,
   LaptopMinimalCheck,
   Logs,
@@ -37,7 +36,7 @@ import { useTranslation } from 'react-i18next';
 import { useParams } from 'umi';
 import AgentCanvas from './canvas';
 import { DropdownProvider } from './canvas/context';
-import { useHandleExportOrImportJsonFile } from './hooks/use-export-json';
+import { useHandleExportJsonFile } from './hooks/use-export-json';
 import { useFetchDataOnMount } from './hooks/use-fetch-data';
 import { useGetBeginNodeDataInputs } from './hooks/use-get-begin-query';
 import {
@@ -46,7 +45,6 @@ import {
   useWatchAgentChange,
 } from './hooks/use-save-graph';
 import { SettingDialog } from './setting-dialog';
-import { UploadAgentDialog } from './upload-agent-dialog';
 import { useAgentHistoryManager } from './use-agent-history-manager';
 import { VersionDialog } from './version-dialog';
 
@@ -71,13 +69,8 @@ export default function Agent() {
   } = useSetModalState();
   const { t } = useTranslation();
   useAgentHistoryManager();
-  const {
-    handleExportJson,
-    handleImportJson,
-    fileUploadVisible,
-    onFileUploadOk,
-    hideFileUploadModal,
-  } = useHandleExportOrImportJsonFile();
+
+  const { handleExportJson } = useHandleExportJsonFile();
   const { saveGraph, loading } = useSaveGraph();
   const { flowDetail: agentDetail } = useFetchDataOnMount();
   const inputs = useGetBeginNodeDataInputs();
@@ -158,11 +151,6 @@ export default function Agent() {
               </Button>
             </DropdownMenuTrigger>
             <DropdownMenuContent>
-              <AgentDropdownMenuItem onClick={handleImportJson}>
-                <Download />
-                {t('flow.import')}
-              </AgentDropdownMenuItem>
-              <DropdownMenuSeparator />
               <AgentDropdownMenuItem onClick={handleExportJson}>
                 <Upload />
                 {t('flow.export')}
@@ -193,12 +181,6 @@ export default function Agent() {
           ></AgentCanvas>
         </DropdownProvider>
       </ReactFlowProvider>
-      {fileUploadVisible && (
-        <UploadAgentDialog
-          hideModal={hideFileUploadModal}
-          onOk={onFileUploadOk}
-        ></UploadAgentDialog>
-      )}
       {embedVisible && (
         <EmbedDialog
           visible={embedVisible}

web/src/pages/agents/agent-templates.tsx

@@ -27,9 +27,11 @@ export default function AgentTemplates() {
   const [selectMenuItem, setSelectMenuItem] = useState<string>(
     MenuItemKey.Recommended,
   );
+
   useEffect(() => {
     setTemplateList(list);
   }, [list]);
+
   const {
     visible: creatingVisible,
     hideModal: hideCreatingModal,
@@ -110,10 +112,9 @@ export default function AgentTemplates() {
 
         <main className="flex-1 bg-text-title-invert/50 h-dvh">
           <div className="grid gap-6 sm:grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 2xl:grid-cols-5 max-h-[94vh] overflow-auto px-8 pt-8">
-            {tempListFilter?.map((x, index) => {
+            {tempListFilter?.map((x) => {
               return (
                 <TemplateCard
-                  isCreate={index === 0}
                   key={x.id}
                   data={x}
                   showModal={showModal}

web/src/pages/agents/constant.ts

@@ -0,0 +1,4 @@
+export enum FlowType {
+  Agent = 'agent',
+  Flow = 'flow',
+}

web/src/pages/agents/create-agent-dialog.tsx

@@ -6,16 +6,18 @@ import {
   DialogHeader,
   DialogTitle,
 } from '@/components/ui/dialog';
-import { IModalProps } from '@/interfaces/common';
 import { TagRenameId } from '@/pages/add-knowledge/constant';
 import { useTranslation } from 'react-i18next';
-import { CreateAgentForm } from './create-agent-form';
+import { CreateAgentForm, CreateAgentFormProps } from './create-agent-form';
+
+type CreateAgentDialogProps = CreateAgentFormProps;
 
 export function CreateAgentDialog({
   hideModal,
   onOk,
   loading,
-}: IModalProps<any>) {
+  shouldChooseAgent,
+}: CreateAgentDialogProps) {
   const { t } = useTranslation();
 
   return (
@@ -24,7 +26,11 @@ export function CreateAgentDialog({
         <DialogHeader>
           <DialogTitle>{t('flow.createGraph')}</DialogTitle>
         </DialogHeader>
-        <CreateAgentForm hideModal={hideModal} onOk={onOk}></CreateAgentForm>
+        <CreateAgentForm
+          hideModal={hideModal}
+          onOk={onOk}
+          shouldChooseAgent={shouldChooseAgent}
+        ></CreateAgentForm>
         <DialogFooter>
           <ButtonLoading type="submit" form={TagRenameId} loading={loading}>
             {t('common.save')}

web/src/pages/agents/create-agent-form.tsx

@@ -4,38 +4,94 @@ import { zodResolver } from '@hookform/resolvers/zod';
 import { useForm } from 'react-hook-form';
 import { z } from 'zod';
 
-import {
-  Form,
-  FormControl,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from '@/components/ui/form';
-import { Input } from '@/components/ui/input';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { Card, CardContent } from '@/components/ui/card';
+import { Form } from '@/components/ui/form';
 import { IModalProps } from '@/interfaces/common';
+import { cn } from '@/lib/utils';
 import { TagRenameId } from '@/pages/add-knowledge/constant';
+import { BrainCircuit, Check, Route } from 'lucide-react';
+import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
+import { FlowType } from './constant';
+import { NameFormField, NameFormSchema } from './name-form-field';
 
-export function CreateAgentForm({ hideModal, onOk }: IModalProps<any>) {
+export type CreateAgentFormProps = IModalProps<any> & {
+  shouldChooseAgent?: boolean;
+};
+
+type FlowTypeCardProps = {
+  value?: FlowType;
+  onChange?: (value: FlowType) => void;
+};
+function FlowTypeCards({ value, onChange }: FlowTypeCardProps) {
+  const handleChange = useCallback(
+    (value: FlowType) => () => {
+      onChange?.(value);
+    },
+    [onChange],
+  );
+
+  return (
+    <section className="flex gap-10">
+      {Object.values(FlowType).map((val) => {
+        const isActive = value === val;
+        return (
+          <Card
+            key={val}
+            className={cn('flex-1 rounded-lg  border bg-transparent', {
+              'border-text-primary': isActive,
+              'border-border-default': !isActive,
+            })}
+          >
+            <CardContent
+              onClick={handleChange(val)}
+              className={cn(
+                'cursor-pointer p-5 text-text-secondary flex justify-between items-center',
+                {
+                  'text-text-primary': isActive,
+                },
+              )}
+            >
+              <div className="flex gap-2">
+                {val === FlowType.Agent ? (
+                  <BrainCircuit className="size-6" />
+                ) : (
+                  <Route className="size-6" />
+                )}
+                <p>{val}</p>
+              </div>
+              {isActive && <Check />}
+            </CardContent>
+          </Card>
+        );
+      })}
+    </section>
+  );
+}
+
+export const FormSchema = z.object({
+  ...NameFormSchema,
+  tag: z.string().trim().optional(),
+  description: z.string().trim().optional(),
+  type: z.nativeEnum(FlowType).optional(),
+});
+
+export type FormSchemaType = z.infer<typeof FormSchema>;
+
+export function CreateAgentForm({
+  hideModal,
+  onOk,
+  shouldChooseAgent = false,
+}: CreateAgentFormProps) {
   const { t } = useTranslation();
-  const FormSchema = z.object({
-    name: z
-      .string()
-      .min(1, {
-        message: t('common.namePlaceholder'),
-      })
-      .trim(),
-    tag: z.string().trim().optional(),
-    description: z.string().trim().optional(),
-  });
 
-  const form = useForm<z.infer<typeof FormSchema>>({
+  const form = useForm<FormSchemaType>({
     resolver: zodResolver(FormSchema),
-    defaultValues: { name: '' },
+    defaultValues: { name: '', type: FlowType.Agent },
   });
 
-  async function onSubmit(data: z.infer<typeof FormSchema>) {
+  async function onSubmit(data: FormSchemaType) {
     const ret = await onOk?.(data);
     if (ret) {
       hideModal?.();
@@ -49,57 +105,12 @@ export function CreateAgentForm({ hideModal, onOk }: IModalProps<any>) {
         className="space-y-6"
         id={TagRenameId}
       >
-        <FormField
-          control={form.control}
-          name="name"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>{t('common.name')}</FormLabel>
-              <FormControl>
-                <Input
-                  placeholder={t('common.namePlaceholder')}
-                  {...field}
-                  autoComplete="off"
-                />
-              </FormControl>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        {/* <FormField
-          control={form.control}
-          name="tag"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>{t('flow.tag')}</FormLabel>
-              <FormControl>
-                <Input
-                  placeholder={t('flow.tagPlaceholder')}
-                  {...field}
-                  autoComplete="off"
-                />
-              </FormControl>
-              <FormMessage />
-            </FormItem>
-          )}
-        />
-        <FormField
-          control={form.control}
-          name="description"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>{t('flow.description')}</FormLabel>
-              <FormControl>
-                <Input
-                  placeholder={t('flow.descriptionPlaceholder')}
-                  {...field}
-                  autoComplete="off"
-                />
-              </FormControl>
-              <FormMessage />
-            </FormItem>
-          )}
-        /> */}
+        {shouldChooseAgent && (
+          <RAGFlowFormItem required name="type" label={t('common.type')}>
+            <FlowTypeCards></FlowTypeCards>
+          </RAGFlowFormItem>
+        )}
+        <NameFormField></NameFormField>
       </form>
     </Form>
   );

web/src/pages/agents/hooks/use-create-agent.ts

@@ -0,0 +1,48 @@
+import { useSetModalState } from '@/hooks/common-hooks';
+import { useSetAgent } from '@/hooks/use-agent-request';
+import { EmptyDsl, useSetDataflow } from '@/hooks/use-dataflow-request';
+import { useCallback } from 'react';
+import { FlowType } from '../constant';
+import { FormSchemaType } from '../create-agent-form';
+
+export function useCreateAgentOrPipeline() {
+  const { loading, setAgent } = useSetAgent();
+  const { loading: dataflowLoading, setDataflow } = useSetDataflow();
+  const {
+    visible: creatingVisible,
+    hideModal: hideCreatingModal,
+    showModal: showCreatingModal,
+  } = useSetModalState();
+
+  const createAgent = useCallback(
+    async (name: string) => {
+      return setAgent({ title: name, dsl: EmptyDsl });
+    },
+    [setAgent],
+  );
+
+  const handleCreateAgentOrPipeline = useCallback(
+    async (data: FormSchemaType) => {
+      if (data.type === FlowType.Agent) {
+        const ret = await createAgent(data.name);
+        if (ret.code === 0) {
+          hideCreatingModal();
+        }
+      } else {
+        setDataflow({
+          title: data.name,
+          dsl: EmptyDsl,
+        });
+      }
+    },
+    [createAgent, hideCreatingModal, setDataflow],
+  );
+
+  return {
+    loading: loading || dataflowLoading,
+    creatingVisible,
+    hideCreatingModal,
+    showCreatingModal,
+    handleCreateAgentOrPipeline,
+  };
+}

web/src/pages/agents/index.tsx

@@ -1,14 +1,24 @@
 import ListFilterBar from '@/components/list-filter-bar';
 import { RenameDialog } from '@/components/rename-dialog';
 import { Button } from '@/components/ui/button';
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
 import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
 import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
 import { useFetchAgentListByPage } from '@/hooks/use-agent-request';
 import { t } from 'i18next';
 import { pick } from 'lodash';
-import { Plus } from 'lucide-react';
+import { Clipboard, ClipboardPlus, FileInput, Plus } from 'lucide-react';
 import { useCallback } from 'react';
 import { AgentCard } from './agent-card';
+import { CreateAgentDialog } from './create-agent-dialog';
+import { useCreateAgentOrPipeline } from './hooks/use-create-agent';
+import { UploadAgentDialog } from './upload-agent-dialog';
+import { useHandleImportJsonFile } from './use-import-json';
 import { useRenameAgent } from './use-rename-agent';
 
 export default function Agents() {
@@ -25,6 +35,21 @@ export default function Agents() {
     showAgentRenameModal,
   } = useRenameAgent();
 
+  const {
+    creatingVisible,
+    hideCreatingModal,
+    showCreatingModal,
+    loading,
+    handleCreateAgentOrPipeline,
+  } = useCreateAgentOrPipeline();
+
+  const {
+    handleImportJson,
+    fileUploadVisible,
+    onFileUploadOk,
+    hideFileUploadModal,
+  } = useHandleImportJsonFile();
+
   const handlePageChange = useCallback(
     (page: number, pageSize?: number) => {
       setPagination({ page, pageSize });
@@ -41,10 +66,37 @@ export default function Agents() {
           onSearchChange={handleInputChange}
           icon="agent"
         >
-          <Button onClick={navigateToAgentTemplates}>
-            <Plus className="mr-2 h-4 w-4" />
-            {t('flow.createGraph')}
-          </Button>
+          <DropdownMenu>
+            <DropdownMenuTrigger>
+              <Button>
+                <Plus className="mr-2 h-4 w-4" />
+                {t('flow.createGraph')}
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent>
+              <DropdownMenuItem
+                justifyBetween={false}
+                onClick={showCreatingModal}
+              >
+                <Clipboard />
+                Create from Blank
+              </DropdownMenuItem>
+              <DropdownMenuItem
+                justifyBetween={false}
+                onClick={navigateToAgentTemplates}
+              >
+                <ClipboardPlus />
+                Create from Template
+              </DropdownMenuItem>
+              <DropdownMenuItem
+                justifyBetween={false}
+                onClick={handleImportJson}
+              >
+                <FileInput />
+                Import json file
+              </DropdownMenuItem>
+            </DropdownMenuContent>
+          </DropdownMenu>
         </ListFilterBar>
       </div>
       <div className="flex-1 overflow-auto">
@@ -75,6 +127,21 @@ export default function Agents() {
           loading={agentRenameLoading}
         ></RenameDialog>
       )}
+      {creatingVisible && (
+        <CreateAgentDialog
+          loading={loading}
+          visible={creatingVisible}
+          hideModal={hideCreatingModal}
+          shouldChooseAgent
+          onOk={handleCreateAgentOrPipeline}
+        ></CreateAgentDialog>
+      )}
+      {fileUploadVisible && (
+        <UploadAgentDialog
+          hideModal={hideFileUploadModal}
+          onOk={onFileUploadOk}
+        ></UploadAgentDialog>
+      )}
     </section>
   );
 }

web/src/pages/agents/name-form-field.tsx

@@ -0,0 +1,28 @@
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { Input } from '@/components/ui/input';
+import i18n from '@/locales/config';
+import { useTranslation } from 'react-i18next';
+import { z } from 'zod';
+
+export const NameFormSchema = {
+  name: z
+    .string()
+    .min(1, {
+      message: i18n.t('common.namePlaceholder'),
+    })
+    .trim(),
+};
+
+export function NameFormField() {
+  const { t } = useTranslation();
+  return (
+    <RAGFlowFormItem
+      name="name"
+      required
+      label={t('common.name')}
+      tooltip={t('flow.sqlStatementTip')}
+    >
+      <Input placeholder={t('common.namePlaceholder')} autoComplete="off" />
+    </RAGFlowFormItem>
+  );
+}

web/src/pages/agents/template-card.tsx

@@ -3,7 +3,6 @@ import { Button } from '@/components/ui/button';
 import { Card, CardContent } from '@/components/ui/card';
 import { IFlowTemplate } from '@/interfaces/database/flow';
 import i18n from '@/locales/config';
-import { Plus } from 'lucide-react';
 import { useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 interface IProps {
@@ -12,7 +11,7 @@ interface IProps {
   showModal(record: IFlowTemplate): void;
 }
 
-export function TemplateCard({ data, showModal, isCreate = false }: IProps) {
+export function TemplateCard({ data, showModal }: IProps) {
   const { t } = useTranslation();
 
   const handleClick = useCallback(() => {
@@ -26,41 +25,24 @@ export function TemplateCard({ data, showModal, isCreate = false }: IProps) {
   return (
     <Card className="border-colors-outline-neutral-standard group relative min-h-40">
       <CardContent className="p-4 ">
-        {isCreate && (
-          <div
-            className="flex flex-col justify-center items-center gap-4 mb-4 absolute top-0 right-0 left-0 bottom-0 cursor-pointer "
+        <div className="flex justify-start items-center gap-4 mb-4">
+          <RAGFlowAvatar
+            className="w-7 h-7"
+            avatar={data.avatar ? data.avatar : 'https://github.com/shadcn.png'}
+            name={data?.title[language] || 'CN'}
+          ></RAGFlowAvatar>
+          <div className="text-[18px] font-bold ">{data?.title[language]}</div>
+        </div>
+        <p className="break-words">{data?.description[language]}</p>
+        <div className="group-hover:bg-gradient-to-t from-black/70 from-10% via-black/0 via-50% to-black/0 w-full h-full group-hover:block absolute top-0 left-0 hidden rounded-xl">
+          <Button
+            variant="default"
+            className="w-1/3 absolute bottom-4 right-4 left-4 justify-center text-center m-auto"
             onClick={handleClick}
           >
-            <Plus size={50} fontWeight={700} />
-            <div>{t('flow.createAgent')}</div>
-          </div>
-        )}
-        {!isCreate && (
-          <>
-            <div className="flex justify-start items-center gap-4 mb-4">
-              <RAGFlowAvatar
-                className="w-7 h-7"
-                avatar={
-                  data.avatar ? data.avatar : 'https://github.com/shadcn.png'
-                }
-                name={data?.title[language] || 'CN'}
-              ></RAGFlowAvatar>
-              <div className="text-[18px] font-bold ">
-                {data?.title[language]}
-              </div>
-            </div>
-            <p className="break-words">{data?.description[language]}</p>
-            <div className="group-hover:bg-gradient-to-t from-black/70 from-10% via-black/0 via-50% to-black/0 w-full h-full group-hover:block absolute top-0 left-0 hidden rounded-xl">
-              <Button
-                variant="default"
-                className="w-1/3 absolute bottom-4 right-4 left-4 justify-center text-center m-auto"
-                onClick={handleClick}
-              >
-                {t('flow.useTemplate')}
-              </Button>
-            </div>
-          </>
-        )}
+            {t('flow.useTemplate')}
+          </Button>
+        </div>
       </CardContent>
     </Card>
   );

web/src/pages/agents/upload-agent-dialog/index.tsx

@@ -1,3 +1,4 @@
+import { ButtonLoading } from '@/components/ui/button';
 import {
   Dialog,
   DialogContent,
@@ -5,7 +6,6 @@ import {
   DialogHeader,
   DialogTitle,
 } from '@/components/ui/dialog';
-import { LoadingButton } from '@/components/ui/loading-button';
 import { IModalProps } from '@/interfaces/common';
 import { TagRenameId } from '@/pages/add-knowledge/constant';
 import { useTranslation } from 'react-i18next';
@@ -26,9 +26,9 @@ export function UploadAgentDialog({
         </DialogHeader>
         <UploadAgentForm hideModal={hideModal} onOk={onOk}></UploadAgentForm>
         <DialogFooter>
-          <LoadingButton type="submit" form={TagRenameId} loading={loading}>
+          <ButtonLoading type="submit" form={TagRenameId} loading={loading}>
             {t('common.save')}
-          </LoadingButton>
+          </ButtonLoading>
         </DialogFooter>
       </DialogContent>
     </Dialog>

web/src/pages/agents/upload-agent-dialog/upload-agent-form.tsx

@@ -13,32 +13,24 @@ import {
   FormLabel,
   FormMessage,
 } from '@/components/ui/form';
-import { FileMimeType, Platform } from '@/constants/common';
+import { FileMimeType } from '@/constants/common';
 import { IModalProps } from '@/interfaces/common';
 import { TagRenameId } from '@/pages/add-knowledge/constant';
-import { useTranslation } from 'react-i18next';
+import { NameFormField, NameFormSchema } from '../name-form-field';
 
-// const options = Object.values(Platform).map((x) => ({ label: x, value: x }));
+export const FormSchema = z.object({
+  fileList: z.array(z.instanceof(File)),
+  ...NameFormSchema,
+});
 
+export type FormSchemaType = z.infer<typeof FormSchema>;
 export function UploadAgentForm({ hideModal, onOk }: IModalProps<any>) {
-  const { t } = useTranslation();
-  const FormSchema = z.object({
-    platform: z
-      .string()
-      .min(1, {
-        message: t('common.namePlaceholder'),
-      })
-      .trim(),
-    fileList: z.array(z.instanceof(File)),
-  });
-
   const form = useForm<z.infer<typeof FormSchema>>({
     resolver: zodResolver(FormSchema),
-    defaultValues: { platform: Platform.RAGFlow },
+    defaultValues: { name: '' },
   });
 
-  async function onSubmit(data: z.infer<typeof FormSchema>) {
-    console.log('🚀 ~ onSubmit ~ data:', data);
+  async function onSubmit(data: FormSchemaType) {
     const ret = await onOk?.(data);
     if (ret) {
       hideModal?.();
@@ -52,12 +44,13 @@ export function UploadAgentForm({ hideModal, onOk }: IModalProps<any>) {
         className="space-y-6"
         id={TagRenameId}
       >
+        <NameFormField></NameFormField>
         <FormField
           control={form.control}
           name="fileList"
           render={({ field }) => (
             <FormItem>
-              <FormLabel>{t('common.name')}</FormLabel>
+              <FormLabel required>DSL</FormLabel>
               <FormControl>
                 <FileUploader
                   value={field.value}
@@ -70,19 +63,6 @@ export function UploadAgentForm({ hideModal, onOk }: IModalProps<any>) {
             </FormItem>
           )}
         />
-        {/* <FormField
-          control={form.control}
-          name="platform"
-          render={({ field }) => (
-            <FormItem>
-              <FormLabel>{t('common.name')}</FormLabel>
-              <FormControl>
-                <RAGFlowSelect {...field} options={options} />
-              </FormControl>
-              <FormMessage />
-            </FormItem>
-          )}
-        /> */}
       </form>
     </Form>
   );

web/src/pages/agents/use-import-json.ts

@@ -0,0 +1,56 @@
+import { useToast } from '@/components/hooks/use-toast';
+import { FileMimeType } from '@/constants/common';
+import { useSetModalState } from '@/hooks/common-hooks';
+import { EmptyDsl, useSetAgent } from '@/hooks/use-agent-request';
+import { message } from 'antd';
+import isEmpty from 'lodash/isEmpty';
+import { useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { FormSchemaType } from './upload-agent-dialog/upload-agent-form';
+
+export const useHandleImportJsonFile = () => {
+  const {
+    visible: fileUploadVisible,
+    hideModal: hideFileUploadModal,
+    showModal: showFileUploadModal,
+  } = useSetModalState();
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  const { loading, setAgent } = useSetAgent();
+
+  const onFileUploadOk = useCallback(
+    async ({ fileList, name }: FormSchemaType) => {
+      if (fileList.length > 0) {
+        const file = fileList[0];
+        if (file.type !== FileMimeType.Json) {
+          toast({ title: t('flow.jsonUploadTypeErrorMessage') });
+          return;
+        }
+
+        const graphStr = await file.text();
+        const errorMessage = t('flow.jsonUploadContentErrorMessage');
+        try {
+          const graph = JSON.parse(graphStr);
+          if (graphStr && !isEmpty(graph) && Array.isArray(graph?.nodes)) {
+            const dsl = { ...EmptyDsl, graph };
+            setAgent({ title: name, dsl });
+            hideFileUploadModal();
+          } else {
+            message.error(errorMessage);
+          }
+        } catch (error) {
+          message.error(errorMessage);
+        }
+      }
+    },
+    [hideFileUploadModal, setAgent, t, toast],
+  );
+
+  return {
+    fileUploadVisible,
+    handleImportJson: showFileUploadModal,
+    hideFileUploadModal,
+    onFileUploadOk,
+    loading,
+  };
+};

web/src/pages/data-flow/canvas/index.tsx

@@ -44,6 +44,7 @@ import { CategorizeNode } from './node/categorize-node';
 import ChunkerNode from './node/chunker-node';
 import { InnerNextStepDropdown } from './node/dropdown/next-step-dropdown';
 import { GenerateNode } from './node/generate-node';
+import { HierarchicalMergerNode } from './node/hierarchical-merger-node';
 import { InvokeNode } from './node/invoke-node';
 import { IterationNode, IterationStartNode } from './node/iteration-node';
 import { KeywordNode } from './node/keyword-node';
@@ -54,6 +55,7 @@ import ParserNode from './node/parser-node';
 import { RelevantNode } from './node/relevant-node';
 import { RetrievalNode } from './node/retrieval-node';
 import { RewriteNode } from './node/rewrite-node';
+import { SplitterNode } from './node/splitter-node';
 import { SwitchNode } from './node/switch-node';
 import { TemplateNode } from './node/template-node';
 import TokenizerNode from './node/tokenizer-node';
@@ -82,6 +84,8 @@ export const nodeTypes: NodeTypes = {
   parserNode: ParserNode,
   chunkerNode: ChunkerNode,
   tokenizerNode: TokenizerNode,
+  splitterNode: SplitterNode,
+  hierarchicalMergerNode: HierarchicalMergerNode,
 };
 
 const edgeTypes = {

web/src/pages/data-flow/canvas/node/dropdown/next-step-dropdown.tsx

@@ -17,9 +17,9 @@ import {
   TooltipTrigger,
 } from '@/components/ui/tooltip';
 import { IModalProps } from '@/interfaces/common';
+import { useGetNodeDescription, useGetNodeName } from '@/pages/data-flow/hooks';
 import { Position } from '@xyflow/react';
 import { t } from 'i18next';
-import { lowerFirst } from 'lodash';
 import {
   PropsWithChildren,
   createContext,
@@ -28,7 +28,6 @@ import {
   useEffect,
   useRef,
 } from 'react';
-import { useTranslation } from 'react-i18next';
 import { Operator } from '../../../constant';
 import { AgentInstanceContext, HandleContext } from '../../../context';
 import OperatorIcon from '../../../operator-icon';
@@ -53,7 +52,9 @@ function OperatorItemList({
   const handleContext = useContext(HandleContext);
   const hideModal = useContext(HideModalContext);
   const onNodeCreated = useContext(OnNodeCreatedContext);
-  const { t } = useTranslation();
+
+  const getNodeName = useGetNodeName();
+  const getNodeDescription = useGetNodeDescription();
 
   const handleClick = (operator: Operator) => {
     const contextData = handleContext || {
@@ -84,7 +85,7 @@ function OperatorItemList({
     const commonContent = (
       <div className="hover:bg-background-card py-1 px-3 cursor-pointer rounded-sm flex gap-2 items-center justify-start">
         <OperatorIcon name={operator} />
-        {t(`flow.${lowerFirst(operator)}`)}
+        {getNodeName(operator)}
       </div>
     );
 
@@ -101,12 +102,12 @@ function OperatorItemList({
               onSelect={() => hideModal?.()}
             >
               <OperatorIcon name={operator} />
-              {t(`flow.${lowerFirst(operator)}`)}
+              {getNodeName(operator)}
             </DropdownMenuItem>
           )}
         </TooltipTrigger>
         <TooltipContent side="right">
-          <p>{t(`flow.${lowerFirst(operator)}Description`)}</p>
+          <p>{getNodeDescription(operator)}</p>
         </TooltipContent>
       </Tooltip>
     );
@@ -140,6 +141,8 @@ function AccordionOperators({
               Operator.Parser,
               Operator.Chunker,
               Operator.Tokenizer,
+              Operator.Splitter,
+              Operator.HierarchicalMerger,
             ]}
             isCustomDropdown={isCustomDropdown}
             mousePosition={mousePosition}

web/src/pages/data-flow/canvas/node/hierarchical-merger-node.tsx

@@ -0,0 +1 @@
+export { RagNode as HierarchicalMergerNode } from './index';

web/src/pages/data-flow/canvas/node/splitter-node.tsx

@@ -0,0 +1 @@
+export { RagNode as SplitterNode } from './index';

web/src/pages/data-flow/constant.tsx

@@ -66,6 +66,8 @@ export enum Operator {
   Parser = 'Parser',
   Chunker = 'Chunker',
   Tokenizer = 'Tokenizer',
+  Splitter = 'Splitter',
+  HierarchicalMerger = 'HierarchicalMerger',
 }
 
 export const SwitchLogicOperatorOptions = ['and', 'or'];
@@ -74,20 +76,6 @@ export const CommonOperatorList = Object.values(Operator).filter(
   (x) => x !== Operator.Note,
 );
 
-export const AgentOperatorList = [
-  Operator.Retrieval,
-  Operator.Categorize,
-  Operator.Message,
-  Operator.RewriteQuestion,
-  Operator.KeywordExtract,
-  Operator.Switch,
-  Operator.Concentrator,
-  Operator.Iteration,
-  Operator.WaitingDialogue,
-  Operator.Note,
-  Operator.Agent,
-];
-
 export const SwitchOperatorOptions = [
   { value: '=', label: 'equal', icon: 'equal' },
   { value: '≠', label: 'notEqual', icon: 'not-equals' },
@@ -388,7 +376,11 @@ export const initialStringTransformValues = {
   },
 };
 
-export const initialParserValues = { outputs: {} };
+export const initialParserValues = { outputs: {}, parser: [] };
+
+export const initialSplitterValues = { outputs: {}, chunk_token_size: 512 };
+
+export const initialHierarchicalMergerValues = { outputs: {} };
 
 export const CategorizeAnchorPointPositions = [
   { top: 1, right: 34 },
@@ -473,6 +465,8 @@ export const NodeMap = {
   [Operator.Parser]: 'parserNode',
   [Operator.Chunker]: 'chunkerNode',
   [Operator.Tokenizer]: 'tokenizerNode',
+  [Operator.Splitter]: 'splitterNode',
+  [Operator.HierarchicalMerger]: 'hierarchicalMergerNode',
 };
 
 export enum BeginQueryType {
@@ -523,3 +517,15 @@ export enum AgentExceptionMethod {
   Comment = 'comment',
   Goto = 'goto',
 }
+
+export enum FileType {
+  PDF = 'pdf',
+  Spreadsheet = 'spreadsheet',
+  Image = 'image',
+  Email = 'email',
+  TextMarkdown = 'text&markdown',
+  Docx = 'docx',
+  PowerPoint = 'ppt',
+  Video = 'video',
+  Audio = 'audio',
+}

web/src/pages/data-flow/form-sheet/form-config-map.tsx

@@ -7,6 +7,7 @@ import CodeForm from '../form/code-form';
 import CrawlerForm from '../form/crawler-form';
 import EmailForm from '../form/email-form';
 import ExeSQLForm from '../form/exesql-form';
+import HierarchicalMergerForm from '../form/hierarchical-merger-form';
 import InvokeForm from '../form/invoke-form';
 import IterationForm from '../form/iteration-form';
 import IterationStartForm from '../form/iteration-start-from';
@@ -16,6 +17,7 @@ import ParserForm from '../form/parser-form';
 import RelevantForm from '../form/relevant-form';
 import RetrievalForm from '../form/retrieval-form/next';
 import RewriteQuestionForm from '../form/rewrite-question-form';
+import SplitterForm from '../form/splitter-form';
 import StringTransformForm from '../form/string-transform-form';
 import SwitchForm from '../form/switch-form';
 import TokenizerForm from '../form/tokenizer-form';
@@ -94,4 +96,10 @@ export const FormConfigMap = {
   [Operator.Tokenizer]: {
     component: TokenizerForm,
   },
+  [Operator.Splitter]: {
+    component: SplitterForm,
+  },
+  [Operator.HierarchicalMerger]: {
+    component: HierarchicalMergerForm,
+  },
 };

web/src/pages/data-flow/form-sheet/next.tsx

@@ -5,13 +5,13 @@ import {
   SheetHeader,
   SheetTitle,
 } from '@/components/ui/sheet';
-import { useTranslate } from '@/hooks/common-hooks';
 import { IModalProps } from '@/interfaces/common';
 import { RAGFlowNodeType } from '@/interfaces/database/flow';
 import { cn } from '@/lib/utils';
 import { lowerFirst } from 'lodash';
 import { Play, X } from 'lucide-react';
 import { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
 import { BeginId, Operator } from '../constant';
 import { AgentFormContext } from '../context';
 import { RunTooltip } from '../flow-tooltip';
@@ -60,7 +60,7 @@ const FormSheet = ({
     );
   }, [clickedToolId, operatorName]);
 
-  const { t } = useTranslate('flow');
+  const { t } = useTranslation();
 
   return (
     <Sheet open={visible} modal={false}>
@@ -80,7 +80,7 @@ const FormSheet = ({
                 <div className="flex-1">MCP Config</div>
               ) : (
                 <div className="flex items-center gap-1 flex-1">
-                  <label htmlFor="">{t('title')}</label>
+                  <label htmlFor="">{t('flow.title')}</label>
                   {node?.id === BeginId ? (
                     <span>{t(BeginId)}</span>
                   ) : (
@@ -106,7 +106,7 @@ const FormSheet = ({
             {isMcp || (
               <span>
                 {t(
-                  `${lowerFirst(operatorName === Operator.Tool ? clickedToolId : operatorName)}Description`,
+                  `dataflow.${lowerFirst(operatorName === Operator.Tool ? clickedToolId : operatorName)}Description`,
                 )}
               </span>
             )}

web/src/pages/data-flow/form/hierarchical-merger-form/index.tsx

@@ -0,0 +1,158 @@
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { BlockButton, Button } from '@/components/ui/button';
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
+import { Form } from '@/components/ui/form';
+import { Input } from '@/components/ui/input';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { X } from 'lucide-react';
+import { memo } from 'react';
+import { useFieldArray, useForm, useFormContext } from 'react-hook-form';
+import { z } from 'zod';
+import { initialHierarchicalMergerValues } from '../../constant';
+import { useFormValues } from '../../hooks/use-form-values';
+import { useWatchFormChange } from '../../hooks/use-watch-form-change';
+import { INextOperatorForm } from '../../interface';
+import { buildOutputList } from '../../utils/build-output-list';
+import { FormWrapper } from '../components/form-wrapper';
+import { Output } from '../components/output';
+
+const outputList = buildOutputList(initialHierarchicalMergerValues.outputs);
+
+enum Hierarchy {
+  H1 = '1',
+  H2 = '2',
+  H3 = '3',
+  H4 = '4',
+  H5 = '5',
+}
+
+const HierarchyOptions = [
+  { label: 'H1', value: Hierarchy.H1 },
+  { label: 'H2', value: Hierarchy.H2 },
+  { label: 'H3', value: Hierarchy.H3 },
+  { label: 'H4', value: Hierarchy.H4 },
+  { label: 'H5', value: Hierarchy.H5 },
+];
+
+export const FormSchema = z.object({
+  hierarchy: z.number(),
+  levels: z.array(
+    z.object({
+      expressions: z.array(z.object({ expression: z.string() })),
+    }),
+  ),
+});
+
+type RegularExpressionsProps = {
+  index: number;
+  parentName: string;
+  removeParent: (index: number) => void;
+};
+
+export function RegularExpressions({
+  index,
+  parentName,
+  removeParent,
+}: RegularExpressionsProps) {
+  const form = useFormContext();
+
+  const name = `${parentName}.${index}.expressions`;
+
+  const { fields, append, remove } = useFieldArray({
+    name: name,
+    control: form.control,
+  });
+
+  return (
+    <Card>
+      <CardHeader className="flex-row justify-between items-center">
+        <CardTitle>H{index}</CardTitle>
+        <Button
+          type="button"
+          variant={'ghost'}
+          onClick={() => removeParent(index)}
+        >
+          <X />
+        </Button>
+      </CardHeader>
+      <CardContent>
+        <section className="space-y-4">
+          {fields.map((field, index) => (
+            <div key={field.id} className="flex items-center gap-2">
+              <div className="space-y-2 flex-1">
+                <RAGFlowFormItem
+                  name={`${name}.${index}.expression`}
+                  label={'expression'}
+                  labelClassName="!hidden"
+                >
+                  <Input className="!m-0"></Input>
+                </RAGFlowFormItem>
+              </div>
+              <Button
+                type="button"
+                variant={'ghost'}
+                onClick={() => remove(index)}
+              >
+                <X />
+              </Button>
+            </div>
+          ))}
+        </section>
+        <BlockButton
+          onClick={() => append({ expression: '' })}
+          className="mt-6"
+        >
+          Add
+        </BlockButton>
+      </CardContent>
+    </Card>
+  );
+}
+
+const HierarchicalMergerForm = ({ node }: INextOperatorForm) => {
+  const defaultValues = useFormValues(initialHierarchicalMergerValues, node);
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    defaultValues,
+    resolver: zodResolver(FormSchema),
+  });
+
+  const name = 'levels';
+
+  const { fields, append, remove } = useFieldArray({
+    name: name,
+    control: form.control,
+  });
+
+  useWatchFormChange(node?.id, form);
+
+  return (
+    <Form {...form}>
+      <FormWrapper>
+        <RAGFlowFormItem name={'hierarchy'} label={'hierarchy'}>
+          <SelectWithSearch options={HierarchyOptions}></SelectWithSearch>
+        </RAGFlowFormItem>
+        {fields.map((field, index) => (
+          <div key={field.id} className="flex items-center gap-2">
+            <div className="space-y-2 flex-1">
+              <RegularExpressions
+                parentName={name}
+                index={index}
+                removeParent={remove}
+              ></RegularExpressions>
+            </div>
+          </div>
+        ))}
+        <BlockButton onClick={() => append({ expressions: [] })}>
+          Add
+        </BlockButton>
+      </FormWrapper>
+      <div className="p-5">
+        <Output list={outputList}></Output>
+      </div>
+    </Form>
+  );
+};
+
+export default memo(HierarchicalMergerForm);

web/src/pages/data-flow/form/parser-form/common-form-fields.tsx

@@ -0,0 +1,69 @@
+import { LayoutRecognizeFormField } from '@/components/layout-recognize-form-field';
+import { LLMFormField } from '@/components/llm-setting-items/llm-form-field';
+import {
+  SelectWithSearch,
+  SelectWithSearchFlagOptionType,
+} from '@/components/originui/select-with-search';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { buildOptions } from '@/utils/form';
+import { useTranslation } from 'react-i18next';
+import { FileType } from '../../constant';
+import { OutputFormatMap } from './constant';
+import { CommonProps } from './interface';
+import { buildFieldNameWithPrefix } from './utils';
+
+function buildOutputOptionsFormatMap() {
+  return Object.entries(OutputFormatMap).reduce<
+    Record<string, SelectWithSearchFlagOptionType[]>
+  >((pre, [key, value]) => {
+    pre[key] = buildOptions(value);
+    return pre;
+  }, {});
+}
+
+export type OutputFormatFormFieldProps = CommonProps & {
+  fileType: FileType;
+};
+
+export function OutputFormatFormField({
+  prefix,
+  fileType,
+}: OutputFormatFormFieldProps) {
+  const { t } = useTranslation();
+  return (
+    <RAGFlowFormItem
+      name={buildFieldNameWithPrefix(`output_format`, prefix)}
+      label={t('dataflow.outputFormat')}
+    >
+      <SelectWithSearch
+        options={buildOutputOptionsFormatMap()[fileType]}
+      ></SelectWithSearch>
+    </RAGFlowFormItem>
+  );
+}
+
+export function ParserMethodFormField({ prefix }: CommonProps) {
+  return (
+    <LayoutRecognizeFormField
+      name={buildFieldNameWithPrefix(`parse_method`, prefix)}
+      horizontal={false}
+    ></LayoutRecognizeFormField>
+  );
+
+  return (
+    <RAGFlowFormItem
+      name={buildFieldNameWithPrefix(`parse_method`, prefix)}
+      label="parse_method"
+    >
+      <SelectWithSearch options={[]}></SelectWithSearch>
+    </RAGFlowFormItem>
+  );
+}
+
+export function LargeModelFormField({ prefix }: CommonProps) {
+  return (
+    <LLMFormField
+      name={buildFieldNameWithPrefix('llm_id', prefix)}
+    ></LLMFormField>
+  );
+}

web/src/pages/data-flow/form/parser-form/constant.ts

@@ -0,0 +1,52 @@
+import { FileType } from '../../constant';
+
+export enum PdfOutputFormat {
+  Json = 'json',
+  Markdown = 'markdown',
+}
+
+export enum SpreadsheetOutputFormat {
+  Json = 'json',
+  Html = 'html',
+}
+
+export enum ImageOutputFormat {
+  Text = 'text',
+}
+
+export enum EmailOutputFormat {
+  Json = 'json',
+  Text = 'text',
+}
+
+export enum TextMarkdownOutputFormat {
+  Text = 'text',
+}
+
+export enum DocxOutputFormat {
+  Markdown = 'markdown',
+}
+
+export enum PptOutputFormat {
+  Json = 'json',
+}
+
+export enum VideoOutputFormat {
+  Json = 'json',
+}
+
+export enum AudioOutputFormat {
+  Text = 'text',
+}
+
+export const OutputFormatMap = {
+  [FileType.PDF]: PdfOutputFormat,
+  [FileType.Spreadsheet]: SpreadsheetOutputFormat,
+  [FileType.Image]: ImageOutputFormat,
+  [FileType.Email]: EmailOutputFormat,
+  [FileType.TextMarkdown]: TextMarkdownOutputFormat,
+  [FileType.Docx]: DocxOutputFormat,
+  [FileType.PowerPoint]: PptOutputFormat,
+  [FileType.Video]: VideoOutputFormat,
+  [FileType.Audio]: AudioOutputFormat,
+};

web/src/pages/data-flow/form/parser-form/email-form-fields.tsx

@@ -0,0 +1,37 @@
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { buildOptions } from '@/utils/form';
+import { useTranslation } from 'react-i18next';
+import { FileType } from '../../constant';
+import { OutputFormatFormField } from './common-form-fields';
+import { CommonProps } from './interface';
+import { buildFieldNameWithPrefix } from './utils';
+
+const options = buildOptions([
+  'from',
+  'to',
+  'cc',
+  'bcc',
+  'date',
+  'subject',
+  'body',
+  'attachments',
+]);
+
+export function EmailFormFields({ prefix }: CommonProps) {
+  const { t } = useTranslation();
+  return (
+    <>
+      <RAGFlowFormItem
+        name={buildFieldNameWithPrefix(`fields`, prefix)}
+        label={t('dataflow.fields')}
+      >
+        <SelectWithSearch options={options}></SelectWithSearch>
+      </RAGFlowFormItem>
+      <OutputFormatFormField
+        prefix={prefix}
+        fileType={FileType.Email}
+      ></OutputFormatFormField>
+    </>
+  );
+}

web/src/pages/data-flow/form/parser-form/image-form-fields.tsx

@@ -0,0 +1,21 @@
+import { FileType } from '../../constant';
+import {
+  LargeModelFormField,
+  OutputFormatFormField,
+  ParserMethodFormField,
+} from './common-form-fields';
+import { CommonProps } from './interface';
+
+export function ImageFormFields({ prefix }: CommonProps) {
+  return (
+    <>
+      <ParserMethodFormField prefix={prefix}></ParserMethodFormField>
+      {/* Multimodal Model */}
+      <LargeModelFormField prefix={prefix}></LargeModelFormField>
+      <OutputFormatFormField
+        prefix={prefix}
+        fileType={FileType.Image}
+      ></OutputFormatFormField>
+    </>
+  );
+}

web/src/pages/data-flow/form/parser-form/index.tsx

@@ -1,135 +1,172 @@
-import { FormContainer } from '@/components/form-container';
-import NumberInput from '@/components/originui/number-input';
 import { SelectWithSearch } from '@/components/originui/select-with-search';
-import {
-  Form,
-  FormControl,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from '@/components/ui/form';
-import { useTranslate } from '@/hooks/common-hooks';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { BlockButton, Button } from '@/components/ui/button';
+import { Form } from '@/components/ui/form';
+import { Separator } from '@/components/ui/separator';
+import { cn } from '@/lib/utils';
+import { buildOptions } from '@/utils/form';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { memo } from 'react';
-import { useForm, useFormContext } from 'react-hook-form';
+import { useHover } from 'ahooks';
+import { Trash2 } from 'lucide-react';
+import { memo, useCallback, useMemo, useRef } from 'react';
+import {
+  UseFieldArrayRemove,
+  useFieldArray,
+  useForm,
+  useFormContext,
+} from 'react-hook-form';
+import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
-import { initialParserValues } from '../../constant';
+import { FileType, initialParserValues } from '../../constant';
 import { useFormValues } from '../../hooks/use-form-values';
 import { useWatchFormChange } from '../../hooks/use-watch-form-change';
 import { INextOperatorForm } from '../../interface';
-import { GoogleCountryOptions, GoogleLanguageOptions } from '../../options';
 import { buildOutputList } from '../../utils/build-output-list';
-import { ApiKeyField } from '../components/api-key-field';
-import { FormWrapper } from '../components/form-wrapper';
 import { Output } from '../components/output';
-import { QueryVariable } from '../components/query-variable';
+import { OutputFormatFormField } from './common-form-fields';
+import { EmailFormFields } from './email-form-fields';
+import { ImageFormFields } from './image-form-fields';
+import { PdfFormFields } from './pdf-form-fields';
+import { buildFieldNameWithPrefix } from './utils';
+import { VideoFormFields } from './video-form-fields';
 
 const outputList = buildOutputList(initialParserValues.outputs);
 
-export const GoogleFormPartialSchema = {
-  api_key: z.string(),
-  country: z.string(),
-  language: z.string(),
+const FileFormatOptions = buildOptions(FileType);
+
+const FileFormatWidgetMap = {
+  [FileType.PDF]: PdfFormFields,
+  [FileType.Video]: VideoFormFields,
+  [FileType.Audio]: VideoFormFields,
+  [FileType.Email]: EmailFormFields,
+  [FileType.Image]: ImageFormFields,
+};
+
+type ParserItemProps = {
+  name: string;
+  index: number;
+  fieldLength: number;
+  remove: UseFieldArrayRemove;
 };
 
 export const FormSchema = z.object({
-  ...GoogleFormPartialSchema,
-  q: z.string(),
-  start: z.number(),
-  num: z.number(),
+  parser: z.array(
+    z.object({
+      fileFormat: z.string().nullish(),
+      output_format: z.string().optional(),
+      parse_method: z.string().optional(),
+      llm_id: z.string().optional(),
+      lang: z.string().optional(),
+      fields: z.array(z.string()).optional(),
+    }),
+  ),
 });
 
-export function GoogleFormWidgets() {
-  const form = useFormContext();
-  const { t } = useTranslate('flow');
+export type FormSchemaType = z.infer<typeof FormSchema>;
 
+function ParserItem({ name, index, fieldLength, remove }: ParserItemProps) {
+  const { t } = useTranslation();
+  const form = useFormContext<FormSchemaType>();
+  const ref = useRef(null);
+  const isHovering = useHover(ref);
+
+  const prefix = `${name}.${index}`;
+  const fileFormat = form.getValues(`parser.${index}.fileFormat`);
+
+  const values = form.getValues();
+  const parserList = values.parser.slice(); // Adding, deleting, or modifying the parser array will not change the reference.
+
+  const filteredFileFormatOptions = useMemo(() => {
+    const otherFileFormatList = parserList
+      .filter((_, idx) => idx !== index)
+      .map((x) => x.fileFormat);
+
+    return FileFormatOptions.filter((x) => {
+      return !otherFileFormatList.includes(x.value);
+    });
+  }, [index, parserList]);
+
+  const Widget =
+    typeof fileFormat === 'string' && fileFormat in FileFormatWidgetMap
+      ? FileFormatWidgetMap[fileFormat as keyof typeof FileFormatWidgetMap]
+      : OutputFormatFormField;
   return (
-    <>
-      <FormField
-        control={form.control}
-        name={`country`}
-        render={({ field }) => (
-          <FormItem className="flex-1">
-            <FormLabel>{t('country')}</FormLabel>
-            <FormControl>
-              <SelectWithSearch
-                {...field}
-                options={GoogleCountryOptions}
-              ></SelectWithSearch>
-            </FormControl>
-            <FormMessage />
-          </FormItem>
+    <section
+      className={cn('space-y-5 px-5 py-2.5 rounded-md', {
+        'bg-state-error-5': isHovering,
+      })}
+    >
+      <div className="flex justify-between items-center">
+        <span className="text-text-primary text-sm font-medium">
+          Parser {index}
+        </span>
+        {index > 0 && (
+          <Button variant={'ghost'} onClick={() => remove(index)} ref={ref}>
+            <Trash2 />
+          </Button>
         )}
-      />
-      <FormField
-        control={form.control}
-        name={`language`}
-        render={({ field }) => (
-          <FormItem className="flex-1">
-            <FormLabel>{t('language')}</FormLabel>
-            <FormControl>
-              <SelectWithSearch
-                {...field}
-                options={GoogleLanguageOptions}
-              ></SelectWithSearch>
-            </FormControl>
-            <FormMessage />
-          </FormItem>
-        )}
-      />
-    </>
+      </div>
+      <RAGFlowFormItem
+        name={buildFieldNameWithPrefix(`fileFormat`, prefix)}
+        label={t('dataflow.fileFormats')}
+      >
+        <SelectWithSearch
+          options={filteredFileFormatOptions}
+        ></SelectWithSearch>
+      </RAGFlowFormItem>
+      <Widget prefix={prefix} fileType={fileFormat as FileType}></Widget>
+      {index < fieldLength - 1 && <Separator />}
+    </section>
   );
 }
 
 const ParserForm = ({ node }: INextOperatorForm) => {
-  const { t } = useTranslate('flow');
+  const { t } = useTranslation();
   const defaultValues = useFormValues(initialParserValues, node);
 
   const form = useForm<z.infer<typeof FormSchema>>({
     defaultValues,
     resolver: zodResolver(FormSchema),
+    shouldUnregister: true,
   });
 
+  const name = 'parser';
+  const { fields, remove, append } = useFieldArray({
+    name,
+    control: form.control,
+  });
+
+  const add = useCallback(() => {
+    append({
+      fileFormat: null,
+      output_format: '',
+      parse_method: '',
+      llm_id: '',
+      lang: '',
+      fields: [],
+    });
+  }, [append]);
+
   useWatchFormChange(node?.id, form);
 
   return (
     <Form {...form}>
-      <FormWrapper>
-        <FormContainer>
-          <QueryVariable name="q"></QueryVariable>
-        </FormContainer>
-        <FormContainer>
-          <ApiKeyField placeholder={t('apiKeyPlaceholder')}></ApiKeyField>
-          <FormField
-            control={form.control}
-            name={`start`}
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>{t('flowStart')}</FormLabel>
-                <FormControl>
-                  <NumberInput {...field} className="w-full"></NumberInput>
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <FormField
-            control={form.control}
-            name={`num`}
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>{t('flowNum')}</FormLabel>
-                <FormControl>
-                  <NumberInput {...field} className="w-full"></NumberInput>
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
-          <GoogleFormWidgets></GoogleFormWidgets>
-        </FormContainer>
-      </FormWrapper>
+      <form className="px-5">
+        {fields.map((field, index) => {
+          return (
+            <ParserItem
+              key={field.id}
+              name={name}
+              index={index}
+              fieldLength={fields.length}
+              remove={remove}
+            ></ParserItem>
+          );
+        })}
+        <BlockButton onClick={add} type="button" className="mt-2.5">
+          {t('dataflow.addParser')}
+        </BlockButton>
+      </form>
       <div className="p-5">
         <Output list={outputList}></Output>
       </div>

web/src/pages/data-flow/form/parser-form/interface.ts

@@ -0,0 +1,3 @@
+export type CommonProps = {
+  prefix: string;
+};

web/src/pages/data-flow/form/parser-form/pdf-form-fields.tsx

@@ -0,0 +1,29 @@
+import { CrossLanguageFormField } from '@/components/cross-language-form-field';
+import { useTranslation } from 'react-i18next';
+import { FileType } from '../../constant';
+import {
+  LargeModelFormField,
+  OutputFormatFormField,
+  ParserMethodFormField,
+} from './common-form-fields';
+import { CommonProps } from './interface';
+import { buildFieldNameWithPrefix } from './utils';
+
+export function PdfFormFields({ prefix }: CommonProps) {
+  const { t } = useTranslation();
+  return (
+    <>
+      <ParserMethodFormField prefix={prefix}></ParserMethodFormField>
+      {/* Multimodal Model */}
+      <LargeModelFormField prefix={prefix}></LargeModelFormField>
+      <CrossLanguageFormField
+        name={buildFieldNameWithPrefix(`lang`, prefix)}
+        label={t('dataflow.lang')}
+      ></CrossLanguageFormField>
+      <OutputFormatFormField
+        prefix={prefix}
+        fileType={FileType.Image}
+      ></OutputFormatFormField>
+    </>
+  );
+}

web/src/pages/data-flow/form/parser-form/utils.ts

@@ -0,0 +1,3 @@
+export function buildFieldNameWithPrefix(name: string, prefix: string) {
+  return `${prefix}.${name}`;
+}

web/src/pages/data-flow/form/parser-form/video-form-fields.tsx

@@ -0,0 +1,21 @@
+import {
+  LargeModelFormField,
+  OutputFormatFormField,
+  OutputFormatFormFieldProps,
+} from './common-form-fields';
+
+export function VideoFormFields({
+  prefix,
+  fileType,
+}: OutputFormatFormFieldProps) {
+  return (
+    <>
+      {/* Multimodal Model */}
+      <LargeModelFormField prefix={prefix}></LargeModelFormField>
+      <OutputFormatFormField
+        prefix={prefix}
+        fileType={fileType}
+      ></OutputFormatFormField>
+    </>
+  );
+}

web/src/pages/data-flow/form/splitter-form/index.tsx

@@ -0,0 +1,95 @@
+import { FormContainer } from '@/components/form-container';
+import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SliderInputFormField } from '@/components/slider-input-form-field';
+import { BlockButton, Button } from '@/components/ui/button';
+import { Form } from '@/components/ui/form';
+import { Input } from '@/components/ui/input';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { X } from 'lucide-react';
+import { memo } from 'react';
+import { useFieldArray, useForm } from 'react-hook-form';
+import { z } from 'zod';
+import { initialChunkerValues, initialSplitterValues } from '../../constant';
+import { useFormValues } from '../../hooks/use-form-values';
+import { useWatchFormChange } from '../../hooks/use-watch-form-change';
+import { INextOperatorForm } from '../../interface';
+import { buildOutputList } from '../../utils/build-output-list';
+import { FormWrapper } from '../components/form-wrapper';
+import { Output } from '../components/output';
+
+const outputList = buildOutputList(initialSplitterValues.outputs);
+
+export const FormSchema = z.object({
+  chunk_token_size: z.number(),
+  delimiters: z.array(
+    z.object({
+      value: z.string().optional(),
+    }),
+  ),
+  overlapped_percent: z.number(), // 0.0 - 0.3
+});
+
+const SplitterForm = ({ node }: INextOperatorForm) => {
+  const defaultValues = useFormValues(initialChunkerValues, node);
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    defaultValues,
+    resolver: zodResolver(FormSchema),
+  });
+  const name = 'delimiters';
+
+  const { fields, append, remove } = useFieldArray({
+    name: name,
+    control: form.control,
+  });
+
+  useWatchFormChange(node?.id, form);
+
+  return (
+    <Form {...form}>
+      <FormWrapper>
+        <FormContainer>
+          <SliderInputFormField
+            name="chunk_token_size"
+            max={2048}
+            label="chunk_token_size"
+          ></SliderInputFormField>
+          <SliderInputFormField
+            name="overlapped_percent"
+            max={0.3}
+            min={0.1}
+            step={0.01}
+            label="overlapped_percent"
+          ></SliderInputFormField>
+          <span>delimiters</span>
+          {fields.map((field, index) => (
+            <div key={field.id} className="flex items-center gap-2">
+              <div className="space-y-2 flex-1">
+                <RAGFlowFormItem
+                  name={`${name}.${index}.value`}
+                  label="delimiter"
+                  labelClassName="!hidden"
+                >
+                  <Input className="!m-0"></Input>
+                </RAGFlowFormItem>
+              </div>
+              <Button
+                type="button"
+                variant={'ghost'}
+                onClick={() => remove(index)}
+              >
+                <X />
+              </Button>
+            </div>
+          ))}
+          <BlockButton onClick={() => append({ value: '' })}>Add</BlockButton>
+        </FormContainer>
+      </FormWrapper>
+      <div className="p-5">
+        <Output list={outputList}></Output>
+      </div>
+    </Form>
+  );
+};
+
+export default memo(SplitterForm);

web/src/pages/data-flow/hooks.tsx

@@ -30,7 +30,16 @@ export const useGetNodeName = () => {
   const { t } = useTranslation();
 
   return (type: string) => {
-    const name = t(`flow.${lowerFirst(type)}`);
+    const name = t(`dataflow.${lowerFirst(type)}`);
+    return name;
+  };
+};
+
+export const useGetNodeDescription = () => {
+  const { t } = useTranslation();
+
+  return (type: string) => {
+    const name = t(`dataflow.${lowerFirst(type)}Description`);
     return name;
   };
 };

web/src/pages/data-flow/hooks/use-add-node.ts

@@ -18,6 +18,7 @@ import {
   initialCrawlerValues,
   initialEmailValues,
   initialExeSqlValues,
+  initialHierarchicalMergerValues,
   initialInvokeValues,
   initialIterationStartValues,
   initialIterationValues,
@@ -28,6 +29,7 @@ import {
   initialRelevantValues,
   initialRetrievalValues,
   initialRewriteQuestionValues,
+  initialSplitterValues,
   initialStringTransformValues,
   initialSwitchValues,
   initialTokenizerValues,
@@ -82,6 +84,8 @@ export const useInitializeOperatorParams = () => {
       [Operator.Parser]: initialParserValues,
       [Operator.Chunker]: initialChunkerValues,
       [Operator.Tokenizer]: initialTokenizerValues,
+      [Operator.Splitter]: initialSplitterValues,
+      [Operator.HierarchicalMerger]: initialHierarchicalMergerValues,
     };
   }, [llmId]);
 
@@ -108,7 +112,7 @@ export const useGetNodeName = () => {
   const { t } = useTranslation();
 
   return (type: string) => {
-    const name = t(`flow.${lowerFirst(type)}`);
+    const name = t(`dataflow.${lowerFirst(type)}`);
     return name;
   };
 };

web/src/pages/dataflow-result/chunker.tsx

@@ -0,0 +1,234 @@
+import message from '@/components/ui/message';
+import {
+  RAGFlowPagination,
+  RAGFlowPaginationType,
+} from '@/components/ui/ragflow-pagination';
+import { Spin } from '@/components/ui/spin';
+import {
+  useFetchNextChunkList,
+  useSwitchChunk,
+} from '@/hooks/use-chunk-request';
+import classNames from 'classnames';
+import { useCallback, useEffect, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import ChunkCard from './components/chunk-card';
+import CreatingModal from './components/chunk-creating-modal';
+import ChunkResultBar from './components/chunk-result-bar';
+import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
+import RerunButton from './components/rerun-button';
+import {
+  useChangeChunkTextMode,
+  useDeleteChunkByIds,
+  useHandleChunkCardClick,
+  useUpdateChunk,
+} from './hooks';
+import styles from './index.less';
+const ChunkerContainer = () => {
+  const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
+  const [isChange, setIsChange] = useState(false);
+  const { t } = useTranslation();
+  const {
+    data: { documentInfo, data = [], total },
+    pagination,
+    loading,
+    searchString,
+    handleInputChange,
+    available,
+    handleSetAvailable,
+  } = useFetchNextChunkList();
+  const { handleChunkCardClick, selectedChunkId } = useHandleChunkCardClick();
+  const isPdf = documentInfo?.type === 'pdf';
+  const {
+    chunkUpdatingLoading,
+    onChunkUpdatingOk,
+    showChunkUpdatingModal,
+    hideChunkUpdatingModal,
+    chunkId,
+    chunkUpdatingVisible,
+    documentId,
+  } = useUpdateChunk();
+  const { removeChunk } = useDeleteChunkByIds();
+  const { changeChunkTextMode, textMode } = useChangeChunkTextMode();
+  const selectAllChunk = useCallback(
+    (checked: boolean) => {
+      setSelectedChunkIds(checked ? data.map((x) => x.chunk_id) : []);
+    },
+    [data],
+  );
+  const showSelectedChunkWarning = useCallback(() => {
+    message.warning(t('message.pleaseSelectChunk'));
+  }, [t]);
+  const { switchChunk } = useSwitchChunk();
+
+  const [chunkList, setChunkList] = useState(data);
+  useEffect(() => {
+    setChunkList(data);
+  }, [data]);
+  const onPaginationChange: RAGFlowPaginationType['onChange'] = (
+    page,
+    size,
+  ) => {
+    setSelectedChunkIds([]);
+    pagination.onChange?.(page, size);
+  };
+
+  const handleSwitchChunk = useCallback(
+    async (available?: number, chunkIds?: string[]) => {
+      let ids = chunkIds;
+      if (!chunkIds) {
+        ids = selectedChunkIds;
+        if (selectedChunkIds.length === 0) {
+          showSelectedChunkWarning();
+          return;
+        }
+      }
+
+      const resCode: number = await switchChunk({
+        chunk_ids: ids,
+        available_int: available,
+        doc_id: documentId,
+      });
+      if (ids?.length && resCode === 0) {
+        chunkList.forEach((x: any) => {
+          if (ids.indexOf(x['chunk_id']) > -1) {
+            x['available_int'] = available;
+          }
+        });
+        setChunkList(chunkList);
+      }
+    },
+    [
+      switchChunk,
+      documentId,
+      selectedChunkIds,
+      showSelectedChunkWarning,
+      chunkList,
+    ],
+  );
+  const handleSingleCheckboxClick = useCallback(
+    (chunkId: string, checked: boolean) => {
+      setSelectedChunkIds((previousIds) => {
+        const idx = previousIds.findIndex((x) => x === chunkId);
+        const nextIds = [...previousIds];
+        if (checked && idx === -1) {
+          nextIds.push(chunkId);
+        } else if (!checked && idx !== -1) {
+          nextIds.splice(idx, 1);
+        }
+        return nextIds;
+      });
+    },
+    [],
+  );
+  const handleRemoveChunk = useCallback(async () => {
+    if (selectedChunkIds.length > 0) {
+      const resCode: number = await removeChunk(selectedChunkIds, documentId);
+      if (resCode === 0) {
+        setSelectedChunkIds([]);
+      }
+    } else {
+      showSelectedChunkWarning();
+    }
+  }, [selectedChunkIds, documentId, removeChunk, showSelectedChunkWarning]);
+
+  const handleChunkEditSave = (e: any) => {
+    setIsChange(true);
+    onChunkUpdatingOk(e);
+  };
+  return (
+    <>
+      {isChange && (
+        <div className=" absolute top-2 right-6">
+          <RerunButton />
+        </div>
+      )}
+      <div
+        className={classNames(
+          { [styles.pagePdfWrapper]: isPdf },
+          'flex flex-col w-3/5',
+        )}
+      >
+        <Spin spinning={loading} className={styles.spin} size="large">
+          <div className="h-[50px] flex flex-row justify-between items-end pb-[5px]">
+            <div>
+              <h2 className="text-[16px]">{t('chunk.chunkResult')}</h2>
+              <div className="text-[12px] text-text-secondary italic">
+                {t('chunk.chunkResultTip')}
+              </div>
+            </div>
+            <ChunkResultBar
+              handleInputChange={handleInputChange}
+              searchString={searchString}
+              changeChunkTextMode={changeChunkTextMode}
+              createChunk={showChunkUpdatingModal}
+              available={available}
+              selectAllChunk={selectAllChunk}
+              handleSetAvailable={handleSetAvailable}
+            />
+          </div>
+          <div className=" rounded-[16px] box-border	mb-2">
+            <div className="pt-[5px] pb-[5px]">
+              <CheckboxSets
+                selectAllChunk={selectAllChunk}
+                switchChunk={handleSwitchChunk}
+                removeChunk={handleRemoveChunk}
+                checked={selectedChunkIds.length === data.length}
+                selectedChunkIds={selectedChunkIds}
+              />
+            </div>
+            <div className="h-[calc(100vh-280px)] overflow-y-auto pr-2 scrollbar-thin">
+              <div
+                className={classNames(
+                  styles.chunkContainer,
+                  {
+                    [styles.chunkOtherContainer]: !isPdf,
+                  },
+                  'flex flex-col gap-4',
+                )}
+              >
+                {chunkList.map((item) => (
+                  <ChunkCard
+                    item={item}
+                    key={item.chunk_id}
+                    editChunk={showChunkUpdatingModal}
+                    checked={selectedChunkIds.some((x) => x === item.chunk_id)}
+                    handleCheckboxClick={handleSingleCheckboxClick}
+                    switchChunk={handleSwitchChunk}
+                    clickChunkCard={handleChunkCardClick}
+                    selected={item.chunk_id === selectedChunkId}
+                    textMode={textMode}
+                  ></ChunkCard>
+                ))}
+              </div>
+            </div>
+            <div className={styles.pageFooter}>
+              <RAGFlowPagination
+                pageSize={pagination.pageSize}
+                current={pagination.current}
+                total={total}
+                onChange={(page, pageSize) => {
+                  onPaginationChange(page, pageSize);
+                }}
+              ></RAGFlowPagination>
+            </div>
+          </div>
+        </Spin>
+      </div>
+      {chunkUpdatingVisible && (
+        <CreatingModal
+          doc_id={documentId}
+          chunkId={chunkId}
+          hideModal={hideChunkUpdatingModal}
+          visible={chunkUpdatingVisible}
+          loading={chunkUpdatingLoading}
+          onOk={(e) => {
+            handleChunkEditSave(e);
+          }}
+          parserId={documentInfo.parser_id}
+        />
+      )}
+    </>
+  );
+};
+
+export { ChunkerContainer };

web/src/pages/dataflow-result/components/chunk-card/index.less

@@ -0,0 +1,36 @@
+.image {
+  width: 100px !important;
+  object-fit: contain;
+}
+
+.imagePreview {
+  max-width: 50vw;
+  max-height: 50vh;
+  object-fit: contain;
+}
+
+.content {
+  flex: 1;
+  .chunkText;
+}
+
+.contentEllipsis {
+  .multipleLineEllipsis(3);
+}
+
+.contentText {
+  word-break: break-all !important;
+}
+
+.chunkCard {
+  width: 100%;
+  padding: 18px 10px;
+}
+
+.cardSelected {
+  background-color: @selectedBackgroundColor;
+}
+
+.cardSelectedDark {
+  background-color: #ffffff2f;
+}

web/src/pages/dataflow-result/components/chunk-card/index.tsx

@@ -0,0 +1,127 @@
+import Image from '@/components/image';
+import { useTheme } from '@/components/theme-provider';
+import { Card } from '@/components/ui/card';
+import { Checkbox } from '@/components/ui/checkbox';
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from '@/components/ui/popover';
+import { Switch } from '@/components/ui/switch';
+import { IChunk } from '@/interfaces/database/knowledge';
+import { CheckedState } from '@radix-ui/react-checkbox';
+import classNames from 'classnames';
+import DOMPurify from 'dompurify';
+import { useEffect, useState } from 'react';
+import { ChunkTextMode } from '../../constant';
+import styles from './index.less';
+
+interface IProps {
+  item: IChunk;
+  checked: boolean;
+  switchChunk: (available?: number, chunkIds?: string[]) => void;
+  editChunk: (chunkId: string) => void;
+  handleCheckboxClick: (chunkId: string, checked: boolean) => void;
+  selected: boolean;
+  clickChunkCard: (chunkId: string) => void;
+  textMode: ChunkTextMode;
+}
+
+const ChunkCard = ({
+  item,
+  checked,
+  handleCheckboxClick,
+  editChunk,
+  switchChunk,
+  selected,
+  clickChunkCard,
+  textMode,
+}: IProps) => {
+  const available = Number(item.available_int);
+  const [enabled, setEnabled] = useState(false);
+  const { theme } = useTheme();
+
+  const onChange = (checked: boolean) => {
+    setEnabled(checked);
+    switchChunk(available === 0 ? 1 : 0, [item.chunk_id]);
+  };
+
+  const handleCheck = (e: CheckedState) => {
+    handleCheckboxClick(item.chunk_id, e === 'indeterminate' ? false : e);
+  };
+
+  const handleContentDoubleClick = () => {
+    editChunk(item.chunk_id);
+  };
+
+  const handleContentClick = () => {
+    clickChunkCard(item.chunk_id);
+  };
+
+  useEffect(() => {
+    setEnabled(available === 1);
+  }, [available]);
+  const [open, setOpen] = useState<boolean>(false);
+  return (
+    <Card
+      className={classNames('rounded-lg w-full py-3 px-3', {
+        'bg-bg-title': selected,
+        'bg-bg-input': !selected,
+      })}
+    >
+      <div className="flex items-start justify-between gap-2">
+        <Checkbox onCheckedChange={handleCheck} checked={checked}></Checkbox>
+        {item.image_id && (
+          <Popover open={open}>
+            <PopoverTrigger
+              asChild
+              onMouseEnter={() => setOpen(true)}
+              onMouseLeave={() => setOpen(false)}
+            >
+              <div>
+                <Image id={item.image_id} className={styles.image}></Image>
+              </div>
+            </PopoverTrigger>
+            <PopoverContent
+              className="p-0"
+              align={'start'}
+              side={'right'}
+              sideOffset={-20}
+            >
+              <div>
+                <Image
+                  id={item.image_id}
+                  className={styles.imagePreview}
+                ></Image>
+              </div>
+            </PopoverContent>
+          </Popover>
+        )}
+        <section
+          onDoubleClick={handleContentDoubleClick}
+          onClick={handleContentClick}
+          className={styles.content}
+        >
+          <div
+            dangerouslySetInnerHTML={{
+              __html: DOMPurify.sanitize(item.content_with_weight),
+            }}
+            className={classNames(styles.contentText, {
+              [styles.contentEllipsis]: textMode === ChunkTextMode.Ellipse,
+            })}
+          ></div>
+        </section>
+        <div>
+          <Switch
+            checked={enabled}
+            onCheckedChange={onChange}
+            aria-readonly
+            className="!m-0"
+          />
+        </div>
+      </div>
+    </Card>
+  );
+};
+
+export default ChunkCard;

web/src/pages/dataflow-result/components/chunk-creating-modal/index.tsx

@@ -0,0 +1,206 @@
+import EditTag from '@/components/edit-tag';
+import Divider from '@/components/ui/divider';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import {
+  HoverCard,
+  HoverCardContent,
+  HoverCardTrigger,
+} from '@/components/ui/hover-card';
+import { Modal } from '@/components/ui/modal/modal';
+import Space from '@/components/ui/space';
+import { Switch } from '@/components/ui/switch';
+import { Textarea } from '@/components/ui/textarea';
+import { useFetchChunk } from '@/hooks/chunk-hooks';
+import { IModalProps } from '@/interfaces/common';
+import { Trash2 } from 'lucide-react';
+import React, { useCallback, useEffect, useState } from 'react';
+import { FieldValues, FormProvider, useForm } from 'react-hook-form';
+import { useTranslation } from 'react-i18next';
+import { useDeleteChunkByIds } from '../../hooks';
+import {
+  transformTagFeaturesArrayToObject,
+  transformTagFeaturesObjectToArray,
+} from '../../utils';
+import { TagFeatureItem } from './tag-feature-item';
+
+interface kFProps {
+  doc_id: string;
+  chunkId: string | undefined;
+  parserId: string;
+}
+
+const ChunkCreatingModal: React.FC<IModalProps<any> & kFProps> = ({
+  doc_id,
+  chunkId,
+  hideModal,
+  onOk,
+  loading,
+  parserId,
+}) => {
+  // const [form] = Form.useForm();
+  // const form = useFormContext();
+  const form = useForm<FieldValues>({
+    defaultValues: {
+      content_with_weight: '',
+      tag_kwd: [],
+      question_kwd: [],
+      important_kwd: [],
+      tag_feas: [],
+    },
+  });
+  const [checked, setChecked] = useState(false);
+  const { removeChunk } = useDeleteChunkByIds();
+  const { data } = useFetchChunk(chunkId);
+  const { t } = useTranslation();
+
+  const isTagParser = parserId === 'tag';
+  const onSubmit = useCallback(
+    (values: FieldValues) => {
+      onOk?.({
+        ...values,
+        tag_feas: transformTagFeaturesArrayToObject(values.tag_feas),
+        available_int: checked ? 1 : 0,
+      });
+    },
+    [checked, onOk],
+  );
+
+  const handleOk = form.handleSubmit(onSubmit);
+
+  const handleRemove = useCallback(() => {
+    if (chunkId) {
+      return removeChunk([chunkId], doc_id);
+    }
+  }, [chunkId, doc_id, removeChunk]);
+
+  const handleCheck = useCallback(() => {
+    setChecked(!checked);
+  }, [checked]);
+
+  useEffect(() => {
+    if (data?.code === 0) {
+      const { available_int, tag_feas } = data.data;
+      form.reset({
+        ...data.data,
+        tag_feas: transformTagFeaturesObjectToArray(tag_feas),
+      });
+
+      setChecked(available_int !== 0);
+    }
+  }, [data, form, chunkId]);
+
+  return (
+    <Modal
+      title={`${chunkId ? t('common.edit') : t('common.create')} ${t('chunk.chunk')}`}
+      open={true}
+      onOk={handleOk}
+      onCancel={hideModal}
+      confirmLoading={loading}
+      destroyOnClose
+    >
+      <Form {...form}>
+        <div className="flex flex-col gap-4">
+          <FormField
+            control={form.control}
+            name="content_with_weight"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('chunk.chunk')}</FormLabel>
+                <FormControl>
+                  <Textarea {...field} autoSize={{ minRows: 4, maxRows: 10 }} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name="important_kwd"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>{t('chunk.keyword')}</FormLabel>
+                <FormControl>
+                  <EditTag {...field} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          <FormField
+            control={form.control}
+            name="question_kwd"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel className="flex justify-start items-start">
+                  <div className="flex items-center gap-0">
+                    <span>{t('chunk.question')}</span>
+                    <HoverCard>
+                      <HoverCardTrigger asChild>
+                        <span className="text-xs mt-[-3px] text-center scale-[90%] font-thin text-primary cursor-pointer rounded-full w-[16px] h-[16px] border-muted-foreground/50 border">
+                          ?
+                        </span>
+                      </HoverCardTrigger>
+                      <HoverCardContent className="w-80" side="top">
+                        {t('chunk.questionTip')}
+                      </HoverCardContent>
+                    </HoverCard>
+                  </div>
+                </FormLabel>
+                <FormControl>
+                  <EditTag {...field} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+
+          {isTagParser && (
+            <FormField
+              control={form.control}
+              name="tag_kwd"
+              render={({ field }) => (
+                <FormItem>
+                  <FormLabel>{t('knowledgeConfiguration.tagName')}</FormLabel>
+                  <FormControl>
+                    <EditTag {...field} />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+          )}
+
+          {!isTagParser && (
+            <FormProvider {...form}>
+              <TagFeatureItem />
+            </FormProvider>
+          )}
+        </div>
+      </Form>
+
+      {chunkId && (
+        <section>
+          <Divider />
+          <Space size={'large'}>
+            <div className="flex items-center gap-2">
+              {t('chunk.enabled')}
+              <Switch checked={checked} onCheckedChange={handleCheck} />
+            </div>
+            <div className="flex items-center gap-1" onClick={handleRemove}>
+              <Trash2 size={16} /> {t('common.delete')}
+            </div>
+          </Space>
+        </section>
+      )}
+    </Modal>
+  );
+};
+export default ChunkCreatingModal;

web/src/pages/dataflow-result/components/chunk-creating-modal/tag-feature-item.tsx

@@ -0,0 +1,136 @@
+import { SelectWithSearch } from '@/components/originui/select-with-search';
+import { Button } from '@/components/ui/button';
+import {
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { NumberInput } from '@/components/ui/input';
+import { useFetchTagListByKnowledgeIds } from '@/hooks/knowledge-hooks';
+import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request';
+import { CircleMinus, Plus } from 'lucide-react';
+import { useCallback, useEffect, useMemo } from 'react';
+import { useFieldArray, useFormContext } from 'react-hook-form';
+import { useTranslation } from 'react-i18next';
+import { FormListItem } from '../../utils';
+
+const FieldKey = 'tag_feas';
+
+export const TagFeatureItem = () => {
+  const { t } = useTranslation();
+  const { setKnowledgeIds, list } = useFetchTagListByKnowledgeIds();
+  const { data: knowledgeConfiguration } = useFetchKnowledgeBaseConfiguration();
+  const form = useFormContext();
+  const tagKnowledgeIds = useMemo(() => {
+    return knowledgeConfiguration?.parser_config?.tag_kb_ids ?? [];
+  }, [knowledgeConfiguration?.parser_config?.tag_kb_ids]);
+
+  const options = useMemo(() => {
+    return list.map((x) => ({
+      value: x[0],
+      label: x[0],
+    }));
+  }, [list]);
+
+  const filterOptions = useCallback(
+    (index: number) => {
+      const tags: FormListItem[] = form.getValues(FieldKey) ?? [];
+
+      // Exclude it's own current data
+      const list = tags
+        .filter((x, idx) => x && index !== idx)
+        .map((x) => x.tag);
+      // Exclude the selected data from other options from one's own options.
+      const resultList = options.filter(
+        (x) => !list.some((y) => x.value === y),
+      );
+      return resultList;
+    },
+    [form, options],
+  );
+
+  useEffect(() => {
+    setKnowledgeIds(tagKnowledgeIds);
+  }, [setKnowledgeIds, tagKnowledgeIds]);
+
+  const { fields, append, remove } = useFieldArray({
+    control: form.control,
+    name: FieldKey,
+  });
+  return (
+    <FormField
+      control={form.control}
+      name={FieldKey as any}
+      render={() => (
+        <FormItem>
+          <FormLabel>{t('knowledgeConfiguration.tags')}</FormLabel>
+          <div>
+            {fields.map((item, name) => {
+              return (
+                <div key={item.id} className="flex gap-3 items-center mb-4">
+                  <div className="flex flex-1 gap-8">
+                    <FormField
+                      control={form.control}
+                      name={`${FieldKey}.${name}.tag` as any}
+                      render={({ field }) => (
+                        <FormItem className="w-2/3">
+                          <FormControl className="w-full">
+                            <div>
+                              <SelectWithSearch
+                                options={filterOptions(name)}
+                                placeholder={t(
+                                  'knowledgeConfiguration.tagName',
+                                )}
+                                value={field.value}
+                                onChange={field.onChange}
+                              />
+                            </div>
+                          </FormControl>
+                          <FormMessage />
+                        </FormItem>
+                      )}
+                    />
+                    <FormField
+                      control={form.control}
+                      name={`${FieldKey}.${name}.frequency`}
+                      render={({ field }) => (
+                        <FormItem>
+                          <FormControl>
+                            <NumberInput
+                              value={field.value}
+                              onChange={field.onChange}
+                              placeholder={t(
+                                'knowledgeConfiguration.frequency',
+                              )}
+                              max={10}
+                              min={0}
+                            />
+                          </FormControl>
+                          <FormMessage />
+                        </FormItem>
+                      )}
+                    />
+                  </div>
+                  <CircleMinus
+                    onClick={() => remove(name)}
+                    className="text-red-500"
+                  />
+                </div>
+              );
+            })}
+            <Button
+              variant="dashed"
+              className="w-full flex items-center justify-center gap-2"
+              onClick={() => append({ tag: '', frequency: 0 })}
+            >
+              <Plus size={16} />
+              {t('knowledgeConfiguration.addTag')}
+            </Button>
+          </div>
+        </FormItem>
+      )}
+    />
+  );
+};

web/src/pages/dataflow-result/components/chunk-result-bar/checkbox-sets.tsx

@@ -0,0 +1,85 @@
+import { Checkbox } from '@/components/ui/checkbox';
+import { Label } from '@/components/ui/label';
+import { Ban, CircleCheck, Trash2 } from 'lucide-react';
+import { useCallback, useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+type ICheckboxSetProps = {
+  selectAllChunk: (e: any) => void;
+  removeChunk: (e?: any) => void;
+  switchChunk: (available: number) => void;
+  checked: boolean;
+  selectedChunkIds: string[];
+};
+export default (props: ICheckboxSetProps) => {
+  const {
+    selectAllChunk,
+    removeChunk,
+    switchChunk,
+    checked,
+    selectedChunkIds,
+  } = props;
+  const { t } = useTranslation();
+  const handleSelectAllCheck = useCallback(
+    (e: any) => {
+      console.log('eee=', e);
+      selectAllChunk(e);
+    },
+    [selectAllChunk],
+  );
+
+  const handleDeleteClick = useCallback(() => {
+    removeChunk();
+  }, [removeChunk]);
+
+  const handleEnabledClick = useCallback(() => {
+    switchChunk(1);
+  }, [switchChunk]);
+
+  const handleDisabledClick = useCallback(() => {
+    switchChunk(0);
+  }, [switchChunk]);
+
+  const isSelected = useMemo(() => {
+    return selectedChunkIds?.length > 0;
+  }, [selectedChunkIds]);
+
+  return (
+    <div className="flex gap-[40px] py-4 px-2">
+      <div className="flex items-center gap-3 cursor-pointer text-muted-foreground hover:text-text-primary">
+        <Checkbox
+          id="all_chunks_checkbox"
+          onCheckedChange={handleSelectAllCheck}
+          checked={checked}
+          className=" data-[state=checked]:bg-text-primary data-[state=checked]:border-text-primary data-[state=checked]:text-bg-base  border-muted-foreground text-muted-foreground hover:text-bg-base hover:border-text-primary "
+        />
+        <Label htmlFor="all_chunks_checkbox">{t('chunk.selectAll')}</Label>
+      </div>
+      {isSelected && (
+        <>
+          <div
+            className="flex items-center cursor-pointer text-muted-foreground hover:text-text-primary"
+            onClick={handleEnabledClick}
+          >
+            <CircleCheck size={16} />
+            <span className="block ml-1">{t('chunk.enable')}</span>
+          </div>
+          <div
+            className="flex items-center cursor-pointer text-muted-foreground hover:text-text-primary"
+            onClick={handleDisabledClick}
+          >
+            <Ban size={16} />
+            <span className="block ml-1">{t('chunk.disable')}</span>
+          </div>
+          <div
+            className="flex items-center cursor-pointer text-red-400 hover:text-red-500"
+            onClick={handleDeleteClick}
+          >
+            <Trash2 size={16} />
+            <span className="block ml-1">{t('chunk.delete')}</span>
+          </div>
+        </>
+      )}
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/chunk-result-bar/index.tsx

@@ -0,0 +1,108 @@
+import { Input } from '@/components/originui/input';
+import { Button } from '@/components/ui/button';
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from '@/components/ui/popover';
+import { Radio } from '@/components/ui/radio';
+import { useTranslate } from '@/hooks/common-hooks';
+import { cn } from '@/lib/utils';
+import { SearchOutlined } from '@ant-design/icons';
+import { ListFilter, Plus } from 'lucide-react';
+import { useState } from 'react';
+import { ChunkTextMode } from '../../constant';
+interface ChunkResultBarProps {
+  changeChunkTextMode: React.Dispatch<React.SetStateAction<string | number>>;
+  available: number | undefined;
+  selectAllChunk: (value: boolean) => void;
+  handleSetAvailable: (value: number | undefined) => void;
+  createChunk: () => void;
+  handleInputChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
+  searchString: string;
+}
+export default ({
+  changeChunkTextMode,
+  available,
+  selectAllChunk,
+  handleSetAvailable,
+  createChunk,
+  handleInputChange,
+  searchString,
+}: ChunkResultBarProps) => {
+  const { t } = useTranslate('chunk');
+  const [textSelectValue, setTextSelectValue] = useState<string | number>(
+    ChunkTextMode.Full,
+  );
+  const handleFilterChange = (e: string | number) => {
+    const value = e === -1 ? undefined : (e as number);
+    selectAllChunk(false);
+    handleSetAvailable(value);
+  };
+  const filterContent = (
+    <div className="w-[200px]">
+      <Radio.Group onChange={handleFilterChange} value={available}>
+        <div className="flex flex-col gap-2 p-4">
+          <Radio value={-1}>{t('all')}</Radio>
+          <Radio value={1}>{t('enabled')}</Radio>
+          <Radio value={0}>{t('disabled')}</Radio>
+        </div>
+      </Radio.Group>
+    </div>
+  );
+  const textSelectOptions = [
+    { label: t(ChunkTextMode.Full), value: ChunkTextMode.Full },
+    { label: t(ChunkTextMode.Ellipse), value: ChunkTextMode.Ellipse },
+  ];
+
+  const changeTextSelectValue = (value: string | number) => {
+    setTextSelectValue(value);
+    changeChunkTextMode(value);
+  };
+  return (
+    <div className="flex gap-2">
+      <div className="flex items-center gap-1 bg-bg-card text-muted-foreground w-fit h-[35px] rounded-md p-1">
+        {textSelectOptions.map((option) => (
+          <div
+            key={option.value}
+            className={cn(
+              'flex items-center cursor-pointer px-4 py-1 rounded-md',
+              {
+                'text-primary bg-bg-base': option.value === textSelectValue,
+                'text-text-primary': option.value !== textSelectValue,
+              },
+            )}
+            onClick={() => changeTextSelectValue(option.value)}
+          >
+            {option.label}
+          </div>
+        ))}
+      </div>
+      <Input
+        className="bg-bg-card text-muted-foreground"
+        style={{ width: 200 }}
+        placeholder={t('search')}
+        icon={<SearchOutlined />}
+        onChange={handleInputChange}
+        value={searchString}
+      />
+      <Popover>
+        <PopoverTrigger asChild>
+          <Button className="bg-bg-card text-muted-foreground hover:bg-card">
+            <ListFilter />
+          </Button>
+        </PopoverTrigger>
+        <PopoverContent className="p-0 w-[200px]">
+          {filterContent}
+        </PopoverContent>
+      </Popover>
+      <Button
+        onClick={() => createChunk()}
+        variant={'secondary'}
+        className="bg-bg-card text-muted-foreground hover:bg-card"
+      >
+        <Plus size={44} />
+      </Button>
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/chunk-toolbar/index.tsx

@@ -0,0 +1,221 @@
+import { ReactComponent as FilterIcon } from '@/assets/filter.svg';
+import { KnowledgeRouteKey } from '@/constants/knowledge';
+import { IChunkListResult, useSelectChunkList } from '@/hooks/chunk-hooks';
+import { useTranslate } from '@/hooks/common-hooks';
+import { useKnowledgeBaseId } from '@/hooks/knowledge-hooks';
+import {
+  ArrowLeftOutlined,
+  CheckCircleOutlined,
+  CloseCircleOutlined,
+  DeleteOutlined,
+  DownOutlined,
+  FilePdfOutlined,
+  PlusOutlined,
+  SearchOutlined,
+} from '@ant-design/icons';
+import {
+  Button,
+  Checkbox,
+  Flex,
+  Input,
+  Menu,
+  MenuProps,
+  Popover,
+  Radio,
+  RadioChangeEvent,
+  Segmented,
+  SegmentedProps,
+  Space,
+  Typography,
+} from 'antd';
+import { useCallback, useMemo, useState } from 'react';
+import { Link } from 'umi';
+import { ChunkTextMode } from '../../constant';
+
+const { Text } = Typography;
+
+interface IProps
+  extends Pick<
+    IChunkListResult,
+    'searchString' | 'handleInputChange' | 'available' | 'handleSetAvailable'
+  > {
+  checked: boolean;
+  selectAllChunk: (checked: boolean) => void;
+  createChunk: () => void;
+  removeChunk: () => void;
+  switchChunk: (available: number) => void;
+  changeChunkTextMode(mode: ChunkTextMode): void;
+}
+
+const ChunkToolBar = ({
+  selectAllChunk,
+  checked,
+  createChunk,
+  removeChunk,
+  switchChunk,
+  changeChunkTextMode,
+  available,
+  handleSetAvailable,
+  searchString,
+  handleInputChange,
+}: IProps) => {
+  const data = useSelectChunkList();
+  const documentInfo = data?.documentInfo;
+  const knowledgeBaseId = useKnowledgeBaseId();
+  const [isShowSearchBox, setIsShowSearchBox] = useState(false);
+  const { t } = useTranslate('chunk');
+
+  const handleSelectAllCheck = useCallback(
+    (e: any) => {
+      selectAllChunk(e.target.checked);
+    },
+    [selectAllChunk],
+  );
+
+  const handleSearchIconClick = () => {
+    setIsShowSearchBox(true);
+  };
+
+  const handleSearchBlur = () => {
+    if (!searchString?.trim()) {
+      setIsShowSearchBox(false);
+    }
+  };
+
+  const handleDelete = useCallback(() => {
+    removeChunk();
+  }, [removeChunk]);
+
+  const handleEnabledClick = useCallback(() => {
+    switchChunk(1);
+  }, [switchChunk]);
+
+  const handleDisabledClick = useCallback(() => {
+    switchChunk(0);
+  }, [switchChunk]);
+
+  const items: MenuProps['items'] = useMemo(() => {
+    return [
+      {
+        key: '1',
+        label: (
+          <>
+            <Checkbox onChange={handleSelectAllCheck} checked={checked}>
+              <b>{t('selectAll')}</b>
+            </Checkbox>
+          </>
+        ),
+      },
+      { type: 'divider' },
+      {
+        key: '2',
+        label: (
+          <Space onClick={handleEnabledClick}>
+            <CheckCircleOutlined />
+            <b>{t('enabledSelected')}</b>
+          </Space>
+        ),
+      },
+      {
+        key: '3',
+        label: (
+          <Space onClick={handleDisabledClick}>
+            <CloseCircleOutlined />
+            <b>{t('disabledSelected')}</b>
+          </Space>
+        ),
+      },
+      { type: 'divider' },
+      {
+        key: '4',
+        label: (
+          <Space onClick={handleDelete}>
+            <DeleteOutlined />
+            <b>{t('deleteSelected')}</b>
+          </Space>
+        ),
+      },
+    ];
+  }, [
+    checked,
+    handleSelectAllCheck,
+    handleDelete,
+    handleEnabledClick,
+    handleDisabledClick,
+    t,
+  ]);
+
+  const content = (
+    <Menu style={{ width: 200 }} items={items} selectable={false} />
+  );
+
+  const handleFilterChange = (e: RadioChangeEvent) => {
+    selectAllChunk(false);
+    handleSetAvailable(e.target.value);
+  };
+
+  const filterContent = (
+    <Radio.Group onChange={handleFilterChange} value={available}>
+      <Space direction="vertical">
+        <Radio value={undefined}>{t('all')}</Radio>
+        <Radio value={1}>{t('enabled')}</Radio>
+        <Radio value={0}>{t('disabled')}</Radio>
+      </Space>
+    </Radio.Group>
+  );
+
+  return (
+    <Flex justify="space-between" align="center">
+      <Space size={'middle'}>
+        <Link
+          to={`/knowledge/${KnowledgeRouteKey.Dataset}?id=${knowledgeBaseId}`}
+        >
+          <ArrowLeftOutlined />
+        </Link>
+        <FilePdfOutlined />
+        <Text ellipsis={{ tooltip: documentInfo?.name }} style={{ width: 150 }}>
+          {documentInfo?.name}
+        </Text>
+      </Space>
+      <Space>
+        <Segmented
+          options={[
+            { label: t(ChunkTextMode.Full), value: ChunkTextMode.Full },
+            { label: t(ChunkTextMode.Ellipse), value: ChunkTextMode.Ellipse },
+          ]}
+          onChange={changeChunkTextMode as SegmentedProps['onChange']}
+        />
+        <Popover content={content} placement="bottom" arrow={false}>
+          <Button>
+            {t('bulk')}
+            <DownOutlined />
+          </Button>
+        </Popover>
+        {isShowSearchBox ? (
+          <Input
+            size="middle"
+            placeholder={t('search')}
+            prefix={<SearchOutlined />}
+            allowClear
+            onChange={handleInputChange}
+            onBlur={handleSearchBlur}
+            value={searchString}
+          />
+        ) : (
+          <Button icon={<SearchOutlined />} onClick={handleSearchIconClick} />
+        )}
+
+        <Popover content={filterContent} placement="bottom" arrow={false}>
+          <Button icon={<FilterIcon />} />
+        </Popover>
+        <Button
+          icon={<PlusOutlined />}
+          type="primary"
+          onClick={() => createChunk()}
+        />
+      </Space>
+    </Flex>
+  );
+};
+
+export default ChunkToolBar;

web/src/pages/dataflow-result/components/document-preview/csv-preview.tsx

@@ -0,0 +1,114 @@
+import message from '@/components/ui/message';
+import { Spin } from '@/components/ui/spin';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import React, { useEffect, useRef, useState } from 'react';
+
+interface CSVData {
+  rows: string[][];
+  headers: string[];
+}
+
+interface FileViewerProps {
+  className?: string;
+  url: string;
+}
+
+const CSVFileViewer: React.FC<FileViewerProps> = ({ url }) => {
+  const [data, setData] = useState<CSVData | null>(null);
+  const [isLoading, setIsLoading] = useState<boolean>(true);
+  const containerRef = useRef<HTMLDivElement>(null);
+  // const url = useGetDocumentUrl();
+  const parseCSV = (csvText: string): CSVData => {
+    console.log('Parsing CSV data:', csvText);
+    const lines = csvText.split('\n');
+    const headers = lines[0].split(',').map((header) => header.trim());
+    const rows = lines
+      .slice(1)
+      .map((line) => line.split(',').map((cell) => cell.trim()));
+
+    return { headers, rows };
+  };
+
+  useEffect(() => {
+    const loadCSV = async () => {
+      try {
+        const res = await request(url, {
+          method: 'GET',
+          responseType: 'blob',
+          onError: () => {
+            message.error('file load failed');
+            setIsLoading(false);
+          },
+        });
+
+        // parse CSV file
+        const reader = new FileReader();
+        reader.readAsText(res.data);
+        reader.onload = () => {
+          const parsedData = parseCSV(reader.result as string);
+          console.log('file loaded successfully', reader.result);
+          setData(parsedData);
+        };
+      } catch (error) {
+        message.error('CSV file parse failed');
+        console.error('Error loading CSV file:', error);
+      } finally {
+        setIsLoading(false);
+      }
+    };
+
+    loadCSV();
+
+    return () => {
+      setData(null);
+    };
+  }, [url]);
+
+  return (
+    <div
+      ref={containerRef}
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md',
+        'overflow-auto max-h-[80vh] p-2',
+      )}
+    >
+      {isLoading ? (
+        <div className="absolute inset-0 flex items-center justify-center">
+          <Spin />
+        </div>
+      ) : data ? (
+        <table className="min-w-full divide-y divide-border-normal">
+          <thead className="bg-background-header-bar">
+            <tr>
+              {data.headers.map((header, index) => (
+                <th
+                  key={`header-${index}`}
+                  className="px-6 py-3 text-left text-sm font-medium text-text-primary"
+                >
+                  {header}
+                </th>
+              ))}
+            </tr>
+          </thead>
+          <tbody className="bg-background-paper divide-y divide-border-normal">
+            {data.rows.map((row, rowIndex) => (
+              <tr key={`row-${rowIndex}`}>
+                {row.map((cell, cellIndex) => (
+                  <td
+                    key={`cell-${rowIndex}-${cellIndex}`}
+                    className="px-6 py-4 whitespace-nowrap text-sm text-text-secondary"
+                  >
+                    {cell || '-'}
+                  </td>
+                ))}
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      ) : null}
+    </div>
+  );
+};
+
+export default CSVFileViewer;

web/src/pages/dataflow-result/components/document-preview/doc-preview.tsx

@@ -0,0 +1,70 @@
+import message from '@/components/ui/message';
+import { Spin } from '@/components/ui/spin';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import mammoth from 'mammoth';
+import { useEffect, useState } from 'react';
+
+interface DocPreviewerProps {
+  className?: string;
+  url: string;
+}
+
+export const DocPreviewer: React.FC<DocPreviewerProps> = ({
+  className,
+  url,
+}) => {
+  // const url = useGetDocumentUrl();
+  const [htmlContent, setHtmlContent] = useState<string>('');
+  const [loading, setLoading] = useState(false);
+  const fetchDocument = async () => {
+    setLoading(true);
+    const res = await request(url, {
+      method: 'GET',
+      responseType: 'blob',
+      onError: () => {
+        message.error('Document parsing failed');
+        console.error('Error loading document:', url);
+      },
+    });
+    try {
+      const arrayBuffer = await res.data.arrayBuffer();
+      const result = await mammoth.convertToHtml(
+        { arrayBuffer },
+        { includeDefaultStyleMap: true },
+      );
+
+      const styledContent = result.value
+        .replace(/<p>/g, '<p class="mb-2">')
+        .replace(/<h(\d)>/g, '<h$1 class="font-semibold mt-4 mb-2">');
+
+      setHtmlContent(styledContent);
+    } catch (err) {
+      message.error('Document parsing failed');
+      console.error('Error parsing document:', err);
+    }
+    setLoading(false);
+  };
+
+  useEffect(() => {
+    if (url) {
+      fetchDocument();
+    }
+  }, [url]);
+  return (
+    <div
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md',
+        className,
+      )}
+    >
+      {loading && (
+        <div className="absolute inset-0 flex items-center justify-center">
+          <Spin />
+        </div>
+      )}
+
+      {!loading && <div dangerouslySetInnerHTML={{ __html: htmlContent }} />}
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/document-preview/document-header.tsx

@@ -0,0 +1,21 @@
+import { formatDate } from '@/utils/date';
+import { formatBytes } from '@/utils/file-util';
+
+type Props = {
+  size: number;
+  name: string;
+  create_date: string;
+};
+
+export default ({ size, name, create_date }: Props) => {
+  const sizeName = formatBytes(size);
+  const dateStr = formatDate(create_date);
+  return (
+    <div>
+      <h2 className="text-[16px]">{name}</h2>
+      <div className="text-text-secondary text-[12px] pt-[5px]">
+        Size：{sizeName} Uploaded Time：{dateStr}
+      </div>
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/document-preview/excel-preview.tsx

@@ -0,0 +1,25 @@
+import { useFetchExcel } from '@/pages/document-viewer/hooks';
+import classNames from 'classnames';
+
+interface ExcelCsvPreviewerProps {
+  className?: string;
+  url: string;
+}
+
+export const ExcelCsvPreviewer: React.FC<ExcelCsvPreviewerProps> = ({
+  className,
+  url,
+}) => {
+  // const url = useGetDocumentUrl();
+  const { containerRef } = useFetchExcel(url);
+
+  return (
+    <div
+      ref={containerRef}
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md excel-csv-previewer',
+        className,
+      )}
+    ></div>
+  );
+};

web/src/pages/dataflow-result/components/document-preview/hooks.ts

@@ -0,0 +1,55 @@
+import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
+import { api_host } from '@/utils/api';
+import { useSize } from 'ahooks';
+import { CustomTextRenderer } from 'node_modules/react-pdf/dist/esm/shared/types';
+import { useCallback, useEffect, useMemo, useState } from 'react';
+
+export const useDocumentResizeObserver = () => {
+  const [containerWidth, setContainerWidth] = useState<number>();
+  const [containerRef, setContainerRef] = useState<HTMLElement | null>(null);
+  const size = useSize(containerRef);
+
+  const onResize = useCallback((width?: number) => {
+    if (width) {
+      setContainerWidth(width);
+    }
+  }, []);
+
+  useEffect(() => {
+    onResize(size?.width);
+  }, [size?.width, onResize]);
+
+  return { containerWidth, setContainerRef };
+};
+
+function highlightPattern(text: string, pattern: string, pageNumber: number) {
+  if (pageNumber === 2) {
+    return `<mark>${text}</mark>`;
+  }
+  if (text.trim() !== '' && pattern.match(text)) {
+    // return pattern.replace(text, (value) => `<mark>${value}</mark>`);
+    return `<mark>${text}</mark>`;
+  }
+  return text.replace(pattern, (value) => `<mark>${value}</mark>`);
+}
+
+export const useHighlightText = (searchText: string = '') => {
+  const textRenderer: CustomTextRenderer = useCallback(
+    (textItem) => {
+      return highlightPattern(textItem.str, searchText, textItem.pageNumber);
+    },
+    [searchText],
+  );
+
+  return textRenderer;
+};
+
+export const useGetDocumentUrl = () => {
+  const { documentId } = useGetKnowledgeSearchParams();
+
+  const url = useMemo(() => {
+    return `${api_host}/document/get/${documentId}`;
+  }, [documentId]);
+
+  return url;
+};

web/src/pages/dataflow-result/components/document-preview/image-preview.tsx

@@ -0,0 +1,73 @@
+import message from '@/components/ui/message';
+import { Spin } from '@/components/ui/spin';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import { useEffect, useState } from 'react';
+
+interface ImagePreviewerProps {
+  className?: string;
+  url: string;
+}
+
+export const ImagePreviewer: React.FC<ImagePreviewerProps> = ({
+  className,
+  url,
+}) => {
+  // const url = useGetDocumentUrl();
+  const [imageSrc, setImageSrc] = useState<string | null>(null);
+  const [isLoading, setIsLoading] = useState<boolean>(true);
+
+  const fetchImage = async () => {
+    setIsLoading(true);
+    const res = await request(url, {
+      method: 'GET',
+      responseType: 'blob',
+      onError: () => {
+        message.error('Failed to load image');
+        setIsLoading(false);
+      },
+    });
+    const objectUrl = URL.createObjectURL(res.data);
+    setImageSrc(objectUrl);
+    setIsLoading(false);
+  };
+  useEffect(() => {
+    if (url) {
+      fetchImage();
+    }
+  }, [url]);
+
+  useEffect(() => {
+    return () => {
+      if (imageSrc) {
+        URL.revokeObjectURL(imageSrc);
+      }
+    };
+  }, [imageSrc]);
+
+  return (
+    <div
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md image-previewer',
+        className,
+      )}
+    >
+      {isLoading && (
+        <div className="absolute inset-0 flex items-center justify-center">
+          <Spin />
+        </div>
+      )}
+
+      {!isLoading && imageSrc && (
+        <div className="max-h-[80vh] overflow-auto p-2">
+          <img
+            src={imageSrc}
+            alt={'image'}
+            className="w-full h-auto max-w-full object-contain"
+            onLoad={() => URL.revokeObjectURL(imageSrc!)}
+          />
+        </div>
+      )}
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/document-preview/index.less

@@ -0,0 +1,13 @@
+.documentContainer {
+  width: 100%;
+  // height: calc(100vh - 284px);
+  height: calc(100vh - 170px);
+  position: relative;
+  :global(.PdfHighlighter) {
+    overflow-x: hidden;
+  }
+  :global(.Highlight--scrolledTo .Highlight__part) {
+    overflow-x: hidden;
+    background-color: rgba(255, 226, 143, 1);
+  }
+}

web/src/pages/dataflow-result/components/document-preview/index.tsx

@@ -0,0 +1,68 @@
+import { memo } from 'react';
+
+import CSVFileViewer from './csv-preview';
+import { DocPreviewer } from './doc-preview';
+import { ExcelCsvPreviewer } from './excel-preview';
+import { ImagePreviewer } from './image-preview';
+import styles from './index.less';
+import PdfPreviewer, { IProps } from './pdf-preview';
+import { PptPreviewer } from './ppt-preview';
+import { TxtPreviewer } from './txt-preview';
+
+type PreviewProps = {
+  fileType: string;
+  className?: string;
+  url: string;
+};
+const Preview = ({
+  fileType,
+  className,
+  highlights,
+  setWidthAndHeight,
+  url,
+}: PreviewProps & Partial<IProps>) => {
+  return (
+    <>
+      {fileType === 'pdf' && highlights && setWidthAndHeight && (
+        <section className={styles.documentPreview}>
+          <PdfPreviewer
+            highlights={highlights}
+            setWidthAndHeight={setWidthAndHeight}
+            url={url}
+          ></PdfPreviewer>
+        </section>
+      )}
+      {['doc', 'docx'].indexOf(fileType) > -1 && (
+        <section>
+          <DocPreviewer className={className} url={url} />
+        </section>
+      )}
+      {['txt', 'md'].indexOf(fileType) > -1 && (
+        <section>
+          <TxtPreviewer className={className} url={url} />
+        </section>
+      )}
+      {['visual'].indexOf(fileType) > -1 && (
+        <section>
+          <ImagePreviewer className={className} url={url} />
+        </section>
+      )}
+      {['pptx'].indexOf(fileType) > -1 && (
+        <section>
+          <PptPreviewer className={className} url={url} />
+        </section>
+      )}
+      {['xlsx'].indexOf(fileType) > -1 && (
+        <section>
+          <ExcelCsvPreviewer className={className} url={url} />
+        </section>
+      )}
+      {['csv'].indexOf(fileType) > -1 && (
+        <section>
+          <CSVFileViewer className={className} url={url} />
+        </section>
+      )}
+    </>
+  );
+};
+export default memo(Preview);

web/src/pages/dataflow-result/components/document-preview/pdf-preview.tsx

@@ -0,0 +1,127 @@
+import { memo, useEffect, useRef } from 'react';
+import {
+  AreaHighlight,
+  Highlight,
+  IHighlight,
+  PdfHighlighter,
+  PdfLoader,
+  Popup,
+} from 'react-pdf-highlighter';
+
+import { useCatchDocumentError } from '@/components/pdf-previewer/hooks';
+import { Spin } from '@/components/ui/spin';
+import FileError from '@/pages/document-viewer/file-error';
+import styles from './index.less';
+
+export interface IProps {
+  highlights: IHighlight[];
+  setWidthAndHeight: (width: number, height: number) => void;
+  url: string;
+}
+const HighlightPopup = ({
+  comment,
+}: {
+  comment: { text: string; emoji: string };
+}) =>
+  comment.text ? (
+    <div className="Highlight__popup">
+      {comment.emoji} {comment.text}
+    </div>
+  ) : null;
+
+// TODO: merge with DocumentPreviewer
+const PdfPreview = ({ highlights: state, setWidthAndHeight, url }: IProps) => {
+  // const url = useGetDocumentUrl();
+
+  const ref = useRef<(highlight: IHighlight) => void>(() => {});
+  const error = useCatchDocumentError(url);
+
+  const resetHash = () => {};
+
+  useEffect(() => {
+    if (state.length > 0) {
+      ref?.current(state[0]);
+    }
+  }, [state]);
+
+  return (
+    <div
+      className={`${styles.documentContainer} rounded-[10px] overflow-hidden	`}
+    >
+      <PdfLoader
+        url={url}
+        beforeLoad={
+          <div className="absolute inset-0 flex items-center justify-center">
+            <Spin />
+          </div>
+        }
+        workerSrc="/pdfjs-dist/pdf.worker.min.js"
+        errorMessage={<FileError>{error}</FileError>}
+      >
+        {(pdfDocument) => {
+          pdfDocument.getPage(1).then((page) => {
+            const viewport = page.getViewport({ scale: 1 });
+            const width = viewport.width;
+            const height = viewport.height;
+            setWidthAndHeight(width, height);
+          });
+
+          return (
+            <PdfHighlighter
+              pdfDocument={pdfDocument}
+              enableAreaSelection={(event) => event.altKey}
+              onScrollChange={resetHash}
+              scrollRef={(scrollTo) => {
+                ref.current = scrollTo;
+              }}
+              onSelectionFinished={() => null}
+              highlightTransform={(
+                highlight,
+                index,
+                setTip,
+                hideTip,
+                viewportToScaled,
+                screenshot,
+                isScrolledTo,
+              ) => {
+                const isTextHighlight = !Boolean(
+                  highlight.content && highlight.content.image,
+                );
+
+                const component = isTextHighlight ? (
+                  <Highlight
+                    isScrolledTo={isScrolledTo}
+                    position={highlight.position}
+                    comment={highlight.comment}
+                  />
+                ) : (
+                  <AreaHighlight
+                    isScrolledTo={isScrolledTo}
+                    highlight={highlight}
+                    onChange={() => {}}
+                  />
+                );
+
+                return (
+                  <Popup
+                    popupContent={<HighlightPopup {...highlight} />}
+                    onMouseOver={(popupContent) =>
+                      setTip(highlight, () => popupContent)
+                    }
+                    onMouseOut={hideTip}
+                    key={index}
+                  >
+                    {component}
+                  </Popup>
+                );
+              }}
+              highlights={state}
+            />
+          );
+        }}
+      </PdfLoader>
+    </div>
+  );
+};
+
+export default memo(PdfPreview);

web/src/pages/dataflow-result/components/document-preview/ppt-preview.tsx

@@ -0,0 +1,70 @@
+import message from '@/components/ui/message';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import { init } from 'pptx-preview';
+import { useEffect, useRef } from 'react';
+interface PptPreviewerProps {
+  className?: string;
+  url: string;
+}
+
+export const PptPreviewer: React.FC<PptPreviewerProps> = ({
+  className,
+  url,
+}) => {
+  // const url = useGetDocumentUrl();
+  const wrapper = useRef<HTMLDivElement>(null);
+  const containerRef = useRef<HTMLDivElement>(null);
+  const fetchDocument = async () => {
+    const res = await request(url, {
+      method: 'GET',
+      responseType: 'blob',
+      onError: () => {
+        message.error('Document parsing failed');
+        console.error('Error loading document:', url);
+      },
+    });
+    console.log(res);
+    try {
+      const arrayBuffer = await res.data.arrayBuffer();
+
+      if (containerRef.current) {
+        let width = 500;
+        let height = 900;
+        if (containerRef.current) {
+          width = containerRef.current.clientWidth - 50;
+          height = containerRef.current.clientHeight - 50;
+        }
+        let pptxPrviewer = init(containerRef.current, {
+          width: width,
+          height: height,
+        });
+        pptxPrviewer.preview(arrayBuffer);
+      }
+    } catch (err) {
+      message.error('ppt parse failed');
+    }
+  };
+
+  useEffect(() => {
+    if (url) {
+      fetchDocument();
+    }
+  }, [url]);
+
+  return (
+    <div
+      ref={containerRef}
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md ppt-previewer',
+        className,
+      )}
+    >
+      <div className="overflow-auto p-2">
+        <div className="flex flex-col gap-4">
+          <div ref={wrapper} />
+        </div>
+      </div>
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/document-preview/txt-preview.tsx

@@ -0,0 +1,56 @@
+import message from '@/components/ui/message';
+import { Spin } from '@/components/ui/spin';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import { useEffect, useState } from 'react';
+
+type TxtPreviewerProps = { className?: string; url: string };
+export const TxtPreviewer = ({ className, url }: TxtPreviewerProps) => {
+  // const url = useGetDocumentUrl();
+  const [loading, setLoading] = useState(false);
+  const [data, setData] = useState<string>('');
+  const fetchTxt = async () => {
+    setLoading(true);
+    const res = await request(url, {
+      method: 'GET',
+      responseType: 'blob',
+      onError: (err: any) => {
+        message.error('Failed to load file');
+        console.error('Error loading file:', err);
+      },
+    });
+    // blob to string
+    const reader = new FileReader();
+    reader.readAsText(res.data);
+    reader.onload = () => {
+      setData(reader.result as string);
+      setLoading(false);
+      console.log('file loaded successfully', reader.result);
+    };
+    console.log('file data:', res);
+  };
+  useEffect(() => {
+    if (url) {
+      fetchTxt();
+    } else {
+      setLoading(false);
+      setData('');
+    }
+  }, [url]);
+  return (
+    <div
+      className={classNames(
+        'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md',
+        className,
+      )}
+    >
+      {loading && (
+        <div className="absolute inset-0 flex items-center justify-center">
+          <Spin />
+        </div>
+      )}
+
+      {!loading && <pre className="whitespace-pre-wrap p-2 ">{data}</pre>}
+    </div>
+  );
+};

web/src/pages/dataflow-result/components/parse-editer/index.tsx

@@ -0,0 +1,48 @@
+import { Textarea } from '@/components/ui/textarea';
+import { cn } from '@/lib/utils';
+import { useState } from 'react';
+
+interface FormatPreserveEditorProps {
+  initialValue: string;
+  onSave: (value: string) => void;
+  className?: string;
+}
+const FormatPreserveEditor = ({
+  initialValue,
+  onSave,
+  className,
+}: FormatPreserveEditorProps) => {
+  const [content, setContent] = useState(initialValue);
+  const [isEditing, setIsEditing] = useState(false);
+
+  const handleEdit = () => setIsEditing(true);
+
+  const handleSave = () => {
+    onSave(content);
+    setIsEditing(false);
+  };
+
+  return (
+    <div className="editor-container">
+      {isEditing ? (
+        <Textarea
+          className={cn(
+            'w-full h-full bg-transparent text-text-secondary',
+            className,
+          )}
+          value={content}
+          onChange={(e) => setContent(e.target.value)}
+          onBlur={handleSave}
+          autoSize={{ maxRows: 100 }}
+          autoFocus
+        />
+      ) : (
+        <pre className="text-text-secondary" onClick={handleEdit}>
+          {content}
+        </pre>
+      )}
+    </div>
+  );
+};
+
+export default FormatPreserveEditor;

web/src/pages/dataflow-result/components/rerun-button/index.tsx

@@ -0,0 +1,29 @@
+import SvgIcon from '@/components/svg-icon';
+import { Button } from '@/components/ui/button';
+import { CircleAlert } from 'lucide-react';
+import { useTranslation } from 'react-i18next';
+import { useRerunDataflow } from '../../hooks';
+interface RerunButtonProps {
+  className?: string;
+}
+const RerunButton = (props: RerunButtonProps) => {
+  const { t } = useTranslation();
+  const { loading } = useRerunDataflow();
+  const clickFunc = () => {
+    console.log('click rerun button');
+  };
+  return (
+    <div className="flex flex-col gap-2">
+      <div className="text-xs text-text-primary flex items-center gap-1">
+        <CircleAlert color="#d29e2d" strokeWidth={1} size={12} />
+        {t('dataflowParser.rerunFromCurrentStepTip')}
+      </div>
+      <Button onClick={clickFunc} disabled={loading}>
+        <SvgIcon name="rerun" width={16} />
+        {t('dataflowParser.rerunFromCurrentStep')}
+      </Button>
+    </div>
+  );
+};
+
+export default RerunButton;

web/src/pages/dataflow-result/components/time-line/index.tsx

@@ -0,0 +1,82 @@
+import { CustomTimeline, TimelineNode } from '@/components/originui/timeline';
+import {
+  CheckLine,
+  FilePlayIcon,
+  Grid3x2,
+  ListPlus,
+  PlayIcon,
+} from 'lucide-react';
+import { useMemo } from 'react';
+export const TimelineNodeObj = {
+  begin: {
+    id: 1,
+    title: 'Begin',
+    icon: <PlayIcon size={13} />,
+    clickable: false,
+  },
+  parser: { id: 2, title: 'Parser', icon: <FilePlayIcon size={13} /> },
+  chunker: { id: 3, title: 'Chunker', icon: <Grid3x2 size={13} /> },
+  indexer: {
+    id: 4,
+    title: 'Indexer',
+    icon: <ListPlus size={13} />,
+    clickable: false,
+  },
+  complete: {
+    id: 5,
+    title: 'Complete',
+    icon: <CheckLine size={13} />,
+    clickable: false,
+  },
+};
+
+export interface TimelineDataFlowProps {
+  activeId: number | string;
+  activeFunc: (id: number | string) => void;
+}
+const TimelineDataFlow = ({ activeFunc, activeId }: TimelineDataFlowProps) => {
+  // const [activeStep, setActiveStep] = useState(2);
+  const timelineNodes: TimelineNode[] = useMemo(() => {
+    const nodes: TimelineNode[] = [];
+    Object.keys(TimelineNodeObj).forEach((key) => {
+      nodes.push({
+        ...TimelineNodeObj[key as keyof typeof TimelineNodeObj],
+        className: 'w-32',
+        completed: false,
+      });
+    });
+    return nodes;
+  }, []);
+
+  const activeStep = useMemo(() => {
+    const index = timelineNodes.findIndex((node) => node.id === activeId);
+    return index > -1 ? index + 1 : 0;
+  }, [activeId, timelineNodes]);
+  const handleStepChange = (step: number, id: string | number) => {
+    // setActiveStep(step);
+    activeFunc?.(id);
+    console.log(step, id);
+  };
+
+  return (
+    <div className="">
+      <div>
+        <CustomTimeline
+          nodes={timelineNodes as TimelineNode[]}
+          activeStep={activeStep}
+          onStepChange={handleStepChange}
+          orientation="horizontal"
+          lineStyle="solid"
+          nodeSize={24}
+          activeStyle={{
+            nodeSize: 30,
+            iconColor: 'var(--accent-primary)',
+            textColor: 'var(--accent-primary)',
+          }}
+        />
+      </div>
+    </div>
+  );
+};
+
+export default TimelineDataFlow;