Add tree_merge for law parsers, significantly outperforming hierarchical_merge (#10202)

### What problem does this PR solve?
Add tree_merge for law parsers, significantly outperforming
hierarchical_merge, solved: #8637
1. Add tree_merge for law parsers, include build_tree and get_tree by
dfs.
2. add Copyright statement for helath_utils
### Type of change

- [x] Documentation Update
- [x] Performance Improvement

rag/app/laws.py

@@ -22,12 +22,15 @@ from docx import Document
 
 from api.db import ParserType
 from deepdoc.parser.utils import get_text
-from rag.nlp import bullets_category, remove_contents_table, hierarchical_merge, \
-    make_colon_as_title, tokenize_chunks, docx_question_level
-from rag.nlp import rag_tokenizer
+from rag.nlp import bullets_category, remove_contents_table, \
+    make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge
+from rag.nlp import rag_tokenizer, Node
 from deepdoc.parser import PdfParser, DocxParser, PlainParser, HtmlParser
 
 
+
+
+
 class Docx(DocxParser):
     def __init__(self):
         pass
@@ -55,49 +58,37 @@ class Docx(DocxParser):
         return [line for line in lines if line]
 
     def __call__(self, filename, binary=None, from_page=0, to_page=100000):
-        self.doc = Document(
-            filename) if not binary else Document(BytesIO(binary))
-        pn = 0
-        lines = []
-        bull = bullets_category([p.text for p in self.doc.paragraphs])
-        for p in self.doc.paragraphs:
-            if pn > to_page:
-                break
-            question_level, p_text = docx_question_level(p, bull)
-            if not p_text.strip("\n"):
-                continue
-            lines.append((question_level, p_text))
-
-            for run in p.runs:
-                if 'lastRenderedPageBreak' in run._element.xml:
-                    pn += 1
-                    continue
-                if 'w:br' in run._element.xml and 'type="page"' in run._element.xml:
-                    pn += 1
-
-        visit = [False for _ in range(len(lines))]
-        sections = []
-        for s in range(len(lines)):
-            e = s + 1
-            while e < len(lines):
-                if lines[e][0] <= lines[s][0]:
+            self.doc = Document(
+                filename) if not binary else Document(BytesIO(binary))
+            pn = 0
+            lines = []
+            level_set = set()
+            bull = bullets_category([p.text for p in self.doc.paragraphs])
+            for p in self.doc.paragraphs:
+                if pn > to_page:
                     break
-                e += 1
-            if e - s == 1 and visit[s]:
-                continue
-            sec = []
-            next_level = lines[s][0] + 1
-            while not sec and next_level < 22:
-                for i in range(s+1, e):
-                    if lines[i][0] != next_level:
+                question_level, p_text = docx_question_level(p, bull)
+                if not p_text.strip("\n"):
+                    continue
+                lines.append((question_level, p_text))
+                level_set.add(question_level)
+                for run in p.runs:
+                    if 'lastRenderedPageBreak' in run._element.xml:
+                        pn += 1
                         continue
-                    sec.append(lines[i][1])
-                    visit[i] = True
-                next_level += 1
-            sec.insert(0, lines[s][1])
+                    if 'w:br' in run._element.xml and 'type="page"' in run._element.xml:
+                        pn += 1
+
+            sorted_levels = sorted(level_set)
+
+            h2_level = sorted_levels[1] if len(sorted_levels) > 1 else 1
+            h2_level = sorted_levels[-2] if h2_level == sorted_levels[-1] and len(sorted_levels) > 2 else h2_level
+
+            root = Node(level=0, depth=h2_level, texts=[])
+            root.build_tree(lines)
+
+            return [("\n").join(element) for element in root.get_tree() if element]
 
-            sections.append("\n".join(sec))
-        return [s for s in sections if s]
 
     def __str__(self) -> str:
         return f'''
@@ -163,7 +154,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
         chunks = Docx()(filename, binary)
         callback(0.7, "Finish parsing.")
         return tokenize_chunks(chunks, doc, eng, None)
-
+    
     elif re.search(r"\.pdf$", filename, re.IGNORECASE):
         pdf_parser = Pdf()
         if parser_config.get("layout_recognize", "DeepDOC") == "Plain Text":
@@ -172,7 +163,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
                                     from_page=from_page, to_page=to_page, callback=callback)[0]:
             sections.append(txt + poss)
 
-    elif re.search(r"\.txt$", filename, re.IGNORECASE):
+    elif re.search(r"\.(txt|md|markdown|mdx)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
         txt = get_text(filename, binary)
         sections = txt.split("\n")
@@ -203,13 +194,16 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
 
     make_colon_as_title(sections)
     bull = bullets_category(sections)
-    chunks = hierarchical_merge(bull, sections, 5)
-    if not chunks:
+    res = tree_merge(bull, sections, 2)
+
+
+    if not res:
         callback(0.99, "No chunk parsed out.")
 
-    return tokenize_chunks(["\n".join(ck)
-                           for ck in chunks], doc, eng, pdf_parser)
+    return tokenize_chunks(res, doc, eng, pdf_parser)
 
+    # chunks = hierarchical_merge(bull, sections, 5)
+    #     return tokenize_chunks(["\n".join(ck)for ck in chunks], doc, eng, pdf_parser)
 
 if __name__ == "__main__":
     import sys