diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py index 868fc5f41..906de8135 100644 --- a/deepdoc/parser/excel_parser.py +++ b/deepdoc/parser/excel_parser.py @@ -41,7 +41,7 @@ class RAGFlowExcelParser: try: file_like_object.seek(0) - df = pd.read_csv(file_like_object) + df = pd.read_csv(file_like_object, on_bad_lines='skip') return RAGFlowExcelParser._dataframe_to_workbook(df) except Exception as e_csv: @@ -164,7 +164,7 @@ class RAGFlowExcelParser: except Exception as e: logging.warning(f"Parse spreadsheet error: {e}, trying to interpret as CSV file") file_like_object.seek(0) - df = pd.read_csv(file_like_object) + df = pd.read_csv(file_like_object, on_bad_lines='skip') df = df.replace(r"^\s*$", "", regex=True) return df.to_markdown(index=False) diff --git a/rag/app/laws.py b/rag/app/laws.py index ba2592833..e09bb4d67 100644 --- a/rag/app/laws.py +++ b/rag/app/laws.py @@ -201,12 +201,23 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, elif re.search(r"\.doc$", filename, re.IGNORECASE): callback(0.1, "Start to parse.") - binary = BytesIO(binary) - doc_parsed = parser.from_buffer(binary) - sections = doc_parsed['content'].split('\n') - sections = [s for s in sections if s] - callback(0.8, "Finish parsing.") + try: + from tika import parser as tika_parser + except Exception as e: + callback(0.8, f"tika not available: {e}. Unsupported .doc parsing.") + logging.warning(f"tika not available: {e}. Unsupported .doc parsing for {filename}.") + return [] + binary = BytesIO(binary) + doc_parsed = tika_parser.from_buffer(binary) + if doc_parsed.get('content', None) is not None: + sections = doc_parsed['content'].split('\n') + sections = [s for s in sections if s] + callback(0.8, "Finish parsing.") + else: + callback(0.8, f"tika.parser got empty content from {filename}.") + logging.warning(f"tika.parser got empty content from {filename}.") + return [] else: raise NotImplementedError( "file type not supported yet(doc, docx, pdf, txt supported)")