Refactor log utils (#10973)

### What problem does this PR solve?

As title.

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

admin/server/admin_server.py

@@ -23,7 +23,7 @@ import traceback
 from werkzeug.serving import run_simple
 from flask import Flask
 from routes import admin_bp
-from api.utils.log_utils import init_root_logger
+from common.log_utils import init_root_logger
 from common.contants import SERVICE_CONF
 from common.config_utils import show_configs
 from api import settings

api/ragflow_server.py

@@ -18,7 +18,7 @@
 # from beartype.claw import beartype_all  # <-- you didn't sign up for this
 # beartype_all(conf=BeartypeConf(violation_type=UserWarning))    # <-- emit warnings from all code
 
-from api.utils.log_utils import init_root_logger
+from common.log_utils import init_root_logger
 from plugin import GlobalPluginManager
 init_root_logger("ragflow_server")
 

api/utils/api_utils.py

@@ -22,14 +22,10 @@ import queue
 import random
 import threading
 import time
-from base64 import b64encode
 from copy import deepcopy
 from functools import wraps
-from hmac import HMAC
 from io import BytesIO
 from typing import Any, Callable, Coroutine, Optional, Type, Union
-from urllib.parse import quote, urlencode
-from uuid import uuid1
 
 import requests
 import trio
@@ -82,46 +78,6 @@ def serialize_for_json(obj):
         return str(obj)
 
 
-def request(**kwargs):
-    sess = requests.Session()
-    stream = kwargs.pop("stream", sess.stream)
-    timeout = kwargs.pop("timeout", None)
-    kwargs["headers"] = {k.replace("_", "-").upper(): v for k, v in kwargs.get("headers", {}).items()}
-    prepped = requests.Request(**kwargs).prepare()
-
-    if settings.CLIENT_AUTHENTICATION and settings.HTTP_APP_KEY and settings.SECRET_KEY:
-        timestamp = str(round(time() * 1000))
-        nonce = str(uuid1())
-        signature = b64encode(
-            HMAC(
-                settings.SECRET_KEY.encode("ascii"),
-                b"\n".join(
-                    [
-                        timestamp.encode("ascii"),
-                        nonce.encode("ascii"),
-                        settings.HTTP_APP_KEY.encode("ascii"),
-                        prepped.path_url.encode("ascii"),
-                        prepped.body if kwargs.get("json") else b"",
-                        urlencode(sorted(kwargs["data"].items()), quote_via=quote, safe="-._~").encode(
-                            "ascii") if kwargs.get("data") and isinstance(kwargs["data"], dict) else b"",
-                    ]
-                ),
-                "sha1",
-            ).digest()
-        ).decode("ascii")
-
-        prepped.headers.update(
-            {
-                "TIMESTAMP": timestamp,
-                "NONCE": nonce,
-                "APP-KEY": settings.HTTP_APP_KEY,
-                "SIGNATURE": signature,
-            }
-        )
-
-    return sess.send(prepped, stream=stream, timeout=timeout)
-
-
 def get_exponential_backoff_interval(retries, full_jitter=False):
     """Calculate the exponential backoff wait time."""
     # Will be zero if factor equals 0

api/utils/health.py

@@ -1,104 +0,0 @@
-from timeit import default_timer as timer
-
-from api import settings
-from api.db.db_models import DB
-from rag.utils.redis_conn import REDIS_CONN
-from rag.utils.storage_factory import STORAGE_IMPL
-
-
-def _ok_nok(ok: bool) -> str:
-    return "ok" if ok else "nok"
-
-
-def check_db() -> tuple[bool, dict]:
-    st = timer()
-    try:
-        # lightweight probe; works for MySQL/Postgres
-        DB.execute_sql("SELECT 1")
-        return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"}
-    except Exception as e:
-        return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
-
-
-def check_redis() -> tuple[bool, dict]:
-    st = timer()
-    try:
-        ok = bool(REDIS_CONN.health())
-        return ok, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"}
-    except Exception as e:
-        return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
-
-
-def check_doc_engine() -> tuple[bool, dict]:
-    st = timer()
-    try:
-        meta = settings.docStoreConn.health()
-        # treat any successful call as ok
-        return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", **(meta or {})}
-    except Exception as e:
-        return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
-
-
-def check_storage() -> tuple[bool, dict]:
-    st = timer()
-    try:
-        STORAGE_IMPL.health()
-        return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"}
-    except Exception as e:
-        return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
-
-
-def check_chat() -> tuple[bool, dict]:
-    st = timer()
-    try:
-        cfg = getattr(settings, "CHAT_CFG", None)
-        ok = bool(cfg and cfg.get("factory"))
-        return ok, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"}
-    except Exception as e:
-        return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)}
-
-
-def run_health_checks() -> tuple[dict, bool]:
-    result: dict[str, str | dict] = {}
-
-    db_ok, db_meta = check_db()
-    chat_ok, chat_meta = check_chat()
-
-    result["db"] = _ok_nok(db_ok)
-    if not db_ok:
-        result.setdefault("_meta", {})["db"] = db_meta
-
-    result["chat"] = _ok_nok(chat_ok)
-    if not chat_ok:
-        result.setdefault("_meta", {})["chat"] = chat_meta
-
-    # Optional probes (do not change minimal contract but exposed for observability)
-    try:
-        redis_ok, redis_meta = check_redis()
-        result["redis"] = _ok_nok(redis_ok)
-        if not redis_ok:
-            result.setdefault("_meta", {})["redis"] = redis_meta
-    except Exception:
-        result["redis"] = "nok"
-
-    try:
-        doc_ok, doc_meta = check_doc_engine()
-        result["doc_engine"] = _ok_nok(doc_ok)
-        if not doc_ok:
-            result.setdefault("_meta", {})["doc_engine"] = doc_meta
-    except Exception:
-        result["doc_engine"] = "nok"
-
-    try:
-        sto_ok, sto_meta = check_storage()
-        result["storage"] = _ok_nok(sto_ok)
-        if not sto_ok:
-            result.setdefault("_meta", {})["storage"] = sto_meta
-    except Exception:
-        result["storage"] = "nok"
-
-    all_ok = (result.get("db") == "ok") and (result.get("chat") == "ok")
-    result["status"] = "ok" if all_ok else "nok"
-    return result, all_ok
-
-

api/utils/log_utils.py

@@ -13,70 +13,3 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import os
-import os.path
-import logging
-from logging.handlers import RotatingFileHandler
-from common.file_utils import get_project_base_directory
-
-initialized_root_logger = False
-
-def init_root_logger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"):
-    global initialized_root_logger
-    if initialized_root_logger:
-        return
-    initialized_root_logger = True
-
-    logger = logging.getLogger()
-    logger.handlers.clear()
-    log_path = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"{logfile_basename}.log"))
-
-    os.makedirs(os.path.dirname(log_path), exist_ok=True)
-    formatter = logging.Formatter(log_format)
-
-    handler1 = RotatingFileHandler(log_path, maxBytes=10*1024*1024, backupCount=5)
-    handler1.setFormatter(formatter)
-    logger.addHandler(handler1)
-
-    handler2 = logging.StreamHandler()
-    handler2.setFormatter(formatter)
-    logger.addHandler(handler2)
-
-    logging.captureWarnings(True)
-
-    LOG_LEVELS = os.environ.get("LOG_LEVELS", "")
-    pkg_levels = {}
-    for pkg_name_level in LOG_LEVELS.split(","):
-        terms = pkg_name_level.split("=")
-        if len(terms)!= 2:
-            continue
-        pkg_name, pkg_level = terms[0], terms[1]
-        pkg_name = pkg_name.strip()
-        pkg_level = logging.getLevelName(pkg_level.strip().upper())
-        if not isinstance(pkg_level, int):
-            pkg_level = logging.INFO
-        pkg_levels[pkg_name] = logging.getLevelName(pkg_level)
-
-    for pkg_name in ['peewee', 'pdfminer']:
-        if pkg_name not in pkg_levels:
-            pkg_levels[pkg_name] = logging.getLevelName(logging.WARNING)
-    if 'root' not in pkg_levels:
-        pkg_levels['root'] = logging.getLevelName(logging.INFO)
-
-    for pkg_name, pkg_level in pkg_levels.items():
-        pkg_logger = logging.getLogger(pkg_name)
-        pkg_logger.setLevel(pkg_level)
-
-    msg = f"{logfile_basename} log path: {log_path}, log levels: {pkg_levels}"
-    logger.info(msg)
-
-
-def log_exception(e, *args):
-    logging.exception(e)
-    for a in args:
-        if hasattr(a, "text"):
-            logging.error(a.text)
-            raise Exception(a.text)
-        else:
-            logging.error(str(a))
-    raise e

common/log_utils.py

@@ -0,0 +1,83 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+import os.path
+import logging
+from logging.handlers import RotatingFileHandler
+from common.file_utils import get_project_base_directory
+
+initialized_root_logger = False
+
+def init_root_logger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"):
+    global initialized_root_logger
+    if initialized_root_logger:
+        return
+    initialized_root_logger = True
+
+    logger = logging.getLogger()
+    logger.handlers.clear()
+    log_path = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"{logfile_basename}.log"))
+
+    os.makedirs(os.path.dirname(log_path), exist_ok=True)
+    formatter = logging.Formatter(log_format)
+
+    handler1 = RotatingFileHandler(log_path, maxBytes=10*1024*1024, backupCount=5)
+    handler1.setFormatter(formatter)
+    logger.addHandler(handler1)
+
+    handler2 = logging.StreamHandler()
+    handler2.setFormatter(formatter)
+    logger.addHandler(handler2)
+
+    logging.captureWarnings(True)
+
+    LOG_LEVELS = os.environ.get("LOG_LEVELS", "")
+    pkg_levels = {}
+    for pkg_name_level in LOG_LEVELS.split(","):
+        terms = pkg_name_level.split("=")
+        if len(terms)!= 2:
+            continue
+        pkg_name, pkg_level = terms[0], terms[1]
+        pkg_name = pkg_name.strip()
+        pkg_level = logging.getLevelName(pkg_level.strip().upper())
+        if not isinstance(pkg_level, int):
+            pkg_level = logging.INFO
+        pkg_levels[pkg_name] = logging.getLevelName(pkg_level)
+
+    for pkg_name in ['peewee', 'pdfminer']:
+        if pkg_name not in pkg_levels:
+            pkg_levels[pkg_name] = logging.getLevelName(logging.WARNING)
+    if 'root' not in pkg_levels:
+        pkg_levels['root'] = logging.getLevelName(logging.INFO)
+
+    for pkg_name, pkg_level in pkg_levels.items():
+        pkg_logger = logging.getLogger(pkg_name)
+        pkg_logger.setLevel(pkg_level)
+
+    msg = f"{logfile_basename} log path: {log_path}, log levels: {pkg_levels}"
+    logger.info(msg)
+
+
+def log_exception(e, *args):
+    logging.exception(e)
+    for a in args:
+        if hasattr(a, "text"):
+            logging.error(a.text)
+            raise Exception(a.text)
+        else:
+            logging.error(str(a))
+    raise e

rag/llm/embedding_model.py

@@ -27,7 +27,7 @@ from ollama import Client
 from openai import OpenAI
 from zhipuai import ZhipuAI
 
-from api.utils.log_utils import log_exception
+from common.log_utils import log_exception
 from common.token_utils import num_tokens_from_string, truncate
 from api import settings
 import logging

rag/llm/rerank_model.py

@@ -22,7 +22,7 @@ import numpy as np
 import requests
 from yarl import URL
 
-from api.utils.log_utils import log_exception
+from common.log_utils import log_exception
 from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response
 
 class Base(ABC):

rag/svr/task_executor.py

@@ -28,7 +28,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.pipeline_operation_log_service import PipelineOperationLogService
 from api.utils.api_utils import timeout
 from common.base64_image import image2id
-from api.utils.log_utils import init_root_logger
+from common.log_utils import init_root_logger
 from common.file_utils import get_project_base_directory
 from common.config_utils import show_configs
 from graphrag.general.index import run_graphrag_for_kb