diff --git a/admin/server/admin_server.py b/admin/server/admin_server.py index 3524e6faa..3dc552399 100644 --- a/admin/server/admin_server.py +++ b/admin/server/admin_server.py @@ -23,7 +23,7 @@ import traceback from werkzeug.serving import run_simple from flask import Flask from routes import admin_bp -from api.utils.log_utils import init_root_logger +from common.log_utils import init_root_logger from common.contants import SERVICE_CONF from common.config_utils import show_configs from api import settings diff --git a/api/ragflow_server.py b/api/ragflow_server.py index 2cdba51e6..0a5bba1b9 100644 --- a/api/ragflow_server.py +++ b/api/ragflow_server.py @@ -18,7 +18,7 @@ # from beartype.claw import beartype_all # <-- you didn't sign up for this # beartype_all(conf=BeartypeConf(violation_type=UserWarning)) # <-- emit warnings from all code -from api.utils.log_utils import init_root_logger +from common.log_utils import init_root_logger from plugin import GlobalPluginManager init_root_logger("ragflow_server") diff --git a/api/utils/api_utils.py b/api/utils/api_utils.py index 46f613598..b768d943d 100644 --- a/api/utils/api_utils.py +++ b/api/utils/api_utils.py @@ -22,14 +22,10 @@ import queue import random import threading import time -from base64 import b64encode from copy import deepcopy from functools import wraps -from hmac import HMAC from io import BytesIO from typing import Any, Callable, Coroutine, Optional, Type, Union -from urllib.parse import quote, urlencode -from uuid import uuid1 import requests import trio @@ -82,46 +78,6 @@ def serialize_for_json(obj): return str(obj) -def request(**kwargs): - sess = requests.Session() - stream = kwargs.pop("stream", sess.stream) - timeout = kwargs.pop("timeout", None) - kwargs["headers"] = {k.replace("_", "-").upper(): v for k, v in kwargs.get("headers", {}).items()} - prepped = requests.Request(**kwargs).prepare() - - if settings.CLIENT_AUTHENTICATION and settings.HTTP_APP_KEY and settings.SECRET_KEY: - timestamp = str(round(time() * 1000)) - nonce = str(uuid1()) - signature = b64encode( - HMAC( - settings.SECRET_KEY.encode("ascii"), - b"\n".join( - [ - timestamp.encode("ascii"), - nonce.encode("ascii"), - settings.HTTP_APP_KEY.encode("ascii"), - prepped.path_url.encode("ascii"), - prepped.body if kwargs.get("json") else b"", - urlencode(sorted(kwargs["data"].items()), quote_via=quote, safe="-._~").encode( - "ascii") if kwargs.get("data") and isinstance(kwargs["data"], dict) else b"", - ] - ), - "sha1", - ).digest() - ).decode("ascii") - - prepped.headers.update( - { - "TIMESTAMP": timestamp, - "NONCE": nonce, - "APP-KEY": settings.HTTP_APP_KEY, - "SIGNATURE": signature, - } - ) - - return sess.send(prepped, stream=stream, timeout=timeout) - - def get_exponential_backoff_interval(retries, full_jitter=False): """Calculate the exponential backoff wait time.""" # Will be zero if factor equals 0 diff --git a/api/utils/health.py b/api/utils/health.py deleted file mode 100644 index 394154b9a..000000000 --- a/api/utils/health.py +++ /dev/null @@ -1,104 +0,0 @@ -from timeit import default_timer as timer - -from api import settings -from api.db.db_models import DB -from rag.utils.redis_conn import REDIS_CONN -from rag.utils.storage_factory import STORAGE_IMPL - - -def _ok_nok(ok: bool) -> str: - return "ok" if ok else "nok" - - -def check_db() -> tuple[bool, dict]: - st = timer() - try: - # lightweight probe; works for MySQL/Postgres - DB.execute_sql("SELECT 1") - return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"} - except Exception as e: - return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)} - - -def check_redis() -> tuple[bool, dict]: - st = timer() - try: - ok = bool(REDIS_CONN.health()) - return ok, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"} - except Exception as e: - return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)} - - -def check_doc_engine() -> tuple[bool, dict]: - st = timer() - try: - meta = settings.docStoreConn.health() - # treat any successful call as ok - return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", **(meta or {})} - except Exception as e: - return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)} - - -def check_storage() -> tuple[bool, dict]: - st = timer() - try: - STORAGE_IMPL.health() - return True, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"} - except Exception as e: - return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)} - - -def check_chat() -> tuple[bool, dict]: - st = timer() - try: - cfg = getattr(settings, "CHAT_CFG", None) - ok = bool(cfg and cfg.get("factory")) - return ok, {"elapsed": f"{(timer() - st) * 1000.0:.1f}"} - except Exception as e: - return False, {"elapsed": f"{(timer() - st) * 1000.0:.1f}", "error": str(e)} - - -def run_health_checks() -> tuple[dict, bool]: - result: dict[str, str | dict] = {} - - db_ok, db_meta = check_db() - chat_ok, chat_meta = check_chat() - - result["db"] = _ok_nok(db_ok) - if not db_ok: - result.setdefault("_meta", {})["db"] = db_meta - - result["chat"] = _ok_nok(chat_ok) - if not chat_ok: - result.setdefault("_meta", {})["chat"] = chat_meta - - # Optional probes (do not change minimal contract but exposed for observability) - try: - redis_ok, redis_meta = check_redis() - result["redis"] = _ok_nok(redis_ok) - if not redis_ok: - result.setdefault("_meta", {})["redis"] = redis_meta - except Exception: - result["redis"] = "nok" - - try: - doc_ok, doc_meta = check_doc_engine() - result["doc_engine"] = _ok_nok(doc_ok) - if not doc_ok: - result.setdefault("_meta", {})["doc_engine"] = doc_meta - except Exception: - result["doc_engine"] = "nok" - - try: - sto_ok, sto_meta = check_storage() - result["storage"] = _ok_nok(sto_ok) - if not sto_ok: - result.setdefault("_meta", {})["storage"] = sto_meta - except Exception: - result["storage"] = "nok" - - all_ok = (result.get("db") == "ok") and (result.get("chat") == "ok") - result["status"] = "ok" if all_ok else "nok" - return result, all_ok - - diff --git a/api/utils/log_utils.py b/api/utils/log_utils.py index 0348f9e09..cd7307f51 100644 --- a/api/utils/log_utils.py +++ b/api/utils/log_utils.py @@ -13,70 +13,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os -import os.path -import logging -from logging.handlers import RotatingFileHandler -from common.file_utils import get_project_base_directory - -initialized_root_logger = False - -def init_root_logger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"): - global initialized_root_logger - if initialized_root_logger: - return - initialized_root_logger = True - - logger = logging.getLogger() - logger.handlers.clear() - log_path = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"{logfile_basename}.log")) - - os.makedirs(os.path.dirname(log_path), exist_ok=True) - formatter = logging.Formatter(log_format) - - handler1 = RotatingFileHandler(log_path, maxBytes=10*1024*1024, backupCount=5) - handler1.setFormatter(formatter) - logger.addHandler(handler1) - - handler2 = logging.StreamHandler() - handler2.setFormatter(formatter) - logger.addHandler(handler2) - - logging.captureWarnings(True) - - LOG_LEVELS = os.environ.get("LOG_LEVELS", "") - pkg_levels = {} - for pkg_name_level in LOG_LEVELS.split(","): - terms = pkg_name_level.split("=") - if len(terms)!= 2: - continue - pkg_name, pkg_level = terms[0], terms[1] - pkg_name = pkg_name.strip() - pkg_level = logging.getLevelName(pkg_level.strip().upper()) - if not isinstance(pkg_level, int): - pkg_level = logging.INFO - pkg_levels[pkg_name] = logging.getLevelName(pkg_level) - - for pkg_name in ['peewee', 'pdfminer']: - if pkg_name not in pkg_levels: - pkg_levels[pkg_name] = logging.getLevelName(logging.WARNING) - if 'root' not in pkg_levels: - pkg_levels['root'] = logging.getLevelName(logging.INFO) - - for pkg_name, pkg_level in pkg_levels.items(): - pkg_logger = logging.getLogger(pkg_name) - pkg_logger.setLevel(pkg_level) - - msg = f"{logfile_basename} log path: {log_path}, log levels: {pkg_levels}" - logger.info(msg) - - -def log_exception(e, *args): - logging.exception(e) - for a in args: - if hasattr(a, "text"): - logging.error(a.text) - raise Exception(a.text) - else: - logging.error(str(a)) - raise e \ No newline at end of file diff --git a/common/log_utils.py b/common/log_utils.py new file mode 100644 index 000000000..e2110ebeb --- /dev/null +++ b/common/log_utils.py @@ -0,0 +1,83 @@ +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import os.path +import logging +from logging.handlers import RotatingFileHandler +from common.file_utils import get_project_base_directory + +initialized_root_logger = False + +def init_root_logger(logfile_basename: str, log_format: str = "%(asctime)-15s %(levelname)-8s %(process)d %(message)s"): + global initialized_root_logger + if initialized_root_logger: + return + initialized_root_logger = True + + logger = logging.getLogger() + logger.handlers.clear() + log_path = os.path.abspath(os.path.join(get_project_base_directory(), "logs", f"{logfile_basename}.log")) + + os.makedirs(os.path.dirname(log_path), exist_ok=True) + formatter = logging.Formatter(log_format) + + handler1 = RotatingFileHandler(log_path, maxBytes=10*1024*1024, backupCount=5) + handler1.setFormatter(formatter) + logger.addHandler(handler1) + + handler2 = logging.StreamHandler() + handler2.setFormatter(formatter) + logger.addHandler(handler2) + + logging.captureWarnings(True) + + LOG_LEVELS = os.environ.get("LOG_LEVELS", "") + pkg_levels = {} + for pkg_name_level in LOG_LEVELS.split(","): + terms = pkg_name_level.split("=") + if len(terms)!= 2: + continue + pkg_name, pkg_level = terms[0], terms[1] + pkg_name = pkg_name.strip() + pkg_level = logging.getLevelName(pkg_level.strip().upper()) + if not isinstance(pkg_level, int): + pkg_level = logging.INFO + pkg_levels[pkg_name] = logging.getLevelName(pkg_level) + + for pkg_name in ['peewee', 'pdfminer']: + if pkg_name not in pkg_levels: + pkg_levels[pkg_name] = logging.getLevelName(logging.WARNING) + if 'root' not in pkg_levels: + pkg_levels['root'] = logging.getLevelName(logging.INFO) + + for pkg_name, pkg_level in pkg_levels.items(): + pkg_logger = logging.getLogger(pkg_name) + pkg_logger.setLevel(pkg_level) + + msg = f"{logfile_basename} log path: {log_path}, log levels: {pkg_levels}" + logger.info(msg) + + +def log_exception(e, *args): + logging.exception(e) + for a in args: + if hasattr(a, "text"): + logging.error(a.text) + raise Exception(a.text) + else: + logging.error(str(a)) + raise e \ No newline at end of file diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py index 82564a056..5424e44b4 100644 --- a/rag/llm/embedding_model.py +++ b/rag/llm/embedding_model.py @@ -27,7 +27,7 @@ from ollama import Client from openai import OpenAI from zhipuai import ZhipuAI -from api.utils.log_utils import log_exception +from common.log_utils import log_exception from common.token_utils import num_tokens_from_string, truncate from api import settings import logging diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py index 15fae3d34..f295266b4 100644 --- a/rag/llm/rerank_model.py +++ b/rag/llm/rerank_model.py @@ -22,7 +22,7 @@ import numpy as np import requests from yarl import URL -from api.utils.log_utils import log_exception +from common.log_utils import log_exception from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response class Base(ABC): diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index e6fef691d..f42523492 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -28,7 +28,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.pipeline_operation_log_service import PipelineOperationLogService from api.utils.api_utils import timeout from common.base64_image import image2id -from api.utils.log_utils import init_root_logger +from common.log_utils import init_root_logger from common.file_utils import get_project_base_directory from common.config_utils import show_configs from graphrag.general.index import run_graphrag_for_kb