From 6878d23a57acb0980d9939732e1324e9e8d5efdb Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 15 Nov 2024 09:29:40 +0800 Subject: [PATCH] Print configs when startup RAGFlow server (#3414) ### What problem does this PR solve? Print configs at the RAGFlow startup phase. ### Type of change - [x] New Feature (non-breaking change which adds functionality) ``` 2024-11-14 21:27:53,090 INFO 962231 Current configs, from /home/weilongma/Documents/development/ragflow/conf/service_conf.yaml: 2024-11-14 21:27:53,090 INFO 962231 ragflow: {'host': '0.0.0.0', 'http_port': 9380} 2024-11-14 21:27:53,090 INFO 962231 mysql: {'name': 'rag_flow', 'user': 'root', 'password': 'infini_rag_flow', 'host': 'mysql', 'port': 5455, 'max_connections': 100, 'stale_timeout': 30} 2024-11-14 21:27:53,090 INFO 962231 minio: {'user': 'rag_flow', 'password': 'infini_rag_flow', 'host': 'minio:9000'} 2024-11-14 21:27:53,090 INFO 962231 es: {'hosts': 'http://es01:1200', 'username': 'elastic', 'password': 'infini_rag_flow'} 2024-11-14 21:27:53,090 INFO 962231 redis: {'db': 1, 'password': 'infini_rag_flow', 'host': 'redis:6379'} ``` Signed-off-by: jinhai --- api/constants.py | 4 +++- api/ragflow_server.py | 10 +++++---- api/utils/__init__.py | 50 +++++++++++++++++++++++++------------------ api/validation.py | 4 ++-- 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/api/constants.py b/api/constants.py index 636c246bc..f571d2157 100644 --- a/api/constants.py +++ b/api/constants.py @@ -15,4 +15,6 @@ NAME_LENGTH_LIMIT = 2 ** 10 -IMG_BASE64_PREFIX = 'data:image/png;base64,' \ No newline at end of file +IMG_BASE64_PREFIX = 'data:image/png;base64,' + +SERVICE_CONF = "service_conf.yaml" \ No newline at end of file diff --git a/api/ragflow_server.py b/api/ragflow_server.py index 7be85c078..091cf9a58 100644 --- a/api/ragflow_server.py +++ b/api/ragflow_server.py @@ -17,6 +17,7 @@ import logging import inspect from api.utils.log_utils import initRootLogger + initRootLogger(inspect.getfile(inspect.currentframe())) for module in ["pdfminer"]: module_logger = logging.getLogger(module) @@ -45,6 +46,7 @@ from api import utils from api.db.db_models import init_database_tables as init_web_db from api.db.init_data import init_web_data from api.versions import get_ragflow_version +from api.utils import show_configs def update_progress(): @@ -71,6 +73,7 @@ if __name__ == '__main__': logging.info( f'project base: {utils.file_utils.get_project_base_directory()}' ) + show_configs() # init db init_web_db() @@ -80,7 +83,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - "--version", default=False, help="rag flow version", action="store_true" + "--version", default=False, help="RAGFlow version", action="store_true" ) parser.add_argument( "--debug", default=False, help="debug mode", action="store_true" @@ -97,9 +100,8 @@ if __name__ == '__main__': RuntimeConfig.init_env() RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT) - - thr = ThreadPoolExecutor(max_workers=1) - thr.submit(update_progress) + thread = ThreadPoolExecutor(max_workers=1) + thread.submit(update_progress) # start http server try: diff --git a/api/utils/__init__.py b/api/utils/__init__.py index edb50577f..9be51f9b3 100644 --- a/api/utils/__init__.py +++ b/api/utils/__init__.py @@ -23,56 +23,63 @@ import socket import time import uuid import requests +import logging from enum import Enum, IntEnum import importlib from Cryptodome.PublicKey import RSA from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5 - from filelock import FileLock +from api.constants import SERVICE_CONF from . import file_utils -SERVICE_CONF = "service_conf.yaml" - def conf_realpath(conf_name): conf_path = f"conf/{conf_name}" return os.path.join(file_utils.get_project_base_directory(), conf_path) -def get_base_config(key, default=None, conf_name=SERVICE_CONF) -> dict: +def read_config(conf_name=SERVICE_CONF): local_config = {} local_path = conf_realpath(f'local.{conf_name}') - if default is None: - default = os.environ.get(key.upper()) + # load local config file if os.path.exists(local_path): local_config = file_utils.load_yaml_conf(local_path) if not isinstance(local_config, dict): raise ValueError(f'Invalid config file: "{local_path}".') - if key is not None and key in local_config: - return local_config[key] + global_config_path = conf_realpath(conf_name) + global_config = file_utils.load_yaml_conf(global_config_path) - config_path = conf_realpath(conf_name) - config = file_utils.load_yaml_conf(config_path) + if not isinstance(global_config, dict): + raise ValueError(f'Invalid config file: "{global_config_path}".') - if not isinstance(config, dict): - raise ValueError(f'Invalid config file: "{config_path}".') + global_config.update(local_config) + return global_config - config.update(local_config) - return config.get(key, default) if key is not None else config + +CONFIGS = read_config() + + +def show_configs(): + logging.info(f"Current configs, from {conf_realpath(SERVICE_CONF)}:") + for k, v in CONFIGS.items(): + logging.info(f"{k}: {v}") + + +def get_base_config(key, default=None): + if key is None: + return None + if default is None: + default = os.environ.get(key.upper()) + return CONFIGS.get(key, default) use_deserialize_safe_module = get_base_config( 'use_deserialize_safe_module', False) -class CoordinationCommunicationProtocol(object): - HTTP = "http" - GRPC = "grpc" - - class BaseType: def to_dict(self): return dict([(k.lstrip("_"), v) for k, v in self.__dict__.items()]) @@ -98,6 +105,7 @@ class BaseType: data = obj return {"type": obj.__class__.__name__, "data": data, "module": module} + return _dict(self) @@ -342,8 +350,8 @@ def download_img(url): return "" response = requests.get(url) return "data:" + \ - response.headers.get('Content-Type', 'image/jpg') + ";" + \ - "base64," + base64.b64encode(response.content).decode("utf-8") + response.headers.get('Content-Type', 'image/jpg') + ";" + \ + "base64," + base64.b64encode(response.content).decode("utf-8") def delta_seconds(date_string: str): diff --git a/api/validation.py b/api/validation.py index 7c0e42a14..39d506a8f 100644 --- a/api/validation.py +++ b/api/validation.py @@ -43,7 +43,7 @@ def download_nltk_data(): try: from multiprocessing import Pool pool = Pool(processes=1) - thr = pool.apply_async(download_nltk_data) - binary = thr.get(timeout=60) + thread = pool.apply_async(download_nltk_data) + binary = thread.get(timeout=60) except Exception as e: print('\x1b[6;37;41m WARNING \x1b[0m' + "Downloading NLTK data failure.", flush=True)