pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 12:32:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move var from rag.settings to common.globals (#11022)

Browse Source 
### What problem does this PR solve?

As title.

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2025-11-05 15:48:50 +08:00
committed by GitHub
parent dddf766470
commit 02d10f8eda
13 changed files with 85 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
rag/settings.py
View File

@ -15,50 +15,12 @@
#
import os
import logging
from common.config_utils import get_base_config, decrypt_database_config
from common.file_utils import get_project_base_directory
from common.misc_utils import pip_install_torch
from common import globals
# Server
RAG_CONF_PATH = os.path.join(get_project_base_directory(), "conf")
# Get storage type and document engine from system environment variables
STORAGE_IMPL_TYPE = os.getenv('STORAGE_IMPL', 'MINIO')
globals.DOC_ENGINE = os.getenv('DOC_ENGINE', 'elasticsearch')
ES = {}
INFINITY = {}
AZURE = {}
S3 = {}
MINIO = {}
OSS = {}
OS = {}
# Initialize the selected configuration data based on environment variables to solve the problem of initialization errors due to lack of configuration
if globals.DOC_ENGINE == 'elasticsearch':
ES = get_base_config("es", {})
elif globals.DOC_ENGINE == 'opensearch':
OS = get_base_config("os", {})
elif globals.DOC_ENGINE == 'infinity':
INFINITY = get_base_config("infinity", {"uri": "infinity:23817"})
if STORAGE_IMPL_TYPE in ['AZURE_SPN', 'AZURE_SAS']:
AZURE = get_base_config("azure", {})
elif STORAGE_IMPL_TYPE == 'AWS_S3':
S3 = get_base_config("s3", {})
elif STORAGE_IMPL_TYPE == 'MINIO':
MINIO = decrypt_database_config(name="minio")
elif STORAGE_IMPL_TYPE == 'OSS':
OSS = get_base_config("oss", {})
try:
REDIS = decrypt_database_config(name="redis")
except Exception:
try:
REDIS = get_base_config("redis", {})
except Exception:
REDIS = {}
DOC_MAXIMUM_SIZE = int(os.environ.get("MAX_CONTENT_LENGTH", 128 * 1024 * 1024))
DOC_BULK_SIZE = int(os.environ.get("DOC_BULK_SIZE", 4))
EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", 16))

6
rag/utils/azure_sas_conn.py
View File

@ -18,17 +18,17 @@ import logging
import os
import time
from io import BytesIO
from rag import settings
from common.decorator import singleton
from azure.storage.blob import ContainerClient
from common import globals
@singleton
class RAGFlowAzureSasBlob:
def __init__(self):
self.conn = None
self.container_url = os.getenv('CONTAINER_URL', settings.AZURE["container_url"])
self.sas_token = os.getenv('SAS_TOKEN', settings.AZURE["sas_token"])
self.container_url = os.getenv('CONTAINER_URL', globals.AZURE["container_url"])
self.sas_token = os.getenv('SAS_TOKEN', globals.AZURE["sas_token"])
self.__open__()
def __open__(self):

12
rag/utils/azure_spn_conn.py
View File

@ -17,21 +17,21 @@
import logging
import os
import time
from rag import settings
from common.decorator import singleton
from azure.identity import ClientSecretCredential, AzureAuthorityHosts
from azure.storage.filedatalake import FileSystemClient
from common import globals
@singleton
class RAGFlowAzureSpnBlob:
def __init__(self):
self.conn = None
self.account_url = os.getenv('ACCOUNT_URL', settings.AZURE["account_url"])
self.client_id = os.getenv('CLIENT_ID', settings.AZURE["client_id"])
self.secret = os.getenv('SECRET', settings.AZURE["secret"])
self.tenant_id = os.getenv('TENANT_ID', settings.AZURE["tenant_id"])
self.container_name = os.getenv('CONTAINER_NAME', settings.AZURE["container_name"])
self.account_url = os.getenv('ACCOUNT_URL', globals.AZURE["account_url"])
self.client_id = os.getenv('CLIENT_ID', globals.AZURE["client_id"])
self.secret = os.getenv('SECRET', globals.AZURE["secret"])
self.tenant_id = os.getenv('TENANT_ID', globals.AZURE["tenant_id"])
self.container_name = os.getenv('CONTAINER_NAME', globals.AZURE["container_name"])
self.__open__()
def __open__(self):

18
rag/utils/es_conn.py
View File

@ -24,7 +24,6 @@ import copy
from elasticsearch import Elasticsearch, NotFoundError
from elasticsearch_dsl import UpdateByQuery, Q, Search, Index
from elastic_transport import ConnectionTimeout
from rag import settings
from rag.settings import TAG_FLD, PAGERANK_FLD
from common.decorator import singleton
from common.file_utils import get_project_base_directory
@ -33,6 +32,7 @@ from rag.utils.doc_store_conn import DocStoreConnection, MatchExpr, OrderByExpr,
FusionExpr
from rag.nlp import is_english, rag_tokenizer
from common.float_utils import get_float
from common import globals
ATTEMPT_TIME = 2
@ -43,17 +43,17 @@ logger = logging.getLogger('ragflow.es_conn')
class ESConnection(DocStoreConnection):
def __init__(self):
self.info = {}
logger.info(f"Use Elasticsearch {settings.ES['hosts']} as the doc engine.")
logger.info(f"Use Elasticsearch {globals.ES['hosts']} as the doc engine.")
for _ in range(ATTEMPT_TIME):
try:
if self._connect():
break
except Exception as e:
logger.warning(f"{str(e)}. Waiting Elasticsearch {settings.ES['hosts']} to be healthy.")
logger.warning(f"{str(e)}. Waiting Elasticsearch {globals.ES['hosts']} to be healthy.")
time.sleep(5)
if not self.es.ping():
msg = f"Elasticsearch {settings.ES['hosts']} is unhealthy in 120s."
msg = f"Elasticsearch {globals.ES['hosts']} is unhealthy in 120s."
logger.error(msg)
raise Exception(msg)
v = self.info.get("version", {"number": "8.11.3"})
@ -68,14 +68,14 @@ class ESConnection(DocStoreConnection):
logger.error(msg)
raise Exception(msg)
self.mapping = json.load(open(fp_mapping, "r"))
logger.info(f"Elasticsearch {settings.ES['hosts']} is healthy.")
logger.info(f"Elasticsearch {globals.ES['hosts']} is healthy.")
def _connect(self):
self.es = Elasticsearch(
settings.ES["hosts"].split(","),
basic_auth=(settings.ES["username"], settings.ES[
"password"]) if "username" in settings.ES and "password" in settings.ES else None,
verify_certs= settings.ES.get("verify_certs", False),
globals.ES["hosts"].split(","),
basic_auth=(globals.ES["username"], globals.ES[
"password"]) if "username" in globals.ES and "password" in globals.ES else None,
verify_certs= globals.ES.get("verify_certs", False),
timeout=600 )
if self.es:
self.info = self.es.info()

6
rag/utils/infinity_conn.py
View File

@ -25,11 +25,11 @@ from infinity.common import ConflictType, InfinityException, SortType
from infinity.index import IndexInfo, IndexType
from infinity.connection_pool import ConnectionPool
from infinity.errors import ErrorCode
from rag import settings
from rag.settings import PAGERANK_FLD, TAG_FLD
from common.decorator import singleton
import pandas as pd
from common.file_utils import get_project_base_directory
from common import globals
from rag.nlp import is_english
from rag.utils.doc_store_conn import (
@ -130,8 +130,8 @@ def concat_dataframes(df_list: list[pd.DataFrame], selectFields: list[str]) -> p
@singleton
class InfinityConnection(DocStoreConnection):
def __init__(self):
self.dbName = settings.INFINITY.get("db_name", "default_db")
infinity_uri = settings.INFINITY["uri"]
self.dbName = globals.INFINITY.get("db_name", "default_db")
infinity_uri = globals.INFINITY["uri"]
if ":" in infinity_uri:
host, port = infinity_uri.split(":")
infinity_uri = infinity.common.NetworkAddress(host, int(port))

10
rag/utils/minio_conn.py
View File

@ -20,8 +20,8 @@ from minio import Minio
from minio.commonconfig import CopySource
from minio.error import S3Error
from io import BytesIO
from rag import settings
from common.decorator import singleton
from common import globals
@singleton
@ -38,14 +38,14 @@ class RAGFlowMinio:
pass
try:
self.conn = Minio(settings.MINIO["host"],
access_key=settings.MINIO["user"],
secret_key=settings.MINIO["password"],
self.conn = Minio(globals.MINIO["host"],
access_key=globals.MINIO["user"],
secret_key=globals.MINIO["password"],
secure=False
)
except Exception:
logging.exception(
"Fail to connect %s " % settings.MINIO["host"])
"Fail to connect %s " % globals.MINIO["host"])
def __close__(self):
del self.conn

16
rag/utils/opensearch_conn.py
View File

@ -24,13 +24,13 @@ import copy
from opensearchpy import OpenSearch, NotFoundError
from opensearchpy import UpdateByQuery, Q, Search, Index
from opensearchpy import ConnectionTimeout
from rag import settings
from rag.settings import TAG_FLD, PAGERANK_FLD
from common.decorator import singleton
from common.file_utils import get_project_base_directory
from rag.utils.doc_store_conn import DocStoreConnection, MatchExpr, OrderByExpr, MatchTextExpr, MatchDenseExpr, \
FusionExpr
from rag.nlp import is_english, rag_tokenizer
from common import globals
ATTEMPT_TIME = 2
@ -41,13 +41,13 @@ logger = logging.getLogger('ragflow.opensearch_conn')
class OSConnection(DocStoreConnection):
def __init__(self):
self.info = {}
logger.info(f"Use OpenSearch {settings.OS['hosts']} as the doc engine.")
logger.info(f"Use OpenSearch {globals.OS['hosts']} as the doc engine.")
for _ in range(ATTEMPT_TIME):
try:
self.os = OpenSearch(
settings.OS["hosts"].split(","),
http_auth=(settings.OS["username"], settings.OS[
"password"]) if "username" in settings.OS and "password" in settings.OS else None,
globals.OS["hosts"].split(","),
http_auth=(globals.OS["username"], globals.OS[
"password"]) if "username" in globals.OS and "password" in globals.OS else None,
verify_certs=False,
timeout=600
)
@ -55,10 +55,10 @@ class OSConnection(DocStoreConnection):
self.info = self.os.info()
break
except Exception as e:
logger.warning(f"{str(e)}. Waiting OpenSearch {settings.OS['hosts']} to be healthy.")
logger.warning(f"{str(e)}. Waiting OpenSearch {globals.OS['hosts']} to be healthy.")
time.sleep(5)
if not self.os.ping():
msg = f"OpenSearch {settings.OS['hosts']} is unhealthy in 120s."
msg = f"OpenSearch {globals.OS['hosts']} is unhealthy in 120s."
logger.error(msg)
raise Exception(msg)
v = self.info.get("version", {"number": "2.18.0"})
@ -73,7 +73,7 @@ class OSConnection(DocStoreConnection):
logger.error(msg)
raise Exception(msg)
self.mapping = json.load(open(fp_mapping, "r"))
logger.info(f"OpenSearch {settings.OS['hosts']} is healthy.")
logger.info(f"OpenSearch {globals.OS['hosts']} is healthy.")
"""
Database operations

4
rag/utils/oss_conn.py
View File

@ -20,14 +20,14 @@ from botocore.config import Config
import time
from io import BytesIO
from common.decorator import singleton
from rag import settings
from common import globals
@singleton
class RAGFlowOSS:
def __init__(self):
self.conn = None
self.oss_config = settings.OSS
self.oss_config = globals.OSS
self.access_key = self.oss_config.get('access_key', None)
self.secret_key = self.oss_config.get('secret_key', None)
self.endpoint_url = self.oss_config.get('endpoint_url', None)

4
rag/utils/redis_conn.py
View File

@ -19,8 +19,8 @@ import json
import uuid
import valkey as redis
from rag import settings
from common.decorator import singleton
from common import globals
from valkey.lock import Lock
import trio
@ -61,7 +61,7 @@ class RedisDB:
def __init__(self):
self.REDIS = None
self.config = settings.REDIS
self.config = globals.REDIS
self.__open__()
def register_scripts(self) -> None:

5
rag/utils/s3_conn.py
View File

@ -21,13 +21,14 @@ from botocore.config import Config
import time
from io import BytesIO
from common.decorator import singleton
from rag import settings
from common import globals
@singleton
class RAGFlowS3:
def __init__(self):
self.conn = None
self.s3_config = settings.S3
self.s3_config = globals.S3
self.access_key = self.s3_config.get('access_key', None)
self.secret_key = self.s3_config.get('secret_key', None)
self.session_token = self.s3_config.get('session_token', None)