pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 20:42:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move api.settings to common.settings (#11036)

Browse Source 
### What problem does this PR solve?

As title

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2025-11-06 09:36:38 +08:00
committed by GitHub
parent 87c9a054d3
commit f98b24c9bf
68 changed files with 675 additions and 718 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
rag/utils/azure_sas_conn.py
View File

@ -20,15 +20,15 @@ import time
from io import BytesIO
from common.decorator import singleton
from azure.storage.blob import ContainerClient
from common import globals
from common import settings
@singleton
class RAGFlowAzureSasBlob:
def __init__(self):
self.conn = None
self.container_url = os.getenv('CONTAINER_URL', globals.AZURE["container_url"])
self.sas_token = os.getenv('SAS_TOKEN', globals.AZURE["sas_token"])
self.container_url = os.getenv('CONTAINER_URL', settings.AZURE["container_url"])
self.sas_token = os.getenv('SAS_TOKEN', settings.AZURE["sas_token"])
self.__open__()
def __open__(self):

12
rag/utils/azure_spn_conn.py
View File

@ -20,18 +20,18 @@ import time
from common.decorator import singleton
from azure.identity import ClientSecretCredential, AzureAuthorityHosts
from azure.storage.filedatalake import FileSystemClient
from common import globals
from common import settings
@singleton
class RAGFlowAzureSpnBlob:
def __init__(self):
self.conn = None
self.account_url = os.getenv('ACCOUNT_URL', globals.AZURE["account_url"])
self.client_id = os.getenv('CLIENT_ID', globals.AZURE["client_id"])
self.secret = os.getenv('SECRET', globals.AZURE["secret"])
self.tenant_id = os.getenv('TENANT_ID', globals.AZURE["tenant_id"])
self.container_name = os.getenv('CONTAINER_NAME', globals.AZURE["container_name"])
self.account_url = os.getenv('ACCOUNT_URL', settings.AZURE["account_url"])
self.client_id = os.getenv('CLIENT_ID', settings.AZURE["client_id"])
self.secret = os.getenv('SECRET', settings.AZURE["secret"])
self.tenant_id = os.getenv('TENANT_ID', settings.AZURE["tenant_id"])
self.container_name = os.getenv('CONTAINER_NAME', settings.AZURE["container_name"])
self.__open__()
def __open__(self):

20
rag/utils/es_conn.py
View File

@ -24,7 +24,6 @@ import copy
from elasticsearch import Elasticsearch, NotFoundError
from elasticsearch_dsl import UpdateByQuery, Q, Search, Index
from elastic_transport import ConnectionTimeout
from rag.settings import TAG_FLD, PAGERANK_FLD
from common.decorator import singleton
from common.file_utils import get_project_base_directory
from common.misc_utils import convert_bytes
@ -32,7 +31,8 @@ from rag.utils.doc_store_conn import DocStoreConnection, MatchExpr, OrderByExpr,
FusionExpr
from rag.nlp import is_english, rag_tokenizer
from common.float_utils import get_float
from common import globals
from common import settings
from common.constants import PAGERANK_FLD, TAG_FLD
ATTEMPT_TIME = 2
@ -43,17 +43,17 @@ logger = logging.getLogger('ragflow.es_conn')
class ESConnection(DocStoreConnection):
def __init__(self):
self.info = {}
logger.info(f"Use Elasticsearch {globals.ES['hosts']} as the doc engine.")
logger.info(f"Use Elasticsearch {settings.ES['hosts']} as the doc engine.")
for _ in range(ATTEMPT_TIME):
try:
if self._connect():
break
except Exception as e:
logger.warning(f"{str(e)}. Waiting Elasticsearch {globals.ES['hosts']} to be healthy.")
logger.warning(f"{str(e)}. Waiting Elasticsearch {settings.ES['hosts']} to be healthy.")
time.sleep(5)
if not self.es.ping():
msg = f"Elasticsearch {globals.ES['hosts']} is unhealthy in 120s."
msg = f"Elasticsearch {settings.ES['hosts']} is unhealthy in 120s."
logger.error(msg)
raise Exception(msg)
v = self.info.get("version", {"number": "8.11.3"})
@ -68,14 +68,14 @@ class ESConnection(DocStoreConnection):
logger.error(msg)
raise Exception(msg)
self.mapping = json.load(open(fp_mapping, "r"))
logger.info(f"Elasticsearch {globals.ES['hosts']} is healthy.")
logger.info(f"Elasticsearch {settings.ES['hosts']} is healthy.")
def _connect(self):
self.es = Elasticsearch(
globals.ES["hosts"].split(","),
basic_auth=(globals.ES["username"], globals.ES[
"password"]) if "username" in globals.ES and "password" in globals.ES else None,
verify_certs= globals.ES.get("verify_certs", False),
settings.ES["hosts"].split(","),
basic_auth=(settings.ES["username"], settings.ES[
"password"]) if "username" in settings.ES and "password" in settings.ES else None,
verify_certs= settings.ES.get("verify_certs", False),
timeout=600 )
if self.es:
self.info = self.es.info()

9
rag/utils/infinity_conn.py
View File

@ -25,13 +25,12 @@ from infinity.common import ConflictType, InfinityException, SortType
from infinity.index import IndexInfo, IndexType
from infinity.connection_pool import ConnectionPool
from infinity.errors import ErrorCode
from rag.settings import PAGERANK_FLD, TAG_FLD
from common.decorator import singleton
import pandas as pd
from common.file_utils import get_project_base_directory
from common import globals
from rag.nlp import is_english
from common.constants import PAGERANK_FLD, TAG_FLD
from common import settings
from rag.utils.doc_store_conn import (
DocStoreConnection,
MatchExpr,
@ -130,8 +129,8 @@ def concat_dataframes(df_list: list[pd.DataFrame], selectFields: list[str]) -> p
@singleton
class InfinityConnection(DocStoreConnection):
def __init__(self):
self.dbName = globals.INFINITY.get("db_name", "default_db")
infinity_uri = globals.INFINITY["uri"]
self.dbName = settings.INFINITY.get("db_name", "default_db")
infinity_uri = settings.INFINITY["uri"]
if ":" in infinity_uri:
host, port = infinity_uri.split(":")
infinity_uri = infinity.common.NetworkAddress(host, int(port))

10
rag/utils/minio_conn.py
View File

@ -21,7 +21,7 @@ from minio.commonconfig import CopySource
from minio.error import S3Error
from io import BytesIO
from common.decorator import singleton
from common import globals
from common import settings
@singleton
@ -38,14 +38,14 @@ class RAGFlowMinio:
pass
try:
self.conn = Minio(globals.MINIO["host"],
access_key=globals.MINIO["user"],
secret_key=globals.MINIO["password"],
self.conn = Minio(settings.MINIO["host"],
access_key=settings.MINIO["user"],
secret_key=settings.MINIO["password"],
secure=False
)
except Exception:
logging.exception(
"Fail to connect %s " % globals.MINIO["host"])
"Fail to connect %s " % settings.MINIO["host"])
def __close__(self):
del self.conn

18
rag/utils/opensearch_conn.py
View File

@ -24,13 +24,13 @@ import copy
from opensearchpy import OpenSearch, NotFoundError
from opensearchpy import UpdateByQuery, Q, Search, Index
from opensearchpy import ConnectionTimeout
from rag.settings import TAG_FLD, PAGERANK_FLD
from common.decorator import singleton
from common.file_utils import get_project_base_directory
from rag.utils.doc_store_conn import DocStoreConnection, MatchExpr, OrderByExpr, MatchTextExpr, MatchDenseExpr, \
FusionExpr
from rag.nlp import is_english, rag_tokenizer
from common import globals
from common.constants import PAGERANK_FLD, TAG_FLD
from common import settings
ATTEMPT_TIME = 2
@ -41,13 +41,13 @@ logger = logging.getLogger('ragflow.opensearch_conn')
class OSConnection(DocStoreConnection):
def __init__(self):
self.info = {}
logger.info(f"Use OpenSearch {globals.OS['hosts']} as the doc engine.")
logger.info(f"Use OpenSearch {settings.OS['hosts']} as the doc engine.")
for _ in range(ATTEMPT_TIME):
try:
self.os = OpenSearch(
globals.OS["hosts"].split(","),
http_auth=(globals.OS["username"], globals.OS[
"password"]) if "username" in globals.OS and "password" in globals.OS else None,
settings.OS["hosts"].split(","),
http_auth=(settings.OS["username"], settings.OS[
"password"]) if "username" in settings.OS and "password" in settings.OS else None,
verify_certs=False,
timeout=600
)
@ -55,10 +55,10 @@ class OSConnection(DocStoreConnection):
self.info = self.os.info()
break
except Exception as e:
logger.warning(f"{str(e)}. Waiting OpenSearch {globals.OS['hosts']} to be healthy.")
logger.warning(f"{str(e)}. Waiting OpenSearch {settings.OS['hosts']} to be healthy.")
time.sleep(5)
if not self.os.ping():
msg = f"OpenSearch {globals.OS['hosts']} is unhealthy in 120s."
msg = f"OpenSearch {settings.OS['hosts']} is unhealthy in 120s."
logger.error(msg)
raise Exception(msg)
v = self.info.get("version", {"number": "2.18.0"})
@ -73,7 +73,7 @@ class OSConnection(DocStoreConnection):
logger.error(msg)
raise Exception(msg)
self.mapping = json.load(open(fp_mapping, "r"))
logger.info(f"OpenSearch {globals.OS['hosts']} is healthy.")
logger.info(f"OpenSearch {settings.OS['hosts']} is healthy.")
"""
Database operations

4
rag/utils/oss_conn.py
View File

@ -20,14 +20,14 @@ from botocore.config import Config
import time
from io import BytesIO
from common.decorator import singleton
from common import globals
from common import settings
@singleton
class RAGFlowOSS:
def __init__(self):
self.conn = None
self.oss_config = globals.OSS
self.oss_config = settings.OSS
self.access_key = self.oss_config.get('access_key', None)
self.secret_key = self.oss_config.get('secret_key', None)
self.endpoint_url = self.oss_config.get('endpoint_url', None)

13
rag/utils/redis_conn.py
View File

@ -20,10 +20,19 @@ import uuid
import valkey as redis
from common.decorator import singleton
from common import globals
from common import settings
from valkey.lock import Lock
import trio
REDIS = {}
try:
REDIS = settings.decrypt_database_config(name="redis")
except Exception:
try:
REDIS = settings.get_base_config("redis", {})
except Exception:
REDIS = {}
class RedisMsg:
def __init__(self, consumer, queue_name, group_name, msg_id, message):
self.__consumer = consumer
@ -61,7 +70,7 @@ class RedisDB:
def __init__(self):
self.REDIS = None
self.config = globals.REDIS
self.config = REDIS
self.__open__()
def register_scripts(self) -> None:

4
rag/utils/s3_conn.py
View File

@ -21,14 +21,14 @@ from botocore.config import Config
import time
from io import BytesIO
from common.decorator import singleton
from common import globals
from common import settings
@singleton
class RAGFlowS3:
def __init__(self):
self.conn = None
self.s3_config = globals.S3
self.s3_config = settings.S3
self.access_key = self.s3_config.get('access_key', None)
self.secret_key = self.s3_config.get('secret_key', None)
self.session_token = self.s3_config.get('session_token', None)

38
rag/utils/storage_factory.py
View File

@ -13,41 +13,3 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
from enum import Enum
from rag.utils.azure_sas_conn import RAGFlowAzureSasBlob
from rag.utils.azure_spn_conn import RAGFlowAzureSpnBlob
from rag.utils.minio_conn import RAGFlowMinio
from rag.utils.opendal_conn import OpenDALStorage
from rag.utils.s3_conn import RAGFlowS3
from rag.utils.oss_conn import RAGFlowOSS
class Storage(Enum):
MINIO = 1
AZURE_SPN = 2
AZURE_SAS = 3
AWS_S3 = 4
OSS = 5
OPENDAL = 6
class StorageFactory:
storage_mapping = {
Storage.MINIO: RAGFlowMinio,
Storage.AZURE_SPN: RAGFlowAzureSpnBlob,
Storage.AZURE_SAS: RAGFlowAzureSasBlob,
Storage.AWS_S3: RAGFlowS3,
Storage.OSS: RAGFlowOSS,
Storage.OPENDAL: OpenDALStorage
}
@classmethod
def create(cls, storage: Storage):
return cls.storage_mapping[storage]()
STORAGE_IMPL_TYPE = os.getenv('STORAGE_IMPL', 'MINIO')
STORAGE_IMPL = StorageFactory.create(Storage[STORAGE_IMPL_TYPE])