Add time utils (#10849)

### What problem does this PR solve?

- Add time utilities and unit tests

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

api/apps/api_app.py

@@ -33,7 +33,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
 from api import settings
-from api.utils import get_uuid, current_timestamp, datetime_format
+from api.utils import get_uuid
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
     generate_confirmation_token
 
@@ -41,6 +41,7 @@ from api.utils.file_utils import filename_type, thumbnail
 from rag.app.tag import label_question
 from rag.prompts.generator import keyword_extraction
 from rag.utils.storage_factory import STORAGE_IMPL
+from common.time_utils import current_timestamp, datetime_format
 
 from api.db.services.canvas_service import UserCanvasService
 from agent.canvas import Canvas

api/apps/system_app.py

@@ -24,7 +24,6 @@ from api.db.services.api_service import APITokenService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import UserTenantService
 from api import settings
-from api.utils import current_timestamp, datetime_format
 from api.utils.api_utils import (
     get_json_result,
     get_data_error_result,
@@ -32,6 +31,7 @@ from api.utils.api_utils import (
     generate_confirmation_token,
 )
 from api.versions import get_ragflow_version
+from common.time_utils import current_timestamp, datetime_format
 from rag.utils.storage_factory import STORAGE_IMPL, STORAGE_IMPL_TYPE
 from timeit import default_timer as timer
 

api/apps/tenant_app.py

@@ -23,7 +23,8 @@ from api.db import UserTenantRole, StatusEnum
 from api.db.db_models import UserTenant
 from api.db.services.user_service import UserTenantService, UserService
 
-from api.utils import get_uuid, delta_seconds
+from api.utils import get_uuid
+from common.time_utils import delta_seconds
 from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 from api.utils.web_utils import send_invite_email
 

api/apps/user_app.py

@@ -34,13 +34,8 @@ from api.db.services.file_service import FileService
 from api.db.services.llm_service import get_init_tenant_llm
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
-from api.utils import (
-    current_timestamp,
-    datetime_format,
-    download_img,
-    get_format_time,
-    get_uuid,
-)
+from common.time_utils import current_timestamp, datetime_format, get_format_time
+from api.utils import download_img, get_uuid
 from api.utils.api_utils import (
     construct_response,
     get_data_error_result,

api/db/db_models.py

@@ -35,6 +35,8 @@ from api.db import ParserType, SerializedType
 from api.utils.json import json_dumps, json_loads
 from api.utils.configs import deserialize_b64, serialize_b64
 
+from common.time_utils import current_timestamp, timestamp_to_date, date_string_to_timestamp
+
 
 def singleton(cls, *args, **kw):
     instances = {}
@@ -189,7 +191,7 @@ class BaseModel(Model):
                         for i, v in enumerate(f_v):
                             if isinstance(v, str) and f_n in auto_date_timestamp_field():
                                 # time type: %Y-%m-%d %H:%M:%S
-                                f_v[i] = utils.date_string_to_timestamp(v)
+                                f_v[i] = date_string_to_timestamp(v)
                         lt_value = f_v[0]
                         gt_value = f_v[1]
                         if lt_value is not None and gt_value is not None:
@@ -218,9 +220,9 @@ class BaseModel(Model):
     @classmethod
     def insert(cls, __data=None, **insert):
         if isinstance(__data, dict) and __data:
-            __data[cls._meta.combined["create_time"]] = utils.current_timestamp()
+            __data[cls._meta.combined["create_time"]] = current_timestamp()
         if insert:
-            insert["create_time"] = utils.current_timestamp()
+            insert["create_time"] = current_timestamp()
 
         return super().insert(__data, **insert)
 
@@ -231,11 +233,11 @@ class BaseModel(Model):
         if not normalized:
             return {}
 
-        normalized[cls._meta.combined["update_time"]] = utils.current_timestamp()
+        normalized[cls._meta.combined["update_time"]] = current_timestamp()
 
         for f_n in AUTO_DATE_TIMESTAMP_FIELD_PREFIX:
             if {f"{f_n}_time", f"{f_n}_date"}.issubset(cls._meta.combined.keys()) and cls._meta.combined[f"{f_n}_time"] in normalized and normalized[cls._meta.combined[f"{f_n}_time"]] is not None:
-                normalized[cls._meta.combined[f"{f_n}_date"]] = utils.timestamp_to_date(normalized[cls._meta.combined[f"{f_n}_time"]])
+                normalized[cls._meta.combined[f"{f_n}_date"]] = timestamp_to_date(normalized[cls._meta.combined[f"{f_n}_time"]])
 
         return normalized
 
@@ -331,9 +333,9 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
                 # 08006: connection_failure
                 # 08003: connection_does_not_exist
                 # 08000: connection_exception
-                error_messages = ['connection', 'server closed', 'connection refused', 
+                error_messages = ['connection', 'server closed', 'connection refused',
                                 'no connection to the server', 'terminating connection']
-                
+
                 should_retry = any(msg in str(e).lower() for msg in error_messages)
 
                 if should_retry and attempt < self.max_retries:
@@ -366,7 +368,7 @@ class RetryingPooledPostgresqlDatabase(PooledPostgresqlDatabase):
             except (OperationalError, InterfaceError) as e:
                 error_messages = ['connection', 'server closed', 'connection refused',
                                 'no connection to the server', 'terminating connection']
-                
+
                 should_retry = any(msg in str(e).lower() for msg in error_messages)
 
                 if should_retry and attempt < self.max_retries:
@@ -394,7 +396,7 @@ class BaseDataBase:
     def __init__(self):
         database_config = settings.DATABASE.copy()
         db_name = database_config.pop("name")
-        
+
         pool_config = {
             'max_retries': 5,
             'retry_delay': 1,

api/db/db_utils.py

@@ -18,7 +18,7 @@ from functools import reduce
 
 from playhouse.pool import PooledMySQLDatabase
 
-from api.utils import current_timestamp, timestamp_to_date
+from common.time_utils import current_timestamp, timestamp_to_date
 
 from api.db.db_models import DB, DataBaseModel
 

api/db/services/api_service.py

@@ -19,7 +19,7 @@ import peewee
 
 from api.db.db_models import DB, API4Conversation, APIToken, Dialog
 from api.db.services.common_service import CommonService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class APITokenService(CommonService):

api/db/services/common_service.py

@@ -19,7 +19,8 @@ import peewee
 from peewee import InterfaceError, OperationalError
 
 from api.db.db_models import DB
-from api.utils import current_timestamp, datetime_format, get_uuid
+from api.utils import get_uuid
+from common.time_utils import current_timestamp, datetime_format
 
 def retry_db_operation(func):
     @retry(

api/db/services/dialog_service.py

@@ -34,7 +34,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.langfuse_service import TenantLangfuseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.tenant_llm_service import TenantLLMService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 from graphrag.general.mind_map_extractor import MindMapExtractor
 from rag.app.resume import forbidden_select_fields4resume
 from rag.app.tag import label_question

api/db/services/document_service.py

@@ -34,7 +34,8 @@ from api.db.db_models import DB, Document, Knowledgebase, Task, Tenant, UserTena
 from api.db.db_utils import bulk_insert_into_db
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils import current_timestamp, get_format_time, get_uuid
+from api.utils import get_uuid
+from common.time_utils import current_timestamp, get_format_time
 from rag.nlp import rag_tokenizer, search
 from rag.settings import get_svr_queue_name, SVR_CONSUMER_GROUP_NAME
 from rag.utils.redis_conn import REDIS_CONN

api/db/services/file2document_service.py

@@ -20,7 +20,7 @@ from api.db.db_models import DB
 from api.db.db_models import File, File2Document
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class File2DocumentService(CommonService):

api/db/services/knowledgebase_service.py

@@ -20,7 +20,7 @@ from peewee import fn, JOIN
 from api.db import StatusEnum, TenantPermission
 from api.db.db_models import DB, Document, Knowledgebase, User, UserTenant, UserCanvas
 from api.db.services.common_service import CommonService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class KnowledgebaseService(CommonService):

api/db/services/langfuse_service.py

@@ -20,7 +20,7 @@ import peewee
 
 from api.db.db_models import DB, TenantLangfuse
 from api.db.services.common_service import CommonService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class TenantLangfuseService(CommonService):

api/db/services/pipeline_operation_log_service.py

@@ -27,7 +27,8 @@ from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import GRAPH_RAPTOR_FAKE_DOC_ID
-from api.utils import current_timestamp, datetime_format, get_uuid
+from api.utils import get_uuid
+from common.time_utils import current_timestamp, datetime_format
 
 
 class PipelineOperationLogService(CommonService):

api/db/services/search_service.py

@@ -20,7 +20,7 @@ from peewee import fn
 from api.db import StatusEnum
 from api.db.db_models import DB, Search, User
 from api.db.services.common_service import CommonService
-from api.utils import current_timestamp, datetime_format
+from common.time_utils import current_timestamp, datetime_format
 
 
 class SearchService(CommonService):

api/db/services/task_service.py

@@ -27,7 +27,8 @@ from api.db import StatusEnum, FileType, TaskStatus
 from api.db.db_models import Task, Document, Knowledgebase, Tenant
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
-from api.utils import current_timestamp, get_uuid
+from api.utils import get_uuid
+from common.time_utils import current_timestamp
 from deepdoc.parser.excel_parser import RAGFlowExcelParser
 from rag.settings import get_svr_queue_name
 from rag.utils.storage_factory import STORAGE_IMPL

api/db/services/user_service.py

@@ -24,7 +24,8 @@ from api.db import UserTenantRole
 from api.db.db_models import DB, UserTenant
 from api.db.db_models import User, Tenant
 from api.db.services.common_service import CommonService
-from api.utils import get_uuid, current_timestamp, datetime_format
+from api.utils import get_uuid
+from common.time_utils import current_timestamp, datetime_format
 from api.db import StatusEnum
 from rag.settings import MINIO
 

api/utils/__init__.py

@@ -14,11 +14,9 @@
 #  limitations under the License.
 #
 import base64
-import datetime
 import hashlib
 import os
 import socket
-import time
 import uuid
 import requests
 
@@ -26,26 +24,6 @@ import importlib
 
 from .common import string_to_bytes
 
-
-def current_timestamp():
-    return int(time.time() * 1000)
-
-
-def timestamp_to_date(timestamp, format_string="%Y-%m-%d %H:%M:%S"):
-    if not timestamp:
-        timestamp = time.time()
-    timestamp = int(timestamp) / 1000
-    time_array = time.localtime(timestamp)
-    str_date = time.strftime(format_string, time_array)
-    return str_date
-
-
-def date_string_to_timestamp(time_str, format_string="%Y-%m-%d %H:%M:%S"):
-    time_array = time.strptime(time_str, format_string)
-    time_stamp = int(time.mktime(time_array) * 1000)
-    return time_stamp
-
-
 def get_lan_ip():
     if os.name != "nt":
         import fcntl
@@ -94,26 +72,6 @@ def get_uuid():
     return uuid.uuid1().hex
 
 
-def datetime_format(date_time: datetime.datetime) -> datetime.datetime:
-    return datetime.datetime(date_time.year, date_time.month, date_time.day,
-                             date_time.hour, date_time.minute, date_time.second)
-
-
-def get_format_time() -> datetime.datetime:
-    return datetime_format(datetime.datetime.now())
-
-
-def str2date(date_time: str):
-    return datetime.datetime.strptime(date_time, '%Y-%m-%d')
-
-
-def elapsed2time(elapsed):
-    seconds = elapsed / 1000
-    minuter, second = divmod(seconds, 60)
-    hour, minuter = divmod(minuter, 60)
-    return '%02d:%02d:%02d' % (hour, minuter, second)
-
-
 def download_img(url):
     if not url:
         return ""
@@ -123,10 +81,5 @@ def download_img(url):
         "base64," + base64.b64encode(response.content).decode("utf-8")
 
 
-def delta_seconds(date_string: str):
-    dt = datetime.datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
-    return (datetime.datetime.now() - dt).total_seconds()
-
-
 def hash_str2int(line: str, mod: int = 10 ** 8) -> int:
     return int(hashlib.sha1(line.encode("utf-8")).hexdigest(), 16) % mod