Remove 'get_lan_ip' and add common misc_utils.py (#10880)

### What problem does this PR solve?

Add get_uuid, download_img and hash_str2int into misc_utils.py

### Type of change

- [x] Refactoring

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>

api/apps/api_app.py

@@ -33,7 +33,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import queue_tasks, TaskService
 from api.db.services.user_service import UserTenantService
 from api import settings
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import server_error_response, get_data_error_result, get_json_result, validate_request, \
     generate_confirmation_token
 

api/apps/canvas_app.py

@@ -35,7 +35,7 @@ from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, Ta
 from api.db.services.user_service import TenantService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.settings import RetCode
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_json_result, server_error_response, validate_request, get_data_error_result
 from agent.canvas import Canvas
 from peewee import MySQLDatabase, PostgresqlDatabase

api/apps/dialog_app.py

@@ -24,7 +24,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService, UserTenantService
 from api import settings
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_json_result
 
 

api/apps/document_app.py

@@ -35,7 +35,7 @@ from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks, queue_dataflow
 from api.db.services.user_service import UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import (
     get_data_error_result,
     get_json_result,

api/apps/file2document_app.py

@@ -23,7 +23,7 @@ from flask import request
 from flask_login import login_required, current_user
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.db import FileType
 from api.db.services.document_service import DocumentService
 from api import settings

api/apps/file_app.py

@@ -26,7 +26,7 @@ from api.common.check_team_permission import check_file_team_permission
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.db import FileType, FileSource
 from api.db.services import duplicate_name
 from api.db.services.file_service import FileService

api/apps/kb_app.py

@@ -31,7 +31,7 @@ from api.db.services.pipeline_operation_log_service import PipelineOperationLogS
 from api.db.services.task_service import TaskService, GRAPH_RAPTOR_FAKE_DOC_ID
 from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import get_error_data_result, server_error_response, get_data_error_result, validate_request, not_allowed_parameters
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.db import PipelineTaskType, StatusEnum, FileSource, VALID_FILE_TYPES, VALID_TASK_STATUS
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.db_models import File

api/apps/mcp_server_app.py

@@ -22,7 +22,7 @@ from api.db.services.mcp_server_service import MCPServerService
 from api.db.services.user_service import TenantService
 from api.settings import RetCode
 
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_json_result, server_error_response, validate_request, \
     get_mcp_tools
 from api.utils.web_utils import get_float, safe_json_parse

api/apps/sdk/agent.py

@@ -20,7 +20,7 @@ from typing import Any, cast
 from api.db.services.canvas_service import UserCanvasService
 from api.db.services.user_canvas_version import UserCanvasVersionService
 from api.settings import RetCode
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_error_data_result, get_json_result, token_required
 from api.utils.api_utils import get_result
 from flask import request

api/apps/sdk/chat.py

@@ -23,7 +23,7 @@ from api.db.services.dialog_service import DialogService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import check_duplicate_ids, get_error_data_result, get_result, token_required
 
 

api/apps/sdk/dataset.py

@@ -28,7 +28,7 @@ from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.user_service import TenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import (
     deep_merge,
     get_error_argument_result,

api/apps/sdk/files.py

@@ -26,7 +26,7 @@ from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import server_error_response, token_required
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.db import FileType
 from api.db.services import duplicate_name
 from api.db.services.file_service import FileService

api/apps/sdk/session.py

@@ -35,7 +35,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import check_duplicate_ids, get_data_openai, get_error_data_result, get_json_result, \
     get_result, server_error_response, token_required, validate_request
 from rag.app.tag import label_question

api/apps/search_app.py

@@ -24,7 +24,7 @@ from api.db.db_models import DB
 from api.db.services import duplicate_name
 from api.db.services.search_service import SearchService
 from api.db.services.user_service import TenantService, UserTenantService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_error_result, get_json_result, not_allowed_parameters, server_error_response, validate_request
 
 

api/apps/tenant_app.py

@@ -23,7 +23,7 @@ from api.db import UserTenantRole, StatusEnum
 from api.db.db_models import UserTenant
 from api.db.services.user_service import UserTenantService, UserService
 
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import delta_seconds
 from api.utils.api_utils import get_json_result, validate_request, server_error_response, get_data_error_result
 from api.utils.web_utils import send_invite_email

api/apps/user_app.py

@@ -35,7 +35,7 @@ from api.db.services.llm_service import get_init_tenant_llm
 from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserService, UserTenantService
 from common.time_utils import current_timestamp, datetime_format, get_format_time
-from api.utils import download_img, get_uuid
+from common.misc_utils import download_img, get_uuid
 from api.utils.api_utils import (
     construct_response,
     get_data_error_result,

api/db/services/canvas_service.py

@@ -22,7 +22,7 @@ from api.db import CanvasCategory, TenantPermission
 from api.db.db_models import DB, CanvasTemplate, User, UserCanvas, API4Conversation
 from api.db.services.api_service import API4ConversationService
 from api.db.services.common_service import CommonService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.api_utils import get_data_openai
 import tiktoken
 from peewee import fn

api/db/services/common_service.py

@@ -19,7 +19,7 @@ import peewee
 from peewee import InterfaceError, OperationalError
 
 from api.db.db_models import DB
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp, datetime_format
 
 def retry_db_operation(func):

api/db/services/conversation_service.py

@@ -20,7 +20,7 @@ from api.db.db_models import Conversation, DB
 from api.db.services.api_service import API4ConversationService
 from api.db.services.common_service import CommonService
 from api.db.services.dialog_service import DialogService, chat
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 import json
 
 from rag.prompts.generator import chunks_format

api/db/services/document_service.py

@@ -34,7 +34,7 @@ from api.db.db_models import DB, Document, Knowledgebase, Task, Tenant, UserTena
 from api.db.db_utils import bulk_insert_into_db
 from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp, get_format_time
 from rag.nlp import rag_tokenizer, search
 from rag.settings import get_svr_queue_name, SVR_CONSUMER_GROUP_NAME

api/db/services/file_service.py

@@ -27,7 +27,7 @@ from api.db.services import duplicate_name
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from api.utils.file_utils import filename_type, read_potential_broken_pdf, thumbnail_img
 from rag.llm.cv_model import GptV4
 from rag.utils.storage_factory import STORAGE_IMPL

api/db/services/pipeline_operation_log_service.py

@@ -27,7 +27,7 @@ from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import GRAPH_RAPTOR_FAKE_DOC_ID
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp, datetime_format
 
 

api/db/services/task_service.py

@@ -27,7 +27,7 @@ from api.db import StatusEnum, FileType, TaskStatus
 from api.db.db_models import Task, Document, Knowledgebase, Tenant
 from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp
 from deepdoc.parser.excel_parser import RAGFlowExcelParser
 from rag.settings import get_svr_queue_name

api/db/services/user_service.py

@@ -24,7 +24,7 @@ from api.db import UserTenantRole
 from api.db.db_models import DB, UserTenant
 from api.db.db_models import User, Tenant
 from api.db.services.common_service import CommonService
-from api.utils import get_uuid
+from common.misc_utils import get_uuid
 from common.time_utils import current_timestamp, datetime_format
 from api.db import StatusEnum
 from rag.settings import MINIO

api/utils/__init__.py

@@ -13,49 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-import base64
-import hashlib
-import os
-import socket
-import uuid
-import requests
-
 import importlib
 
-from .common import string_to_bytes
-
-def get_lan_ip():
-    if os.name != "nt":
-        import fcntl
-        import struct
-
-        def get_interface_ip(ifname):
-            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            return socket.inet_ntoa(
-                fcntl.ioctl(s.fileno(), 0x8915, struct.pack('256s', string_to_bytes(ifname[:15])))[20:24])
-
-    ip = socket.gethostbyname(socket.getfqdn())
-    if ip.startswith("127.") and os.name != "nt":
-        interfaces = [
-            "bond1",
-            "eth0",
-            "eth1",
-            "eth2",
-            "wlan0",
-            "wlan1",
-            "wifi0",
-            "ath0",
-            "ath1",
-            "ppp0",
-        ]
-        for ifname in interfaces:
-            try:
-                ip = get_interface_ip(ifname)
-                break
-            except IOError:
-                pass
-    return ip or ''
-
 
 def from_dict_hook(in_dict: dict):
     if "type" in in_dict and "data" in in_dict:
@@ -66,20 +25,3 @@ def from_dict_hook(in_dict: dict):
                 in_dict["module"]), in_dict["type"])(**in_dict["data"])
     else:
         return in_dict
-
-
-def get_uuid():
-    return uuid.uuid1().hex
-
-
-def download_img(url):
-    if not url:
-        return ""
-    response = requests.get(url)
-    return "data:" + \
-        response.headers.get('Content-Type', 'image/jpg') + ";" + \
-        "base64," + base64.b64encode(response.content).decode("utf-8")
-
-
-def hash_str2int(line: str, mod: int = 10 ** 8) -> int:
-    return int(hashlib.sha1(line.encode("utf-8")).hexdigest(), 16) % mod