pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2026-02-02 00:25:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move token related functions to common (#10942)

Browse Source 
### What problem does this PR solve?

As title

### Type of change

- [x] Refactoring

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2025-11-03 08:50:05 +08:00
committed by GitHub
parent 44f2d6f5da
commit 360f5c1179
25 changed files with 529 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rag/llm/chat_model.py
View File

@ -36,7 +36,7 @@ from zhipuai import ZhipuAI
from rag.llm import FACTORY_DEFAULT_BASE_URL, LITELLM_PROVIDER_PREFIX, SupportedLiteLLMProvider
from rag.nlp import is_chinese, is_english
from rag.utils import num_tokens_from_string, total_token_count_from_response
from common.token_utils import num_tokens_from_string, total_token_count_from_response
# Error message constants

2
rag/llm/cv_model.py
View File

@ -30,7 +30,7 @@ from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI
from rag.nlp import is_english
from rag.prompts.generator import vision_llm_describe_prompt
from rag.utils import num_tokens_from_string, total_token_count_from_response
from common.token_utils import num_tokens_from_string, total_token_count_from_response
class Base(ABC):

2
rag/llm/embedding_model.py
View File

@ -28,7 +28,7 @@ from openai import OpenAI
from zhipuai import ZhipuAI
from api.utils.log_utils import log_exception
from rag.utils import num_tokens_from_string, truncate
from common.token_utils import num_tokens_from_string, truncate
from api import settings
import logging

2
rag/llm/rerank_model.py
View File

@ -23,7 +23,7 @@ import requests
from yarl import URL
from api.utils.log_utils import log_exception
from rag.utils import num_tokens_from_string, truncate, total_token_count_from_response
from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response
class Base(ABC):
def __init__(self, key, model_name, **kwargs):

2
rag/llm/sequence2txt_model.py
View File

@ -24,7 +24,7 @@ import requests
from openai import OpenAI
from openai.lib.azure import AzureOpenAI
from rag.utils import num_tokens_from_string
from common.token_utils import num_tokens_from_string
class Base(ABC):

2
rag/llm/tts_model.py
View File

@ -36,7 +36,7 @@ import requests
import websocket
from pydantic import BaseModel, conint
from rag.utils import num_tokens_from_string
from common.token_utils import num_tokens_from_string
class ServeReferenceAudio(BaseModel):