mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-08 20:42:30 +08:00
truncate text to fitin embedding model (#692)
### What problem does this PR solve? ### Type of change - [x] Refactoring
This commit is contained in:
@ -63,3 +63,7 @@ def num_tokens_from_string(string: str) -> int:
|
||||
num_tokens = len(encoder.encode(string))
|
||||
return num_tokens
|
||||
|
||||
|
||||
def truncate(string: str, max_len: int) -> int:
|
||||
"""Returns truncated text if the length of text exceed max_len."""
|
||||
return encoder.decode(encoder.encode(string)[:max_len])
|
||||
|
||||
Reference in New Issue
Block a user