truncate text to fitin embedding model (#692)

### What problem does this PR solve?


### Type of change

- [x] Refactoring
This commit is contained in:
KevinHuSh
2024-05-09 11:35:08 +08:00
committed by GitHub
parent bca63ad571
commit 4153a36683
2 changed files with 11 additions and 6 deletions

View File

@ -63,3 +63,7 @@ def num_tokens_from_string(string: str) -> int:
num_tokens = len(encoder.encode(string))
return num_tokens
def truncate(string: str, max_len: int) -> int:
"""Returns truncated text if the length of text exceed max_len."""
return encoder.decode(encoder.encode(string)[:max_len])