style: fix typo and format code (#2618)

### What problem does this PR solve?

- Fix typo
- Remove unused import
- Format code

### Type of change

- [x] Other (please describe): typo and format
This commit is contained in:
yqkcn
2024-09-27 13:17:25 +08:00
committed by GitHub
parent 4c0b79c4f6
commit 34abcf7704
5 changed files with 12 additions and 16 deletions

View File

@ -78,11 +78,9 @@ encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")
def num_tokens_from_string(string: str) -> int:
"""Returns the number of tokens in a text string."""
try:
num_tokens = len(encoder.encode(string))
return num_tokens
except Exception as e:
pass
return 0
return len(encoder.encode(string))
except Exception:
return 0
def truncate(string: str, max_len: int) -> str: