style: fix typo and format code (#2618)

### What problem does this PR solve?

- Fix typo
- Remove unused import
- Format code

### Type of change

- [x] Other (please describe): typo and format
This commit is contained in:
yqkcn
2024-09-27 13:17:25 +08:00
committed by GitHub
parent 4c0b79c4f6
commit 34abcf7704
5 changed files with 12 additions and 16 deletions

View File

@ -1,6 +1,6 @@
import re
from graphrag.index import build_knowlege_graph_chunks
from graphrag.index import build_knowledge_graph_chunks
from rag.app import naive
from rag.nlp import rag_tokenizer, tokenize_chunks
@ -15,9 +15,9 @@ def chunk(filename, binary, tenant_id, from_page=0, to_page=100000,
parser_config["layout_recognize"] = False
sections = naive.chunk(filename, binary, from_page=from_page, to_page=to_page, section_only=True,
parser_config=parser_config, callback=callback)
chunks = build_knowlege_graph_chunks(tenant_id, sections, callback,
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
)
chunks = build_knowledge_graph_chunks(tenant_id, sections, callback,
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
)
for c in chunks: c["docnm_kwd"] = filename
doc = {

View File

@ -20,7 +20,6 @@ from abc import ABC
from openai import OpenAI
import openai
from ollama import Client
from volcengine.maas.v2 import MaasService
from rag.nlp import is_english
from rag.utils import num_tokens_from_string
from groq import Groq
@ -29,6 +28,7 @@ import json
import requests
import asyncio
class Base(ABC):
def __init__(self, key, model_name, base_url):
self.client = OpenAI(api_key=key, base_url=base_url)

View File

@ -78,11 +78,9 @@ encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")
def num_tokens_from_string(string: str) -> int:
"""Returns the number of tokens in a text string."""
try:
num_tokens = len(encoder.encode(string))
return num_tokens
except Exception as e:
pass
return 0
return len(encoder.encode(string))
except Exception:
return 0
def truncate(string: str, max_len: int) -> str: