mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-12-08 20:42:30 +08:00
style: fix typo and format code (#2618)
### What problem does this PR solve? - Fix typo - Remove unused import - Format code ### Type of change - [x] Other (please describe): typo and format
This commit is contained in:
@ -1,6 +1,6 @@
|
||||
import re
|
||||
|
||||
from graphrag.index import build_knowlege_graph_chunks
|
||||
from graphrag.index import build_knowledge_graph_chunks
|
||||
from rag.app import naive
|
||||
from rag.nlp import rag_tokenizer, tokenize_chunks
|
||||
|
||||
@ -15,9 +15,9 @@ def chunk(filename, binary, tenant_id, from_page=0, to_page=100000,
|
||||
parser_config["layout_recognize"] = False
|
||||
sections = naive.chunk(filename, binary, from_page=from_page, to_page=to_page, section_only=True,
|
||||
parser_config=parser_config, callback=callback)
|
||||
chunks = build_knowlege_graph_chunks(tenant_id, sections, callback,
|
||||
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
|
||||
)
|
||||
chunks = build_knowledge_graph_chunks(tenant_id, sections, callback,
|
||||
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
|
||||
)
|
||||
for c in chunks: c["docnm_kwd"] = filename
|
||||
|
||||
doc = {
|
||||
|
||||
@ -20,7 +20,6 @@ from abc import ABC
|
||||
from openai import OpenAI
|
||||
import openai
|
||||
from ollama import Client
|
||||
from volcengine.maas.v2 import MaasService
|
||||
from rag.nlp import is_english
|
||||
from rag.utils import num_tokens_from_string
|
||||
from groq import Groq
|
||||
@ -29,6 +28,7 @@ import json
|
||||
import requests
|
||||
import asyncio
|
||||
|
||||
|
||||
class Base(ABC):
|
||||
def __init__(self, key, model_name, base_url):
|
||||
self.client = OpenAI(api_key=key, base_url=base_url)
|
||||
|
||||
@ -78,11 +78,9 @@ encoder = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
||||
def num_tokens_from_string(string: str) -> int:
|
||||
"""Returns the number of tokens in a text string."""
|
||||
try:
|
||||
num_tokens = len(encoder.encode(string))
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
pass
|
||||
return 0
|
||||
return len(encoder.encode(string))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def truncate(string: str, max_len: int) -> str:
|
||||
|
||||
Reference in New Issue
Block a user