add support for Voyage AI (#2159)

### What problem does this PR solve?

#1853  #2138 add support for Voyage AI

### Type of change
- [x] New Feature (non-breaking change which adds functionality)

---------

Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
This commit is contained in:
黄腾
2024-08-29 16:14:49 +08:00
committed by GitHub
parent 15b78bd894
commit 99993e5026
9 changed files with 141 additions and 5 deletions

View File

@ -623,3 +623,24 @@ class BaiduYiyanEmbed(Base):
np.array([r["embedding"] for r in res["data"]]),
res["usage"]["total_tokens"],
)
class VoyageEmbed(Base):
def __init__(self, key, model_name, base_url=None):
import voyageai
self.client = voyageai.Client(api_key=key)
self.model_name = model_name
def encode(self, texts: list, batch_size=32):
res = self.client.embed(
texts=texts, model=self.model_name, input_type="document"
)
return np.array(res.embeddings), res.total_tokens
def encode_queries(self, text):
res = self.client.embed
res = self.client.embed(
texts=text, model=self.model_name, input_type="query"
)
return np.array(res.embeddings), res.total_tokens