Integration with Infinity (#2894)

### What problem does this PR solve?

Integration with Infinity

- Replaced ELASTICSEARCH with dataStoreConn
- Renamed deleteByQuery with delete
- Renamed bulk to upsertBulk
- getHighlight, getAggregation
- Fix KGSearch.search
- Moved Dealer.sql_retrieval to es_conn.py


### Type of change

- [x] Refactoring
This commit is contained in:
Zhichang Yu
2024-11-12 14:59:41 +08:00
committed by GitHub
parent 00b6000b76
commit f4c52371ab
42 changed files with 2647 additions and 1878 deletions

View File

@ -50,8 +50,8 @@ class Document(Base):
return res.content
def list_chunks(self,page=1, page_size=30, keywords="", id:str=None):
data={"keywords": keywords,"page":page,"page_size":page_size,"id":id}
def list_chunks(self,page=1, page_size=30, keywords=""):
data={"keywords": keywords,"page":page,"page_size":page_size}
res = self.get(f'/datasets/{self.dataset_id}/documents/{self.id}/chunks', data)
res = res.json()
if res.get("code") == 0:

View File

@ -126,6 +126,7 @@ def test_delete_chunk_with_success(get_api_key_fixture):
docs = ds.upload_documents(documents)
doc = docs[0]
chunk = doc.add_chunk(content="This is a chunk addition test")
sleep(5)
doc.delete_chunks([chunk.id])
@ -146,6 +147,8 @@ def test_update_chunk_content(get_api_key_fixture):
docs = ds.upload_documents(documents)
doc = docs[0]
chunk = doc.add_chunk(content="This is a chunk addition test")
# For ElasticSearch, the chunk is not searchable in shot time (~2s).
sleep(3)
chunk.update({"content":"This is a updated content"})
def test_update_chunk_available(get_api_key_fixture):
@ -165,7 +168,9 @@ def test_update_chunk_available(get_api_key_fixture):
docs = ds.upload_documents(documents)
doc = docs[0]
chunk = doc.add_chunk(content="This is a chunk addition test")
chunk.update({"available":False})
# For ElasticSearch, the chunk is not searchable in shot time (~2s).
sleep(3)
chunk.update({"available":0})
def test_retrieve_chunks(get_api_key_fixture):