Integration with Infinity (#2894)

### What problem does this PR solve?

Integration with Infinity

- Replaced ELASTICSEARCH with dataStoreConn
- Renamed deleteByQuery with delete
- Renamed bulk to upsertBulk
- getHighlight, getAggregation
- Fix KGSearch.search
- Moved Dealer.sql_retrieval to es_conn.py


### Type of change

- [x] Refactoring

docker/.env

@@ -19,6 +19,11 @@ KIBANA_PASSWORD=infini_rag_flow
 # Update it according to the available memory in the host machine.
 MEM_LIMIT=8073741824
 
+# Port to expose Infinity API to the host
+INFINITY_THRIFT_PORT=23817
+INFINITY_HTTP_PORT=23820
+INFINITY_PSQL_PORT=5432
+
 # The password for MySQL. 
 # When updated, you must revise the `mysql.password` entry in service_conf.yaml.
 MYSQL_PASSWORD=infini_rag_flow

docker/docker-compose-base.yml

@@ -6,6 +6,7 @@ services:
       - esdata01:/usr/share/elasticsearch/data
     ports:
       - ${ES_PORT}:9200
+    env_file: .env
     environment:
       - node.name=es01
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
@@ -27,12 +28,40 @@ services:
       retries: 120
     networks:
       - ragflow
-    restart: always
+    restart: on-failure
+
+  # infinity:
+  #   container_name: ragflow-infinity
+  #   image: infiniflow/infinity:v0.5.0-dev2
+  #   volumes:
+  #     - infinity_data:/var/infinity
+  #   ports:
+  #     - ${INFINITY_THRIFT_PORT}:23817
+  #     - ${INFINITY_HTTP_PORT}:23820
+  #     - ${INFINITY_PSQL_PORT}:5432
+  #   env_file: .env
+  #   environment:
+  #     - TZ=${TIMEZONE}
+  #   mem_limit: ${MEM_LIMIT}
+  #   ulimits:
+  #     nofile:
+  #       soft: 500000
+  #       hard: 500000
+  #   networks:
+  #     - ragflow
+  #   healthcheck:
+  #     test: ["CMD", "curl", "http://localhost:23820/admin/node/current"]
+  #     interval: 10s
+  #     timeout: 10s
+  #     retries: 120
+  #   restart: on-failure
+
 
   mysql:
     # mysql:5.7 linux/arm64 image is unavailable.
     image: mysql:8.0.39
     container_name: ragflow-mysql
+    env_file: .env
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD}
       - TZ=${TIMEZONE}
@@ -55,7 +84,7 @@ services:
       interval: 10s
       timeout: 10s
       retries: 3
-    restart: always
+    restart: on-failure
 
   minio:
     image: quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z
@@ -64,6 +93,7 @@ services:
     ports:
       - ${MINIO_PORT}:9000
       - ${MINIO_CONSOLE_PORT}:9001
+    env_file: .env
     environment:
       - MINIO_ROOT_USER=${MINIO_USER}
       - MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}
@@ -72,25 +102,28 @@ services:
       - minio_data:/data
     networks:
       - ragflow
-    restart: always
+    restart: on-failure
 
   redis:
     image: valkey/valkey:8
     container_name: ragflow-redis
     command: redis-server --requirepass ${REDIS_PASSWORD} --maxmemory 128mb --maxmemory-policy allkeys-lru
+    env_file: .env
     ports:
       - ${REDIS_PORT}:6379
     volumes:
       - redis_data:/data
     networks:
       - ragflow
-    restart: always
+    restart: on-failure
 
 
 
 volumes:
   esdata01:
     driver: local
+  infinity_data:
+    driver: local
   mysql_data:
     driver: local
   minio_data:

docker/docker-compose.yml

@@ -1,6 +1,5 @@
 include:
-  - path: ./docker-compose-base.yml
-    env_file: ./.env
+  - ./docker-compose-base.yml
 
 services:
   ragflow:
@@ -15,19 +14,21 @@ services:
       - ${SVR_HTTP_PORT}:9380
       - 80:80
       - 443:443
-      - 5678:5678
     volumes:
       - ./service_conf.yaml:/ragflow/conf/service_conf.yaml
       - ./ragflow-logs:/ragflow/logs
       - ./nginx/ragflow.conf:/etc/nginx/conf.d/ragflow.conf
       - ./nginx/proxy.conf:/etc/nginx/proxy.conf
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+    env_file: .env
     environment:
       - TZ=${TIMEZONE}
       - HF_ENDPOINT=${HF_ENDPOINT}
       - MACOS=${MACOS}
     networks:
       - ragflow
-    restart: always
+    restart: on-failure
+    # https://docs.docker.com/engine/daemon/prometheus/#create-a-prometheus-configuration
+    # If you're using Docker Desktop, the --add-host flag is optional. This flag makes sure that the host's internal IP gets exposed to the Prometheus container. 
     extra_hosts:
       - "host.docker.internal:host-gateway"