From 320cbf545f043ff8bcb3b5744377c910a7db7fbc Mon Sep 17 00:00:00 2001 From: KevinHuSh Date: Wed, 28 Feb 2024 18:57:43 +0800 Subject: [PATCH] docker debugging modification (#82) --- Dockerfile | 16 ++++--- api/db/init_data.py | 2 +- docker/docker-compose.yml | 29 +++++++----- docker/entrypoint.sh | 24 +++++++--- docker/nginx.conf | 92 +++++++++++++++++++-------------------- docker/service_conf.yaml | 8 ++-- rag/svr/task_executor.py | 2 +- 7 files changed, 99 insertions(+), 74 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09ee6e14e..b49434fe7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,20 @@ FROM infiniflow/ragflow-base:v1.0 +USER root WORKDIR /ragflow -COPY . ./ -RUN cd ./web && npm i && npm build +ADD ./web ./web +RUN cd ./web && npm i && npm run build -ENV PYTHONPATH=/ragflow +ADD ./api ./api +ADD ./conf ./conf +ADD ./deepdoc ./deepdoc +ADD ./rag ./rag + +ENV PYTHONPATH=/ragflow/ ENV HF_ENDPOINT=https://hf-mirror.com -COPY docker/entrypoint.sh ./ +ADD docker/entrypoint.sh ./entrypoint.sh RUN chmod +x ./entrypoint.sh -ENTRYPOINT ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["./entrypoint.sh"] diff --git a/api/db/init_data.py b/api/db/init_data.py index efa2d8a2a..b0f0b2f25 100644 --- a/api/db/init_data.py +++ b/api/db/init_data.py @@ -66,7 +66,7 @@ def init_superuser(): chat_mdl = LLMBundle(tenant["id"], LLMType.CHAT, tenant["llm_id"]) msg = chat_mdl.chat(system="", history=[{"role": "user", "content": "Hello!"}], gen_conf={}) if msg.find("ERROR: ") == 0: - print("\33[91m【ERROR】\33[0m: ", "'{}' dosen't work. {}".format(tenant["llm_id"]), msg) + print("\33[91m【ERROR】\33[0m: ", "'{}' dosen't work. {}".format(tenant["llm_id"], msg)) embd_mdl = LLMBundle(tenant["id"], LLMType.EMBEDDING, tenant["embd_id"]) v, c = embd_mdl.encode(["Hello!"]) if c == 0: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1cd7f0313..aa3042a06 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -19,13 +19,19 @@ services: memlock: soft: -1 hard: -1 + healthcheck: + test: ["CMD-SHELL", "curl http://localhost:9200"] + interval: 10s + timeout: 10s + retries: 120 networks: - ragflow restart: always kibana: - depends_on: - - es01 + depends_on: + es01: + condition: service_healthy image: docker.elastic.co/kibana/kibana:${STACK_VERSION} container_name: ragflow-kibana volumes: @@ -60,10 +66,10 @@ services: networks: - ragflow healthcheck: - test: [ "CMD-SHELL", "curl --silent localhost:3306 >/dev/null || exit 1" ] - interval: 10s - timeout: 10s - retries: 3 + test: ["CMD", "mysqladmin" ,"ping", "-uroot", "-p${MYSQL_PASSWORD}"] + interval: 10s + timeout: 10s + retries: 3 restart: always @@ -86,17 +92,20 @@ services: ragflow: depends_on: - - es01 - - mysql - - minio + mysql: + condition: service_healthy + es01: + condition: service_healthy image: infiniflow/ragflow:v1.0 container_name: ragflow-server ports: - ${SVR_HTTP_PORT}:9380 + - 80:80 + - 443:443 volumes: - ./service_conf.yaml:/ragflow/conf/service_conf.yaml - - ./nginx.conf:/etc/nginx/nginx.conf - ./ragflow-logs:/ragflow/logs + - ./nginx.conf:/etc/nginx/conf.d/ragflow.conf networks: - ragflow restart: always diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 99661f14e..b75b578ca 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,24 +1,34 @@ #!/bin/bash -python rag/svr/task_broker.py & +/usr/sbin/nginx + +export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/ + +PY=/root/miniconda3/envs/py11/bin/python + + function task_exe(){ - while [ 1 -eq 1 ];do mpirun -n 2 python rag/svr/task_executor.py ; done + sleep 60; + while [ 1 -eq 1 ];do mpirun -n 2 --allow-run-as-root $PY rag/svr/task_executor.py ; done } function watch_broker(){ - while [ 1 -eq 1];do + while [ 1 -eq 1 ];do C=`ps aux|grep "task_broker.py"|grep -v grep|wc -l`; if [ $C -lt 1 ];then - python rag/svr/task_broker.py & + $PY rag/svr/task_broker.py & fi sleep 5; done } +function task_bro(){ + sleep 60; + watch_broker; +} +task_bro & task_exe & -sleep 10; -watch_broker & -python api/ragflow_server.py \ No newline at end of file +$PY api/ragflow_server.py \ No newline at end of file diff --git a/docker/nginx.conf b/docker/nginx.conf index af83a6142..d81596017 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -1,46 +1,46 @@ -upstream api_service { - server 127.0.0.1:9380; - keepalive 2000; -} - -server { - listen 5101; - # server_name xxx.com; - - # gzip config - gzip on; - gzip_min_length 1k; - gzip_comp_level 9; - gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; - gzip_vary on; - gzip_disable "MSIE [1-6]\."; - - - # remove api server header - proxy_hide_header Access-Control-Allow-Origin; - proxy_hide_header Access-Control-Allow-Methods; - proxy_hide_header Access-Control-Allow-Headers; - - root /ragflow/web/dist; - - location / { - index index.html; - try_files $uri $uri/ /index.html; - } - - # Cache-Control: max-age、Expires - location ~ ^/static/(css|js|media)/ { - expires 10y; - access_log off; - } - - # api proxy - location ^~/v1 { - rewrite ^/v1/(.*)$ /$1 break; - proxy_pass http://api_service/; - proxy_set_header Host $http_host; - proxy_set_header Connection close; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Server $host; - } -} \ No newline at end of file +upstream api_service { + server ragflow:9380; + keepalive 2000; +} + +server { + listen 80; + server_name ragflow.io; + + # gzip config + gzip on; + gzip_min_length 1k; + gzip_comp_level 9; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + + # remove api server header + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header Access-Control-Allow-Headers; + + root /ragflow/web/dist; + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Cache-Control: max-age、Expires + location ~ ^/static/(css|js|media)/ { + expires 10y; + access_log off; + } + + # api proxy + location ^~/v1 { + rewrite ^/v1/(.*)$ /$1 break; + proxy_pass http://api_service/; + proxy_set_header Host $http_host; + proxy_set_header Connection close; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Server $host; + } +} diff --git a/docker/service_conf.yaml b/docker/service_conf.yaml index 5bf7b782a..fdffebf54 100644 --- a/docker/service_conf.yaml +++ b/docker/service_conf.yaml @@ -5,16 +5,16 @@ mysql: name: 'rag_flow' user: 'root' password: 'infini_rag_flow' - host: '127.0.0.1' - port: 5455 + host: 'mysql' + port: 3306 max_connections: 100 stale_timeout: 30 minio: user: 'rag_flow' password: 'infini_rag_flow' - host: '127.0.0.1:9000' + host: 'minio:9000' es: - hosts: 'http://127.0.0.1:1200' + hosts: 'http://es01:9200' user_default_llm: factory: '通义千问' api_key: 'sk-xxxxxxxxxxxxx' diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index 042e2d42e..0d0b4e490 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -48,7 +48,7 @@ from api.utils.file_utils import get_project_base_directory BATCH_SIZE = 64 FACTORY = { - ParserType.GENERAL.value: manual, + ParserType.GENERAL.value: laws, ParserType.PAPER.value: paper, ParserType.BOOK.value: book, ParserType.PRESENTATION.value: presentation,