diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 74aeeee9f..934005ede 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -202,8 +202,11 @@ jobs: echo -e "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> docker/.env echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV} + # Patch entrypoint.sh for coverage + sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' docker/entrypoint.sh + sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d - uv sync --python 3.12 --group test --frozen && uv pip install sdk/python + uv sync --python 3.12 --group test --frozen && uv pip install -e sdk/python - name: Run sdk tests against Elasticsearch run: | @@ -212,7 +215,7 @@ jobs: echo "Waiting for service to be available..." sleep 5 done - source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log + source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log - name: Run web api tests against Elasticsearch run: | @@ -221,7 +224,7 @@ jobs: echo "Waiting for service to be available..." sleep 5 done - source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api/ 2>&1 | tee es_web_api_test.log + source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api 2>&1 | tee es_web_api_test.log - name: Run http api tests against Elasticsearch run: | @@ -232,7 +235,7 @@ jobs: done source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee es_http_api_test.log - - name: RAGFlow CLI retrieval test + - name: RAGFlow CLI retrieval test Elasticsearch env: PYTHONPATH: ${{ github.workspace }} run: | @@ -333,7 +336,43 @@ jobs: run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync" run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'" - - name: Collect ragflow log + - name: Stop ragflow to save coverage Elasticsearch + if: ${{ !cancelled() }} + run: | + # Send SIGINT to ragflow_server.py to trigger coverage save + PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1) + if [ -n "$PID" ]; then + echo "Sending SIGINT to ragflow_server.py (PID: $PID)..." + sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID + # Wait for process to exit and coverage file to be written + sleep 10 + else + echo "ragflow_server.py not found!" + fi + sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} stop + + - name: Generate server coverage report Elasticsearch + if: ${{ !cancelled() }} + run: | + # .coverage file should be in docker/ragflow-logs/.coverage + if [ -f docker/ragflow-logs/.coverage ]; then + echo "Found .coverage file" + cp docker/ragflow-logs/.coverage .coverage + source .venv/bin/activate + # Create .coveragerc to map container paths to host paths + echo "[paths]" > .coveragerc + echo "source =" >> .coveragerc + echo " ." >> .coveragerc + echo " /ragflow" >> .coveragerc + coverage xml -o coverage-es-server.xml + rm .coveragerc + # Clean up for next run + sudo rm docker/ragflow-logs/.coverage + else + echo ".coverage file not found!" + fi + + - name: Collect ragflow log Elasticsearch if: ${{ !cancelled() }} run: | if [ -d docker/ragflow-logs ]; then @@ -362,7 +401,7 @@ jobs: echo "Waiting for service to be available..." sleep 5 done - source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log + source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log - name: Run web api tests against Infinity run: | @@ -371,7 +410,7 @@ jobs: echo "Waiting for service to be available..." sleep 5 done - source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api/ 2>&1 | tee infinity_web_api_test.log + source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api/test_api_app 2>&1 | tee infinity_web_api_test.log - name: Run http api tests against Infinity run: | @@ -382,7 +421,7 @@ jobs: done source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee infinity_http_api_test.log - - name: RAGFlow CLI retrieval test (Infinity) + - name: RAGFlow CLI retrieval test Infinity env: PYTHONPATH: ${{ github.workspace }} run: | @@ -483,6 +522,47 @@ jobs: run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync" run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'" + - name: Stop ragflow to save coverage Infinity + if: ${{ !cancelled() }} + run: | + # Send SIGINT to ragflow_server.py to trigger coverage save + PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1) + if [ -n "$PID" ]; then + echo "Sending SIGINT to ragflow_server.py (PID: $PID)..." + sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID + # Wait for process to exit and coverage file to be written + sleep 10 + else + echo "ragflow_server.py not found!" + fi + sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} stop + + - name: Generate server coverage report Infinity + if: ${{ !cancelled() }} + run: | + # .coverage file should be in docker/ragflow-logs/.coverage + if [ -f docker/ragflow-logs/.coverage ]; then + echo "Found .coverage file" + cp docker/ragflow-logs/.coverage .coverage + source .venv/bin/activate + # Create .coveragerc to map container paths to host paths + echo "[paths]" > .coveragerc + echo "source =" >> .coveragerc + echo " ." >> .coveragerc + echo " /ragflow" >> .coveragerc + coverage xml -o coverage-infinity-server.xml + rm .coveragerc + else + echo ".coverage file not found!" + fi + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + - name: Collect ragflow log if: ${{ !cancelled() }} run: | @@ -493,6 +573,7 @@ jobs: echo "No docker/ragflow-logs directory found; skipping log collection" fi sudo rm -rf docker/ragflow-logs || true + - name: Stop ragflow:nightly if: always() # always run this step even if previous steps failed run: | diff --git a/pyproject.toml b/pyproject.toml index 4ae26ae99..aada385ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,6 +172,7 @@ test = [ "requests>=2.32.2", "requests-toolbelt>=1.0.0", "pycryptodomex==3.20.0", + "codecov>=2.1.13", ] [[tool.uv.index]] diff --git a/uv.lock b/uv.lock index a1b0323b2..fcb72e077 100644 --- a/uv.lock +++ b/uv.lock @@ -1096,6 +1096,19 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/e1/3714a2f371985215c219c2a70953d38e3eed81ef165aed061d21de0e998b/cobble-0.1.4-py3-none-any.whl", hash = "sha256:36c91b1655e599fd428e2b95fdd5f0da1ca2e9f1abb0bc871dec21a0e78a2b44", size = 3984, upload-time = "2024-06-01T18:11:07.911Z" }, ] +[[package]] +name = "codecov" +version = "2.1.13" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "coverage" }, + { name = "requests" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/bb/594b26d2c85616be6195a64289c578662678afa4910cef2d3ce8417cf73e/codecov-2.1.13.tar.gz", hash = "sha256:2362b685633caeaf45b9951a9b76ce359cd3581dd515b430c6c3f5dfb4d92a8c", size = 21416, upload-time = "2023-04-17T23:11:39.779Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/af/02/18785edcdf6266cdd6c6dc7635f1cbeefd9a5b4c3bb8aff8bd681e9dd095/codecov-2.1.13-py2.py3-none-any.whl", hash = "sha256:c2ca5e51bba9ebb43644c43d0690148a55086f7f5e6fd36170858fa4206744d5", size = 16512, upload-time = "2023-04-17T23:11:37.344Z" }, +] + [[package]] name = "cohere" version = "5.6.2" @@ -6223,6 +6236,7 @@ dependencies = [ [package.dev-dependencies] test = [ + { name = "codecov" }, { name = "hypothesis" }, { name = "openpyxl" }, { name = "pillow" }, @@ -6357,6 +6371,7 @@ requires-dist = [ [package.metadata.requires-dev] test = [ + { name = "codecov", specifier = ">=2.1.13" }, { name = "hypothesis", specifier = ">=6.132.0" }, { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pillow", specifier = ">=10.4.0,<13.0.0" },