langflow/.github/workflows/python_test.yml

name: Python tests

on:
  workflow_call:
    secrets:
      OPENAI_API_KEY:
        required: true
      ANTHROPIC_API_KEY:
        required: true
      CODECOV_TOKEN:
        required: false
    inputs:
      python-versions:
        description: "(Optional) Python versions to test"
        required: true
        type: string
        default: "['3.10', '3.11', '3.12', '3.13']"
      ref:
        description: "(Optional) ref to checkout"
        required: false
        type: string
      nightly:
        description: "Whether run is from the nightly build"
        required: false
        type: boolean
        default: false
      runs-on:
        description: "Runner to use for the tests"
        required: false
        type: string
        default: "ubuntu-latest"
  workflow_dispatch:
    inputs:
      python-versions:
        description: "(Optional) Python versions to test"
        required: true
        type: string
        default: "['3.10', '3.11', '3.12', '3.13']"
      runs-on:
        description: "Runner to use for the tests"
        required: false
        type: choice
        options:
          - ubuntu-latest
          - self-hosted
          - '["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral"]'
        default: ubuntu-latest
env:
  POETRY_VERSION: "1.8.2"
  NODE_VERSION: "22"
  PYTEST_RUN_PATH: "src/backend/tests"
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

jobs:
  build:
    name: Unit Tests - Python ${{ matrix.python-version }} - Group ${{ matrix.group }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13"]' ) }}
        splitCount: [5]
        group: [1, 2, 3, 4, 5]
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Setup Node.js
        uses: actions/setup-node@v6
        id: setup-node
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false

      - name: Install the project
        run: uv sync

      - name: Generate dynamic coverage configuration
        run: |
          echo "Generating dynamic coverage configuration..."
          python3 scripts/generate_coverage_config.py
          echo "Generated .coveragerc with the following exclusions:"
          echo "Bundled components and legacy files excluded from coverage"

      - name: Run unit tests
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 15
          max_attempts: 2
          command: make unit_tests args="-x -vv --splits ${{ matrix.splitCount }} --group ${{ matrix.group }} --reruns 5 --cov --cov-config=src/backend/.coveragerc --cov-report=xml --cov-report=html"

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        if: matrix.python-version == '3.10'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: backend
          name: backend-coverage-group-${{ matrix.group }}
          fail_ci_if_error: false
          directory: ./

      - name: Upload coverage artifacts
        uses: actions/upload-artifact@v6
        if: matrix.python-version == '3.10'
        with:
          name: backend-coverage-report-group-${{ matrix.group }}
          path: |
            coverage.xml
            htmlcov/
          retention-days: 30

  integration-tests:
    name: Integration Tests - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13"]' ) }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false
      - name: Install the project
        run: uv sync
      - name: Run integration tests
        run: make integration_tests_no_api_keys
        env:
          PYLEAK_LOG_LEVEL: debug # enable pyleak logging
          DO_NOT_TRACK: true # disable telemetry reporting

  lfx-tests:
    name: LFX Tests - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13"]' ) }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false
      - name: Run lfx tests
        run: make lfx_tests
        env:
          DO_NOT_TRACK: true # disable telemetry reporting

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        if: matrix.python-version == '3.10'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./src/lfx/coverage.xml
          flags: lfx
          name: lfx-coverage
          fail_ci_if_error: false
          directory: ./src/lfx/

      - name: Upload coverage artifacts
        uses: actions/upload-artifact@v6
        if: matrix.python-version == '3.10'
        with:
          name: lfx-coverage-report
          path: |
            src/lfx/coverage.xml
            src/lfx/htmlcov/
          retention-days: 30

  test-cli:
    name: Test CLI - Python ${{ matrix.python-version }}
    runs-on: ${{ (inputs['runs-on'] && startsWith(format('{0}', inputs['runs-on']), '[') && fromJSON(inputs['runs-on'])) || inputs['runs-on'] || github.event.inputs['runs-on'] || 'ubuntu-latest' }}
    strategy:
      matrix:
        python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12", "3.13"]') }}
    steps:
      - name: Check out the code at a specific ref
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.ref || github.ref }}
      - name: "Setup Environment"
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
          python-version: ${{ matrix.python-version }}
          prune-cache: false

      - name: Check Version
        id: check-version
        # We need to print $3 because langflow-base is a dependency of langflow
        # For langlow we'd use print $2
        run: |
          version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1)
          url="https://pypi.org/pypi/langflow-base/json"
          if [ ${{ inputs.nightly }} == true ]; then
            url="https://pypi.org/pypi/langflow-base-nightly/json"
          fi

          last_released_version=$(curl -s $url | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
          if [ "$version" != "$last_released_version" ]; then
            echo "Version $version has not been released yet. Skipping the rest of the job."
            echo skipped=true >> $GITHUB_OUTPUT
            exit 0
          else
            echo version=$version >> $GITHUB_OUTPUT
            echo skipped=false >> $GITHUB_OUTPUT
          fi
      - name: Build wheel
        if: steps.check-version.outputs.skipped == 'false'
        run: |
          make build main=true
      - name: Install wheel and Test CLI
        if: steps.check-version.outputs.skipped == 'false'
        run: |
          uv venv new-venv
          source new-venv/bin/activate
          uv pip install dist/*.whl
      - name: Test CLI
        if: steps.check-version.outputs.skipped == 'false'
        run: |
          source new-venv/bin/activate
          python -m langflow run --host localhost --port 7860 --backend-only &
          SERVER_PID=$!
          # Wait for the server to start
          timeout 120 bash -c 'until curl -f http://localhost:7860/api/v1/auto_login; do sleep 5; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
          # Terminate the server
          kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
          sleep 20 # give the server some time to terminate
          # Check if the server is still running
          if kill -0 $SERVER_PID 2>/dev/null; then
            echo "Failed to terminate the server"
            exit 0
          else
            echo "Server terminated successfully"
          fi