From 8b70a2deeb0d10f4378aae1eb30e8a4bd457b96b Mon Sep 17 00:00:00 2001 From: ogabrielluiz Date: Mon, 20 Apr 2026 08:46:53 -0300 Subject: [PATCH] docs(bench): post-fix snapshot numbers and narrative (VAL-01) - Add .planning/benchmarks/post-2026-04-20.md with before/after comparison table covering the 4 authoritative scenarios + 2 sentinel rows (one promoted to authoritative per D-12 amendment), MEAS-03 six-checkpoint breakdown for the 3 checkpoint-emitting scenarios, narrative summary of wins (Phase 3 / 5 / 5.5 / 4 / 6 IMP-11), Phase 3 ROADMAP reconciliation per D-17, known gaps, and CI gate evidence pointers. - Add machine-readable companion .planning/benchmarks/post-2026-04-20.json with schema_version=1, captured_ref, snapshot_run_id=24642673292, and scenarios + checkpoints_seconds maps. - Commit authoritative thresholds.json captured from snapshot run 24642673292 (captured_on=2026-04-20, captured_ref=12750/merge@eb2272ccc711768ecb11a3c0982aa419852bb17a). --- .planning/benchmarks/post-2026-04-20.json | 20 +++++ .planning/benchmarks/post-2026-04-20.md | 77 ++++++++++++++++++++ src/backend/tests/benchmarks/thresholds.json | 42 ++++++++--- 3 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 .planning/benchmarks/post-2026-04-20.json create mode 100644 .planning/benchmarks/post-2026-04-20.md diff --git a/.planning/benchmarks/post-2026-04-20.json b/.planning/benchmarks/post-2026-04-20.json new file mode 100644 index 0000000000..04d7d02134 --- /dev/null +++ b/.planning/benchmarks/post-2026-04-20.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "captured_on": "2026-04-20", + "captured_ref": "12750/merge@eb2272ccc711768ecb11a3c0982aa419852bb17a", + "snapshot_run_id": "24642673292", + "before_anchor": ".planning/benchmarks/baseline-2026-04-17.md", + "scenarios": { + "lfx_bare": {"before_ms": 17820, "after_ms": 10550, "delta_ms": -7270, "delta_pct": -40.8, "runs": 10, "stddev_ms": 99.0}, + "lfx_with_flow": {"before_ms": 18920, "after_ms": 16013, "delta_ms": -2907, "delta_pct": -15.4, "runs": 10, "stddev_ms": 228.6}, + "lfx_with_flow_prebaked": {"before_ms": 9520, "after_ms": 8425, "delta_ms": -1095, "delta_pct": -11.5, "runs": 10, "stddev_ms": 99.9}, + "langflow_run_no_change_restart": {"before_ms": null, "after_ms": 11325, "delta_ms": null, "delta_pct": null, "runs": 5, "stddev_ms": 83.7, "note": "New scenario in Phase 4; no pre-Phase-4 baseline"}, + "langflow_run_http_ready": {"before_ms": null, "after_ms": 0, "delta_ms": null, "delta_pct": null, "runs": 0, "stddev_ms": 0.0, "note": "Sentinel; pre-existing structlog marker issue deferred"}, + "lfx_reference_image": {"before_ms": null, "after_ms": 2404, "delta_ms": null, "delta_pct": null, "runs": 10, "stddev_ms": 20.9, "note": "New in Phase 5 plan 05-02; promoted from sentinel per D-12 amendment"} + }, + "checkpoints_seconds": { + "lfx_bare": {"process-start": 0.00, "after-imports": 0.58, "after-initialize-services": 0.93, "before-run-flow": 0.95, "after-component-index": 7.88, "after-run-flow": 8.71}, + "lfx_with_flow": {"process-start": 0.00, "after-imports": 8.22, "after-initialize-services": 8.58, "before-run-flow": 8.60, "after-component-index": 12.16, "after-run-flow": 13.85}, + "lfx_with_flow_prebaked": {"process-start": 0.00, "after-imports": 3.84, "after-initialize-services": 3.93, "before-run-flow": 3.93, "after-component-index": 5.72, "after-run-flow": 6.49} + } +} diff --git a/.planning/benchmarks/post-2026-04-20.md b/.planning/benchmarks/post-2026-04-20.md new file mode 100644 index 0000000000..c31bc7c1e6 --- /dev/null +++ b/.planning/benchmarks/post-2026-04-20.md @@ -0,0 +1,77 @@ +# Cold-Start Post-Fix Numbers - 2026-04-20 (Authoritative, Linux CI) + +**Source:** GitHub Actions build 24642673292 on `cold-start-improvements-v2` @ `11470f81079fe0157c52b0ee333720057d7db74b` +**Runner:** ubuntu-latest +**Python:** 3.13 +**Trigger:** `run-benchmark-snapshot` label on PR #12750 +**Before anchor:** `.planning/benchmarks/baseline-2026-04-17.md` (build 24541900208) +**Status:** Post-fix snapshot capturing the cumulative effect of Phases 2, 3, 4, 5, and 5.5. + +## Comparison table + +| Scenario | Before (ms) | After (ms) | Delta ms | Delta % | Runs | Stddev (ms) | Notes | +|----------|-------------|------------|----------|---------|------|-------------|-------| +| lfx_bare | 17820 | 10550 | -7270 | -40.8 | 10 | 99.0 | | +| lfx_with_flow | 18920 | 16013 | -2907 | -15.4 | 10 | 228.6 | | +| lfx_with_flow_prebaked | 9520 | 8425 | -1095 | -11.5 | 10 | 99.9 | | +| langflow_run_no_change_restart | -- | 11325 | -- | -- | 5 | 83.7 | New scenario in Phase 4 (plan 04-05); no pre-Phase-4 baseline. | +| langflow_run_http_ready | -- | 0 (sentinel) | -- | -- | 0 | 0 | Pre-existing sentinel; structlog marker issue deferred. See Known gaps. | +| lfx_reference_image | new in Phase 5 plan 05-02; no pre-Phase-5 baseline | 2404 | -- | -- | 10 | 20.9 | Authoritative this phase (promoted from sentinel per D-12 amendment). | + +## MEAS-03 checkpoints all fire + +All 6 checkpoints emit in the expected order (times in seconds from process start): + +| Checkpoint | lfx_bare | lfx_with_flow | lfx_with_flow_prebaked | +|-----------|---------|---------------|------------------------| +| process-start | 0.00 | 0.00 | 0.00 | +| after-imports | 0.58 | 8.22 | 3.84 | +| after-initialize-services | 0.93 | 8.58 | 3.93 | +| before-run-flow | 0.95 | 8.60 | 3.93 | +| after-component-index | 7.88 | 12.16 | 5.72 | +| after-run-flow | 8.71 | 13.85 | 6.49 | + +Scenario `langflow_run_no_change_restart` does not emit lfx._bench checkpoints (the scenario measures a full `langflow run --backend-only` process via TCP readiness probe, not the lfx checkpoint instrumentation). Full-process mean is reported in the Comparison table above. + +## Summary of wins + +Numbers below are sourced only from the authoritative `thresholds.json` runs. Internal checkpoint deltas (for example Phase 3 Graph cold-import 1440ms -> 356ms) are narrative color, not rows in the Comparison table. + +1. **Phase 3 Graph cold-import -75% (1440ms -> 356ms).** Deferred pandas, numpy, PIL, langchain_core, langchain_classic, langchain_text_splitters, and networkx off the Graph hot path. Absence verified by `src/lfx/tests/unit/test_import_absence.py`. The whole-scenario impact on `lfx_bare` (17.82s -> 10.55s) is dominated by this bucket. +2. **Phase 5 UV_COMPILE_BYTECODE=1 + multi-stage image.** `src/lfx/docker/Dockerfile` sets `ENV UV_COMPILE_BYTECODE=1` and splits deps and source into separate layers. The `lfx_with_flow` vs `lfx_with_flow_prebaked` delta (9.40s before Phase 5 Dockerfile, 7.59s in this snapshot) reflects the bytecode-compile cost pre-baked into the image layer. +3. **Phase 5.5 component-index cache-hit <500ms vs ~3.4s rebuild.** When the persisted `ComponentCache` on disk matches the installed `lfx` version, the full package walk is skipped in `_read_component_index`. Parity with the miss path is locked by `test_parity_cache_hit_vs_miss` + the `<500ms` perf assertion in `test_component_index.py::TestIDX08CacheHit`. +4. **Phase 4 SVC-01 hash-gate + SVC-02 asyncio.gather + SVC-03 asyncio.Event.** `langflow_run_no_change_restart` captures the combined effect: starter-projects skip via content-hash (SVC-01), two waves of lifespan tasks run concurrently (SVC-02), MCP startup is event-driven rather than `asyncio.sleep(10.0)` (SVC-03). The scenario mean is 11325ms (5 runs, stddev 83.7ms). +5. **Phase 6 IMP-11 lazy globals regression fix (commit 11470f8107).** While validating on CI, Phase 6 discovered and fixed an eager `importlib.import_module` regression in `validate.prepare_global_scope` - the helper was importing `langchain_*` module prefixes at scope-prep time rather than on attribute access. Replaced with a `_LazyExecGlobals` proxy that defers the import until an `eval`/`exec` call actually references the symbol. This is the reason `lfx_with_flow` shows a full -15.4% improvement vs the baseline; eager `langchain_classic` imports during component instantiation had been accounting for roughly 3s of the cost masked by the earlier interim snapshot. + +## Phase 3 ROADMAP reconciliation + +The 2026-04-17 scope change narrowed Phase 3 active scope to the Graph hot path. Langflow-run-specific plans (03-03 IMP-04, 03-04 IMP-05, 03-05 IMP-06, 03-06 IMP-01) were deferred to v2. Plans 03-02 (IMP-03 PIL), 03-07 (IMP-07 field_typing PEP 562), 03-08a through 03-08e (IMP-08 langchain_core Graph-path subset), and 03-09 (phase close) were delivered on the Graph path. ROADMAP.md checkboxes for these four remain `[ ]` as a bookkeeping gap that this document reconciles; authoritative status is STATE.md §"Phase 3 close notes" (Graph cold-import 1440ms -> 356ms, -75%; 7 commits: 288aa05814, 3490da3009, 3c279d56e1, 62bc53e642, fccce205c2, 01954dbe99, 943012d72a; pandas / numpy / PIL / langchain_core / langchain_classic / langchain_text_splitters / networkx absence verified by 0 modules on the Graph path). + +The published Phase 3 win is the Graph cold-import -75% captured in the "Summary of wins" bucket 1 above. The deferred Langflow-run plans are logged in REQUIREMENTS.md as v2 scope and are explicitly out of scope for the cold-start improvements milestone. + +## Known gaps + +One matrix cell remains sentinel per CONTEXT.md D-12 and is carried forward to a future milestone: + +- **`langflow_run_http_ready`**: `mean_ms: 0`, `runs: 0`. The langflow structlog processor pipeline does not route uvicorn's `Application startup complete.` INFO log to stdout in a form `_langflow_supervisor.py` `READY_MARKER` can detect, so the scenario times out. Fix candidates are listed in `.planning/benchmarks/baseline-2026-04-17.md` §"Known gap". Deferred. + +This scenario is `continue-on-error: true` in the workflow matrix and does not block the VAL-03 green-run gate. + +## CI gate evidence + +The VAL-03 verify-mode green run on the final Phase 6 PR is documented in `.planning/phases/06-validation-and-publication/06-03-PLAN.md` execution output and is cited by run ID here once available: + +- Verify-mode run: https://github.com/langflow-ai/langflow/actions/runs/ +- Snapshot-mode run (this document's source): https://github.com/langflow-ai/langflow/actions/runs/24642673292 + +Both IDs are cross-referenced in `.planning/benchmarks/parity-confirmation-YYYY-MM-DD.md` under "Umbrella CI run". + +## Artifacts + +Full artifacts (per-scenario hyperfine JSONs, pyinstrument HTML, -X importtime traces, checkpoints JSONs) available via: + +``` +gh run download 24642673292 --name cold-start-benchmark-reports --dir /tmp/bench-snapshot-24642673292 +``` + +`src/backend/tests/benchmarks/thresholds.json` in the worktree is the committed, authoritative per-scenario summary. diff --git a/src/backend/tests/benchmarks/thresholds.json b/src/backend/tests/benchmarks/thresholds.json index a79519d060..7fd1aa20e8 100644 --- a/src/backend/tests/benchmarks/thresholds.json +++ b/src/backend/tests/benchmarks/thresholds.json @@ -1,18 +1,42 @@ { "schema_version": 1, "measurement_mode": "bytecode_compile_delta", - "captured_on": "2026-04-18", - "captured_ref": "cold-start-improvements-v2 @ 4d2820ae73 (build 24603653799)", + "captured_on": "2026-04-20", + "captured_ref": "12750/merge@eb2272ccc711768ecb11a3c0982aa419852bb17a", "captured_runner": "ubuntu-latest (GitHub Actions)", "python_version": "3.13", "allowed_regression_pct": 15, - "_note": "Captured on Linux CI via run-benchmark-snapshot label trigger (build 24603653799). Phase 3 import-time deferrals dropped lfx_bare by ~42% vs the Phase 1 baseline (17824.74ms -> 10275.94ms). langflow_run_no_change_restart is the Phase 4 SVC-01 D-03 scenario: total `langflow run --backend-only` wall-clock when the starter-projects hash matches; the 11s mean is the whole process (Python startup + imports + alembic + service init + lifespan + uvicorn bind), with the starter-sync block itself gated to ms by the hash check. langflow_run_http_ready remains sentinel: structlog swallows uvicorn's startup marker; no_change_restart sidesteps this by probing TCP 127.0.0.1:7860 instead. measurement_mode is bytecode_compile_delta per D-11a.", + "_note": "Captured on Linux CI via the cold-start-benchmark matrix workflow. langflow_run_http_ready may be sentinel if its supervisor marker is still unresolved. measurement_mode is bytecode_compile_delta per D-11a.", "scenarios": { - "lfx_bare": {"mean_ms": 10275.94, "stddev_ms": 33.6, "runs": 10}, - "lfx_with_flow": {"mean_ms": 17961.33, "stddev_ms": 242.2, "runs": 10}, - "lfx_with_flow_prebaked": {"mean_ms": 8031.26, "stddev_ms": 267.43, "runs": 10}, - "langflow_run_http_ready": {"mean_ms": 0, "stddev_ms": 0, "runs": 0}, - "langflow_run_no_change_restart": {"mean_ms": 11078.05, "stddev_ms": 23.39, "runs": 5}, - "lfx_reference_image": {"mean_ms": 0, "stddev_ms": 0, "runs": 0} + "lfx_bare": { + "mean_ms": 10549.93, + "stddev_ms": 98.96, + "runs": 10 + }, + "lfx_with_flow": { + "mean_ms": 16012.93, + "stddev_ms": 228.58, + "runs": 10 + }, + "lfx_with_flow_prebaked": { + "mean_ms": 8424.65, + "stddev_ms": 99.93, + "runs": 10 + }, + "langflow_run_http_ready": { + "mean_ms": 0, + "stddev_ms": 0, + "runs": 0 + }, + "langflow_run_no_change_restart": { + "mean_ms": 11324.7, + "stddev_ms": 83.73, + "runs": 5 + }, + "lfx_reference_image": { + "mean_ms": 2404.12, + "stddev_ms": 20.93, + "runs": 10 + } } }