From 3d415c2ad0c2eb89a2783457b5577dc314778687 Mon Sep 17 00:00:00 2001 From: Alek Date: Sun, 12 Jul 2026 17:23:42 -0700 Subject: [PATCH 01/13] Add event loop performance benchmarks --- tests/benchmarks/loop_probe.py | 100 +++++++++++++++++ tests/benchmarks/test_event_processing.py | 129 +++++++++++++++++++--- tests/units/benchmarks/test_loop_probe.py | 60 ++++++++++ 3 files changed, 275 insertions(+), 14 deletions(-) create mode 100644 tests/benchmarks/loop_probe.py create mode 100644 tests/units/benchmarks/test_loop_probe.py diff --git a/tests/benchmarks/loop_probe.py b/tests/benchmarks/loop_probe.py new file mode 100644 index 00000000000..9d7d53f8e93 --- /dev/null +++ b/tests/benchmarks/loop_probe.py @@ -0,0 +1,100 @@ +"""Wall-time probes for asyncio performance benchmarks.""" + +import asyncio +import contextlib +from dataclasses import dataclass, field +from types import TracebackType + + +@dataclass +class EventLoopProbe: + """Sample event-loop lag and live task count during a workload.""" + + interval: float = 0.01 + lag_samples: list[float] = field(default_factory=list, init=False) + peak_tasks: int = field(default=0, init=False) + _task: asyncio.Task[None] | None = field(default=None, init=False, repr=False) + + async def __aenter__(self) -> "EventLoopProbe": + """Start sampling and allow the sampler task to initialize. + + Returns: + The running probe. + """ + if self.interval <= 0: + msg = "Event-loop probe interval must be greater than zero." + raise ValueError(msg) + self.lag_samples.clear() + self.peak_tasks = 0 + self._task = asyncio.create_task( + self._sample(), + name="reflex_benchmark_event_loop_probe", + ) + await asyncio.sleep(0) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Stop the sampler. + + Args: + exc_type: Exception type raised by the measured workload, if any. + exc_value: Exception raised by the measured workload, if any. + traceback: Traceback raised by the measured workload, if any. + """ + if self._task is None: + return + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _sample(self) -> None: + """Record scheduling delay and the number of live tasks.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + self.interval + while True: + await asyncio.sleep(max(0.0, deadline - loop.time())) + now = loop.time() + self.lag_samples.append(max(0.0, now - deadline)) + self.peak_tasks = max(self.peak_tasks, len(asyncio.all_tasks(loop))) + deadline = max(deadline + self.interval, now + self.interval) + + def summary(self) -> dict[str, float | int]: + """Return stable summary fields for benchmark JSON output. + + Returns: + Sample count, peak tasks, and event-loop lag percentiles in milliseconds. + """ + return { + "sample_count": len(self.lag_samples), + "peak_tasks": self.peak_tasks, + "lag_p50_ms": self._percentile(50) * 1000, + "lag_p95_ms": self._percentile(95) * 1000, + "lag_p99_ms": self._percentile(99) * 1000, + "lag_max_ms": max(self.lag_samples, default=0.0) * 1000, + } + + def _percentile(self, percentile: float) -> float: + """Calculate a linearly interpolated lag percentile. + + Args: + percentile: Percentile in the inclusive range 0 through 100. + + Returns: + The percentile lag in seconds, or zero when there are no samples. + """ + if not self.lag_samples: + return 0.0 + values = sorted(self.lag_samples) + position = (len(values) - 1) * percentile / 100 + lower_index = int(position) + upper_index = min(lower_index + 1, len(values) - 1) + fraction = position - lower_index + return ( + values[lower_index] + (values[upper_index] - values[lower_index]) * fraction + ) diff --git a/tests/benchmarks/test_event_processing.py b/tests/benchmarks/test_event_processing.py index 15acf8094d4..9e34dd67719 100644 --- a/tests/benchmarks/test_event_processing.py +++ b/tests/benchmarks/test_event_processing.py @@ -7,7 +7,7 @@ import asyncio import traceback -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any from unittest import mock @@ -20,6 +20,7 @@ from reflex_base.utils.format import format_event_handler from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import BaseStateToken from .fixtures import BenchmarkState @@ -33,8 +34,8 @@ async def event_processing_harness(): enqueued directly and deltas are collected via the emit callback. Yields: - An async callable that enqueues the given number of events - and waits for all expected deltas. + An async event runner and a helper that purges token state between + explicitly cold benchmark rounds. """ emitted_deltas: list[tuple[str, Mapping[str, Mapping[str, Any]]]] = [] @@ -67,7 +68,6 @@ def handle_backend_exception(ex: Exception) -> None: ) processor._root_context = root_context - token = "benchmark-token" handler_name = format_event_handler(BenchmarkState.event_handlers["increment"]) event = Event( name=handler_name, @@ -78,23 +78,33 @@ def handle_backend_exception(ex: Exception) -> None: payload={}, ) - async def run_events(num_events: int, num_expected_deltas: int) -> None: - """Enqueue events and wait for all deltas to be emitted. + async def run_events(tokens: Sequence[str]) -> None: + """Enqueue events for the given tokens and wait for their deltas. Args: - num_events: Number of increment events to enqueue. - num_expected_deltas: How many deltas to wait for. + tokens: Client tokens to enqueue one increment event for. """ emitted_deltas.clear() async with processor as p: async for _ in asyncio.as_completed([ - await p.enqueue(token, event) for _ in range(num_events) + await p.enqueue(token, event) for token in tokens ]): pass - assert len(emitted_deltas) == num_expected_deltas + assert len(emitted_deltas) == len(tokens) - yield run_events + def purge_tokens(tokens: Sequence[str]) -> None: + """Remove benchmark tokens from the in-memory state manager. + + Args: + tokens: Client tokens whose state and lock bookkeeping should be removed. + """ + for token in tokens: + state_manager._purge_token( # pyright: ignore [reportPrivateUsage] + BaseStateToken(ident=token, cls=BenchmarkState) + ) + + yield run_events, purge_tokens await state_manager.close() @@ -109,14 +119,105 @@ def test_process_event( the existing state (warm path). Only event processing is timed. Args: - event_processing_harness: The run_events async callable. + event_processing_harness: The event runner and token purge helpers. benchmark: The codspeed benchmark fixture. """ - run_events = event_processing_harness + run_events, _ = event_processing_harness loop = asyncio.get_event_loop() # Each event handler (increment) does a single state mutation with # no yields, so we expect 1 delta per event = 3 total. @benchmark def _(): - loop.run_until_complete(run_events(num_events=3, num_expected_deltas=3)) + loop.run_until_complete(run_events(["benchmark-token"] * 3)) + + +def test_process_event_cold( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark one event whose token has no existing state. + + Args: + event_processing_harness: The event runner and token purge helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, purge_tokens = event_processing_harness + loop = asyncio.get_event_loop() + iteration = 0 + + def setup(): + """Return a unique cold token for the next measured round.""" + nonlocal iteration + iteration += 1 + return ((f"cold-token-{iteration}",), {}) + + def run_cold(token: str) -> None: + """Process one event with no state cached for its token.""" + loop.run_until_complete(run_events([token])) + + def teardown(token: str) -> None: + """Discard the measured token so cold rounds do not retain state.""" + purge_tokens([token]) + + benchmark.pedantic(run_cold, setup=setup, teardown=teardown) + + +def test_process_event_warm( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark one event for a token whose state is already initialized. + + Args: + event_processing_harness: The event runner and token purge helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _ = event_processing_harness + loop = asyncio.get_event_loop() + token = "warm-token" + loop.run_until_complete(run_events([token])) + + @benchmark + def _(): + loop.run_until_complete(run_events([token])) + + +def test_process_event_burst_same_token( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark ten queued events serialized through one token. + + Args: + event_processing_harness: The event runner and token purge helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _ = event_processing_harness + loop = asyncio.get_event_loop() + tokens = ["burst-token"] * 10 + loop.run_until_complete(run_events([tokens[0]])) + + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) + + +def test_process_event_burst_independent_tokens( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark ten queued events distributed across independent tokens. + + Args: + event_processing_harness: The event runner and token purge helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _ = event_processing_harness + loop = asyncio.get_event_loop() + tokens = [f"independent-token-{index}" for index in range(10)] + loop.run_until_complete(run_events(tokens)) + + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) diff --git a/tests/units/benchmarks/test_loop_probe.py b/tests/units/benchmarks/test_loop_probe.py new file mode 100644 index 00000000000..a86b3c2140f --- /dev/null +++ b/tests/units/benchmarks/test_loop_probe.py @@ -0,0 +1,60 @@ +"""Tests for event-loop benchmark probes.""" + +import asyncio +import time + +import pytest + +from tests.benchmarks.loop_probe import EventLoopProbe + + +@pytest.mark.asyncio +async def test_event_loop_probe_detects_blocking_work(): + """A synchronous pause is reported as event-loop scheduling lag.""" + interval = 0.001 + async with EventLoopProbe(interval=interval) as probe: + await asyncio.sleep(interval * 2) + time.sleep(0.02) # noqa: ASYNC251 - intentional event-loop blocking + await asyncio.sleep(interval * 2) + + summary = probe.summary() + assert summary["sample_count"] >= 2 + assert summary["peak_tasks"] >= 2 + assert summary["lag_max_ms"] >= 10 + + +@pytest.mark.asyncio +async def test_event_loop_probe_resets_between_runs(): + """Reusing a probe does not mix observations from separate workloads.""" + probe = EventLoopProbe(interval=0.001) + async with probe: + await asyncio.sleep(0.005) + first_sample_count = len(probe.lag_samples) + + async with probe: + await asyncio.sleep(0.003) + + assert first_sample_count > len(probe.lag_samples) + + +def test_event_loop_probe_summary_percentiles(): + """Summary values use linearly interpolated lag percentiles.""" + probe = EventLoopProbe() + probe.lag_samples.extend([0.001, 0.002, 0.003, 0.004, 0.005]) + + assert probe.summary() == { + "sample_count": 5, + "peak_tasks": 0, + "lag_p50_ms": 3.0, + "lag_p95_ms": 4.8, + "lag_p99_ms": 4.96, + "lag_max_ms": 5.0, + } + + +@pytest.mark.asyncio +async def test_event_loop_probe_rejects_non_positive_interval(): + """Sampling intervals must advance the event-loop clock.""" + with pytest.raises(ValueError, match="greater than zero"): + async with EventLoopProbe(interval=0): + pass From ee5561b7f6f9d2716c2fda50bab3ba5e80f69884 Mon Sep 17 00:00:00 2001 From: Alek Date: Sun, 12 Jul 2026 18:22:49 -0700 Subject: [PATCH 02/13] Expand event loop performance coverage --- .github/CODEOWNERS | 3 + .github/workflows/performance.yml | 16 + PERFORMANCE.md | 60 ++ pyproject.toml | 3 + tests/benchmarks/support/__init__.py | 15 + tests/benchmarks/support/apps.py | 95 +++ tests/benchmarks/support/diagnostics.py | 60 ++ tests/benchmarks/support/events.py | 91 +++ tests/benchmarks/{ => support}/loop_probe.py | 66 +- tests/benchmarks/support/pipeline_trace.py | 109 ++++ tests/benchmarks/support/redis.py | 44 ++ tests/benchmarks/support/report.py | 238 ++++++++ tests/benchmarks/support/socket_client.py | 143 +++++ tests/benchmarks/support/states.py | 164 +++++ tests/benchmarks/test_event_loop.py | 233 +++++++ tests/benchmarks/test_event_payload.py | 43 ++ tests/benchmarks/test_event_processing.py | 14 +- tests/benchmarks/test_routes.py | 72 +++ tests/benchmarks/test_serialization.py | 61 ++ tests/benchmarks/test_socket_headers.py | 22 + tests/benchmarks/test_state_delta.py | 214 +++++++ tests/benchmarks/test_state_manager.py | 118 ++++ tests/benchmarks/test_state_proxy.py | 64 ++ tests/benchmarks/test_var_operations.py | 39 ++ tests/performance/__init__.py | 1 + tests/performance/conftest.py | 77 +++ tests/performance/test_browser.py | 164 +++++ tests/performance/test_compiler_lifecycle.py | 199 ++++++ tests/performance/test_event_load.py | 112 ++++ tests/performance/test_event_loop.py | 607 +++++++++++++++++++ tests/performance/test_memory.py | 119 ++++ tests/performance/test_redis_latency.py | 236 +++++++ tests/units/benchmarks/test_loop_probe.py | 2 +- tests/units/benchmarks/test_support.py | 74 +++ 34 files changed, 3542 insertions(+), 36 deletions(-) create mode 100644 PERFORMANCE.md create mode 100644 tests/benchmarks/support/__init__.py create mode 100644 tests/benchmarks/support/apps.py create mode 100644 tests/benchmarks/support/diagnostics.py create mode 100644 tests/benchmarks/support/events.py rename tests/benchmarks/{ => support}/loop_probe.py (60%) create mode 100644 tests/benchmarks/support/pipeline_trace.py create mode 100644 tests/benchmarks/support/redis.py create mode 100644 tests/benchmarks/support/report.py create mode 100644 tests/benchmarks/support/socket_client.py create mode 100644 tests/benchmarks/support/states.py create mode 100644 tests/benchmarks/test_event_loop.py create mode 100644 tests/benchmarks/test_event_payload.py create mode 100644 tests/benchmarks/test_routes.py create mode 100644 tests/benchmarks/test_serialization.py create mode 100644 tests/benchmarks/test_socket_headers.py create mode 100644 tests/benchmarks/test_state_delta.py create mode 100644 tests/benchmarks/test_state_manager.py create mode 100644 tests/benchmarks/test_var_operations.py create mode 100644 tests/performance/__init__.py create mode 100644 tests/performance/conftest.py create mode 100644 tests/performance/test_browser.py create mode 100644 tests/performance/test_compiler_lifecycle.py create mode 100644 tests/performance/test_event_load.py create mode 100644 tests/performance/test_event_loop.py create mode 100644 tests/performance/test_memory.py create mode 100644 tests/performance/test_redis_latency.py create mode 100644 tests/units/benchmarks/test_support.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 94dc268f2f3..1586fdb1422 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,6 @@ /docs/ @Alek99 @reflex-dev/reflex-team /README.md @Alek99 @reflex-dev/reflex-team +/PERFORMANCE.md @Alek99 @reflex-dev/reflex-team +/tests/benchmarks/ @Alek99 @reflex-dev/reflex-team +/tests/performance/ @Alek99 @reflex-dev/reflex-team diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 571811116ba..46417ef7551 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,6 +45,14 @@ jobs: mode: instrumentation run: uv run pytest -v tests/benchmarks --codspeed + - name: Run event-loop smoke + run: >- + uv run pytest tests/performance/test_event_loop.py + --run-performance + --performance-scale smoke + --performance-output .pytest-tmp/performance + -q + lighthouse: name: Run Lighthouse benchmark runs-on: ubuntu-22.04 @@ -72,6 +80,14 @@ jobs: mkdir -p .pytest-tmp/lighthouse uv run pytest tests/integration/test_lighthouse.py -q -s --tb=no --basetemp=.pytest-tmp/lighthouse + - name: Run browser performance smoke and bundle budget + run: >- + uv run pytest tests/performance/test_browser.py + --run-performance + --performance-scale smoke + --performance-output .pytest-tmp/performance + -q + - name: Upload Lighthouse artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 00000000000..21948f7d802 --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,60 @@ +# Performance benchmark guide + +Reflex uses separate performance tiers because deterministic CPU work, asynchronous wall time, service load, memory, compiler lifecycle, and browser behavior have different noise and cost profiles. + +## Benchmark tiers + +| Tier | Location | Runs | Purpose | +| --- | --- | --- | --- | +| Deterministic CPU | `tests/benchmarks/` | Every pull request | CodSpeed regression detection and flamegraphs | +| Component wall time | `tests/performance/` | Manual | Event-loop, task, queue, Redis, and allocation behavior | +| Load and lifecycle | `tests/performance/` | Manual | Tail latency, saturation, compiles, hot reload, browser metrics | + +Do not add sleep, network, filesystem, browser, or subprocess timings to a CodSpeed instrumentation benchmark. Put those in a wall-time suite and emit a versioned JSON report. + +## Naming and fixture rules + +- Name benchmarks after the user-visible path they protect, not an implementation detail. +- Separate cold and warm paths. +- Keep setup outside the measured call unless setup is the behavior being measured. +- Use `time.perf_counter_ns()` for wall-time observations. +- Use explicit warmup and measurement counts. +- Batch sub-microsecond operations. +- Preserve raw observations in scheduled-suite artifacts. +- Include small representative and stress cases, but reserve full parameter matrices for scheduled runs. +- Validate correctness—event ordering, final state, update count, errors—inside every performance scenario. + +Shared fixtures and reporting helpers live in `tests/benchmarks/support/`. Scheduled reports use schema version 1 and include the commit, branch, Python, OS, CPU, parameters, raw observations, latency summaries, and suite-specific metrics. + +## Commands + +```console +uv run pytest tests/benchmarks --codspeed +uv run pytest tests/performance -m performance --run-performance +``` + +Individual scheduled suites are controlled by explicit command-line options and environment variables documented in their tests and workflows. They are excluded from ordinary unit and integration discovery. + +## Regression policy + +Thresholds become merge-blocking only after at least 20 comparable baseline runs establish their variance and a benchmark owner accepts the budget. Initial review levels are: + +- More than 5% deterministic CodSpeed CPU regression when statistically supported. +- More than 10% stable component wall-time regression. +- More than 15% p95 or p99 load latency regression at equal offered load. +- More than 10% throughput loss at the established latency objective. +- Any event-ordering failure, lost update, orphan task, or monotonic retained-memory growth. + +Use an absolute floor as well as a percentage. Environment mismatches make comparisons informational unless explicitly approved. + +## Adding or changing a benchmark + +1. State the behavior and regression it protects. +2. Choose the correct tier. +3. Add correctness assertions. +4. Record parameters, warmups, iterations, and environment metadata. +5. Run the focused benchmark and its support tests. +6. Review variance before adding a threshold. +7. Update this guide if the suite, schema, or policy changes. + +Benchmark fixtures are reviewed quarterly. Remove tests that no longer represent supported behavior, but preserve historical benchmark names when a time series remains meaningful. diff --git a/pyproject.toml b/pyproject.toml index 437e2fe5f20..099d6108ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -256,6 +256,9 @@ preview = true filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" +markers = [ + "performance: scheduled wall-time, service, memory, lifecycle, or browser benchmark", +] [tool.codespell] skip = "*.html, examples, *.pyi, */nba.csv, */olympic-winners.json, uv.lock, */bun.lock, node_modules" diff --git a/tests/benchmarks/support/__init__.py b/tests/benchmarks/support/__init__.py new file mode 100644 index 00000000000..e3047e6eb64 --- /dev/null +++ b/tests/benchmarks/support/__init__.py @@ -0,0 +1,15 @@ +"""Shared support utilities for Reflex performance benchmarks.""" + +from .diagnostics import capture_async_diagnostics +from .loop_probe import EventLoopProbe +from .pipeline_trace import PipelineTrace +from .report import BenchmarkResult, PerformanceReport, percentile + +__all__ = [ + "BenchmarkResult", + "EventLoopProbe", + "PerformanceReport", + "PipelineTrace", + "capture_async_diagnostics", + "percentile", +] diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py new file mode 100644 index 00000000000..9f3c543495d --- /dev/null +++ b/tests/benchmarks/support/apps.py @@ -0,0 +1,95 @@ +"""Representative Reflex application sources for lifecycle and load tests.""" + +from __future__ import annotations + + +def lifecycle_app_source(rows: int = 100, pages: int = 1) -> str: + """Create a deterministic multi-page application source. + + Args: + rows: Number of stateful rows per page. + pages: Number of routes. + + Returns: + Python source for a Reflex application. + """ + page_registrations = "\n".join( + f'app.add_page(lambda route="page-{index}": index(route), route="/page-{index}")' + for index in range(pages) + ) + return f"""import reflex as rx + +class State(rx.State): + count: int = 0 + + def increment(self): + self.count += 1 + +def row(index: int): + return rx.hstack(rx.text(index), rx.text(State.count)) + +def index(label: str = "index"): + return rx.vstack( + rx.heading(label), + rx.button("increment", on_click=State.increment), + *[row(index) for index in range({rows})], + ) + +app = rx.App() +app.add_page(index, route="/") +{page_registrations} +""" + + +def load_app_source() -> str: + """Create an app covering the load-test handler scenarios. + + Returns: + Python source for a production-mode Reflex load app. + """ + return """import asyncio +import time +import reflex as rx + +class State(rx.State): + count: int = 0 + values: list[int] = [] + + def increment(self): + self.count += 1 + + async def async_io(self): + await asyncio.sleep(0.01) + self.count += 1 + + def blocking(self): + time.sleep(0.02) + self.count += 1 + + def stream(self): + for _ in range(3): + self.count += 1 + yield + + def append_large(self): + self.values.extend(range(1000)) + +def index(): + return rx.vstack( + rx.text(State.count, id="count"), + rx.button("increment", id="increment", on_click=State.increment), + rx.button("async", id="async-io", on_click=State.async_io), + rx.button("blocking", id="blocking", on_click=State.blocking), + rx.button("stream", id="stream", on_click=State.stream), + rx.button("large", id="append-large", on_click=State.append_large), + rx.foreach(State.values, lambda value: rx.text(value, class_name="value-row")), + rx.link("second", href="/second", id="second-link"), + ) + +def second(): + return rx.heading("Second page", id="second-heading") + +app = rx.App() +app.add_page(index) +app.add_page(second, route="/second") +""" diff --git a/tests/benchmarks/support/diagnostics.py b/tests/benchmarks/support/diagnostics.py new file mode 100644 index 00000000000..8348ff28243 --- /dev/null +++ b/tests/benchmarks/support/diagnostics.py @@ -0,0 +1,60 @@ +"""Diagnostic artifacts captured after severe event-loop lag.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import threading +import traceback +from pathlib import Path +from typing import Any + + +def capture_async_diagnostics( + path: str | Path, + *, + metadata: dict[str, Any] | None = None, +) -> Path: + """Capture task and thread stacks in a JSON artifact. + + Args: + path: Destination path. + metadata: Scenario-specific queue, stage, or lag metadata. + + Returns: + Resolved artifact path. + """ + tasks = [] + for task in asyncio.all_tasks(): + tasks.append({ + "name": task.get_name(), + "done": task.done(), + "cancelled": task.cancelled(), + "stack": [ + line + for frame in task.get_stack() + for line in traceback.format_stack(frame) + ], + }) + frames = sys._current_frames() # pyright: ignore [reportPrivateUsage] + threads = [] + for thread in threading.enumerate(): + frame = frames.get(thread.ident) if thread.ident is not None else None + threads.append({ + "name": thread.name, + "ident": thread.ident, + "daemon": thread.daemon, + "stack": traceback.format_stack(frame) if frame is not None else [], + }) + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps( + {"metadata": metadata or {}, "tasks": tasks, "threads": threads}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return destination diff --git a/tests/benchmarks/support/events.py b/tests/benchmarks/support/events.py new file mode 100644 index 00000000000..4989024a298 --- /dev/null +++ b/tests/benchmarks/support/events.py @@ -0,0 +1,91 @@ +"""Representative event payloads for performance benchmarks.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import uuid +from typing import Any + +from pydantic import BaseModel + + +class PayloadKind(enum.StrEnum): + """Payload shapes exercised by serialization and event benchmarks.""" + + SCALAR = "scalar" + LIST = "list" + MAPPING = "mapping" + DATACLASS = "dataclass" + MODEL = "model" + + +@dataclasses.dataclass(frozen=True) +class PayloadRow: + """Nested dataclass payload row.""" + + identifier: int + label: str + values: tuple[int, ...] + + +class PayloadModel(BaseModel): + """Nested Pydantic payload model.""" + + identifier: int + label: str + values: list[int] + + +def make_payload(kind: PayloadKind, rows: int) -> Any: + """Build a deterministic payload with the requested shape. + + Args: + kind: Payload representation. + rows: Number of repeated rows or scalar characters. + + Returns: + A representative payload. + """ + if kind is PayloadKind.SCALAR: + return "x" * rows + if kind is PayloadKind.LIST: + return list(range(rows)) + if kind is PayloadKind.MAPPING: + return {f"key_{index}": index for index in range(rows)} + if kind is PayloadKind.DATACLASS: + return [ + PayloadRow(index, f"row-{index}", (index, index + 1)) + for index in range(rows) + ] + if kind is PayloadKind.MODEL: + return [ + PayloadModel( + identifier=index, + label=f"row-{index}", + values=[index, index + 1], + ) + for index in range(rows) + ] + msg = f"Unhandled payload kind: {kind}" + raise AssertionError(msg) + + +def wire_edge_cases() -> dict[str, Any]: + """Return values that exercise the complete socket serialization contract. + + Returns: + A mapping containing special numeric and structured values. + """ + return { + "unicode": "Reflex ⚡", + "date": datetime.date(2026, 1, 2), + "datetime": datetime.datetime(2026, 1, 2, 3, 4, 5), + "decimal": decimal.Decimal("123.45"), + "uuid": uuid.UUID("12345678-1234-5678-1234-567812345678"), + "set": {1, 2, 3}, + "large_integer": 2**80, + "dataclass": PayloadRow(1, "row", (1, 2)), + } diff --git a/tests/benchmarks/loop_probe.py b/tests/benchmarks/support/loop_probe.py similarity index 60% rename from tests/benchmarks/loop_probe.py rename to tests/benchmarks/support/loop_probe.py index 9d7d53f8e93..5cd4b1e9019 100644 --- a/tests/benchmarks/loop_probe.py +++ b/tests/benchmarks/support/loop_probe.py @@ -1,21 +1,37 @@ """Wall-time probes for asyncio performance benchmarks.""" +from __future__ import annotations + import asyncio import contextlib +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from types import TracebackType +from .report import percentile + @dataclass class EventLoopProbe: - """Sample event-loop lag and live task count during a workload.""" + """Sample event-loop lag, tasks, and benchmark-specific gauges.""" interval: float = 0.01 + gauges: Mapping[str, Callable[[], int]] = field(default_factory=dict) lag_samples: list[float] = field(default_factory=list, init=False) - peak_tasks: int = field(default=0, init=False) + task_samples: list[int] = field(default_factory=list, init=False) + gauge_samples: dict[str, list[int]] = field(default_factory=dict, init=False) _task: asyncio.Task[None] | None = field(default=None, init=False, repr=False) - async def __aenter__(self) -> "EventLoopProbe": + @property + def peak_tasks(self) -> int: + """Return the largest observed live-task count. + + Returns: + Peak live tasks. + """ + return max(self.task_samples, default=0) + + async def __aenter__(self) -> EventLoopProbe: """Start sampling and allow the sampler task to initialize. Returns: @@ -25,7 +41,8 @@ async def __aenter__(self) -> "EventLoopProbe": msg = "Event-loop probe interval must be greater than zero." raise ValueError(msg) self.lag_samples.clear() - self.peak_tasks = 0 + self.task_samples.clear() + self.gauge_samples = {name: [] for name in self.gauges} self._task = asyncio.create_task( self._sample(), name="reflex_benchmark_event_loop_probe", @@ -54,47 +71,34 @@ async def __aexit__( self._task = None async def _sample(self) -> None: - """Record scheduling delay and the number of live tasks.""" + """Record scheduling delay, tasks, and custom gauges.""" loop = asyncio.get_running_loop() deadline = loop.time() + self.interval while True: await asyncio.sleep(max(0.0, deadline - loop.time())) now = loop.time() self.lag_samples.append(max(0.0, now - deadline)) - self.peak_tasks = max(self.peak_tasks, len(asyncio.all_tasks(loop))) + self.task_samples.append(len(asyncio.all_tasks(loop))) + for name, gauge in self.gauges.items(): + self.gauge_samples[name].append(gauge()) deadline = max(deadline + self.interval, now + self.interval) def summary(self) -> dict[str, float | int]: """Return stable summary fields for benchmark JSON output. Returns: - Sample count, peak tasks, and event-loop lag percentiles in milliseconds. + Sample count, peak tasks, custom gauge peaks, and lag percentiles. """ - return { + summary: dict[str, float | int] = { "sample_count": len(self.lag_samples), "peak_tasks": self.peak_tasks, - "lag_p50_ms": self._percentile(50) * 1000, - "lag_p95_ms": self._percentile(95) * 1000, - "lag_p99_ms": self._percentile(99) * 1000, + "lag_p50_ms": percentile(self.lag_samples, 50) * 1000, + "lag_p95_ms": percentile(self.lag_samples, 95) * 1000, + "lag_p99_ms": percentile(self.lag_samples, 99) * 1000, "lag_max_ms": max(self.lag_samples, default=0.0) * 1000, } - - def _percentile(self, percentile: float) -> float: - """Calculate a linearly interpolated lag percentile. - - Args: - percentile: Percentile in the inclusive range 0 through 100. - - Returns: - The percentile lag in seconds, or zero when there are no samples. - """ - if not self.lag_samples: - return 0.0 - values = sorted(self.lag_samples) - position = (len(values) - 1) * percentile / 100 - lower_index = int(position) - upper_index = min(lower_index + 1, len(values) - 1) - fraction = position - lower_index - return ( - values[lower_index] + (values[upper_index] - values[lower_index]) * fraction - ) + summary.update({ + f"peak_{name}": max(samples, default=0) + for name, samples in self.gauge_samples.items() + }) + return summary diff --git a/tests/benchmarks/support/pipeline_trace.py b/tests/benchmarks/support/pipeline_trace.py new file mode 100644 index 00000000000..2c7ee2d7e65 --- /dev/null +++ b/tests/benchmarks/support/pipeline_trace.py @@ -0,0 +1,109 @@ +"""Benchmark-only event pipeline tracing.""" + +from __future__ import annotations + +import dataclasses +import json +import time +from collections import defaultdict +from collections.abc import Iterable, Sequence +from pathlib import Path +from typing import Any + + +@dataclasses.dataclass(frozen=True) +class StageEvent: + """One monotonic pipeline stage observation.""" + + token: str + stage: str + timestamp_ns: int + + +@dataclasses.dataclass +class PipelineTrace: + """Collect per-token pipeline stages without changing runtime code.""" + + events: list[StageEvent] = dataclasses.field(default_factory=list) + + def record(self, token: str, stage: str) -> None: + """Record a pipeline stage using a monotonic clock. + + Args: + token: Event token or transaction identifier. + stage: Stable stage name. + """ + self.events.append( + StageEvent(token=token, stage=stage, timestamp_ns=time.perf_counter_ns()) + ) + + def durations_ms(self, stages: Sequence[tuple[str, str]]) -> dict[str, list[float]]: + """Calculate named adjacent or non-adjacent stage durations. + + Args: + stages: Start/end stage pairs. + + Returns: + Durations grouped under ``start_to_end`` keys. + """ + grouped: dict[str, list[StageEvent]] = defaultdict(list) + for event in self.events: + grouped[event.token].append(event) + + durations: dict[str, list[float]] = { + f"{start}_to_{end}": [] for start, end in stages + } + for token_events in grouped.values(): + timestamps = {event.stage: event.timestamp_ns for event in token_events} + for start, end in stages: + if start in timestamps and end in timestamps: + durations[f"{start}_to_{end}"].append( + (timestamps[end] - timestamps[start]) / 1_000_000 + ) + return durations + + def chrome_trace(self) -> dict[str, Any]: + """Convert observations to Chrome instant trace events. + + Returns: + A Chrome trace JSON mapping. + """ + return { + "traceEvents": [ + { + "name": event.stage, + "cat": "reflex.event_pipeline", + "ph": "i", + "s": "t", + "pid": 1, + "tid": event.token, + "ts": event.timestamp_ns / 1000, + } + for event in self.events + ] + } + + def write_chrome_trace(self, path: str | Path) -> Path: + """Write Chrome trace JSON. + + Args: + path: Destination path. + + Returns: + The resolved destination path. + """ + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(self.chrome_trace(), indent=2) + "\n", + encoding="utf-8", + ) + return destination + + def extend(self, events: Iterable[StageEvent]) -> None: + """Append existing observations. + + Args: + events: Observations to append. + """ + self.events.extend(events) diff --git a/tests/benchmarks/support/redis.py b/tests/benchmarks/support/redis.py new file mode 100644 index 00000000000..63fb855533f --- /dev/null +++ b/tests/benchmarks/support/redis.py @@ -0,0 +1,44 @@ +"""Real-Redis helpers for scheduled performance suites.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from redis.asyncio import Redis + +from reflex.istate.manager.redis import StateManagerRedis + +REDIS_URL_ENV = "REFLEX_PERF_REDIS_URL" + + +def performance_redis_url() -> str: + """Return the isolated Redis database used by performance tests. + + Returns: + Redis connection URL. + """ + return os.environ.get(REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + + +@asynccontextmanager +async def real_redis_state_manager() -> AsyncIterator[StateManagerRedis]: + """Create an isolated state manager backed by a reachable Redis server. + + Yields: + A state manager whose database is empty at entry and exit. + + Raises: + ConnectionError: If the configured Redis server is unreachable. + """ + redis = Redis.from_url(performance_redis_url()) + await redis.ping() # pyright: ignore [reportGeneralTypeIssues] + await redis.flushdb() + manager = StateManagerRedis(redis=redis) + try: + yield manager + finally: + await manager.close() + await redis.flushdb() + await redis.aclose() diff --git a/tests/benchmarks/support/report.py b/tests/benchmarks/support/report.py new file mode 100644 index 00000000000..2cd8c105386 --- /dev/null +++ b/tests/benchmarks/support/report.py @@ -0,0 +1,238 @@ +"""Versioned result schema for wall-time and load benchmarks.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import platform +import subprocess +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 + + +def percentile(values: Sequence[float], value: float) -> float: + """Return a linearly interpolated percentile. + + Args: + values: Numeric observations. + value: Percentile in the inclusive range 0 through 100. + + Returns: + The interpolated percentile, or zero for an empty sequence. + + Raises: + ValueError: If the requested percentile is outside the valid range. + """ + if not 0 <= value <= 100: + msg = "Percentile must be between 0 and 100." + raise ValueError(msg) + if not values: + return 0.0 + ordered = sorted(values) + position = (len(ordered) - 1) * value / 100 + lower_index = int(position) + upper_index = min(lower_index + 1, len(ordered) - 1) + fraction = position - lower_index + return ( + ordered[lower_index] + (ordered[upper_index] - ordered[lower_index]) * fraction + ) + + +def _git_value(*args: str) -> str | None: + """Read a value from the current Git checkout when available. + + Args: + args: Arguments following ``git``. + + Returns: + Stripped command output, or ``None`` outside a Git checkout. + """ + try: + result = subprocess.run( + ["git", *args], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout.strip() or None + + +@dataclasses.dataclass(frozen=True) +class BenchmarkEnvironment: + """Runtime and repository metadata attached to a performance report.""" + + commit: str | None + branch: str | None + python: str + implementation: str + operating_system: str + machine: str + processor: str + cpu_count: int | None + ci: bool + + @classmethod + def detect(cls) -> BenchmarkEnvironment: + """Detect the current performance environment. + + Returns: + Environment metadata suitable for cross-run validation. + """ + return cls( + commit=os.environ.get("GITHUB_SHA") or _git_value("rev-parse", "HEAD"), + branch=os.environ.get("GITHUB_HEAD_REF") + or os.environ.get("GITHUB_REF_NAME") + or _git_value("branch", "--show-current"), + python=platform.python_version(), + implementation=platform.python_implementation(), + operating_system=platform.platform(), + machine=platform.machine(), + processor=platform.processor(), + cpu_count=os.cpu_count(), + ci=os.environ.get("CI", "").lower() == "true", + ) + + +@dataclasses.dataclass(frozen=True) +class BenchmarkResult: + """One parameterized performance benchmark result.""" + + name: str + parameters: Mapping[str, Any] + observations_ms: Sequence[float] + metrics: Mapping[str, float | int] + warmup_iterations: int = 0 + measurement_iterations: int = 1 + + def summary(self) -> dict[str, float | int]: + """Summarize the benchmark's latency observations. + + Returns: + Count and common latency percentiles in milliseconds. + """ + values = self.observations_ms + return { + "count": len(values), + "mean_ms": sum(values) / len(values) if values else 0.0, + "p50_ms": percentile(values, 50), + "p90_ms": percentile(values, 90), + "p95_ms": percentile(values, 95), + "p99_ms": percentile(values, 99), + "max_ms": max(values, default=0.0), + } + + def as_dict(self) -> dict[str, Any]: + """Serialize this result. + + Returns: + A JSON-compatible result mapping. + """ + return { + "name": self.name, + "parameters": dict(self.parameters), + "warmup_iterations": self.warmup_iterations, + "measurement_iterations": self.measurement_iterations, + "observations_ms": list(self.observations_ms), + "summary": self.summary(), + "metrics": dict(self.metrics), + } + + +@dataclasses.dataclass +class PerformanceReport: + """Collection of results produced in one benchmark environment.""" + + suite: str + results: list[BenchmarkResult] = dataclasses.field(default_factory=list) + environment: BenchmarkEnvironment = dataclasses.field( + default_factory=BenchmarkEnvironment.detect + ) + metadata: dict[str, Any] = dataclasses.field(default_factory=dict) + + def add(self, result: BenchmarkResult) -> None: + """Append a benchmark result. + + Args: + result: Result to append. + """ + self.results.append(result) + + def as_dict(self) -> dict[str, Any]: + """Serialize the report. + + Returns: + A JSON-compatible report mapping. + """ + return { + "schema_version": SCHEMA_VERSION, + "suite": self.suite, + "environment": dataclasses.asdict(self.environment), + "metadata": self.metadata, + "results": [result.as_dict() for result in self.results], + } + + def write(self, path: str | Path) -> Path: + """Write the report as deterministic formatted JSON. + + Args: + path: Destination path. + + Returns: + The resolved destination path. + """ + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(self.as_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return destination + + +def current_process_metrics() -> dict[str, int]: + """Return portable process resource metrics. + + Returns: + Resident memory, thread count, and process ID. + """ + import psutil + + process = psutil.Process() + return { + "pid": process.pid, + "rss_bytes": process.memory_info().rss, + "threads": process.num_threads(), + } + + +def runtime_metadata() -> dict[str, str]: + """Return additional runtime identifiers when installed. + + Returns: + Node and Redis version strings when their commands are available. + """ + metadata = {"python_executable": sys.executable} + for name, command in { + "node": ["node", "--version"], + "redis": ["redis-server", "--version"], + }.items(): + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + continue + metadata[name] = (result.stdout or result.stderr).strip() + return metadata diff --git a/tests/benchmarks/support/socket_client.py b/tests/benchmarks/support/socket_client.py new file mode 100644 index 00000000000..324973663f9 --- /dev/null +++ b/tests/benchmarks/support/socket_client.py @@ -0,0 +1,143 @@ +"""Protocol-aware Socket.IO client used by scheduled load tests.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.parse import quote + +import socketio +from socketio.exceptions import TimeoutError as SocketTimeoutError + + +@dataclasses.dataclass(frozen=True) +class ClientLoadResult: + """Latency observations and failures for one load client.""" + + token: str + latencies_ms: tuple[float, ...] + errors: tuple[str, ...] + + +async def run_socket_client( + url: str, + token: str, + payload: Mapping[str, Any], + events: int, + *, + event_name: str = "event", + response_name: str = "event", + namespace: str = "/_event", + timeout: float = 10, +) -> ClientLoadResult: + """Send events sequentially and measure matching socket responses. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted for each operation. + events: Number of operations. + event_name: Socket event name used for requests. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum response wait per operation. + + Returns: + Per-operation latency and error observations. + """ + return await asyncio.to_thread( + _run_socket_client_sync, + url, + token, + payload, + events, + event_name, + response_name, + namespace, + timeout, + ) + + +def _run_socket_client_sync( + url: str, + token: str, + payload: Mapping[str, Any], + events: int, + event_name: str, + response_name: str, + namespace: str, + timeout: float, +) -> ClientLoadResult: + """Run a load client with the websocket-client transport. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted for each operation. + events: Number of operations. + event_name: Socket event name used for requests. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum response wait per operation. + + Returns: + Per-operation latency and error observations. + """ + client = socketio.SimpleClient(logger=False) + latencies: list[float] = [] + errors: list[str] = [] + + try: + separator = "&" if "?" in url else "?" + client.connect( + f"{url}{separator}token={quote(token)}", + transports=["websocket"], + socketio_path="_event", + headers={"Origin": url}, + namespace=namespace, + wait_timeout=max(1, int(timeout)), + ) + for _ in range(events): + started = time.perf_counter_ns() + client.emit(event_name, dict(payload)) + try: + response = client.receive(timeout=timeout) + while response and response[0] != response_name: + response = client.receive(timeout=timeout) + except SocketTimeoutError: + errors.append("response_timeout") + continue + latencies.append((time.perf_counter_ns() - started) / 1_000_000) + except Exception as err: + errors.append(f"{type(err).__name__}: {err}") + finally: + if client.connected: + client.disconnect() + + return ClientLoadResult(token, tuple(latencies), tuple(errors)) + + +async def run_clients( + clients: int, + factory: Callable[[int], Any], +) -> list[Any]: + """Run a parameterized set of clients concurrently. + + Args: + clients: Number of clients. + factory: Callable returning an awaitable for a client index. + + Returns: + Client results in index order. + """ + loop = asyncio.get_running_loop() + executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(1, clients)) + loop.set_default_executor(executor) + try: + return list(await asyncio.gather(*(factory(index) for index in range(clients)))) + finally: + executor.shutdown(wait=True) diff --git a/tests/benchmarks/support/states.py b/tests/benchmarks/support/states.py new file mode 100644 index 00000000000..8c952091f64 --- /dev/null +++ b/tests/benchmarks/support/states.py @@ -0,0 +1,164 @@ +"""Scalable state fixtures shared by performance suites.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import cast + +from reflex_base.registry import RegistrationContext + +import reflex as rx +from reflex.state import BaseState + + +class PerformanceState(rx.State): + """Representative state with mutable and computed values.""" + + counter: rx.Field[int] = rx.field(0) + numbers: rx.Field[list[int]] = rx.field(default_factory=list) + mapping: rx.Field[dict[str, int]] = rx.field(default_factory=dict) + + @rx.event + def increment(self): + """Increment the scalar counter.""" + self.counter += 1 + + @rx.event + def append_number(self, value: int): + """Append a number. + + Args: + value: Value to append. + """ + self.numbers.append(value) + + @rx.var(cache=True) + def total(self) -> int: + """Return the sum of numbers and the counter. + + Returns: + Current aggregate. + """ + return self.counter + sum(self.numbers) + + @rx.var(cache=True) + def doubled_total(self) -> int: + """Return a computed value depending on another computed value. + + Returns: + Twice the current aggregate. + """ + return self.total * 2 + + +def initialized_state(size: int) -> PerformanceState: + """Create a state with deterministic mutable values. + + Args: + size: Number of list and mapping elements. + + Returns: + Initialized performance state. + """ + state = PerformanceState( + _reflex_internal_init=True # pyright: ignore [reportCallIssue] + ) + state.numbers = list(range(size)) + state.mapping = {f"key_{index}": index for index in range(size)} + state._clean() + return state + + +@contextmanager +def computed_fanout_state(fanout: int) -> Iterator[tuple[BaseState, tuple[str, ...]]]: + """Create a state whose scalar value feeds a requested number of vars. + + Args: + fanout: Number of cached computed vars depending on ``counter``. + + Yields: + State instance and the generated computed-var names. + """ + with RegistrationContext(): + namespace: dict[str, object] = { + "__module__": __name__, + "__annotations__": {"counter": rx.Field[int]}, + "counter": rx.field(0), + } + names = tuple(f"derived_{index}" for index in range(fanout)) + for index, name in enumerate(names): + + def derived(self: BaseState, offset: int = index) -> int: + """Return the source counter offset by a stable value.""" + return cast(PerformanceState, self).counter + offset + + derived.__name__ = name + namespace[name] = rx.var(cache=True)(derived) + state_type = cast( + type[BaseState], + type(f"PerformanceFanout{fanout}", (rx.State,), namespace), + ) + yield ( + state_type( # pyright: ignore [reportCallIssue] + _reflex_internal_init=True + ), + names, + ) + + +@contextmanager +def state_tree(shape: str) -> Iterator[BaseState]: + """Build an isolated registered state hierarchy. + + Args: + shape: ``depth_10``, ``width_10``, or ``three_by_three``. + + Yields: + Root state containing the requested substate hierarchy. + + Raises: + ValueError: If the shape is unknown. + """ + with RegistrationContext(): + RegistrationContext.register_base_state(PerformanceState) + if shape == "depth_10": + parent: type[BaseState] = PerformanceState + for index in range(10): + parent = cast( + type[BaseState], + type( + f"PerformanceDepth{index}", + (parent,), + {"__module__": __name__}, + ), + ) + elif shape == "width_10": + for index in range(10): + type( + f"PerformanceWide{index}", + (PerformanceState,), + {"__module__": __name__}, + ) + elif shape == "three_by_three": + for parent_index in range(3): + parent = cast( + type[BaseState], + type( + f"PerformanceBranch{parent_index}", + (PerformanceState,), + {"__module__": __name__}, + ), + ) + for child_index in range(3): + type( + f"PerformanceBranch{parent_index}Child{child_index}", + (parent,), + {"__module__": __name__}, + ) + else: + msg = f"Unknown performance state-tree shape: {shape}" + raise ValueError(msg) + yield PerformanceState.get_root_state()( # pyright: ignore [reportCallIssue] + _reflex_internal_init=True + ) diff --git a/tests/benchmarks/test_event_loop.py b/tests/benchmarks/test_event_loop.py new file mode 100644 index 00000000000..86b37a8e5c6 --- /dev/null +++ b/tests/benchmarks/test_event_loop.py @@ -0,0 +1,233 @@ +"""Deterministic benchmarks for event-processor scheduling and lifecycle paths.""" + +import asyncio +from collections.abc import Mapping +from types import SimpleNamespace +from typing import Any, cast + +from pytest_codspeed import BenchmarkFixture +from reflex_base.event.context import EventContext +from reflex_base.event.processor.event_processor import EventProcessor +from reflex_base.registry import RegistrationContext + +from reflex.app import EventNamespace +from reflex.event import Event, EventHandler +from reflex.state import StateUpdate + + +async def _noop_handler() -> None: + """Complete without application work.""" + + +async def _chain_handler() -> None: + """Enqueue a child event from the active event context.""" + await EventContext.get().enqueue(Event.from_event_type(NOOP_EVENT())[0]) + + +async def _slow_handler() -> None: + """Remain pending until shutdown cancellation.""" + await asyncio.sleep(60) + + +async def _yield_handler() -> None: + """Emit rapid intermediate deltas in order.""" + context = EventContext.get() + await context.emit_delta({"state": {"value": 1}}) + await context.emit_delta({"state": {"value": 2}}) + + +NOOP_EVENT = EventHandler(fn=_noop_handler) +CHAIN_EVENT = EventHandler(fn=_chain_handler) +SLOW_EVENT = EventHandler(fn=_slow_handler) +YIELD_EVENT = EventHandler(fn=_yield_handler) + + +def _register_handlers() -> RegistrationContext: + """Create an attached registry containing benchmark handlers. + + Returns: + Attached registration context. + """ + context = RegistrationContext() + context.__enter__() + for handler in (NOOP_EVENT, CHAIN_EVENT, SLOW_EVENT, YIELD_EVENT): + RegistrationContext.register_event_handler(handler) + return context + + +async def _process(handler: EventHandler, token: str = "token") -> None: + """Process one event through a configured processor. + + Args: + handler: Registered handler. + token: Client token. + """ + processor = EventProcessor(graceful_shutdown_timeout=2) + processor.configure() + async with processor: + future = await processor.enqueue(token, Event.from_event_type(handler())[0]) + await future.wait_all() + + +def test_event_queue_dispatch(benchmark: BenchmarkFixture): + """Benchmark enqueue, dispatch, task completion, and clean shutdown. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + try: + benchmark(lambda: loop.run_until_complete(_process(NOOP_EVENT))) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_future_tree(benchmark: BenchmarkFixture): + """Benchmark a parent event that enqueues and waits for one child. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + try: + benchmark(lambda: loop.run_until_complete(_process(CHAIN_EVENT))) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_shutdown_drain(benchmark: BenchmarkFixture): + """Benchmark graceful draining of ten queued no-op events. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + + async def drain() -> None: + """Enqueue a burst and let processor shutdown drain it.""" + processor = EventProcessor(graceful_shutdown_timeout=2) + processor.configure() + async with processor: + for _ in range(10): + await processor.enqueue("token", Event.from_event_type(NOOP_EVENT())[0]) + + try: + benchmark(lambda: loop.run_until_complete(drain())) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_shutdown_cancel(benchmark: BenchmarkFixture): + """Benchmark forced cancellation during zero-grace shutdown. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + + async def cancel() -> bool: + """Start a slow task and stop without a drain period. + + Returns: + Whether shutdown cancelled its future. + """ + processor = EventProcessor(graceful_shutdown_timeout=0) + processor.configure() + async with processor: + future = await processor.enqueue( + "token", Event.from_event_type(SLOW_EVENT())[0] + ) + await asyncio.sleep(0) + return future.cancelled() + + try: + assert benchmark(lambda: loop.run_until_complete(cancel())) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_yield_intermediate_update(benchmark: BenchmarkFixture): + """Benchmark ordered streaming of rapid intermediate deltas. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + + async def collect() -> list[Mapping[str, Any]]: + """Stream all deltas from the yielding event. + + Returns: + Ordered emitted deltas. + """ + processor = EventProcessor(graceful_shutdown_timeout=2) + processor.configure() + async with processor: + return [ + delta + async for delta in processor.enqueue_stream_delta( + "token", Event.from_event_type(YIELD_EVENT())[0] + ) + ] + + try: + deltas = benchmark(lambda: loop.run_until_complete(collect())) + assert deltas == [ + {"state": {"value": 1}}, + {"state": {"value": 2}}, + ] + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_emit_update(benchmark: BenchmarkFixture): + """Benchmark the real emit-update await and socket-flush scheduling path. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + emitted: list[StateUpdate] = [] + + async def emit( # noqa: RUF029 + _event: str, update: StateUpdate, *, to: str + ) -> None: + """Record a stand-in Socket.IO write. + + Args: + _event: Socket event name. + update: State update. + to: Socket ID. + """ + assert to == "sid" + emitted.append(update) + + token_manager = SimpleNamespace( + instance_id="instance", + token_to_socket={"token": SimpleNamespace(instance_id="instance", sid="sid")}, + ) + namespace = cast( + EventNamespace, + SimpleNamespace(_token_manager=token_manager, emit=emit), + ) + update = StateUpdate(delta={"state": {"value": 1}}) + + try: + benchmark( + lambda: loop.run_until_complete( + EventNamespace.emit_update(namespace, update, "token") + ) + ) + assert emitted + finally: + loop.close() diff --git a/tests/benchmarks/test_event_payload.py b/tests/benchmarks/test_event_payload.py new file mode 100644 index 00000000000..f250010e076 --- /dev/null +++ b/tests/benchmarks/test_event_payload.py @@ -0,0 +1,43 @@ +"""Benchmarks for chained-event payload snapshotting.""" + +from typing import Any + +from pytest_codspeed import BenchmarkFixture + +from reflex.event import Event, EventHandler + +from .support.states import initialized_state + + +def _receive_payload(payload: Any) -> None: + """Define a handler accepting arbitrary representative payloads. + + Args: + payload: Event payload. + """ + + +PAYLOAD_EVENT = EventHandler(fn=_receive_payload) + + +def test_event_payload_plain_collection(benchmark: BenchmarkFixture): + """Benchmark snapshotting a large proxy-free event payload. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + payload = [{"index": index, "label": f"row-{index}"} for index in range(10_000)] + events = benchmark(lambda: Event.from_event_type(PAYLOAD_EVENT(payload))) + assert len(events[0].payload["payload"]) == len(payload) + + +def test_event_payload_mutable_proxy(benchmark: BenchmarkFixture): + """Benchmark the required detachment of a state-bound mutable proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + state = initialized_state(10_000) + payload = state.numbers + events = benchmark(lambda: Event.from_event_type(PAYLOAD_EVENT(payload))) + assert events[0].payload["payload"] == list(range(10_000)) diff --git a/tests/benchmarks/test_event_processing.py b/tests/benchmarks/test_event_processing.py index 9e34dd67719..fcdfaaa5474 100644 --- a/tests/benchmarks/test_event_processing.py +++ b/tests/benchmarks/test_event_processing.py @@ -183,19 +183,22 @@ def _(): loop.run_until_complete(run_events([token])) +@pytest.mark.parametrize("num_events", [1, 10, 100]) def test_process_event_burst_same_token( + num_events: int, event_processing_harness, benchmark: BenchmarkFixture, ): - """Benchmark ten queued events serialized through one token. + """Benchmark queued events serialized through one token. Args: + num_events: Number of events in the burst. event_processing_harness: The event runner and token purge helpers. benchmark: The codspeed benchmark fixture. """ run_events, _ = event_processing_harness loop = asyncio.get_event_loop() - tokens = ["burst-token"] * 10 + tokens = ["burst-token"] * num_events loop.run_until_complete(run_events([tokens[0]])) @benchmark @@ -203,19 +206,22 @@ def _(): loop.run_until_complete(run_events(tokens)) +@pytest.mark.parametrize("num_events", [1, 10, 100]) def test_process_event_burst_independent_tokens( + num_events: int, event_processing_harness, benchmark: BenchmarkFixture, ): - """Benchmark ten queued events distributed across independent tokens. + """Benchmark queued events distributed across independent tokens. Args: + num_events: Number of independent tokens and events. event_processing_harness: The event runner and token purge helpers. benchmark: The codspeed benchmark fixture. """ run_events, _ = event_processing_harness loop = asyncio.get_event_loop() - tokens = [f"independent-token-{index}" for index in range(10)] + tokens = [f"independent-token-{index}" for index in range(num_events)] loop.run_until_complete(run_events(tokens)) @benchmark diff --git a/tests/benchmarks/test_routes.py b/tests/benchmarks/test_routes.py new file mode 100644 index 00000000000..9585ebb6b74 --- /dev/null +++ b/tests/benchmarks/test_routes.py @@ -0,0 +1,72 @@ +"""Benchmarks for route construction and hot-path matching.""" + +import pytest +from pytest_codspeed import BenchmarkFixture +from reflex_base import constants + +from reflex.route import get_route_args, get_router + + +def _routes(count: int) -> list[str]: + """Build representative static and dynamic routes. + + Args: + count: Approximate route count. + + Returns: + Route templates. + """ + routes = ["index", "posts/[slug]", "files/[[...splat]]"] + routes.extend(f"section-{index}/item-{index}" for index in range(count - 3)) + return routes + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_route_matching(count: int, benchmark: BenchmarkFixture): + """Benchmark repeated matching through representative route tables. + + Args: + count: Number of routes. + benchmark: The CodSpeed benchmark fixture. + """ + router = get_router(_routes(count)) + paths = ["/", "/posts/reflex", f"/section-{count - 4}/item-{count - 4}"] + + def match_paths() -> tuple[str | None, ...]: + """Match the fixed path set. + + Returns: + Matched route templates. + """ + return tuple(router(path) for path in paths) + + result = benchmark(match_paths) + assert result == ("index", "posts/[slug]", paths[-1].removeprefix("/")) + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_route_table_construction(count: int, benchmark: BenchmarkFixture): + """Benchmark cold regex construction and route ordering. + + Args: + count: Number of routes. + benchmark: The CodSpeed benchmark fixture. + """ + routes = _routes(count) + router = benchmark(lambda: get_router(routes)) + assert router("/posts/reflex") == "posts/[slug]" + + +def test_route_argument_extraction(benchmark: BenchmarkFixture): + """Benchmark parsing dynamic and catch-all route arguments. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + route = "/org/[org_id]/project/[project_id]/files/[[...splat]]" + result = benchmark(lambda: get_route_args(route)) + assert result == { + "org_id": constants.RouteArgType.SINGLE, + "project_id": constants.RouteArgType.SINGLE, + "splat": constants.RouteArgType.LIST, + } diff --git a/tests/benchmarks/test_serialization.py b/tests/benchmarks/test_serialization.py new file mode 100644 index 00000000000..de613f00cea --- /dev/null +++ b/tests/benchmarks/test_serialization.py @@ -0,0 +1,61 @@ +"""Benchmarks for state-update and wire serialization.""" + +import pytest +from pytest_codspeed import BenchmarkFixture +from reflex_base.utils.format import json_dumps + +from reflex.state import StateUpdate, serialize_state_update + +from .support.events import PayloadKind, make_payload, wire_edge_cases + + +@pytest.mark.parametrize( + ("kind", "rows"), + [ + pytest.param(PayloadKind.SCALAR, 100, id="scalar_100b"), + pytest.param(PayloadKind.SCALAR, 10_000, id="scalar_10kb"), + pytest.param(PayloadKind.SCALAR, 1_000_000, id="scalar_1mb"), + pytest.param(PayloadKind.MAPPING, 100, id="mapping_100"), + pytest.param(PayloadKind.DATACLASS, 1000, id="dataclass_1000"), + pytest.param(PayloadKind.MODEL, 1000, id="model_1000"), + ], +) +def test_state_update_wire_serialization( + kind: PayloadKind, + rows: int, + benchmark: BenchmarkFixture, +): + """Benchmark encoding representative state updates to socket JSON. + + Args: + kind: Payload shape. + rows: Payload size parameter. + benchmark: The CodSpeed benchmark fixture. + """ + update = StateUpdate( + delta={"performance_state": {"payload": make_payload(kind, rows)}} + ) + encoded = benchmark(lambda: json_dumps(update)) + assert encoded + + +def test_state_update_dataclass_serialization(benchmark: BenchmarkFixture): + """Benchmark conversion of a state update into its transport dictionary. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + update = StateUpdate(delta={"performance_state": {"payload": list(range(10_000))}}) + result = benchmark(lambda: serialize_state_update(update)) + assert result["delta"] + + +def test_wire_edge_case_serialization(benchmark: BenchmarkFixture): + """Benchmark the complete set of supported special wire values. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + payload = wire_edge_cases() + encoded = benchmark(lambda: json_dumps(payload)) + assert "Reflex" in encoded diff --git a/tests/benchmarks/test_socket_headers.py b/tests/benchmarks/test_socket_headers.py new file mode 100644 index 00000000000..fb64309be02 --- /dev/null +++ b/tests/benchmarks/test_socket_headers.py @@ -0,0 +1,22 @@ +"""Benchmarks for per-event ASGI header decoding.""" + +import pytest +from pytest_codspeed import BenchmarkFixture + + +@pytest.mark.parametrize("count", [0, 4, 16, 64]) +def test_decode_event_headers(count: int, benchmark: BenchmarkFixture): + """Benchmark decoding the ASGI header representation used by socket events. + + Args: + count: Number of headers. + benchmark: The CodSpeed benchmark fixture. + """ + headers = [ + (f"x-performance-{index}".encode(), f"value-{index}".encode()) + for index in range(count) + ] + decoded = benchmark( + lambda: {key.decode("utf-8"): value.decode("utf-8") for key, value in headers} + ) + assert len(decoded) == count diff --git a/tests/benchmarks/test_state_delta.py b/tests/benchmarks/test_state_delta.py new file mode 100644 index 00000000000..513344f368b --- /dev/null +++ b/tests/benchmarks/test_state_delta.py @@ -0,0 +1,214 @@ +"""Benchmarks for dirty propagation and state delta generation.""" + +import pytest +from pytest_codspeed import BenchmarkFixture + +from .support.states import ( + PerformanceState, + computed_fanout_state, + initialized_state, + state_tree, +) + + +@pytest.mark.parametrize("size", [10, 100, 1000, 10_000]) +def test_state_delta_scalar_mutation(size: int, benchmark: BenchmarkFixture): + """Benchmark a scalar mutation and delta extraction across state sizes. + + Args: + size: Mutable collection size carried by the state. + benchmark: The CodSpeed benchmark fixture. + """ + state = initialized_state(size) + + def mutate_and_get_delta(): + """Mutate one field, calculate its delta, and reset dirtiness. + + Returns: + State delta. + """ + state.counter += 1 + delta = state.get_delta() + state._clean() + return delta + + delta = benchmark(mutate_and_get_delta) + assert delta + + +@pytest.mark.parametrize("appends", [1, 10, 100]) +def test_dirty_computed_var_propagation( + appends: int, + benchmark: BenchmarkFixture, +): + """Benchmark repeated mutable updates with computed dependencies. + + Args: + appends: Number of mutations in one measured operation. + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a stable state for one measured round. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = initialized_state(10) + _ = state.doubled_total + state._clean() + return ((state,), {}) + + def append_and_recompute(state: PerformanceState) -> int: + """Append values and resolve the dependent computed var. + + Args: + state: Fresh state for this measured round. + + Returns: + Recomputed aggregate. + """ + for value in range(appends): + state.numbers.append(value) + result = state.doubled_total + state._clean() + return result + + assert benchmark.pedantic(append_and_recompute, setup=setup) >= 0 + + +@pytest.mark.parametrize("fanout", [0, 2, 10, 50]) +def test_computed_dependency_fanout(fanout: int, benchmark: BenchmarkFixture): + """Benchmark dirty propagation across a controlled dependency fan-out. + + Args: + fanout: Number of cached vars directly depending on one source field. + benchmark: The CodSpeed benchmark fixture. + """ + + def mutate_and_delta(state): + """Mutate the shared dependency and return its resulting delta. + + Returns: + The updated source value. + """ + state.counter += 1 + state.get_delta() + state._clean() + return state.counter + + with computed_fanout_state(fanout) as (state, names): + for name in names: + getattr(state, name) + state._clean() + assert benchmark(mutate_and_delta, state) >= 1 + + +def test_resolved_delta(benchmark: BenchmarkFixture): + """Benchmark the async resolved-delta path used by event processing. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + import asyncio + + state = initialized_state(1000) + loop = asyncio.new_event_loop() + + def resolve_delta(): + """Mutate and resolve the full delta. + + Returns: + Resolved state delta. + """ + state.counter += 1 + delta = loop.run_until_complete(state._get_resolved_delta()) + state._clean() + return delta + + try: + assert benchmark(resolve_delta) + finally: + loop.close() + + +@pytest.mark.parametrize("shape", ["depth_10", "width_10", "three_by_three"]) +def test_state_tree_delta(shape: str, benchmark: BenchmarkFixture): + """Benchmark delta traversal through representative substate shapes. + + Args: + shape: Registered hierarchy shape. + benchmark: The CodSpeed benchmark fixture. + """ + contexts = [] + + def setup(): + """Create an isolated hierarchy for one measured round. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + context = state_tree(shape) + root = context.__enter__() + contexts.append(context) + return ((root,), {}) + + def mutate_and_delta(root): + """Mutate every performance substate and calculate one root delta. + + Args: + root: Root state. + + Returns: + Root delta. + """ + stack = [root] + while stack: + state = stack.pop() + stack.extend(state.substates.values()) + if hasattr(state, "counter"): + state.counter += 1 + return root.get_delta() + + def teardown(_root) -> None: + """Detach the isolated registration context.""" + contexts.pop().__exit__(None, None, None) + + assert benchmark.pedantic( + mutate_and_delta, + setup=setup, + teardown=teardown, + ) + + +@pytest.mark.parametrize("size", [10, 1000, 100_000]) +def test_state_collection_assignment(size: int, benchmark: BenchmarkFixture): + """Benchmark state assignment and its type-validation hot path. + + Args: + size: Assigned list size. + benchmark: The CodSpeed benchmark fixture. + """ + values = list(range(size)) + + def setup(): + """Create a fresh state for one assignment. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + return ((initialized_state(0),), {}) + + def assign(state: PerformanceState) -> int: + """Assign the representative collection. + + Args: + state: Fresh state. + + Returns: + Assigned collection size. + """ + state.numbers = values + return len(state.numbers) + + assert benchmark.pedantic(assign, setup=setup) == size diff --git a/tests/benchmarks/test_state_manager.py b/tests/benchmarks/test_state_manager.py new file mode 100644 index 00000000000..e6516e76492 --- /dev/null +++ b/tests/benchmarks/test_state_manager.py @@ -0,0 +1,118 @@ +"""Deterministic benchmarks for in-memory state-manager operations.""" + +import asyncio + +from pytest_codspeed import BenchmarkFixture + +from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import StateToken + +from .support.states import PerformanceState + + +def test_state_manager_memory_cold_get(benchmark: BenchmarkFixture): + """Benchmark state construction for an uncached token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + iteration = 0 + + def setup(): + """Return a unique token for one cold measurement.""" + nonlocal iteration + iteration += 1 + token = StateToken(ident=f"cold-{iteration}", cls=PerformanceState) + return ((token,), {}) + + def get_state(token: StateToken[PerformanceState]) -> PerformanceState: + """Fetch a state through the async manager API. + + Returns: + Managed state. + """ + return loop.run_until_complete(manager.get_state(token)) + + def teardown(token: StateToken[PerformanceState]) -> None: + """Purge the measured state.""" + manager._purge_token(token) # pyright: ignore [reportPrivateUsage] + + try: + benchmark.pedantic(get_state, setup=setup, teardown=teardown) + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_warm_get(benchmark: BenchmarkFixture): + """Benchmark state lookup for a cached token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = StateToken(ident="warm", cls=PerformanceState) + loop.run_until_complete(manager.get_state(token)) + + try: + state = benchmark(lambda: loop.run_until_complete(manager.get_state(token))) + assert isinstance(state, PerformanceState) + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_modify(benchmark: BenchmarkFixture): + """Benchmark lock acquisition, mutation, and release for one token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = StateToken(ident="modify", cls=PerformanceState) + + async def modify() -> int: + """Increment one state under the manager lock. + + Returns: + Updated counter. + """ + async with manager.modify_state(token) as state: + state.counter += 1 + return state.counter + + try: + assert benchmark(lambda: loop.run_until_complete(modify())) > 0 + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_read_only_modify(benchmark: BenchmarkFixture): + """Benchmark a read-only lock context that should not produce a write. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = StateToken(ident="read-only", cls=PerformanceState) + + async def read_only() -> int: + """Read one field under the manager lock. + + Returns: + Current counter. + """ + async with manager.modify_state(token) as state: + return state.counter + + try: + assert benchmark(lambda: loop.run_until_complete(read_only())) == 0 + finally: + loop.run_until_complete(manager.close()) + loop.close() diff --git a/tests/benchmarks/test_state_proxy.py b/tests/benchmarks/test_state_proxy.py index 455cb8376ab..859a2e0a5fe 100644 --- a/tests/benchmarks/test_state_proxy.py +++ b/tests/benchmarks/test_state_proxy.py @@ -93,3 +93,67 @@ def test_var_access(access_fn, benchmark: BenchmarkFixture): """ state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] benchmark(lambda: access_fn(state)) + + +def test_proxy_list_mutation(benchmark: BenchmarkFixture): + """Benchmark a batch of writes through a mutable list proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a fresh state for one measured write batch. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] + return ((state,), {}) + + def mutate(state: ProxyBenchmarkState) -> int: + """Append one hundred values through the proxy. + + Args: + state: Fresh benchmark state. + + Returns: + Final list length. + """ + for value in range(100): + state.numbers.append(value) + return len(state.numbers) + + assert benchmark.pedantic(mutate, setup=setup) == N + 100 + + +def test_proxy_mapping_mutation(benchmark: BenchmarkFixture): + """Benchmark a batch of writes through a mutable mapping proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a fresh state for one measured write batch. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] + return ((state,), {}) + + def mutate(state: ProxyBenchmarkState) -> int: + """Update one hundred values through the proxy. + + Args: + state: Fresh benchmark state. + + Returns: + Final mapping length. + """ + for value in range(100): + state.mapping[value] = value + return len(state.mapping) + + assert benchmark.pedantic(mutate, setup=setup) == N diff --git a/tests/benchmarks/test_var_operations.py b/tests/benchmarks/test_var_operations.py new file mode 100644 index 00000000000..aa8edaa4e89 --- /dev/null +++ b/tests/benchmarks/test_var_operations.py @@ -0,0 +1,39 @@ +"""Benchmarks for Var construction and type dispatch.""" + +from pytest_codspeed import BenchmarkFixture + +from reflex.vars import Var + + +def test_var_arithmetic_chain(benchmark: BenchmarkFixture): + """Benchmark construction of a representative arithmetic expression. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + left = Var.create(1) + right = Var.create(2) + result = benchmark(lambda: ((left + right) * right - left) / right) + assert result._js_expr + + +def test_var_to_dispatch(benchmark: BenchmarkFixture): + """Benchmark conversion through the Var subclass registry. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + value = Var(_js_expr="value", _var_type=int) + result = benchmark(lambda: value.to(float)) + assert result._var_type is float + + +def test_var_guess_type_dispatch(benchmark: BenchmarkFixture): + """Benchmark inferred dispatch through the Var subclass registry. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + value = Var(_js_expr="value", _var_type=list[int]) + result = benchmark(value.guess_type) + assert result._var_type == list[int] diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 00000000000..c4e2b92c0cc --- /dev/null +++ b/tests/performance/__init__.py @@ -0,0 +1 @@ +"""Scheduled performance suites excluded from ordinary test runs.""" diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py new file mode 100644 index 00000000000..4208f238b44 --- /dev/null +++ b/tests/performance/conftest.py @@ -0,0 +1,77 @@ +"""Configuration for scheduled performance suites.""" + +from pathlib import Path + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Register explicit performance-suite options. + + Args: + parser: Pytest command-line parser. + """ + group = parser.getgroup("performance") + group.addoption( + "--run-performance", + action="store_true", + default=False, + help="run scheduled performance tests", + ) + group.addoption( + "--performance-output", + default=".pytest-tmp/performance", + help="directory for performance JSON, traces, and profiles", + ) + group.addoption( + "--performance-scale", + choices=("smoke", "release"), + default="smoke", + help="performance scenario scale", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, + items: list[pytest.Item], +) -> None: + """Skip scheduled performance tests unless explicitly enabled. + + Args: + config: Active Pytest configuration. + items: Collected tests. + """ + if config.getoption("--run-performance"): + return + skip = pytest.mark.skip(reason="pass --run-performance to execute this suite") + for item in items: + if "performance" in item.keywords: + item.add_marker(skip) + + +@pytest.fixture +def performance_output(request: pytest.FixtureRequest) -> Path: + """Return and create the performance artifact directory. + + Args: + request: Pytest fixture request. + + Returns: + Artifact directory. + """ + output = Path(request.config.getoption("--performance-output")).resolve() + output.mkdir(parents=True, exist_ok=True) + return output + + +@pytest.fixture +def performance_scale(request: pytest.FixtureRequest) -> str: + """Return the selected scenario scale. + + Args: + request: Pytest fixture request. + + Returns: + ``smoke`` or ``release``. + """ + return request.config.getoption("--performance-scale") diff --git a/tests/performance/test_browser.py b/tests/performance/test_browser.py new file mode 100644 index 00000000000..e8a0712ea04 --- /dev/null +++ b/tests/performance/test_browser.py @@ -0,0 +1,164 @@ +"""Production browser, bundle, hydration, and DOM-update benchmarks.""" + +from __future__ import annotations + +import gzip +import time +from collections.abc import Generator +from pathlib import Path + +import pytest +from playwright.sync_api import Page, expect + +from reflex.testing import AppHarness, AppHarnessProd +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.apps import load_app_source + +MAX_JAVASCRIPT_GZIP_BYTES = 14_000_000 + + +@pytest.fixture(scope="module") +def performance_browser_app(tmp_path_factory) -> Generator[AppHarness, None, None]: + """Build and serve the representative application in production mode. + + Args: + tmp_path_factory: Pytest temporary directory factory. + + Yields: + Running production app harness. + """ + with AppHarnessProd.create( + root=tmp_path_factory.mktemp("performance_browser"), + app_source=load_app_source(), + app_name="performance_browser", + ) as harness: + yield harness + + +def _bundle_metrics(root: Path) -> dict[str, int]: + """Calculate bundle module and compressed-size metrics. + + Args: + root: Reflex application root. + + Returns: + JavaScript module, raw byte, and gzip byte counts. + """ + files = [ + path + for path in (root / ".web").rglob("*") + if path.is_file() and path.suffix in {".js", ".mjs"} + ] + return { + "javascript_modules": len(files), + "javascript_bytes": sum(path.stat().st_size for path in files), + "javascript_gzip_bytes": sum( + len(gzip.compress(path.read_bytes())) for path in files + ), + } + + +def _browser_heap(page: Page) -> int: + """Read Chromium's JavaScript heap metric. + + Args: + page: Playwright page. + + Returns: + Used JavaScript heap bytes, or zero if unavailable. + """ + session = page.context.new_cdp_session(page) + try: + session.send("Performance.enable") + metrics = session.send("Performance.getMetrics")["metrics"] + finally: + session.detach() + values = {metric["name"]: metric["value"] for metric in metrics} + return int(values.get("JSHeapUsedSize", 0)) + + +@pytest.mark.performance +def test_browser_report( + performance_browser_app: AppHarness, + page: Page, + performance_output: Path, + performance_scale: str, +): + """Measure hydration, state-to-DOM updates, large lists, navigation, and heap. + + Args: + performance_browser_app: Running production app. + page: Playwright browser page. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + assert performance_browser_app.frontend_url is not None + page.add_init_script( + """ + window.__reflexLongTasks = []; + new PerformanceObserver((list) => { + window.__reflexLongTasks.push(...list.getEntries().map((entry) => entry.duration)); + }).observe({entryTypes: ['longtask']}); + """ + ) + started = time.perf_counter_ns() + page.goto(performance_browser_app.frontend_url) + expect(page.locator("#count")).to_have_text("0") + hydration_ms = (time.perf_counter_ns() - started) / 1_000_000 + heap_before = _browser_heap(page) + + updates = {"smoke": 10, "release": 1000}[performance_scale] + started = time.perf_counter_ns() + for _ in range(updates): + page.locator("#increment").click() + expect(page.locator("#count")).to_have_text(str(updates)) + update_ms = (time.perf_counter_ns() - started) / 1_000_000 + + started = time.perf_counter_ns() + page.locator("#append-large").click() + expect(page.locator(".value-row")).to_have_count(1000) + large_list_ms = (time.perf_counter_ns() - started) / 1_000_000 + + started = time.perf_counter_ns() + page.locator("#second-link").click() + expect(page.locator("#second-heading")).to_have_text("Second page") + navigation_ms = (time.perf_counter_ns() - started) / 1_000_000 + heap_after = _browser_heap(page) + long_tasks = page.evaluate("window.__reflexLongTasks || []") + assert isinstance(long_tasks, list) + + metrics: dict[str, float | int] = dict( + _bundle_metrics(performance_browser_app.app_path) + ) + metrics.update({ + "heap_before_bytes": heap_before, + "heap_after_bytes": heap_after, + "heap_growth_bytes": heap_after - heap_before, + "long_task_count": len(long_tasks), + "long_task_total_ms": sum(float(value) for value in long_tasks), + "javascript_gzip_budget_bytes": MAX_JAVASCRIPT_GZIP_BYTES, + }) + report = PerformanceReport( + "browser", + metadata={"scale": performance_scale}, + ) + for name, observations, parameters in [ + ("hydration", [hydration_ms], {}), + ("state_update_to_dom", [update_ms / updates], {"updates": updates}), + ("render_1000_rows", [large_list_ms], {"rows": 1000}), + ("route_navigation", [navigation_ms], {}), + ]: + report.add( + BenchmarkResult( + name, + parameters, + observations, + metrics, + measurement_iterations=len(observations), + ) + ) + report.write(performance_output / "browser.json") + + assert metrics["javascript_modules"] > 0 + assert metrics["javascript_gzip_bytes"] <= MAX_JAVASCRIPT_GZIP_BYTES + assert heap_after > 0 diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py new file mode 100644 index 00000000000..b7f1cfd09e7 --- /dev/null +++ b/tests/performance/test_compiler_lifecycle.py @@ -0,0 +1,199 @@ +"""Developer lifecycle benchmarks for init, compile, edits, and export.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.apps import lifecycle_app_source + + +def _run(command: list[str], cwd: Path, timeout: float = 300) -> float: + """Run a lifecycle command and return elapsed milliseconds. + + Args: + command: Command and arguments. + cwd: Working directory. + timeout: Command timeout. + + Returns: + Elapsed wall time in milliseconds. + """ + started = time.perf_counter_ns() + subprocess.run( + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=timeout, + env=os.environ | {"REFLEX_TELEMETRY_ENABLED": "false"}, + ) + return (time.perf_counter_ns() - started) / 1_000_000 + + +def _generated_metrics(root: Path) -> dict[str, int]: + """Count generated frontend files and bytes. + + Args: + root: Application root. + + Returns: + Generated file, JavaScript module, and byte counts. + """ + web = root / ".web" + files = [path for path in web.rglob("*") if path.is_file()] if web.exists() else [] + javascript = [ + path for path in files if path.suffix in {".js", ".jsx", ".ts", ".tsx"} + ] + return { + "generated_files": len(files), + "javascript_modules": len(javascript), + "generated_bytes": sum(path.stat().st_size for path in files), + "javascript_bytes": sum(path.stat().st_size for path in javascript), + } + + +@pytest.mark.performance +def test_compiler_lifecycle_report( + tmp_path: Path, + performance_output: Path, + performance_scale: str, +): + """Measure import, init, cold/warm compiles, edits, reload cycles, and export. + + Args: + tmp_path: Temporary application parent. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + sizes = { + "smoke": (10, 1, 2), + "release": (1000, 25, 100), + } + rows, pages, reload_cycles = sizes[performance_scale] + app_root = tmp_path / "lifecycle_app" + app_root.mkdir() + executable = sys.executable + report = PerformanceReport( + "compiler-lifecycle", + metadata={"scale": performance_scale, "rows": rows, "pages": pages}, + ) + + import_ms = _run([executable, "-c", "import reflex"], app_root) + report.add( + BenchmarkResult( + "import_reflex", + {}, + [import_ms], + {}, + measurement_iterations=1, + ) + ) + + init_ms = _run( + [ + executable, + "-m", + "reflex", + "init", + "--name", + "lifecycle_app", + "--template", + "blank", + "--no-agents", + ], + app_root, + ) + app_source = app_root / "lifecycle_app" / "lifecycle_app.py" + app_source.write_text(lifecycle_app_source(rows, pages), encoding="utf-8") + report.add(BenchmarkResult("init", {}, [init_ms], {}, measurement_iterations=1)) + + compile_command = [executable, "-m", "reflex", "compile", "--no-rich"] + cold_ms = _run(compile_command, app_root) + warm_ms = _run(compile_command, app_root) + report.add( + BenchmarkResult( + "compile_cold", + {"rows": rows, "pages": pages}, + [cold_ms], + _generated_metrics(app_root), + ) + ) + report.add( + BenchmarkResult( + "compile_warm_no_change", + {"rows": rows, "pages": pages}, + [warm_ms], + _generated_metrics(app_root), + ) + ) + + source = app_source.read_text() + page_source = source.replace( + "rx.heading(label)", + 'rx.heading(f"edited {label}")', + ) + app_source.write_text(page_source, encoding="utf-8") + page_edit_ms = _run(compile_command, app_root) + report.add(BenchmarkResult("compile_page_edit", {}, [page_edit_ms], {})) + + shared_source = page_source.replace( + "rx.text(index)", + 'rx.text(index, class_name="shared-row-label")', + ) + app_source.write_text(shared_source, encoding="utf-8") + shared_edit_ms = _run(compile_command, app_root) + report.add( + BenchmarkResult( + "compile_shared_component_edit", + {}, + [shared_edit_ms], + {}, + ) + ) + + source = shared_source.replace("count: int = 0", "count: int = 1") + app_source.write_text(source, encoding="utf-8") + state_edit_ms = _run(compile_command, app_root) + report.add(BenchmarkResult("compile_state_edit", {}, [state_edit_ms], {})) + + reload_observations = [] + for cycle in range(reload_cycles): + app_source.write_text( + source + f"\n# reload cycle {cycle}\n", + encoding="utf-8", + ) + reload_observations.append(_run(compile_command, app_root)) + report.add( + BenchmarkResult( + "hot_reload_cycles", + {"cycles": reload_cycles}, + reload_observations, + _generated_metrics(app_root), + measurement_iterations=reload_cycles, + ) + ) + + export_ms = _run( + [ + executable, + "-m", + "reflex", + "export", + "--backend-only", + "--no-zip", + ], + app_root, + ) + report.add(BenchmarkResult("export_backend", {}, [export_ms], {})) + report.write(performance_output / "compiler-lifecycle.json") + + assert report.results + assert _generated_metrics(app_root)["generated_files"] > 0 diff --git a/tests/performance/test_event_load.py b/tests/performance/test_event_load.py new file mode 100644 index 00000000000..ee9a31bf3f0 --- /dev/null +++ b/tests/performance/test_event_load.py @@ -0,0 +1,112 @@ +"""Production Socket.IO concurrency, throughput, and tail-latency benchmarks.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Generator +from pathlib import Path + +import pytest +from reflex_base.config import get_config +from reflex_base.registry import RegistrationContext + +from reflex.testing import AppHarness, AppHarnessProd +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.apps import load_app_source +from tests.benchmarks.support.socket_client import run_clients, run_socket_client + + +@pytest.fixture(scope="module") +def performance_load_app(tmp_path_factory) -> Generator[AppHarness, None, None]: + """Build and run the representative production load application. + + Args: + tmp_path_factory: Pytest temporary directory factory. + + Yields: + Running production app harness. + """ + with AppHarnessProd.create( + root=tmp_path_factory.mktemp("performance_load"), + app_source=load_app_source(), + app_name="performance_load", + ) as harness: + yield harness + + +@pytest.mark.performance +def test_event_load_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure production Socket.IO latency and throughput across client counts. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + scales = { + "smoke": ([1, 5], 3), + "release": ([1, 10, 50, 200], 50), + } + client_counts, events_per_client = scales[performance_scale] + backend_url = get_config().api_url.rstrip("/") + handler_name = next( + name + for name in RegistrationContext.get().event_handlers + if name.endswith(".increment") and "performance_load" in name + ) + payload = { + "name": handler_name, + "payload": {}, + "router_data": {"path": "/", "pathname": "/", "query": {}}, + } + report = PerformanceReport( + "event-load", + metadata={"scale": performance_scale, "backend_url": backend_url}, + ) + + for clients in client_counts: + started = time.perf_counter_ns() + results = asyncio.run( + run_clients( + clients, + lambda index, clients=clients: run_socket_client( + backend_url, + f"load-token-{clients}-{index}", + payload, + events_per_client, + ), + ) + ) + elapsed_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 + latencies = [latency for result in results for latency in result.latencies_ms] + errors = [error for result in results for error in result.errors] + per_client_max = [max(result.latencies_ms, default=0) for result in results] + expected = clients * events_per_client + report.add( + BenchmarkResult( + name=f"unique_tokens_{clients}_clients", + parameters={ + "clients": clients, + "events_per_client": events_per_client, + }, + observations_ms=latencies, + metrics={ + "throughput_events_per_second": len(latencies) / elapsed_seconds, + "errors": len(errors), + "expected_updates": expected, + "received_updates": len(latencies), + "fairness_spread_ms": max(per_client_max, default=0) + - min(per_client_max, default=0), + }, + measurement_iterations=len(latencies), + ) + ) + assert not errors, errors + assert len(latencies) == expected + + report.write(performance_output / "event-load.json") diff --git a/tests/performance/test_event_loop.py b/tests/performance/test_event_loop.py new file mode 100644 index 00000000000..f999a55b96e --- /dev/null +++ b/tests/performance/test_event_loop.py @@ -0,0 +1,607 @@ +"""Wall-time event-loop, queueing, ordering, and blocking benchmarks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import time +from collections import defaultdict +from collections.abc import AsyncIterator, Mapping +from pathlib import Path +from typing import Any + +import pytest +from reflex_base.event.context import EventContext +from reflex_base.event.processor import BaseStateEventProcessor +from reflex_base.event.processor.event_processor import EventProcessor, EventQueueEntry +from reflex_base.registry import RegisteredEventHandler, RegistrationContext +from typing_extensions import Unpack, override + +import reflex as rx +from reflex.event import Event, EventHandler +from reflex.istate.manager import StateModificationContext +from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import TOKEN_TYPE, StateToken +from reflex.state import BaseState +from tests.benchmarks.support import ( + BenchmarkResult, + EventLoopProbe, + PerformanceReport, + PipelineTrace, + capture_async_diagnostics, +) +from tests.benchmarks.support.pipeline_trace import StageEvent +from tests.benchmarks.support.report import ( + current_process_metrics, + percentile, + runtime_metadata, +) + +_ORDER_LOG: dict[str, list[int]] = defaultdict(list) +_ACTIVE_STATE_TRACE: PipelineTrace | None = None + + +def _record_state_stage(stage: str) -> None: + """Record a stage for the active state event. + + Args: + stage: Pipeline stage name. + """ + trace = _ACTIVE_STATE_TRACE + assert trace is not None + trace.record(EventContext.get().txid, stage) + + +class AttributionState(rx.State): + """State whose handler exposes precise benchmark-only boundaries.""" + + value: rx.Field[int] = rx.field(0) + + @rx.event + def increment(self): + """Record handler and pre-delta boundaries around a mutation.""" + _record_state_stage("handler_started") + self.value += 1 + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def cpu_heavy(self): + """Perform synchronous CPU work on the event loop.""" + _record_state_stage("handler_started") + self.value += sum(range(100_000)) + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def blocking(self): + """Perform a deliberately blocking synchronous wait.""" + _record_state_stage("handler_started") + time.sleep(0.005) + self.value += 1 + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + async def thread_offload(self): + """Move CPU work off the event-loop thread.""" + _record_state_stage("handler_started") + self.value += await asyncio.to_thread(sum, range(100_000)) + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def sync_generator(self): + """Yield three synchronous intermediate state updates.""" + _record_state_stage("handler_started") + for _ in range(3): + self.value += 1 + _record_state_stage("delta_started") + yield + _record_state_stage("handler_finished") + + @rx.event + async def async_generator(self): + """Yield three cooperative asynchronous state updates.""" + _record_state_stage("handler_started") + for _ in range(3): + await asyncio.sleep(0) + self.value += 1 + _record_state_stage("delta_started") + yield + _record_state_stage("handler_finished") + + @rx.event + def failure(self): + """Raise an exception after recording handler entry.""" + _record_state_stage("handler_started") + msg = "intentional performance scenario failure" + raise RuntimeError(msg) + + +class TracingStateManagerMemory(StateManagerMemory): + """Memory manager recording lock and state-flush boundaries.""" + + def __init__(self, trace: PipelineTrace): + """Initialize the manager. + + Args: + trace: Pipeline trace receiving observations. + """ + super().__init__() + self.trace = trace + + @override + @contextlib.asynccontextmanager + async def modify_state( + self, + token: StateToken[TOKEN_TYPE], + **context: Unpack[StateModificationContext], + ) -> AsyncIterator[TOKEN_TYPE]: + """Record lock wait, hold, and flush stages. + + Args: + token: Managed state token. + context: State modification context. + + Yields: + Managed state. + """ + transaction = EventContext.get().txid + self.trace.record(transaction, "lock_wait_started") + async with super().modify_state(token, **context) as state: + self.trace.record(transaction, "lock_acquired") + try: + yield state + finally: + self.trace.record(transaction, "state_flush_started") + self.trace.record(transaction, "state_flush_finished") + self.trace.record(transaction, "lock_released") + + +class AttributionEventProcessor(BaseStateEventProcessor): + """State processor that omits first-connect hydration from attribution.""" + + @override + async def _rehydrate(self, root_state: BaseState) -> None: + """Skip hydration so the scenario isolates one application event. + + Args: + root_state: Root state, intentionally unused. + """ + + +async def _cooperative_handler(sequence: int) -> None: + """Yield to the loop before recording event completion. + + Args: + sequence: Per-token event sequence. + """ + await asyncio.sleep(0.001) + _ORDER_LOG[EventContext.get().token].append(sequence) + + +async def _blocking_handler(sequence: int) -> None: # noqa: RUF029 + """Deliberately block the loop before recording completion. + + Args: + sequence: Per-token event sequence. + """ + time.sleep(0.005) # noqa: ASYNC251 - intentional benchmark scenario + _ORDER_LOG[EventContext.get().token].append(sequence) + + +COOPERATIVE_EVENT = EventHandler(fn=_cooperative_handler) +BLOCKING_EVENT = EventHandler(fn=_blocking_handler) + + +class TracingEventProcessor(EventProcessor): + """Event processor that records benchmark-only task pipeline stages.""" + + def __init__(self, graceful_shutdown_timeout: float | None = None): + """Initialize the processor and its isolated trace. + + Args: + graceful_shutdown_timeout: Maximum graceful drain time. + """ + super().__init__(graceful_shutdown_timeout=graceful_shutdown_timeout) + self.trace = PipelineTrace() + + async def enqueue(self, token: str, event: Event, ev_ctx=None): + """Record ingress and queue completion around normal enqueue. + + Args: + token: Client token. + event: Event to enqueue. + ev_ctx: Optional existing event context. + + Returns: + Tracked event future. + """ + started = time.perf_counter_ns() + future = await super().enqueue(token, event, ev_ctx) + self.trace.extend([ + _stage(future.txid, "received", started), + _stage(future.txid, "enqueued", time.perf_counter_ns()), + ]) + return future + + def _enqueue_for_token( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> None: + """Record shared-queue dequeue and per-token queue entry.""" + self.trace.record(entry.ctx.txid, "dequeued") + self.trace.record(entry.ctx.txid, "token_queued") + super()._enqueue_for_token( + entry=entry, + registered_handler=registered_handler, + ) + + def _create_event_task( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> asyncio.Task: + """Record when per-token ordering permits task creation. + + Returns: + Created event task. + """ + self.trace.record(entry.ctx.txid, "token_ready") + self.trace.record(entry.ctx.txid, "task_created") + return super()._create_event_task( + entry=entry, + registered_handler=registered_handler, + ) + + async def _process_event_queue_entry( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> None: + """Record scheduling, handler, and completion boundaries.""" + self.trace.record(entry.ctx.txid, "task_started") + self.trace.record(entry.ctx.txid, "handler_started") + try: + await super()._process_event_queue_entry( + entry=entry, + registered_handler=registered_handler, + ) + finally: + self.trace.record(entry.ctx.txid, "handler_finished") + self.trace.record(entry.ctx.txid, "completed") + + +def _stage(token: str, stage: str, timestamp_ns: int) -> StageEvent: + """Create a pipeline stage without another clock read. + + Args: + token: Transaction identifier. + stage: Stage name. + timestamp_ns: Existing monotonic timestamp. + + Returns: + Stage event. + """ + return StageEvent(token, stage, timestamp_ns) + + +async def _run_scenario( + handler: EventHandler, + *, + tokens: int, + events_per_token: int, + probe_interval: float, + hot_token_fraction: float = 0, +) -> tuple[list[float], dict[str, float | int], PipelineTrace]: + """Run one event-loop scenario and collect latency and resource metrics. + + Args: + handler: Registered workload handler. + tokens: Number of independent client tokens. + events_per_token: Sequential events per token. + probe_interval: Loop-lag sample interval. + hot_token_fraction: Fraction of all work assigned to token zero. + + Returns: + Latencies, probe/resource metrics, and pipeline trace. + """ + _ORDER_LOG.clear() + processor = TracingEventProcessor(graceful_shutdown_timeout=2) + processor.configure() + + def queue_depth() -> int: + """Return shared queue depth.""" + return processor._queue.qsize() if processor._queue is not None else 0 + + def token_queue_depth() -> int: + """Return total queued per-token work.""" + return sum(len(queue) for queue in processor._token_queues.values()) + + probe = EventLoopProbe( + interval=probe_interval, + gauges={ + "queue_depth": queue_depth, + "token_queue_depth": token_queue_depth, + "processor_tasks": lambda: len(processor._tasks), + }, + ) + latencies: list[float] = [] + started: dict[str, int] = {} + expected_order: dict[str, list[int]] = defaultdict(list) + if hot_token_fraction: + total_events = tokens * events_per_token + hot_events = int(total_events * hot_token_fraction) + token_events = [("token-0", sequence) for sequence in range(hot_events)] + for offset in range(total_events - hot_events): + token_index = 1 + offset % max(1, tokens - 1) + token = f"token-{token_index}" + token_events.append((token, len(expected_order[token]))) + expected_order[token].append(len(expected_order[token])) + expected_order["token-0"] = list(range(hot_events)) + else: + token_events = [ + (f"token-{token_index}", sequence) + for token_index in range(tokens) + for sequence in range(events_per_token) + ] + for token_index in range(tokens): + expected_order[f"token-{token_index}"] = list(range(events_per_token)) + + async with probe, processor: + futures = [] + for token, sequence in token_events: + future = await processor.enqueue( + token, + Event.from_event_type(handler(sequence))[0], + ) + started[future.txid] = time.perf_counter_ns() + future.add_done_callback( + lambda completed, txid=future.txid: latencies.append( + (time.perf_counter_ns() - started[txid]) / 1_000_000 + ) + ) + futures.append(future) + await asyncio.gather(*(future.wait_all() for future in futures)) + await asyncio.sleep(probe_interval * 2) + + for token, expected in expected_order.items(): + assert _ORDER_LOG[token] == expected + assert not processor._tasks + assert not processor._futures + durations = processor.trace.durations_ms([ + ("received", "enqueued"), + ("enqueued", "dequeued"), + ("dequeued", "task_started"), + ("token_queued", "token_ready"), + ("task_created", "task_started"), + ("handler_started", "handler_finished"), + ]) + stage_metrics = { + f"{name}_p95_ms": percentile(observations, 95) + for name, observations in durations.items() + } + metrics = ( + probe.summary() + | current_process_metrics() + | stage_metrics + | { + "tasks_created": sum( + event.stage == "task_created" for event in processor.trace.events + ), + "orphan_tasks": len(processor._tasks), + "orphan_futures": len(processor._futures), + } + ) + return latencies, metrics, processor.trace + + +async def _run_state_pipeline( + handler: Any, +) -> tuple[float, dict[str, float | int], PipelineTrace]: + """Run a full state mutation, delta, flush, and emit pipeline. + + Returns: + End-to-end latency, stage metrics, and trace. + """ + global _ACTIVE_STATE_TRACE + trace = PipelineTrace() + _ACTIVE_STATE_TRACE = trace + manager = TracingStateManagerMemory(trace) + processor = AttributionEventProcessor(graceful_shutdown_timeout=2) + processor.configure(state_manager=manager) + assert processor._root_context is not None # pyright: ignore [reportPrivateUsage] + + async def emit_delta( + _token: str, + _delta: Mapping[str, Mapping[str, Any]], + ) -> None: + """Record delta completion and simulated socket flush boundaries.""" + transaction = EventContext.get().txid + trace.record(transaction, "delta_finished") + trace.record(transaction, "emit_started") + await asyncio.sleep(0) + trace.record(transaction, "emit_finished") + + processor._root_context = dataclasses.replace( # pyright: ignore [reportPrivateUsage] + processor._root_context, # pyright: ignore [reportPrivateUsage] + emit_delta_impl=emit_delta, + ) + event = Event.from_event_type(handler())[0] + event = dataclasses.replace(event, router_data={"path": "/", "query": {}}) + started = time.perf_counter_ns() + try: + async with processor: + received = time.perf_counter_ns() + future = await processor.enqueue("attribution-token", event) + trace.extend([ + StageEvent(future.txid, "received", received), + StageEvent(future.txid, "enqueued", time.perf_counter_ns()), + ]) + await future.wait_all() + finally: + await manager.close() + _ACTIVE_STATE_TRACE = None + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + durations = trace.durations_ms([ + ("received", "enqueued"), + ("lock_wait_started", "lock_acquired"), + ("handler_started", "handler_finished"), + ("delta_started", "delta_finished"), + ("state_flush_started", "state_flush_finished"), + ("emit_started", "emit_finished"), + ("lock_acquired", "lock_released"), + ]) + metrics: dict[str, float | int] = {} + for name, observations in durations.items(): + metrics[f"{name}_p95_ms"] = percentile(observations, 95) + return elapsed, metrics, trace + + +async def _run_failed_state_pipeline() -> None: + """Verify exception cleanup leaves no processor tasks or futures.""" + global _ACTIVE_STATE_TRACE + trace = PipelineTrace() + _ACTIVE_STATE_TRACE = trace + manager = TracingStateManagerMemory(trace) + exceptions: list[Exception] = [] + processor = AttributionEventProcessor( + graceful_shutdown_timeout=2, + backend_exception_handler=exceptions.append, + ) + processor.configure(state_manager=manager) + event = Event.from_event_type(AttributionState.failure())[0] + event = dataclasses.replace(event, router_data={"path": "/", "query": {}}) + try: + async with processor: + future = await processor.enqueue("failure-token", event) + with contextlib.suppress(RuntimeError): + await future.wait_all() + finally: + await manager.close() + _ACTIVE_STATE_TRACE = None + assert exceptions + assert not processor._tasks + assert not processor._futures + + +@pytest.mark.performance +async def test_event_loop_component_report( + performance_output: Path, + performance_scale: str, +): + """Measure cooperative, contended, concurrent, and blocking event workloads. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + scales = { + "smoke": (5, 4, 10), + "release": (200, 50, 1000), + } + independent_tokens, independent_events, same_token_events = scales[ + performance_scale + ] + scenarios = [ + ("same_token", COOPERATIVE_EVENT, 1, same_token_events, 0), + ( + "independent_tokens", + COOPERATIVE_EVENT, + independent_tokens, + independent_events, + 0, + ), + ( + "hot_token_80_20", + COOPERATIVE_EVENT, + max(5, independent_tokens), + independent_events, + 0.8, + ), + ("blocking_handler", BLOCKING_EVENT, 2, 5, 0), + ] + registry = RegistrationContext() + with registry: + for handler in (COOPERATIVE_EVENT, BLOCKING_EVENT): + RegistrationContext.register_event_handler(handler) + RegistrationContext.register_base_state(AttributionState) + + report = PerformanceReport( + "event-loop", + metadata=runtime_metadata() | {"scale": performance_scale}, + ) + traces: list[PipelineTrace] = [] + for name, handler, tokens, events_per_token, hot_fraction in scenarios: + latencies, metrics, trace = await _run_scenario( + handler, + tokens=tokens, + events_per_token=events_per_token, + probe_interval=0.001, + hot_token_fraction=hot_fraction, + ) + report.add( + BenchmarkResult( + name=name, + parameters={ + "tokens": tokens, + "events_per_token": events_per_token, + "hot_token_fraction": hot_fraction, + }, + observations_ms=latencies, + metrics=metrics, + measurement_iterations=len(latencies), + ) + ) + traces.append(trace) + + for state_name, state_handler in [ + ("state_pipeline_attribution", AttributionState.increment), + ("cpu_heavy_handler", AttributionState.cpu_heavy), + ("blocking_state_handler", AttributionState.blocking), + ("thread_offload_handler", AttributionState.thread_offload), + ("sync_generator_handler", AttributionState.sync_generator), + ("async_generator_handler", AttributionState.async_generator), + ]: + state_latency, state_metrics, state_trace = await _run_state_pipeline( + state_handler + ) + report.add( + BenchmarkResult( + name=state_name, + parameters={"state_manager": "memory"}, + observations_ms=[state_latency], + metrics=state_metrics, + measurement_iterations=1, + ) + ) + traces.append(state_trace) + await _run_failed_state_pipeline() + + report.write(performance_output / "event-loop.json") + combined_trace = PipelineTrace() + for trace in traces: + combined_trace.extend(trace.events) + combined_trace.write_chrome_trace(performance_output / "event-loop.trace.json") + + assert report.results + blocking = next( + result for result in report.results if result.name == "blocking_handler" + ) + assert blocking.metrics["lag_max_ms"] >= 2 + capture_async_diagnostics( + performance_output / "event-loop-blocking-diagnostics.json", + metadata={ + "scenario": blocking.name, + "metrics": dict(blocking.metrics), + "last_stages": { + event.token: event.stage for event in combined_trace.events + }, + }, + ) diff --git a/tests/performance/test_memory.py b/tests/performance/test_memory.py new file mode 100644 index 00000000000..b723bb661a7 --- /dev/null +++ b/tests/performance/test_memory.py @@ -0,0 +1,119 @@ +"""Allocation and retained-memory performance benchmarks.""" + +from __future__ import annotations + +import gc +import itertools +import time +import tracemalloc +from collections.abc import Callable +from pathlib import Path + +import pytest +from reflex_base.utils.format import json_dumps + +from reflex.compiler import compiler +from reflex.state import StateUpdate +from reflex.vars import Var +from tests.benchmarks.fixtures import _complicated_page +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.states import initialized_state + + +def _compile_workload() -> None: + """Build and compile a representative component tree.""" + assert compiler._compile_page(_complicated_page()) + + +def _state_delta_workload() -> None: + """Mutate and serialize a state carrying large mutable values.""" + state = initialized_state(1000) + state.counter += 1 + assert json_dumps(StateUpdate(delta=state.get_delta())) + + +def _var_workload() -> None: + """Construct a batch of Var operations and release them.""" + left = Var.create(1) + right = Var.create(2) + for _ in range(1000): + assert ((left + right) * right - left)._js_expr + + +def _measure_allocations( + workload: Callable[[], None], + cycles: int, +) -> tuple[list[float], dict[str, float | int]]: + """Measure elapsed time, peak allocation, and retained-memory trend. + + Args: + workload: Workload invoked once per cycle. + cycles: Measurement cycles after one warmup. + + Returns: + Cycle latencies and allocation metrics. + """ + workload() + gc.collect() + tracemalloc.start() + baseline, _ = tracemalloc.get_traced_memory() + retained: list[int] = [] + latencies: list[float] = [] + peak = baseline + try: + for _ in range(cycles): + started = time.perf_counter_ns() + workload() + latencies.append((time.perf_counter_ns() - started) / 1_000_000) + gc.collect() + current, current_peak = tracemalloc.get_traced_memory() + retained.append(current - baseline) + peak = max(peak, current_peak) + finally: + tracemalloc.stop() + + monotonic_growth = all( + current > previous for previous, current in itertools.pairwise(retained) + ) + return latencies, { + "peak_traced_bytes": peak - baseline, + "retained_bytes": retained[-1] if retained else 0, + "retained_growth_bytes": retained[-1] - retained[0] if len(retained) > 1 else 0, + "monotonic_growth": int(monotonic_growth), + } + + +@pytest.mark.performance +def test_memory_report(performance_output: Path, performance_scale: str): + """Profile allocation and retention across compiler and runtime workloads. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + cycles = {"smoke": 3, "release": 100}[performance_scale] + report = PerformanceReport("memory", metadata={"scale": performance_scale}) + for name, workload in { + "component_compile": _compile_workload, + "state_delta_serialization": _state_delta_workload, + "var_operations": _var_workload, + }.items(): + observations, metrics = _measure_allocations(workload, cycles) + report.add( + BenchmarkResult( + name=name, + parameters={"cycles": cycles}, + observations_ms=observations, + metrics=metrics, + warmup_iterations=1, + measurement_iterations=cycles, + ) + ) + + report.write(performance_output / "memory.json") + assert all(result.metrics["peak_traced_bytes"] >= 0 for result in report.results) + assert not any( + result.metrics["monotonic_growth"] + and result.metrics["retained_growth_bytes"] > 1_000_000 + for result in report.results + ) diff --git a/tests/performance/test_redis_latency.py b/tests/performance/test_redis_latency.py new file mode 100644 index 00000000000..7179289560b --- /dev/null +++ b/tests/performance/test_redis_latency.py @@ -0,0 +1,236 @@ +"""Real-Redis state-manager latency and command benchmarks.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any + +import pytest + +from reflex.istate.manager.token import StateToken +from tests.benchmarks.support import BenchmarkResult, EventLoopProbe, PerformanceReport +from tests.benchmarks.support.redis import real_redis_state_manager +from tests.benchmarks.support.states import PerformanceState + + +async def _measure( + operation: Callable[[], Awaitable[Any]], + iterations: int, +) -> list[float]: + """Measure sequential async operations. + + Args: + operation: Async operation factory. + iterations: Number of calls. + + Returns: + Per-call latencies in milliseconds. + """ + observations = [] + for _ in range(iterations): + started = time.perf_counter_ns() + await operation() + observations.append((time.perf_counter_ns() - started) / 1_000_000) + return observations + + +def _command_calls(info: dict[str, Any]) -> int: + """Sum Redis command calls from ``INFO commandstats``. + + Args: + info: Redis commandstats mapping. + + Returns: + Total command invocations. + """ + return sum( + int(value.get("calls", 0)) + for key, value in info.items() + if key.startswith("cmdstat_") and isinstance(value, dict) + ) + + +def _command_counts(info: dict[str, Any]) -> dict[str, int]: + """Extract per-command call counts. + + Args: + info: Redis commandstats mapping. + + Returns: + Command name to call count. + """ + return { + key.removeprefix("cmdstat_"): int(value.get("calls", 0)) + for key, value in info.items() + if key.startswith("cmdstat_") and isinstance(value, dict) + } + + +@pytest.mark.performance +async def test_redis_state_manager_report( + performance_output: Path, + performance_scale: str, +): + """Measure cold/warm state access, writes, contention, and large payloads. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + iterations = {"smoke": 5, "release": 200}[performance_scale] + concurrency = {"smoke": 5, "release": 200}[performance_scale] + report = PerformanceReport("redis", metadata={"scale": performance_scale}) + + async with real_redis_state_manager() as manager: + before_info = await manager.redis.info("commandstats") + cold_iteration = 0 + + async def cold_get() -> PerformanceState: + """Fetch a unique uncached state. + + Returns: + Newly created state. + """ + nonlocal cold_iteration + cold_iteration += 1 + token = StateToken( + ident=f"cold-{cold_iteration}", + cls=PerformanceState, + ) + return await manager.get_state(token) + + cold = await _measure(cold_get, iterations) + warm_token = StateToken(ident="warm", cls=PerformanceState) + await manager.get_state(warm_token) + warm = await _measure(lambda: manager.get_state(warm_token), iterations) + + async def modify(token: StateToken[PerformanceState]) -> int: + """Modify one token under its Redis lock. + + Args: + token: State token. + + Returns: + Updated counter. + """ + async with manager.modify_state(token) as state: + state.counter += 1 + return state.counter + + async def diagnose_modify() -> dict[str, float | int]: + """Attribute lock/read, serialization, and flush/write costs. + + Returns: + Stage timings and serialized byte count. + """ + context = manager.modify_state(warm_token) + lock_started = time.perf_counter_ns() + state = await context.__aenter__() + lock_read_ms = (time.perf_counter_ns() - lock_started) / 1_000_000 + state.counter += 1 + pickle_started = time.perf_counter_ns() + serialized = StateToken.serialize(state) + pickle_ms = (time.perf_counter_ns() - pickle_started) / 1_000_000 + flush_started = time.perf_counter_ns() + await context.__aexit__(None, None, None) + flush_write_ms = (time.perf_counter_ns() - flush_started) / 1_000_000 + return { + "lock_and_read_ms": lock_read_ms, + "pickle_ms": pickle_ms, + "serialized_bytes": len(serialized), + "flush_and_write_ms": flush_write_ms, + } + + stage_metrics = await diagnose_modify() + writes = await _measure(lambda: modify(warm_token), iterations) + + async def concurrent(tokens: list[StateToken[PerformanceState]]) -> list[float]: + """Measure a concurrent batch. + + Args: + tokens: Token for each concurrent modification. + + Returns: + Per-operation completion latencies. + """ + started = time.perf_counter_ns() + + async def timed(token: StateToken[PerformanceState]) -> float: + """Measure one concurrent modification. + + Args: + token: State token. + + Returns: + Completion latency from batch start. + """ + await modify(token) + return (time.perf_counter_ns() - started) / 1_000_000 + + return list(await asyncio.gather(*(timed(token) for token in tokens))) + + probe = EventLoopProbe(interval=0.001) + async with probe: + same_token = await concurrent([warm_token] * concurrency) + independent_tokens = await concurrent([ + StateToken(ident=f"independent-{index}", cls=PerformanceState) + for index in range(concurrency) + ]) + + large_token = StateToken(ident="large", cls=PerformanceState) + + async def large_write() -> int: + """Write a state large enough to exercise serialization offloading. + + Returns: + Number of stored elements. + """ + async with manager.modify_state(large_token) as state: + state.numbers = list(range(100_000)) + return len(state.numbers) + + large = await _measure(large_write, 1 if performance_scale == "smoke" else 3) + after_info = await manager.redis.info("commandstats") + + command_delta = _command_calls(after_info) - _command_calls(before_info) + before_commands = _command_counts(before_info) + after_commands = _command_counts(after_info) + command_metrics = { + f"redis_command_{command}": count - before_commands.get(command, 0) + for command, count in after_commands.items() + if count - before_commands.get(command, 0) > 0 + } + common_metrics = ( + probe.summary() + | stage_metrics + | command_metrics + | {"redis_commands": command_delta} + ) + for name, observations, parameters in [ + ("cold_get", cold, {"iterations": iterations}), + ("warm_get", warm, {"iterations": iterations}), + ("modify", writes, {"iterations": iterations}), + ("same_token_contention", same_token, {"concurrency": concurrency}), + ( + "independent_token_concurrency", + independent_tokens, + {"concurrency": concurrency}, + ), + ("large_state_write", large, {"elements": 100_000}), + ]: + report.add( + BenchmarkResult( + name, + parameters, + observations, + common_metrics, + measurement_iterations=len(observations), + ) + ) + report.write(performance_output / "redis.json") + + assert command_delta > 0 + assert len(same_token) == len(independent_tokens) == concurrency diff --git a/tests/units/benchmarks/test_loop_probe.py b/tests/units/benchmarks/test_loop_probe.py index a86b3c2140f..fee0c2c8d9f 100644 --- a/tests/units/benchmarks/test_loop_probe.py +++ b/tests/units/benchmarks/test_loop_probe.py @@ -5,7 +5,7 @@ import pytest -from tests.benchmarks.loop_probe import EventLoopProbe +from tests.benchmarks.support.loop_probe import EventLoopProbe @pytest.mark.asyncio diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py new file mode 100644 index 00000000000..6c1938d45cd --- /dev/null +++ b/tests/units/benchmarks/test_support.py @@ -0,0 +1,74 @@ +"""Tests for shared performance benchmark support.""" + +import json + +import pytest + +from tests.benchmarks.support.pipeline_trace import PipelineTrace, StageEvent +from tests.benchmarks.support.report import ( + BenchmarkEnvironment, + BenchmarkResult, + PerformanceReport, + percentile, +) + + +def test_percentile_interpolates_and_validates(): + """Percentiles interpolate and reject values outside the valid range.""" + assert percentile([], 50) == 0 + assert percentile([1, 2, 3, 4, 5], 95) == pytest.approx(4.8) + with pytest.raises(ValueError, match="between 0 and 100"): + percentile([1], 101) + + +def test_performance_report_round_trip(tmp_path): + """Reports retain observations, summaries, metrics, and environment data.""" + environment = BenchmarkEnvironment( + commit="abc", + branch="branch", + python="3.14", + implementation="CPython", + operating_system="test", + machine="machine", + processor="processor", + cpu_count=4, + ci=True, + ) + report = PerformanceReport("event-loop", environment=environment) + report.add( + BenchmarkResult( + name="warm-event", + parameters={"tokens": 1}, + observations_ms=[1, 2, 3], + metrics={"peak_tasks": 4}, + warmup_iterations=1, + measurement_iterations=3, + ) + ) + + output = report.write(tmp_path / "report.json") + payload = json.loads(output.read_text()) + + assert payload["schema_version"] == 1 + assert payload["environment"]["commit"] == "abc" + assert payload["results"][0]["summary"]["p50_ms"] == 2 + assert payload["results"][0]["metrics"]["peak_tasks"] == 4 + + +def test_pipeline_trace_durations_and_chrome_output(tmp_path): + """Pipeline traces calculate durations and produce Chrome trace events.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token", "enqueued", 1_000_000), + StageEvent("token", "dequeued", 2_500_000), + ]) + + assert trace.durations_ms([("enqueued", "dequeued")]) == { + "enqueued_to_dequeued": [1.5] + } + path = trace.write_chrome_trace(tmp_path / "trace.json") + payload = json.loads(path.read_text()) + assert [event["name"] for event in payload["traceEvents"]] == [ + "enqueued", + "dequeued", + ] From 0d11d2c9190ff6c7b72591ae1e81846e1d25d24d Mon Sep 17 00:00:00 2001 From: Alek Date: Sun, 12 Jul 2026 18:25:24 -0700 Subject: [PATCH 03/13] Correct event loop probe sampling --- tests/benchmarks/support/loop_probe.py | 28 +++++++++++++++++++---- tests/units/benchmarks/test_loop_probe.py | 20 +++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/benchmarks/support/loop_probe.py b/tests/benchmarks/support/loop_probe.py index 5cd4b1e9019..366a65bcf9c 100644 --- a/tests/benchmarks/support/loop_probe.py +++ b/tests/benchmarks/support/loop_probe.py @@ -21,6 +21,7 @@ class EventLoopProbe: task_samples: list[int] = field(default_factory=list, init=False) gauge_samples: dict[str, list[int]] = field(default_factory=dict, init=False) _task: asyncio.Task[None] | None = field(default=None, init=False, repr=False) + _deadline: float | None = field(default=None, init=False, repr=False) @property def peak_tasks(self) -> int: @@ -43,6 +44,7 @@ async def __aenter__(self) -> EventLoopProbe: self.lag_samples.clear() self.task_samples.clear() self.gauge_samples = {name: [] for name in self.gauges} + self._deadline = None self._task = asyncio.create_task( self._sample(), name="reflex_benchmark_event_loop_probe", @@ -65,6 +67,9 @@ async def __aexit__( """ if self._task is None: return + loop = asyncio.get_running_loop() + if self._deadline is not None and loop.time() >= self._deadline: + self._record_sample(loop, self._deadline) self._task.cancel() with contextlib.suppress(asyncio.CancelledError): await self._task @@ -75,14 +80,27 @@ async def _sample(self) -> None: loop = asyncio.get_running_loop() deadline = loop.time() + self.interval while True: + self._deadline = deadline await asyncio.sleep(max(0.0, deadline - loop.time())) - now = loop.time() - self.lag_samples.append(max(0.0, now - deadline)) - self.task_samples.append(len(asyncio.all_tasks(loop))) - for name, gauge in self.gauges.items(): - self.gauge_samples[name].append(gauge()) + now = self._record_sample(loop, deadline) deadline = max(deadline + self.interval, now + self.interval) + def _record_sample(self, loop: asyncio.AbstractEventLoop, deadline: float) -> float: + """Record one probe sample and return its observation time. + + Returns: + Event-loop clock time used for the sample. + """ + now = loop.time() + self.lag_samples.append(max(0.0, now - deadline)) + tasks = asyncio.all_tasks(loop) + if self._task is not None: + tasks.discard(self._task) + self.task_samples.append(len(tasks)) + for name, gauge in self.gauges.items(): + self.gauge_samples[name].append(gauge()) + return now + def summary(self) -> dict[str, float | int]: """Return stable summary fields for benchmark JSON output. diff --git a/tests/units/benchmarks/test_loop_probe.py b/tests/units/benchmarks/test_loop_probe.py index fee0c2c8d9f..0f487a91106 100644 --- a/tests/units/benchmarks/test_loop_probe.py +++ b/tests/units/benchmarks/test_loop_probe.py @@ -19,10 +19,28 @@ async def test_event_loop_probe_detects_blocking_work(): summary = probe.summary() assert summary["sample_count"] >= 2 - assert summary["peak_tasks"] >= 2 + assert summary["peak_tasks"] >= 1 assert summary["lag_max_ms"] >= 10 +@pytest.mark.asyncio +async def test_event_loop_probe_detects_terminal_blocking_work(): + """A block immediately before context exit still produces a lag sample.""" + async with EventLoopProbe(interval=0.001) as probe: + time.sleep(0.02) # noqa: ASYNC251 - intentional event-loop blocking + + assert probe.summary()["lag_max_ms"] >= 10 + + +@pytest.mark.asyncio +async def test_event_loop_probe_excludes_sampler_task(): + """Task metrics exclude the probe's own sampler task.""" + async with EventLoopProbe(interval=0.001) as probe: + await asyncio.sleep(0.003) + + assert probe.peak_tasks == 1 + + @pytest.mark.asyncio async def test_event_loop_probe_resets_between_runs(): """Reusing a probe does not mix observations from separate workloads.""" From 00d9a4a338fb98e7f2cb9fc3b3c7f86f062aa111 Mon Sep 17 00:00:00 2001 From: Alek Date: Sun, 12 Jul 2026 19:29:27 -0700 Subject: [PATCH 04/13] Fix performance benchmark validity --- tests/benchmarks/support/apps.py | 14 +- tests/benchmarks/support/pipeline_trace.py | 17 +- tests/benchmarks/support/socket_client.py | 39 +++-- tests/benchmarks/support/states.py | 15 ++ tests/benchmarks/test_state_manager.py | 30 ++-- tests/performance/test_compiler_lifecycle.py | 157 ++++++++++++++++++- tests/performance/test_event_load.py | 3 +- tests/performance/test_redis_latency.py | 57 ++++--- tests/units/benchmarks/test_loop_probe.py | 5 +- tests/units/benchmarks/test_support.py | 31 ++++ 10 files changed, 305 insertions(+), 63 deletions(-) diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py index 9f3c543495d..bb645086bf1 100644 --- a/tests/benchmarks/support/apps.py +++ b/tests/benchmarks/support/apps.py @@ -3,12 +3,17 @@ from __future__ import annotations -def lifecycle_app_source(rows: int = 100, pages: int = 1) -> str: +def lifecycle_app_source( + rows: int = 100, + pages: int = 1, + reload_version: int = 0, +) -> str: """Create a deterministic multi-page application source. Args: rows: Number of stateful rows per page. pages: Number of routes. + reload_version: Value exposed by the backend reload probe. Returns: Python source for a Reflex application. @@ -18,6 +23,9 @@ def lifecycle_app_source(rows: int = 100, pages: int = 1) -> str: for index in range(pages) ) return f"""import reflex as rx +from starlette.responses import JSONResponse + +RELOAD_VERSION = {reload_version} class State(rx.State): count: int = 0 @@ -35,9 +43,13 @@ def index(label: str = "index"): *[row(index) for index in range({rows})], ) +async def reload_version(_request): + return JSONResponse({{"version": RELOAD_VERSION}}) + app = rx.App() app.add_page(index, route="/") {page_registrations} +app._api.add_route("/reload-version", reload_version) """ diff --git a/tests/benchmarks/support/pipeline_trace.py b/tests/benchmarks/support/pipeline_trace.py index 2c7ee2d7e65..2cf31ffaa26 100644 --- a/tests/benchmarks/support/pipeline_trace.py +++ b/tests/benchmarks/support/pipeline_trace.py @@ -5,7 +5,7 @@ import dataclasses import json import time -from collections import defaultdict +from collections import defaultdict, deque from collections.abc import Iterable, Sequence from pathlib import Path from typing import Any @@ -54,12 +54,17 @@ def durations_ms(self, stages: Sequence[tuple[str, str]]) -> dict[str, list[floa f"{start}_to_{end}": [] for start, end in stages } for token_events in grouped.values(): - timestamps = {event.stage: event.timestamp_ns for event in token_events} + ordered_events = sorted(token_events, key=lambda event: event.timestamp_ns) for start, end in stages: - if start in timestamps and end in timestamps: - durations[f"{start}_to_{end}"].append( - (timestamps[end] - timestamps[start]) / 1_000_000 - ) + pending_starts: deque[int] = deque() + for event in ordered_events: + if event.stage == start: + pending_starts.append(event.timestamp_ns) + elif event.stage == end and pending_starts: + started_at = pending_starts.popleft() + durations[f"{start}_to_{end}"].append( + (event.timestamp_ns - started_at) / 1_000_000 + ) return durations def chrome_trace(self) -> dict[str, Any]: diff --git a/tests/benchmarks/support/socket_client.py b/tests/benchmarks/support/socket_client.py index 324973663f9..1099c03b654 100644 --- a/tests/benchmarks/support/socket_client.py +++ b/tests/benchmarks/support/socket_client.py @@ -5,6 +5,7 @@ import asyncio import concurrent.futures import dataclasses +import functools import time from collections.abc import Callable, Mapping from typing import Any @@ -33,6 +34,7 @@ async def run_socket_client( response_name: str = "event", namespace: str = "/_event", timeout: float = 10, + executor: concurrent.futures.Executor | None = None, ) -> ClientLoadResult: """Send events sequentially and measure matching socket responses. @@ -45,20 +47,25 @@ async def run_socket_client( response_name: Socket event name carrying state updates. namespace: Reflex Socket.IO namespace. timeout: Maximum response wait per operation. + executor: Optional executor that owns the blocking socket client. Returns: Per-operation latency and error observations. """ - return await asyncio.to_thread( - _run_socket_client_sync, - url, - token, - payload, - events, - event_name, - response_name, - namespace, - timeout, + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + executor, + functools.partial( + _run_socket_client_sync, + url, + token, + payload, + events, + event_name, + response_name, + namespace, + timeout, + ), ) @@ -123,21 +130,23 @@ def _run_socket_client_sync( async def run_clients( clients: int, - factory: Callable[[int], Any], + factory: Callable[[int, concurrent.futures.Executor], Any], ) -> list[Any]: """Run a parameterized set of clients concurrently. Args: clients: Number of clients. - factory: Callable returning an awaitable for a client index. + factory: Callable returning an awaitable for a client index and executor. Returns: Client results in index order. """ - loop = asyncio.get_running_loop() executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(1, clients)) - loop.set_default_executor(executor) try: - return list(await asyncio.gather(*(factory(index) for index in range(clients)))) + return list( + await asyncio.gather( + *(factory(index, executor) for index in range(clients)) + ) + ) finally: executor.shutdown(wait=True) diff --git a/tests/benchmarks/support/states.py b/tests/benchmarks/support/states.py index 8c952091f64..f229871cefa 100644 --- a/tests/benchmarks/support/states.py +++ b/tests/benchmarks/support/states.py @@ -70,6 +70,21 @@ def initialized_state(size: int) -> PerformanceState: return state +def get_performance_state(root_state: BaseState) -> PerformanceState: + """Return the performance state from a managed root hierarchy. + + Args: + root_state: Root state returned by a state manager. + + Returns: + Performance substate used by benchmark workloads. + """ + return cast( + PerformanceState, + root_state.get_substate(PerformanceState.get_full_name().split(".")), + ) + + @contextmanager def computed_fanout_state(fanout: int) -> Iterator[tuple[BaseState, tuple[str, ...]]]: """Create a state whose scalar value feeds a requested number of vars. diff --git a/tests/benchmarks/test_state_manager.py b/tests/benchmarks/test_state_manager.py index e6516e76492..93ab7d9aaaa 100644 --- a/tests/benchmarks/test_state_manager.py +++ b/tests/benchmarks/test_state_manager.py @@ -5,9 +5,9 @@ from pytest_codspeed import BenchmarkFixture from reflex.istate.manager.memory import StateManagerMemory -from reflex.istate.manager.token import StateToken +from reflex.istate.manager.token import BaseStateToken -from .support.states import PerformanceState +from .support.states import PerformanceState, get_performance_state def test_state_manager_memory_cold_get(benchmark: BenchmarkFixture): @@ -24,18 +24,18 @@ def setup(): """Return a unique token for one cold measurement.""" nonlocal iteration iteration += 1 - token = StateToken(ident=f"cold-{iteration}", cls=PerformanceState) + token = BaseStateToken(ident=f"cold-{iteration}", cls=PerformanceState) return ((token,), {}) - def get_state(token: StateToken[PerformanceState]) -> PerformanceState: + def get_state(token: BaseStateToken) -> PerformanceState: """Fetch a state through the async manager API. Returns: Managed state. """ - return loop.run_until_complete(manager.get_state(token)) + return get_performance_state(loop.run_until_complete(manager.get_state(token))) - def teardown(token: StateToken[PerformanceState]) -> None: + def teardown(token: BaseStateToken) -> None: """Purge the measured state.""" manager._purge_token(token) # pyright: ignore [reportPrivateUsage] @@ -54,11 +54,15 @@ def test_state_manager_memory_warm_get(benchmark: BenchmarkFixture): """ manager = StateManagerMemory() loop = asyncio.new_event_loop() - token = StateToken(ident="warm", cls=PerformanceState) + token = BaseStateToken(ident="warm", cls=PerformanceState) loop.run_until_complete(manager.get_state(token)) try: - state = benchmark(lambda: loop.run_until_complete(manager.get_state(token))) + state = benchmark( + lambda: get_performance_state( + loop.run_until_complete(manager.get_state(token)) + ) + ) assert isinstance(state, PerformanceState) finally: loop.run_until_complete(manager.close()) @@ -73,7 +77,7 @@ def test_state_manager_memory_modify(benchmark: BenchmarkFixture): """ manager = StateManagerMemory() loop = asyncio.new_event_loop() - token = StateToken(ident="modify", cls=PerformanceState) + token = BaseStateToken(ident="modify", cls=PerformanceState) async def modify() -> int: """Increment one state under the manager lock. @@ -81,7 +85,8 @@ async def modify() -> int: Returns: Updated counter. """ - async with manager.modify_state(token) as state: + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) state.counter += 1 return state.counter @@ -100,7 +105,7 @@ def test_state_manager_memory_read_only_modify(benchmark: BenchmarkFixture): """ manager = StateManagerMemory() loop = asyncio.new_event_loop() - token = StateToken(ident="read-only", cls=PerformanceState) + token = BaseStateToken(ident="read-only", cls=PerformanceState) async def read_only() -> int: """Read one field under the manager lock. @@ -108,7 +113,8 @@ async def read_only() -> int: Returns: Current counter. """ - async with manager.modify_state(token) as state: + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) return state.counter try: diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py index b7f1cfd09e7..006207453e2 100644 --- a/tests/performance/test_compiler_lifecycle.py +++ b/tests/performance/test_compiler_lifecycle.py @@ -2,12 +2,18 @@ from __future__ import annotations +import contextlib +import json import os +import socket import subprocess import sys import time from pathlib import Path +from typing import TextIO +from urllib.request import urlopen +import psutil import pytest from tests.benchmarks.support import BenchmarkResult, PerformanceReport @@ -60,6 +66,119 @@ def _generated_metrics(root: Path) -> dict[str, int]: } +def _free_port() -> int: + """Reserve and release an available local TCP port. + + Returns: + Available port number for the development backend. + """ + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def _start_dev_backend( + executable: str, + app_root: Path, + port: int, +) -> tuple[subprocess.Popen[str], TextIO, Path]: + """Start a persistent reload-capable development backend. + + Args: + executable: Python executable running Reflex. + app_root: Generated application root. + port: Backend port. + + Returns: + Backend process, open log stream, and log path. + """ + log_path = app_root / "hot-reload.log" + log_stream = log_path.open("w+", encoding="utf-8") + try: + process = subprocess.Popen( + [ + executable, + "-m", + "reflex", + "run", + "--backend-only", + "--backend-port", + str(port), + "--loglevel", + "error", + ], + cwd=app_root, + env=os.environ | {"REFLEX_TELEMETRY_ENABLED": "false"}, + stdout=log_stream, + stderr=subprocess.STDOUT, + text=True, + ) + except OSError: + log_stream.close() + raise + return process, log_stream, log_path + + +def _wait_for_reload_version( + process: subprocess.Popen[str], + log_path: Path, + port: int, + expected: int, + timeout: float = 120, +) -> None: + """Wait until the live backend exposes an expected source version. + + Args: + process: Development backend process. + log_path: Backend log used for failure context. + port: Backend port. + expected: Source version expected after reload. + timeout: Maximum wait in seconds. + + Raises: + RuntimeError: If the backend exits before serving the version. + TimeoutError: If the reload does not complete in time. + """ + deadline = time.monotonic() + timeout + url = f"http://127.0.0.1:{port}/reload-version" + while time.monotonic() < deadline: + if process.poll() is not None: + log = log_path.read_text(encoding="utf-8")[-4000:] + msg = f"Development backend exited during reload:\n{log}" + raise RuntimeError(msg) + try: + with urlopen(url, timeout=0.5) as response: + payload = json.load(response) + if payload.get("version") == expected: + return + except (OSError, ValueError): + pass + time.sleep(0.05) + log = log_path.read_text(encoding="utf-8")[-4000:] + msg = f"Timed out waiting for reload version {expected}:\n{log}" + raise TimeoutError(msg) + + +def _stop_dev_backend(process: subprocess.Popen[str]) -> None: + """Terminate a development backend and all reload workers. + + Args: + process: Development backend process. + """ + try: + parent = psutil.Process(process.pid) + processes = [parent, *parent.children(recursive=True)] + except psutil.NoSuchProcess: + return + for running in reversed(processes): + with contextlib.suppress(psutil.NoSuchProcess): + running.terminate() + _, alive = psutil.wait_procs(processes, timeout=10) + for running in alive: + with contextlib.suppress(psutil.NoSuchProcess): + running.kill() + + @pytest.mark.performance def test_compiler_lifecycle_report( tmp_path: Path, @@ -165,16 +284,40 @@ def test_compiler_lifecycle_report( report.add(BenchmarkResult("compile_state_edit", {}, [state_edit_ms], {})) reload_observations = [] - for cycle in range(reload_cycles): - app_source.write_text( - source + f"\n# reload cycle {cycle}\n", - encoding="utf-8", + backend_port = _free_port() + backend, backend_log, backend_log_path = _start_dev_backend( + executable, + app_root, + backend_port, + ) + try: + _wait_for_reload_version( + backend, + backend_log_path, + backend_port, + expected=0, ) - reload_observations.append(_run(compile_command, app_root)) + for cycle in range(1, reload_cycles + 1): + cycle_source = source.replace( + "RELOAD_VERSION = 0", + f"RELOAD_VERSION = {cycle}", + ) + started = time.perf_counter_ns() + app_source.write_text(cycle_source, encoding="utf-8") + _wait_for_reload_version( + backend, + backend_log_path, + backend_port, + expected=cycle, + ) + reload_observations.append((time.perf_counter_ns() - started) / 1_000_000) + finally: + _stop_dev_backend(backend) + backend_log.close() report.add( BenchmarkResult( - "hot_reload_cycles", - {"cycles": reload_cycles}, + "backend_hot_reload", + {"cycles": reload_cycles, "watcher": "reflex run"}, reload_observations, _generated_metrics(app_root), measurement_iterations=reload_cycles, diff --git a/tests/performance/test_event_load.py b/tests/performance/test_event_load.py index ee9a31bf3f0..9cbf9d8bca2 100644 --- a/tests/performance/test_event_load.py +++ b/tests/performance/test_event_load.py @@ -74,11 +74,12 @@ def test_event_load_report( results = asyncio.run( run_clients( clients, - lambda index, clients=clients: run_socket_client( + lambda index, executor, clients=clients: run_socket_client( backend_url, f"load-token-{clients}-{index}", payload, events_per_client, + executor=executor, ), ) ) diff --git a/tests/performance/test_redis_latency.py b/tests/performance/test_redis_latency.py index 7179289560b..a5054398466 100644 --- a/tests/performance/test_redis_latency.py +++ b/tests/performance/test_redis_latency.py @@ -10,10 +10,10 @@ import pytest -from reflex.istate.manager.token import StateToken +from reflex.istate.manager.token import BaseStateToken from tests.benchmarks.support import BenchmarkResult, EventLoopProbe, PerformanceReport from tests.benchmarks.support.redis import real_redis_state_manager -from tests.benchmarks.support.states import PerformanceState +from tests.benchmarks.support.states import PerformanceState, get_performance_state async def _measure( @@ -96,18 +96,32 @@ async def cold_get() -> PerformanceState: """ nonlocal cold_iteration cold_iteration += 1 - token = StateToken( + token = BaseStateToken( ident=f"cold-{cold_iteration}", cls=PerformanceState, ) - return await manager.get_state(token) + return get_performance_state(await manager.get_state(token)) cold = await _measure(cold_get, iterations) - warm_token = StateToken(ident="warm", cls=PerformanceState) - await manager.get_state(warm_token) - warm = await _measure(lambda: manager.get_state(warm_token), iterations) + warm_token = BaseStateToken(ident="warm", cls=PerformanceState) + warm_root = await manager.get_state(warm_token) + warm_state = get_performance_state(warm_root) + warm_state.counter += 1 + await manager.set_state(warm_token, warm_root) + warm_key = str(warm_token.with_cls(type(warm_state))) + assert await manager.redis.exists(warm_key) - async def modify(token: StateToken[PerformanceState]) -> int: + async def warm_get() -> PerformanceState: + """Fetch and resolve an existing persisted state. + + Returns: + Persisted performance state. + """ + return get_performance_state(await manager.get_state(warm_token)) + + warm = await _measure(warm_get, iterations) + + async def modify(token: BaseStateToken) -> int: """Modify one token under its Redis lock. Args: @@ -116,7 +130,8 @@ async def modify(token: StateToken[PerformanceState]) -> int: Returns: Updated counter. """ - async with manager.modify_state(token) as state: + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) state.counter += 1 return state.counter @@ -126,20 +141,21 @@ async def diagnose_modify() -> dict[str, float | int]: Returns: Stage timings and serialized byte count. """ - context = manager.modify_state(warm_token) + context = manager.modify_state_with_links(warm_token) lock_started = time.perf_counter_ns() - state = await context.__aenter__() + root_state = await context.__aenter__() lock_read_ms = (time.perf_counter_ns() - lock_started) / 1_000_000 + state = get_performance_state(root_state) state.counter += 1 - pickle_started = time.perf_counter_ns() - serialized = StateToken.serialize(state) - pickle_ms = (time.perf_counter_ns() - pickle_started) / 1_000_000 + serialize_started = time.perf_counter_ns() + serialized = BaseStateToken.serialize(state) + serialize_ms = (time.perf_counter_ns() - serialize_started) / 1_000_000 flush_started = time.perf_counter_ns() await context.__aexit__(None, None, None) flush_write_ms = (time.perf_counter_ns() - flush_started) / 1_000_000 return { "lock_and_read_ms": lock_read_ms, - "pickle_ms": pickle_ms, + "state_serialize_ms": serialize_ms, "serialized_bytes": len(serialized), "flush_and_write_ms": flush_write_ms, } @@ -147,7 +163,7 @@ async def diagnose_modify() -> dict[str, float | int]: stage_metrics = await diagnose_modify() writes = await _measure(lambda: modify(warm_token), iterations) - async def concurrent(tokens: list[StateToken[PerformanceState]]) -> list[float]: + async def concurrent(tokens: list[BaseStateToken]) -> list[float]: """Measure a concurrent batch. Args: @@ -158,7 +174,7 @@ async def concurrent(tokens: list[StateToken[PerformanceState]]) -> list[float]: """ started = time.perf_counter_ns() - async def timed(token: StateToken[PerformanceState]) -> float: + async def timed(token: BaseStateToken) -> float: """Measure one concurrent modification. Args: @@ -176,11 +192,11 @@ async def timed(token: StateToken[PerformanceState]) -> float: async with probe: same_token = await concurrent([warm_token] * concurrency) independent_tokens = await concurrent([ - StateToken(ident=f"independent-{index}", cls=PerformanceState) + BaseStateToken(ident=f"independent-{index}", cls=PerformanceState) for index in range(concurrency) ]) - large_token = StateToken(ident="large", cls=PerformanceState) + large_token = BaseStateToken(ident="large", cls=PerformanceState) async def large_write() -> int: """Write a state large enough to exercise serialization offloading. @@ -188,7 +204,8 @@ async def large_write() -> int: Returns: Number of stored elements. """ - async with manager.modify_state(large_token) as state: + async with manager.modify_state_with_links(large_token) as root_state: + state = get_performance_state(root_state) state.numbers = list(range(100_000)) return len(state.numbers) diff --git a/tests/units/benchmarks/test_loop_probe.py b/tests/units/benchmarks/test_loop_probe.py index 0f487a91106..6c9ad5057c5 100644 --- a/tests/units/benchmarks/test_loop_probe.py +++ b/tests/units/benchmarks/test_loop_probe.py @@ -48,11 +48,14 @@ async def test_event_loop_probe_resets_between_runs(): async with probe: await asyncio.sleep(0.005) first_sample_count = len(probe.lag_samples) + probe.lag_samples.append(-1) async with probe: await asyncio.sleep(0.003) - assert first_sample_count > len(probe.lag_samples) + assert first_sample_count > 0 + assert probe.lag_samples + assert -1 not in probe.lag_samples def test_event_loop_probe_summary_percentiles(): diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py index 6c1938d45cd..22b518cd20e 100644 --- a/tests/units/benchmarks/test_support.py +++ b/tests/units/benchmarks/test_support.py @@ -1,5 +1,6 @@ """Tests for shared performance benchmark support.""" +import asyncio import json import pytest @@ -11,6 +12,7 @@ PerformanceReport, percentile, ) +from tests.benchmarks.support.socket_client import run_clients def test_percentile_interpolates_and_validates(): @@ -72,3 +74,32 @@ def test_pipeline_trace_durations_and_chrome_output(tmp_path): "enqueued", "dequeued", ] + + +def test_pipeline_trace_preserves_repeated_stage_pairs(): + """Repeated streaming stages produce one duration per occurrence.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token", "delta_started", 1_000_000), + StageEvent("token", "delta_finished", 2_000_000), + StageEvent("token", "delta_started", 3_000_000), + StageEvent("token", "delta_finished", 5_000_000), + ]) + + assert trace.durations_ms([("delta_started", "delta_finished")]) == { + "delta_started_to_delta_finished": [1.0, 2.0] + } + + +@pytest.mark.asyncio +async def test_run_clients_leaves_default_executor_usable(): + """The load helper does not close the event loop's default executor.""" + results = await run_clients( + 2, + lambda index, executor: asyncio.get_running_loop().run_in_executor( + executor, lambda: index + ), + ) + + assert results == [0, 1] + assert await asyncio.to_thread(lambda: "still-usable") == "still-usable" From 1d88639608a5b9291f6ffa9d2661695b84ade892 Mon Sep 17 00:00:00 2001 From: Farhan Date: Mon, 13 Jul 2026 21:08:05 +0500 Subject: [PATCH 05/13] test(performance): add wire-size, overhead, memory, and saturation benchmarks Close the measurement gaps identified against peer frameworks (Phoenix LiveView, Blazor Server, Streamlit, Gradio, krausest): - wire-size suite recording serialized delta bytes per canonical interaction, plus received payload bytes in the load client - framework-overhead baseline against a bare Starlette + python-socketio echo server with byte-matched responses - latency-throughput curve with saturation-knee detection in event load - reconnect-storm scenario (simultaneous reconnect after all clients drop) - per-session backend memory with clients held in a subprocess - keyed-list browser ops (create/partial-update/select/swap) timed through paint on a new /rows page - compare_performance_results.py for baseline-vs-current regression gating with percentage thresholds and absolute floors - nightly release-scale workflow with a rolling cached baseline The production app harness is now a session fixture shared by the load, memory, and browser suites to avoid repeated production builds. --- .github/workflows/performance-nightly.yml | 83 ++++++ .github/workflows/performance.yml | 4 +- PERFORMANCE.md | 13 +- pyproject.toml | 1 + scripts/compare_performance_results.py | 253 +++++++++++++++++ tests/benchmarks/support/apps.py | 39 +++ tests/benchmarks/support/baseline_server.py | 114 ++++++++ tests/benchmarks/support/idle_clients.py | 48 ++++ tests/benchmarks/support/socket_client.py | 159 ++++++++++- tests/performance/conftest.py | 25 ++ tests/performance/test_browser.py | 120 ++++++-- tests/performance/test_event_load.py | 256 ++++++++++++++++-- tests/performance/test_session_memory.py | 155 +++++++++++ tests/performance/test_wire_size.py | 88 ++++++ .../units/test_compare_performance_results.py | 157 +++++++++++ 15 files changed, 1445 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/performance-nightly.yml create mode 100644 scripts/compare_performance_results.py create mode 100644 tests/benchmarks/support/baseline_server.py create mode 100644 tests/benchmarks/support/idle_clients.py create mode 100644 tests/performance/test_session_memory.py create mode 100644 tests/performance/test_wire_size.py create mode 100644 tests/units/test_compare_performance_results.py diff --git a/.github/workflows/performance-nightly.yml b/.github/workflows/performance-nightly.yml new file mode 100644 index 00000000000..69185a7e1a8 --- /dev/null +++ b/.github/workflows/performance-nightly.yml @@ -0,0 +1,83 @@ +name: performance-nightly +permissions: + contents: read + +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +env: + REFLEX_TELEMETRY_ENABLED: false + NODE_OPTIONS: "--max_old_space_size=8192" + APP_HARNESS_HEADLESS: 1 + PYTHONUNBUFFERED: 1 + +jobs: + release-scale: + name: Release-scale performance suite + runs-on: ubuntu-22.04 + timeout-minutes: 120 + services: + redis: + image: redis:7.4 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-tags: true + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/setup_build_env + with: + python-version: "3.14" + node-version: "22" + run-uv-sync: true + + - name: Install playwright + run: uv run playwright install chromium --only-shell + + - name: Restore previous baseline + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .performance-baseline + key: performance-baseline-${{ github.run_id }} + restore-keys: performance-baseline- + + - name: Run release-scale performance suite + run: >- + uv run pytest tests/performance + --run-performance + --performance-scale release + --performance-output .pytest-tmp/performance + -q + + - name: Compare against previous nightly + if: ${{ hashFiles('.performance-baseline/**') != '' }} + continue-on-error: true + run: >- + uv run python scripts/compare_performance_results.py + .performance-baseline + .pytest-tmp/performance + + - name: Promote current results to baseline + run: | + rm -rf .performance-baseline + mkdir -p .performance-baseline + cp .pytest-tmp/performance/*.json .performance-baseline/ + + - name: Save baseline + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .performance-baseline + key: performance-baseline-${{ github.run_id }} + + - name: Upload performance artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-release-scale + path: .pytest-tmp/performance/** + if-no-files-found: warn diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 46417ef7551..84946210b48 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,9 +45,9 @@ jobs: mode: instrumentation run: uv run pytest -v tests/benchmarks --codspeed - - name: Run event-loop smoke + - name: Run event-loop and wire-size smoke run: >- - uv run pytest tests/performance/test_event_loop.py + uv run pytest tests/performance/test_event_loop.py tests/performance/test_wire_size.py --run-performance --performance-scale smoke --performance-output .pytest-tmp/performance diff --git a/PERFORMANCE.md b/PERFORMANCE.md index 21948f7d802..d94374ef81d 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -7,8 +7,17 @@ Reflex uses separate performance tiers because deterministic CPU work, asynchron | Tier | Location | Runs | Purpose | | --- | --- | --- | --- | | Deterministic CPU | `tests/benchmarks/` | Every pull request | CodSpeed regression detection and flamegraphs | -| Component wall time | `tests/performance/` | Manual | Event-loop, task, queue, Redis, and allocation behavior | -| Load and lifecycle | `tests/performance/` | Manual | Tail latency, saturation, compiles, hot reload, browser metrics | +| Component wall time | `tests/performance/` | Pull-request smoke, nightly release scale | Event-loop, task, queue, Redis, and allocation behavior | +| Load and lifecycle | `tests/performance/` | Nightly release scale | Tail latency, saturation curve, compiles, hot reload, browser metrics | + +Scheduled coverage beyond timing: + +- `test_wire_size.py` records serialized delta bytes (raw and gzip) per canonical interaction — wire-size regressions are invisible to local-network timing but dominate update latency on real links. +- `test_event_load.py` publishes the latency-throughput curve with a saturation knee, a framework-overhead comparison against a bare Starlette + python-socketio echo server, and a reconnect-storm scenario (simultaneous reconnection after all clients drop). +- `test_session_memory.py` reports marginal backend memory per connected session at small and large per-session state, holding clients in a subprocess so client memory stays out of the measurement. +- `test_browser.py` includes keyed-list DOM operations (create, partial update, select row, swap rows) timed through the following paint. + +The nightly workflow (`performance-nightly.yml`) runs the whole `tests/performance/` suite at release scale and compares against the previous night via `scripts/compare_performance_results.py`, which warns at +25% and fails at +50% with absolute floors (5 ms latency, 1 KB deterministic bytes, 5 MB process-memory metrics). Cross-environment comparisons are informational only. Do not add sleep, network, filesystem, browser, or subprocess timings to a CodSpeed instrumentation benchmark. Put those in a wall-time suite and emit a versioned JSON report. diff --git a/pyproject.toml b/pyproject.toml index 099d6108ca6..8998d560e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -251,6 +251,7 @@ preview = true "*/blank.py" = ["I001"] "docs/package/scripts/*.py" = ["INP001"] "scripts/check_min_deps.py" = ["T201"] +"scripts/compare_performance_results.py" = ["T201"] [tool.pytest.ini_options] filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning" diff --git a/scripts/compare_performance_results.py b/scripts/compare_performance_results.py new file mode 100644 index 00000000000..55558cb544d --- /dev/null +++ b/scripts/compare_performance_results.py @@ -0,0 +1,253 @@ +"""Compare performance report JSON files against a baseline run. + +Matches results across two directories of ``PerformanceReport`` JSON files by +suite, benchmark name, and parameters, then flags latency-percentile and byte +regressions with warn/fail thresholds. Absolute floors keep tiny baselines +from flagging noise: a change must exceed both the percentage threshold and +the floor to count. Comparisons across materially different environments are +reported as informational and never fail. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import sys +from pathlib import Path +from typing import Any + +DEFAULT_WARN_PCT = 25.0 +DEFAULT_FAIL_PCT = 50.0 +DEFAULT_FLOOR_MS = 5.0 +DEFAULT_FLOOR_BYTES = 1_024 +# Process and heap measurements are inherently noisy; require a much larger +# absolute change than deterministic byte metrics like wire or bundle size. +MEMORY_FLOOR_BYTES = 5_242_880 +MEMORY_METRIC_PREFIXES = ("rss_", "heap_") + +COMPARED_PERCENTILES = ("p50_ms", "p95_ms", "p99_ms") +ENVIRONMENT_KEYS = ("python", "implementation", "operating_system", "machine") + + +@dataclasses.dataclass(frozen=True) +class Finding: + """One compared metric and its regression status.""" + + benchmark: str + metric: str + baseline: float + current: float + change_pct: float + status: str + + def render(self) -> str: + """Format the finding for terminal output. + + Returns: + One aligned report line. + """ + return ( + f"{self.status.upper():<6} {self.benchmark} {self.metric}: " + f"{self.baseline:.2f} -> {self.current:.2f} ({self.change_pct:+.1f}%)" + ) + + +def load_reports(directory: Path) -> dict[str, dict[str, Any]]: + """Index benchmark results from every report file in a directory. + + Args: + directory: Directory containing ``PerformanceReport`` JSON files. + + Returns: + Results keyed by suite, benchmark name, and canonical parameters. + """ + index: dict[str, dict[str, Any]] = {} + for path in sorted(directory.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if not isinstance(data, dict) or "schema_version" not in data: + continue + for result in data.get("results", []): + key = "|".join([ + data.get("suite", path.stem), + result["name"], + json.dumps(result.get("parameters", {}), sort_keys=True), + ]) + index[key] = result | {"environment": data.get("environment", {})} + return index + + +def environments_match( + baseline: dict[str, Any], + current: dict[str, Any], +) -> bool: + """Check whether two results were produced in comparable environments. + + Args: + baseline: Baseline result with attached environment. + current: Current result with attached environment. + + Returns: + Whether the interpreter and platform identifiers all match. + """ + baseline_env = baseline.get("environment", {}) + current_env = current.get("environment", {}) + return all( + baseline_env.get(key) == current_env.get(key) for key in ENVIRONMENT_KEYS + ) + + +def _classify( + baseline: float, + current: float, + floor: float, + warn_pct: float, + fail_pct: float, +) -> tuple[float, str]: + """Classify one metric change; lower values are always better. + + Args: + baseline: Baseline value. + current: Current value. + floor: Minimum absolute increase considered meaningful. + warn_pct: Percentage increase that warns. + fail_pct: Percentage increase that fails. + + Returns: + Percentage change and status (``ok``, ``warn``, or ``fail``). + """ + if baseline <= 0: + return 0.0, "ok" + change_pct = (current - baseline) / baseline * 100 + if current - baseline < floor: + return change_pct, "ok" + if change_pct >= fail_pct: + return change_pct, "fail" + if change_pct >= warn_pct: + return change_pct, "warn" + return change_pct, "ok" + + +def compare( + baseline: dict[str, dict[str, Any]], + current: dict[str, dict[str, Any]], + *, + warn_pct: float = DEFAULT_WARN_PCT, + fail_pct: float = DEFAULT_FAIL_PCT, + floor_ms: float = DEFAULT_FLOOR_MS, + floor_bytes: float = DEFAULT_FLOOR_BYTES, +) -> list[Finding]: + """Compare matched benchmark results between two runs. + + Latency percentiles are compared for results with observations; metrics + named ``*_bytes`` are compared with the byte floor, raised to + ``MEMORY_FLOOR_BYTES`` for noisy process-memory metrics. Counts, ratios, + and unmatched benchmarks are ignored. + + Args: + baseline: Indexed baseline results. + current: Indexed current results. + warn_pct: Percentage increase that warns. + fail_pct: Percentage increase that fails. + floor_ms: Minimum meaningful latency increase in milliseconds. + floor_bytes: Minimum meaningful byte increase. + + Returns: + Findings for every compared metric, in report order. + """ + findings: list[Finding] = [] + for key in sorted(baseline.keys() & current.keys()): + baseline_result, current_result = baseline[key], current[key] + comparisons: list[tuple[str, float, float, float]] = [] + if baseline_result.get("observations_ms") and current_result.get( + "observations_ms" + ): + comparisons.extend( + ( + name, + baseline_result["summary"][name], + current_result["summary"][name], + floor_ms, + ) + for name in COMPARED_PERCENTILES + ) + baseline_metrics = baseline_result.get("metrics", {}) + current_metrics = current_result.get("metrics", {}) + comparisons.extend( + ( + name, + baseline_metrics[name], + current_metrics[name], + max(floor_bytes, MEMORY_FLOOR_BYTES) + if name.startswith(MEMORY_METRIC_PREFIXES) + else floor_bytes, + ) + for name in sorted(baseline_metrics.keys() & current_metrics.keys()) + if name.endswith("_bytes") + ) + for metric, baseline_value, current_value, floor in comparisons: + change_pct, status = _classify( + baseline_value, current_value, floor, warn_pct, fail_pct + ) + findings.append( + Finding(key, metric, baseline_value, current_value, change_pct, status) + ) + return findings + + +def main(argv: list[str] | None = None) -> int: + """Compare two performance result directories. + + Args: + argv: Command-line arguments, defaulting to ``sys.argv``. + + Returns: + Process exit code: 1 when comparable environments show a failure. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path, help="baseline results directory") + parser.add_argument("current", type=Path, help="current results directory") + parser.add_argument("--warn-pct", type=float, default=DEFAULT_WARN_PCT) + parser.add_argument("--fail-pct", type=float, default=DEFAULT_FAIL_PCT) + parser.add_argument("--floor-ms", type=float, default=DEFAULT_FLOOR_MS) + parser.add_argument("--floor-bytes", type=float, default=DEFAULT_FLOOR_BYTES) + args = parser.parse_args(argv) + + baseline = load_reports(args.baseline) + current = load_reports(args.current) + if not baseline or not current: + print("No comparable reports found; skipping comparison.") + return 0 + + findings = compare( + baseline, + current, + warn_pct=args.warn_pct, + fail_pct=args.fail_pct, + floor_ms=args.floor_ms, + floor_bytes=args.floor_bytes, + ) + comparable = all( + environments_match(baseline[key], current[key]) + for key in baseline.keys() & current.keys() + ) + for finding in findings: + if finding.status != "ok": + print(finding.render()) + regressions = [finding for finding in findings if finding.status == "fail"] + print( + f"Compared {len(findings)} metrics: " + f"{len(regressions)} fail, " + f"{sum(finding.status == 'warn' for finding in findings)} warn." + ) + if not comparable: + print("Environments differ; comparison is informational only.") + return 0 + return 1 if regressions else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py index bb645086bf1..b7b50c15c67 100644 --- a/tests/benchmarks/support/apps.py +++ b/tests/benchmarks/support/apps.py @@ -101,7 +101,46 @@ def index(): def second(): return rx.heading("Second page", id="second-heading") +class RowState(rx.State): + rows: list[str] = [] + selected: int = -1 + + def create_rows(self): + self.rows = [f"row {index}" for index in range(1000)] + + def partial_update(self): + for index in range(0, len(self.rows), 10): + self.rows[index] += " !!!" + + def select_row(self, index: int): + self.selected = index + + def swap_rows(self): + self.rows[1], self.rows[998] = self.rows[998], self.rows[1] + +def rows(): + return rx.vstack( + rx.text( + rx.cond(RowState.is_hydrated, "hydrated", "loading"), + id="rows-hydrated", + ), + rx.text(RowState.selected, id="selected-row"), + rx.button("create", id="create-rows", on_click=RowState.create_rows), + rx.button("partial", id="partial-update", on_click=RowState.partial_update), + rx.button("swap", id="swap-rows", on_click=RowState.swap_rows), + rx.foreach( + RowState.rows, + lambda row, index: rx.text( + row, + id=f"row-{index}", + class_name=rx.cond(index == RowState.selected, "row selected", "row"), + on_click=RowState.select_row(index), + ), + ), + ) + app = rx.App() app.add_page(index) app.add_page(second, route="/second") +app.add_page(rows, route="/rows") """ diff --git a/tests/benchmarks/support/baseline_server.py b/tests/benchmarks/support/baseline_server.py new file mode 100644 index 00000000000..554d04a9cd3 --- /dev/null +++ b/tests/benchmarks/support/baseline_server.py @@ -0,0 +1,114 @@ +"""Bare Starlette + Socket.IO server mirroring Reflex's websocket substrate. + +Serves the same stack Reflex runs on (uvicorn, Starlette mount at ``/_event``, +python-socketio ``AsyncServer`` on the ``/_event`` namespace) with a handler +that answers every event with a canned state-update payload. Round-trip +latency against this server isolates the substrate cost, so the difference +against a real Reflex app is the framework's own event-pipeline overhead. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any + +import socketio +import uvicorn +from starlette.applications import Starlette +from starlette.routing import Mount + + +class _EchoNamespace(socketio.AsyncNamespace): + """Namespace answering every event with a fixed update payload.""" + + def __init__(self, namespace: str, response: dict[str, Any]): + """Initialize the namespace. + + Args: + namespace: Socket.IO namespace, matching Reflex's event namespace. + response: Payload emitted back for every received event. + """ + super().__init__(namespace) + self._response = response + + async def on_connect(self, sid: str, environ: dict) -> None: + """Accept every connection. + + Args: + sid: Socket.IO session id. + environ: Connection request information. + """ + + async def on_event(self, sid: str, data: Any) -> None: + """Answer an event with the canned update. + + Args: + sid: Socket.IO session id. + data: Received event payload, intentionally unused. + """ + await self.emit("event", self._response, to=sid) + + +class BaselineSocketServer: + """Context manager serving the bare substrate on an ephemeral port.""" + + def __init__(self, response: dict[str, Any]): + """Configure the server without starting it. + + Args: + response: Payload emitted back for every received event. + """ + sio = socketio.AsyncServer( + async_mode="asgi", + cors_allowed_origins="*", + cors_credentials=True, + allow_upgrades=False, + transports=["websocket"], + ) + sio.register_namespace(_EchoNamespace("/_event", response)) + socket_app = socketio.ASGIApp(sio, socketio_path="") + self._server = uvicorn.Server( + uvicorn.Config( + app=Starlette(routes=[Mount("/_event", app=socket_app)]), + host="127.0.0.1", + port=0, + log_level="warning", + ) + ) + self._thread = threading.Thread(target=self._server.run, daemon=True) + self.url = "" + + def __enter__(self) -> BaselineSocketServer: + """Start the server and resolve its ephemeral URL. + + Returns: + The running server. + + Raises: + TimeoutError: If the server does not bind within ten seconds. + """ + self._thread.start() + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if ( + self._server.started + and (servers := self._server.servers) + and (sockets := servers[0].sockets) + ): + host, port = sockets[0].getsockname()[:2] + self.url = f"http://{host}:{port}" + return self + time.sleep(0.01) + self._server.should_exit = True + msg = "Baseline socket server did not start within 10 seconds." + raise TimeoutError(msg) + + def __exit__(self, *exc_info) -> None: + """Stop the server and join its thread. + + Args: + exc_info: Exception information, intentionally unused. + """ + self._server.should_exit = True + self._thread.join(timeout=10) diff --git a/tests/benchmarks/support/idle_clients.py b/tests/benchmarks/support/idle_clients.py new file mode 100644 index 00000000000..f21321d13b8 --- /dev/null +++ b/tests/benchmarks/support/idle_clients.py @@ -0,0 +1,48 @@ +"""Hold idle Socket.IO sessions from a separate process. + +Memory-per-session benchmarks run the backend inside the test process, so +client-side allocations would pollute resident-memory measurements. This +module connects a number of sessions, primes each with one event so the +backend materializes its state, reports readiness on stdout, and holds the +connections until stdin closes. +""" + +from __future__ import annotations + +import json +import sys + +import socketio + +from tests.benchmarks.support.socket_client import _connect + + +def main() -> int: + """Connect, prime, and hold the requested sessions. + + Returns: + Process exit code. + """ + url, token_prefix, count, payload_json = sys.argv[1:5] + payload = json.loads(payload_json) + clients: list[socketio.SimpleClient] = [] + try: + for index in range(int(count)): + client = socketio.SimpleClient(logger=False) + _connect(client, url, f"{token_prefix}-{index}", "/_event", timeout=10) + client.emit("event", payload) + response = client.receive(timeout=10) + while response and response[0] != "event": + response = client.receive(timeout=10) + clients.append(client) + print("ready", flush=True) + sys.stdin.read() + finally: + for client in clients: + if client.connected: + client.disconnect() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/benchmarks/support/socket_client.py b/tests/benchmarks/support/socket_client.py index 1099c03b654..6f0e497c3c1 100644 --- a/tests/benchmarks/support/socket_client.py +++ b/tests/benchmarks/support/socket_client.py @@ -6,6 +6,7 @@ import concurrent.futures import dataclasses import functools +import json import time from collections.abc import Callable, Mapping from typing import Any @@ -22,6 +23,57 @@ class ClientLoadResult: token: str latencies_ms: tuple[float, ...] errors: tuple[str, ...] + payload_bytes: tuple[int, ...] = () + + +@dataclasses.dataclass(frozen=True) +class ReconnectResult: + """Connection and first-response observations for one reconnect client.""" + + token: str + connect_ms: float + first_response_ms: float + errors: tuple[str, ...] + + +def _payload_size(response: Any) -> int: + """Approximate the wire size of a received update payload. + + Args: + response: Decoded ``[event, payload]`` message from the client. + + Returns: + Compact JSON byte length of the payload, excluding Socket.IO framing. + """ + payload = response[1] if len(response) > 1 else None + return len(json.dumps(payload, separators=(",", ":"), default=str)) + + +def _connect( + client: socketio.SimpleClient, + url: str, + token: str, + namespace: str, + timeout: float, +) -> None: + """Connect a client the way the Reflex frontend does. + + Args: + client: Blocking Socket.IO client. + url: Backend Socket.IO URL. + token: Reflex client token. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection wait. + """ + separator = "&" if "?" in url else "?" + client.connect( + f"{url}{separator}token={quote(token)}", + transports=["websocket"], + socketio_path="_event", + headers={"Origin": url}, + namespace=namespace, + wait_timeout=max(1, int(timeout)), + ) async def run_socket_client( @@ -96,18 +148,11 @@ def _run_socket_client_sync( """ client = socketio.SimpleClient(logger=False) latencies: list[float] = [] + payload_sizes: list[int] = [] errors: list[str] = [] try: - separator = "&" if "?" in url else "?" - client.connect( - f"{url}{separator}token={quote(token)}", - transports=["websocket"], - socketio_path="_event", - headers={"Origin": url}, - namespace=namespace, - wait_timeout=max(1, int(timeout)), - ) + _connect(client, url, token, namespace, timeout) for _ in range(events): started = time.perf_counter_ns() client.emit(event_name, dict(payload)) @@ -119,13 +164,107 @@ def _run_socket_client_sync( errors.append("response_timeout") continue latencies.append((time.perf_counter_ns() - started) / 1_000_000) + payload_sizes.append(_payload_size(response)) + except Exception as err: + errors.append(f"{type(err).__name__}: {err}") + finally: + if client.connected: + client.disconnect() + + return ClientLoadResult( + token, tuple(latencies), tuple(errors), tuple(payload_sizes) + ) + + +async def run_reconnect_client( + url: str, + token: str, + payload: Mapping[str, Any], + *, + event_name: str = "event", + response_name: str = "event", + namespace: str = "/_event", + timeout: float = 10, + executor: concurrent.futures.Executor | None = None, +) -> ReconnectResult: + """Connect, send one event, and measure connect and first-response time. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted after connecting. + event_name: Socket event name used for the request. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection and response wait. + executor: Optional executor that owns the blocking socket client. + + Returns: + Connect and first-response observations. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + executor, + functools.partial( + _run_reconnect_client_sync, + url, + token, + payload, + event_name, + response_name, + namespace, + timeout, + ), + ) + + +def _run_reconnect_client_sync( + url: str, + token: str, + payload: Mapping[str, Any], + event_name: str, + response_name: str, + namespace: str, + timeout: float, +) -> ReconnectResult: + """Run one reconnect client with the websocket-client transport. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted after connecting. + event_name: Socket event name used for the request. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection and response wait. + + Returns: + Connect and first-response observations. + """ + client = socketio.SimpleClient(logger=False) + connect_ms = 0.0 + first_response_ms = 0.0 + errors: list[str] = [] + + try: + started = time.perf_counter_ns() + _connect(client, url, token, namespace, timeout) + connect_ms = (time.perf_counter_ns() - started) / 1_000_000 + started = time.perf_counter_ns() + client.emit(event_name, dict(payload)) + response = client.receive(timeout=timeout) + while response and response[0] != response_name: + response = client.receive(timeout=timeout) + first_response_ms = (time.perf_counter_ns() - started) / 1_000_000 + except SocketTimeoutError: + errors.append("response_timeout") except Exception as err: errors.append(f"{type(err).__name__}: {err}") finally: if client.connected: client.disconnect() - return ClientLoadResult(token, tuple(latencies), tuple(errors)) + return ReconnectResult(token, connect_ms, first_response_ms, tuple(errors)) async def run_clients( diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py index 4208f238b44..4f2b30b49da 100644 --- a/tests/performance/conftest.py +++ b/tests/performance/conftest.py @@ -1,9 +1,13 @@ """Configuration for scheduled performance suites.""" +from collections.abc import Generator from pathlib import Path import pytest +from reflex.testing import AppHarness, AppHarnessProd +from tests.benchmarks.support.apps import load_app_source + def pytest_addoption(parser: pytest.Parser) -> None: """Register explicit performance-suite options. @@ -64,6 +68,27 @@ def performance_output(request: pytest.FixtureRequest) -> Path: return output +@pytest.fixture(scope="session") +def performance_load_app(tmp_path_factory) -> Generator[AppHarness, None, None]: + """Build and run the representative production load application once. + + Shared session-wide because a production build takes minutes; consumers + isolate through distinct client tokens. + + Args: + tmp_path_factory: Pytest temporary directory factory. + + Yields: + Running production app harness. + """ + with AppHarnessProd.create( + root=tmp_path_factory.mktemp("performance_load"), + app_source=load_app_source(), + app_name="performance_load", + ) as harness: + yield harness + + @pytest.fixture def performance_scale(request: pytest.FixtureRequest) -> str: """Return the selected scenario scale. diff --git a/tests/performance/test_browser.py b/tests/performance/test_browser.py index e8a0712ea04..18f5eea6000 100644 --- a/tests/performance/test_browser.py +++ b/tests/performance/test_browser.py @@ -4,37 +4,17 @@ import gzip import time -from collections.abc import Generator from pathlib import Path import pytest from playwright.sync_api import Page, expect -from reflex.testing import AppHarness, AppHarnessProd +from reflex.testing import AppHarness from tests.benchmarks.support import BenchmarkResult, PerformanceReport -from tests.benchmarks.support.apps import load_app_source MAX_JAVASCRIPT_GZIP_BYTES = 14_000_000 -@pytest.fixture(scope="module") -def performance_browser_app(tmp_path_factory) -> Generator[AppHarness, None, None]: - """Build and serve the representative application in production mode. - - Args: - tmp_path_factory: Pytest temporary directory factory. - - Yields: - Running production app harness. - """ - with AppHarnessProd.create( - root=tmp_path_factory.mktemp("performance_browser"), - app_source=load_app_source(), - app_name="performance_browser", - ) as harness: - yield harness - - def _bundle_metrics(root: Path) -> dict[str, int]: """Calculate bundle module and compressed-size metrics. @@ -79,7 +59,7 @@ def _browser_heap(page: Page) -> int: @pytest.mark.performance def test_browser_report( - performance_browser_app: AppHarness, + performance_load_app: AppHarness, page: Page, performance_output: Path, performance_scale: str, @@ -87,12 +67,12 @@ def test_browser_report( """Measure hydration, state-to-DOM updates, large lists, navigation, and heap. Args: - performance_browser_app: Running production app. + performance_load_app: Running production app. page: Playwright browser page. performance_output: Artifact directory. performance_scale: Selected scenario scale. """ - assert performance_browser_app.frontend_url is not None + assert performance_load_app.frontend_url is not None page.add_init_script( """ window.__reflexLongTasks = []; @@ -102,7 +82,7 @@ def test_browser_report( """ ) started = time.perf_counter_ns() - page.goto(performance_browser_app.frontend_url) + page.goto(performance_load_app.frontend_url) expect(page.locator("#count")).to_have_text("0") hydration_ms = (time.perf_counter_ns() - started) / 1_000_000 heap_before = _browser_heap(page) @@ -128,7 +108,7 @@ def test_browser_report( assert isinstance(long_tasks, list) metrics: dict[str, float | int] = dict( - _bundle_metrics(performance_browser_app.app_path) + _bundle_metrics(performance_load_app.app_path) ) metrics.update({ "heap_before_bytes": heap_before, @@ -162,3 +142,91 @@ def test_browser_report( assert metrics["javascript_modules"] > 0 assert metrics["javascript_gzip_bytes"] <= MAX_JAVASCRIPT_GZIP_BYTES assert heap_after > 0 + + +def _timed_through_paint(page: Page, action, expectation) -> float: + """Time an interaction from click through the following paint. + + Args: + page: Playwright browser page. + action: Callable triggering the interaction. + expectation: Callable asserting the resulting DOM change. + + Returns: + Elapsed milliseconds including a double requestAnimationFrame flush. + """ + started = time.perf_counter_ns() + action() + expectation() + page.evaluate( + "() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))" + ) + return (time.perf_counter_ns() - started) / 1_000_000 + + +@pytest.mark.performance +def test_browser_rows_report( + performance_load_app: AppHarness, + page: Page, + performance_output: Path, + performance_scale: str, +): + """Measure keyed-list DOM operations: create, partial update, select, swap. + + The discriminating js-framework-benchmark operations for a keyed list + rendered through ``rx.foreach``: full create, every-tenth-row update, a + selection that must not re-render other rows, and a two-row swap that + stresses keyed reconciliation. Durations include the following paint. + + Args: + performance_load_app: Running production app. + page: Playwright browser page. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + assert performance_load_app.frontend_url is not None + page.goto(performance_load_app.frontend_url.rstrip("/") + "/rows") + expect(page.locator("#rows-hydrated")).to_have_text("hydrated") + expect(page.locator("#selected-row")).to_have_text("-1") + + create_ms = _timed_through_paint( + page, + lambda: page.locator("#create-rows").click(), + lambda: expect(page.locator(".row")).to_have_count(1000), + ) + partial_ms = _timed_through_paint( + page, + lambda: page.locator("#partial-update").click(), + lambda: expect(page.locator("#row-990")).to_have_text("row 990 !!!"), + ) + select_ms = _timed_through_paint( + page, + lambda: page.locator("#row-500").click(), + lambda: expect(page.locator("#selected-row")).to_have_text("500"), + ) + swap_ms = _timed_through_paint( + page, + lambda: page.locator("#swap-rows").click(), + lambda: expect(page.locator("#row-1")).to_have_text("row 998"), + ) + + report = PerformanceReport( + "browser-rows", + metadata={"scale": performance_scale, "rows": 1000}, + ) + for name, duration in [ + ("create_1000_rows", create_ms), + ("partial_update_every_10th", partial_ms), + ("select_row", select_ms), + ("swap_rows", swap_ms), + ]: + report.add( + BenchmarkResult( + name, + {"rows": 1000}, + [duration], + {}, + measurement_iterations=1, + ) + ) + report.write(performance_output / "browser-rows.json") diff --git a/tests/performance/test_event_load.py b/tests/performance/test_event_load.py index 9cbf9d8bca2..38b633674c1 100644 --- a/tests/performance/test_event_load.py +++ b/tests/performance/test_event_load.py @@ -3,36 +3,56 @@ from __future__ import annotations import asyncio +import json import time -from collections.abc import Generator from pathlib import Path +from typing import Any import pytest from reflex_base.config import get_config from reflex_base.registry import RegistrationContext -from reflex.testing import AppHarness, AppHarnessProd -from tests.benchmarks.support import BenchmarkResult, PerformanceReport -from tests.benchmarks.support.apps import load_app_source -from tests.benchmarks.support.socket_client import run_clients, run_socket_client +from reflex.testing import AppHarness +from tests.benchmarks.support import BenchmarkResult, PerformanceReport, percentile +from tests.benchmarks.support.baseline_server import BaselineSocketServer +from tests.benchmarks.support.report import current_process_metrics +from tests.benchmarks.support.socket_client import ( + run_clients, + run_reconnect_client, + run_socket_client, +) +# A load level saturates once p99 exceeds the single-client baseline by this +# factor and by this absolute margin, so tiny baselines do not flag noise. +KNEE_P99_FACTOR = 3.0 +KNEE_P99_MARGIN_MS = 25.0 -@pytest.fixture(scope="module") -def performance_load_app(tmp_path_factory) -> Generator[AppHarness, None, None]: - """Build and run the representative production load application. - Args: - tmp_path_factory: Pytest temporary directory factory. +def _increment_payload() -> dict[str, Any]: + """Build the increment event payload for the running load app. - Yields: - Running production app harness. + Returns: + Socket.IO event payload targeting the app's increment handler. """ - with AppHarnessProd.create( - root=tmp_path_factory.mktemp("performance_load"), - app_source=load_app_source(), - app_name="performance_load", - ) as harness: - yield harness + handler_name = next( + name + for name in RegistrationContext.get().event_handlers + if name.endswith(".increment") and "performance_load" in name + ) + return { + "name": handler_name, + "payload": {}, + "router_data": {"path": "/", "pathname": "/", "query": {}}, + } + + +def _backend_url() -> str: + """Return the running backend's Socket.IO URL. + + Returns: + Backend URL without a trailing slash. + """ + return get_config().api_url.rstrip("/") @pytest.mark.performance @@ -43,6 +63,10 @@ def test_event_load_report( ): """Measure production Socket.IO latency and throughput across client counts. + Reports the full latency-throughput curve and the saturation knee: the + first load level whose p99 exceeds the single-client baseline by both + ``KNEE_P99_FACTOR`` and ``KNEE_P99_MARGIN_MS``. + Args: performance_load_app: Running production app. performance_output: Artifact directory. @@ -53,22 +77,14 @@ def test_event_load_report( "release": ([1, 10, 50, 200], 50), } client_counts, events_per_client = scales[performance_scale] - backend_url = get_config().api_url.rstrip("/") - handler_name = next( - name - for name in RegistrationContext.get().event_handlers - if name.endswith(".increment") and "performance_load" in name - ) - payload = { - "name": handler_name, - "payload": {}, - "router_data": {"path": "/", "pathname": "/", "query": {}}, - } + backend_url = _backend_url() + payload = _increment_payload() report = PerformanceReport( "event-load", metadata={"scale": performance_scale, "backend_url": backend_url}, ) + curve: list[dict[str, float | int]] = [] for clients in client_counts: started = time.perf_counter_ns() results = asyncio.run( @@ -85,9 +101,18 @@ def test_event_load_report( ) elapsed_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 latencies = [latency for result in results for latency in result.latencies_ms] + payload_sizes = [size for result in results for size in result.payload_bytes] errors = [error for result in results for error in result.errors] per_client_max = [max(result.latencies_ms, default=0) for result in results] expected = clients * events_per_client + throughput = len(latencies) / elapsed_seconds + p99 = percentile(latencies, 99) + curve.append({ + "clients": clients, + "throughput_events_per_second": throughput, + "p50_ms": percentile(latencies, 50), + "p99_ms": p99, + }) report.add( BenchmarkResult( name=f"unique_tokens_{clients}_clients", @@ -97,12 +122,16 @@ def test_event_load_report( }, observations_ms=latencies, metrics={ - "throughput_events_per_second": len(latencies) / elapsed_seconds, + "throughput_events_per_second": throughput, "errors": len(errors), "expected_updates": expected, "received_updates": len(latencies), "fairness_spread_ms": max(per_client_max, default=0) - min(per_client_max, default=0), + "payload_bytes_mean": sum(payload_sizes) / len(payload_sizes) + if payload_sizes + else 0, + "payload_bytes_max": max(payload_sizes, default=0), }, measurement_iterations=len(latencies), ) @@ -110,4 +139,171 @@ def test_event_load_report( assert not errors, errors assert len(latencies) == expected + baseline_p99 = curve[0]["p99_ms"] + knee_threshold = max( + baseline_p99 * KNEE_P99_FACTOR, baseline_p99 + KNEE_P99_MARGIN_MS + ) + knee_clients = next( + (level["clients"] for level in curve[1:] if level["p99_ms"] > knee_threshold), + 0, + ) + report.metadata["latency_curve"] = curve + report.metadata["knee_threshold_ms"] = knee_threshold + report.metadata["knee_clients"] = knee_clients + report.write(performance_output / "event-load.json") + + +@pytest.mark.performance +def test_framework_overhead_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Compare Reflex round-trips against the bare Starlette+Socket.IO substrate. + + Runs the same client, event payload, and response size against a bare + ``uvicorn`` + Starlette + python-socketio echo server, isolating how much + of the round-trip is Reflex's event pipeline versus the substrate. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + events, warmup = {"smoke": (25, 5), "release": (500, 20)}[performance_scale] + payload = _increment_payload() + + reflex_result = asyncio.run( + run_socket_client(_backend_url(), "overhead-reflex", payload, events + warmup) + ) + assert not reflex_result.errors, reflex_result.errors + + # Pad the canned response so both servers answer with equal payload bytes. + response_bytes = max(reflex_result.payload_bytes, default=0) + canned: dict[str, Any] = {"delta": {"state": {"count": 1}}, "final": True} + base_bytes = len(json.dumps(canned | {"pad": ""}, separators=(",", ":"))) + canned["pad"] = "x" * max(0, response_bytes - base_bytes) + with BaselineSocketServer(canned) as baseline: + baseline_result = asyncio.run( + run_socket_client( + baseline.url, "overhead-baseline", payload, events + warmup + ) + ) + assert not baseline_result.errors, baseline_result.errors + + report = PerformanceReport( + "framework-overhead", + metadata={"scale": performance_scale, "response_bytes": response_bytes}, + ) + reflex_latencies = reflex_result.latencies_ms[warmup:] + baseline_latencies = baseline_result.latencies_ms[warmup:] + for name, latencies in [ + ("reflex_round_trip", reflex_latencies), + ("bare_starlette_round_trip", baseline_latencies), + ]: + report.add( + BenchmarkResult( + name=name, + parameters={"events": events}, + observations_ms=list(latencies), + metrics={}, + warmup_iterations=warmup, + measurement_iterations=len(latencies), + ) + ) + reflex_p50 = percentile(reflex_latencies, 50) + baseline_p50 = percentile(baseline_latencies, 50) + report.add( + BenchmarkResult( + name="framework_overhead", + parameters={"events": events}, + observations_ms=[], + metrics={ + "overhead_p50_ms": reflex_p50 - baseline_p50, + "overhead_p99_ms": percentile(reflex_latencies, 99) + - percentile(baseline_latencies, 99), + "overhead_ratio_p50": reflex_p50 / baseline_p50 if baseline_p50 else 0, + }, + ) + ) + report.write(performance_output / "framework-overhead.json") + assert reflex_p50 > 0 + assert baseline_p50 > 0 + + +@pytest.mark.performance +def test_reconnect_storm_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure simultaneous reconnection after all clients drop. + + Primes per-token state, then reconnects every client at once — the + deploy-restart storm — measuring connect and first-update latency + percentiles and resident-memory growth. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + clients = {"smoke": 5, "release": 100}[performance_scale] + backend_url = _backend_url() + payload = _increment_payload() + tokens = [f"storm-token-{index}" for index in range(clients)] + + # Prime each token so the storm exercises state restore, not creation. + prime_results = asyncio.run( + run_clients( + clients, + lambda index, executor: run_socket_client( + backend_url, tokens[index], payload, 1, executor=executor + ), + ) + ) + assert not any(result.errors for result in prime_results) + # Token disconnect cleanup is fire-and-forget; let it finish so the storm + # exercises reconnection instead of duplicate-tab handling. + time.sleep(1) + + rss_before = current_process_metrics()["rss_bytes"] + started = time.perf_counter_ns() + storm_results = asyncio.run( + run_clients( + clients, + lambda index, executor: run_reconnect_client( + backend_url, tokens[index], payload, executor=executor + ), + ) + ) + storm_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 + rss_after = current_process_metrics()["rss_bytes"] + + errors = [error for result in storm_results for error in result.errors] + connects = [result.connect_ms for result in storm_results] + first_responses = [result.first_response_ms for result in storm_results] + report = PerformanceReport( + "reconnect-storm", + metadata={"scale": performance_scale, "backend_url": backend_url}, + ) + report.add( + BenchmarkResult( + name=f"reconnect_storm_{clients}_clients", + parameters={"clients": clients}, + observations_ms=first_responses, + metrics={ + "connect_p50_ms": percentile(connects, 50), + "connect_p99_ms": percentile(connects, 99), + "connect_max_ms": max(connects, default=0), + "storm_seconds": storm_seconds, + "reconnects_per_second": clients / storm_seconds, + "rss_growth_bytes": rss_after - rss_before, + "errors": len(errors), + }, + measurement_iterations=clients, + ) + ) + report.write(performance_output / "reconnect-storm.json") + assert not errors, errors diff --git a/tests/performance/test_session_memory.py b/tests/performance/test_session_memory.py new file mode 100644 index 00000000000..f052e5c7d22 --- /dev/null +++ b/tests/performance/test_session_memory.py @@ -0,0 +1,155 @@ +"""Backend memory cost per connected session as state size grows.""" + +from __future__ import annotations + +import contextlib +import gc +import json +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from reflex_base.config import get_config +from reflex_base.registry import RegistrationContext + +from reflex.testing import AppHarness +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.report import current_process_metrics + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _handler_payload(suffix: str) -> dict[str, Any]: + """Build an event payload for a load-app handler. + + Args: + suffix: Handler method name. + + Returns: + Socket.IO event payload targeting the handler. + """ + handler_name = next( + name + for name in RegistrationContext.get().event_handlers + if name.endswith(f".{suffix}") and "performance_load" in name + ) + return { + "name": handler_name, + "payload": {}, + "router_data": {"path": "/", "pathname": "/", "query": {}}, + } + + +@contextlib.contextmanager +def _held_sessions( + url: str, + token_prefix: str, + count: int, + payload: dict[str, Any], +) -> Iterator[None]: + """Hold primed sessions from a subprocess so client memory stays external. + + Args: + url: Backend Socket.IO URL. + token_prefix: Unique token prefix for this batch. + count: Number of sessions to hold. + payload: Priming event payload sent once per session. + + Yields: + While the sessions are connected and primed. + """ + process = subprocess.Popen( + [ + sys.executable, + "-m", + "tests.benchmarks.support.idle_clients", + url, + token_prefix, + str(count), + json.dumps(payload), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + cwd=REPO_ROOT, + ) + try: + assert process.stdout is not None + assert process.stdin is not None + ready = process.stdout.readline().strip() + assert ready == "ready", f"session holder failed: {ready!r}" + yield + finally: + if process.stdin is not None: + with contextlib.suppress(OSError): + process.stdin.close() + process.wait(timeout=30) + + +def _settled_rss() -> int: + """Collect garbage, let the loop settle, and read resident memory. + + Returns: + Resident set size in bytes. + """ + gc.collect() + time.sleep(0.2) + return current_process_metrics()["rss_bytes"] + + +@pytest.mark.performance +def test_session_memory_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure marginal backend memory per connected, primed session. + + The capacity-planning number: bytes of backend memory per session at a + small and a large per-session state, measured between two held batches so + interpreter warmup does not inflate the marginal cost. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + batch = {"smoke": 5, "release": 50}[performance_scale] + backend_url = get_config().api_url.rstrip("/") + report = PerformanceReport( + "session-memory", + metadata={"scale": performance_scale, "batch": batch}, + ) + + for variant, handler in [ + ("small_state", "increment"), + ("large_state", "append_large"), + ]: + payload = _handler_payload(handler) + rss_start = _settled_rss() + with _held_sessions(backend_url, f"mem-{variant}-a", batch, payload): + rss_first = _settled_rss() + with _held_sessions(backend_url, f"mem-{variant}-b", batch, payload): + rss_second = _settled_rss() + report.add( + BenchmarkResult( + name=f"session_memory_{variant}", + parameters={"sessions_per_batch": batch, "handler": handler}, + observations_ms=[], + metrics={ + "rss_start_bytes": rss_start, + "rss_first_batch_bytes": rss_first, + "rss_second_batch_bytes": rss_second, + "first_batch_bytes_per_session": (rss_first - rss_start) / batch, + "marginal_bytes_per_session": (rss_second - rss_first) / batch, + }, + measurement_iterations=batch, + ) + ) + + report.write(performance_output / "session-memory.json") + assert len(report.results) == 2 diff --git a/tests/performance/test_wire_size.py b/tests/performance/test_wire_size.py new file mode 100644 index 00000000000..27c91143e87 --- /dev/null +++ b/tests/performance/test_wire_size.py @@ -0,0 +1,88 @@ +"""Wire-size benchmarks: serialized delta bytes per canonical interaction.""" + +from __future__ import annotations + +import gzip +from collections.abc import Callable +from pathlib import Path + +import pytest +from reflex_base.utils.format import json_dumps + +from reflex.state import StateUpdate +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.states import PerformanceState, initialized_state + +STATE_SIZE = 1000 + + +def _scalar_change(state: PerformanceState) -> None: + """Increment the scalar counter.""" + state.counter += 1 + + +def _list_append(state: PerformanceState) -> None: + """Append one element to the list value.""" + state.numbers.append(STATE_SIZE) + + +def _partial_row_update(state: PerformanceState) -> None: + """Mutate a single element in the middle of the list value.""" + state.numbers[STATE_SIZE // 2] += 1 + + +def _mapping_update(state: PerformanceState) -> None: + """Update one existing mapping entry.""" + state.mapping["key_0"] += 1 + + +def _large_append(state: PerformanceState) -> None: + """Extend the list value by a thousand elements.""" + state.numbers.extend(range(STATE_SIZE)) + + +INTERACTIONS: dict[str, Callable[[PerformanceState], None]] = { + "scalar_change": _scalar_change, + "list_append": _list_append, + "partial_row_update": _partial_row_update, + "mapping_update": _mapping_update, + "large_append": _large_append, +} + + +@pytest.mark.performance +def test_wire_size_report(performance_output: Path, performance_scale: str): + """Record serialized state-update bytes for canonical interactions. + + Wire bytes regress silently: they are invisible to timing benchmarks on a + local network but dominate update latency on real links. Each interaction + mutates a warm state and serializes the resulting update exactly as the + event pipeline would. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + report = PerformanceReport("wire-size", metadata={"scale": performance_scale}) + sizes: dict[str, int] = {} + for name, interaction in INTERACTIONS.items(): + state = initialized_state(STATE_SIZE) + interaction(state) + wire = json_dumps(StateUpdate(delta=state.get_delta())).encode() + sizes[name] = len(wire) + report.add( + BenchmarkResult( + name=name, + parameters={"state_size": STATE_SIZE}, + observations_ms=[], + metrics={ + "wire_bytes": len(wire), + "wire_gzip_bytes": len(gzip.compress(wire)), + }, + ) + ) + + report.write(performance_output / "wire-size.json") + assert all(size > 0 for size in sizes.values()) + # A scalar change must stay far below a whole-collection resend. + assert sizes["scalar_change"] < sizes["large_append"] diff --git a/tests/units/test_compare_performance_results.py b/tests/units/test_compare_performance_results.py new file mode 100644 index 00000000000..93771e5f930 --- /dev/null +++ b/tests/units/test_compare_performance_results.py @@ -0,0 +1,157 @@ +"""Tests for the performance report comparison script.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from scripts.compare_performance_results import ( + compare, + environments_match, + load_reports, + main, +) + +ENVIRONMENT = { + "python": "3.14.0", + "implementation": "CPython", + "operating_system": "Linux", + "machine": "x86_64", +} + + +def _report( + directory: Path, + filename: str, + observations: list[float], + metrics: dict[str, Any] | None = None, + environment: dict[str, Any] | None = None, +) -> None: + """Write a minimal schema-compatible performance report. + + Args: + directory: Destination directory. + filename: Report file name. + observations: Latency observations in milliseconds. + metrics: Optional extra metrics for the single result. + environment: Optional environment override. + """ + ordered = sorted(observations) + summary = { + "count": len(ordered), + "p50_ms": ordered[len(ordered) // 2] if ordered else 0.0, + "p95_ms": ordered[-1] if ordered else 0.0, + "p99_ms": ordered[-1] if ordered else 0.0, + } + directory.joinpath(filename).write_text( + json.dumps({ + "schema_version": 1, + "suite": "demo", + "environment": environment or ENVIRONMENT, + "metadata": {}, + "results": [ + { + "name": "round_trip", + "parameters": {"clients": 1}, + "observations_ms": observations, + "summary": summary, + "metrics": metrics or {}, + } + ], + }) + ) + + +def test_load_reports_ignores_non_reports(tmp_path: Path): + """Files without the report schema are skipped.""" + tmp_path.joinpath("junk.json").write_text('{"unrelated": true}') + tmp_path.joinpath("broken.json").write_text("{") + _report(tmp_path, "real.json", [1.0, 2.0]) + reports = load_reports(tmp_path) + assert len(reports) == 1 + assert next(iter(reports)).startswith("demo|round_trip|") + + +def test_compare_flags_latency_regression(tmp_path: Path): + """A doubled p50 above the floor fails; small drifts stay ok.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [10.0] * 10) + _report(current_dir, "demo.json", [21.0] * 10) + findings = compare(load_reports(baseline_dir), load_reports(current_dir)) + assert {finding.status for finding in findings} == {"fail"} + + _report(current_dir, "demo.json", [10.5] * 10) + findings = compare(load_reports(baseline_dir), load_reports(current_dir)) + assert {finding.status for finding in findings} == {"ok"} + + +def test_compare_respects_absolute_floor(tmp_path: Path): + """A large percentage change under the absolute floor stays ok.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [0.5] * 10) + _report(current_dir, "demo.json", [1.5] * 10) + findings = compare(load_reports(baseline_dir), load_reports(current_dir)) + assert {finding.status for finding in findings} == {"ok"} + + +def test_compare_flags_byte_metrics(tmp_path: Path): + """Metrics named *_bytes are compared with the byte floor.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [], metrics={"wire_bytes": 100_000}) + _report(current_dir, "demo.json", [], metrics={"wire_bytes": 140_000}) + findings = compare(load_reports(baseline_dir), load_reports(current_dir)) + assert [finding.status for finding in findings] == ["warn"] + assert findings[0].metric == "wire_bytes" + + +def test_compare_memory_metrics_use_larger_floor(tmp_path: Path): + """Noisy process-memory metrics need a multi-megabyte absolute change.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [], metrics={"rss_growth_bytes": 1_000_000}) + _report(current_dir, "demo.json", [], metrics={"rss_growth_bytes": 2_000_000}) + findings = compare(load_reports(baseline_dir), load_reports(current_dir)) + assert [finding.status for finding in findings] == ["ok"] + + +def test_environment_mismatch_is_informational(tmp_path: Path, capsys): + """A failing comparison across environments exits zero.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [10.0] * 10) + _report( + current_dir, + "demo.json", + [30.0] * 10, + environment=ENVIRONMENT | {"python": "3.13.0"}, + ) + assert not environments_match( + next(iter(load_reports(baseline_dir).values())), + next(iter(load_reports(current_dir).values())), + ) + assert main([str(baseline_dir), str(current_dir)]) == 0 + assert "informational" in capsys.readouterr().out + + +def test_main_exit_codes(tmp_path: Path): + """Matching environments exit one on failure and zero when clean.""" + baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" + baseline_dir.mkdir() + current_dir.mkdir() + _report(baseline_dir, "demo.json", [10.0] * 10) + _report(current_dir, "demo.json", [30.0] * 10) + assert main([str(baseline_dir), str(current_dir)]) == 1 + + _report(current_dir, "demo.json", [10.0] * 10) + assert main([str(baseline_dir), str(current_dir)]) == 0 + + assert main([str(tmp_path / "missing"), str(current_dir)]) == 0 From 43a1fde6eaa8786d08d14d256b941bd7decaaff6 Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 11:12:00 -0700 Subject: [PATCH 06/13] Fix performance benchmark review findings --- .github/workflows/performance-nightly.yml | 83 ------ PERFORMANCE.md | 14 +- pyproject.toml | 1 - scripts/compare_performance_results.py | 253 ------------------ tests/benchmarks/support/events.py | 2 +- tests/performance/conftest.py | 8 +- tests/units/benchmarks/test_support.py | 17 ++ .../units/test_compare_performance_results.py | 157 ----------- 8 files changed, 28 insertions(+), 507 deletions(-) delete mode 100644 .github/workflows/performance-nightly.yml delete mode 100644 scripts/compare_performance_results.py delete mode 100644 tests/units/test_compare_performance_results.py diff --git a/.github/workflows/performance-nightly.yml b/.github/workflows/performance-nightly.yml deleted file mode 100644 index 69185a7e1a8..00000000000 --- a/.github/workflows/performance-nightly.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: performance-nightly -permissions: - contents: read - -on: - schedule: - - cron: "0 3 * * *" - workflow_dispatch: - -env: - REFLEX_TELEMETRY_ENABLED: false - NODE_OPTIONS: "--max_old_space_size=8192" - APP_HARNESS_HEADLESS: 1 - PYTHONUNBUFFERED: 1 - -jobs: - release-scale: - name: Release-scale performance suite - runs-on: ubuntu-22.04 - timeout-minutes: 120 - services: - redis: - image: redis:7.4 - ports: - - 6379:6379 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-tags: true - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/setup_build_env - with: - python-version: "3.14" - node-version: "22" - run-uv-sync: true - - - name: Install playwright - run: uv run playwright install chromium --only-shell - - - name: Restore previous baseline - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: .performance-baseline - key: performance-baseline-${{ github.run_id }} - restore-keys: performance-baseline- - - - name: Run release-scale performance suite - run: >- - uv run pytest tests/performance - --run-performance - --performance-scale release - --performance-output .pytest-tmp/performance - -q - - - name: Compare against previous nightly - if: ${{ hashFiles('.performance-baseline/**') != '' }} - continue-on-error: true - run: >- - uv run python scripts/compare_performance_results.py - .performance-baseline - .pytest-tmp/performance - - - name: Promote current results to baseline - run: | - rm -rf .performance-baseline - mkdir -p .performance-baseline - cp .pytest-tmp/performance/*.json .performance-baseline/ - - - name: Save baseline - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: .performance-baseline - key: performance-baseline-${{ github.run_id }} - - - name: Upload performance artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: performance-release-scale - path: .pytest-tmp/performance/** - if-no-files-found: warn diff --git a/PERFORMANCE.md b/PERFORMANCE.md index d94374ef81d..7ae5da08479 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -7,18 +7,16 @@ Reflex uses separate performance tiers because deterministic CPU work, asynchron | Tier | Location | Runs | Purpose | | --- | --- | --- | --- | | Deterministic CPU | `tests/benchmarks/` | Every pull request | CodSpeed regression detection and flamegraphs | -| Component wall time | `tests/performance/` | Pull-request smoke, nightly release scale | Event-loop, task, queue, Redis, and allocation behavior | -| Load and lifecycle | `tests/performance/` | Nightly release scale | Tail latency, saturation curve, compiles, hot reload, browser metrics | +| Component wall time | `tests/performance/` | Pull-request smoke, manual release scale | Event-loop, task, queue, Redis, and allocation behavior | +| Load and lifecycle | `tests/performance/` | Manual release scale | Tail latency, saturation curve, compiles, hot reload, browser metrics | -Scheduled coverage beyond timing: +Opt-in coverage beyond timing: - `test_wire_size.py` records serialized delta bytes (raw and gzip) per canonical interaction — wire-size regressions are invisible to local-network timing but dominate update latency on real links. - `test_event_load.py` publishes the latency-throughput curve with a saturation knee, a framework-overhead comparison against a bare Starlette + python-socketio echo server, and a reconnect-storm scenario (simultaneous reconnection after all clients drop). - `test_session_memory.py` reports marginal backend memory per connected session at small and large per-session state, holding clients in a subprocess so client memory stays out of the measurement. - `test_browser.py` includes keyed-list DOM operations (create, partial update, select row, swap rows) timed through the following paint. -The nightly workflow (`performance-nightly.yml`) runs the whole `tests/performance/` suite at release scale and compares against the previous night via `scripts/compare_performance_results.py`, which warns at +25% and fails at +50% with absolute floors (5 ms latency, 1 KB deterministic bytes, 5 MB process-memory metrics). Cross-environment comparisons are informational only. - Do not add sleep, network, filesystem, browser, or subprocess timings to a CodSpeed instrumentation benchmark. Put those in a wall-time suite and emit a versioned JSON report. ## Naming and fixture rules @@ -30,10 +28,10 @@ Do not add sleep, network, filesystem, browser, or subprocess timings to a CodSp - Use explicit warmup and measurement counts. - Batch sub-microsecond operations. - Preserve raw observations in scheduled-suite artifacts. -- Include small representative and stress cases, but reserve full parameter matrices for scheduled runs. +- Include small representative and stress cases, but reserve full parameter matrices for manual release-scale runs. - Validate correctness—event ordering, final state, update count, errors—inside every performance scenario. -Shared fixtures and reporting helpers live in `tests/benchmarks/support/`. Scheduled reports use schema version 1 and include the commit, branch, Python, OS, CPU, parameters, raw observations, latency summaries, and suite-specific metrics. +Shared fixtures and reporting helpers live in `tests/benchmarks/support/`. Performance reports use schema version 1 and include the commit, branch, Python, OS, CPU, parameters, raw observations, latency summaries, and suite-specific metrics. ## Commands @@ -42,7 +40,7 @@ uv run pytest tests/benchmarks --codspeed uv run pytest tests/performance -m performance --run-performance ``` -Individual scheduled suites are controlled by explicit command-line options and environment variables documented in their tests and workflows. They are excluded from ordinary unit and integration discovery. +Individual performance suites are controlled by explicit command-line options and environment variables documented in their tests. They are excluded from ordinary unit and integration discovery. ## Regression policy diff --git a/pyproject.toml b/pyproject.toml index 8998d560e86..099d6108ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -251,7 +251,6 @@ preview = true "*/blank.py" = ["I001"] "docs/package/scripts/*.py" = ["INP001"] "scripts/check_min_deps.py" = ["T201"] -"scripts/compare_performance_results.py" = ["T201"] [tool.pytest.ini_options] filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning" diff --git a/scripts/compare_performance_results.py b/scripts/compare_performance_results.py deleted file mode 100644 index 55558cb544d..00000000000 --- a/scripts/compare_performance_results.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Compare performance report JSON files against a baseline run. - -Matches results across two directories of ``PerformanceReport`` JSON files by -suite, benchmark name, and parameters, then flags latency-percentile and byte -regressions with warn/fail thresholds. Absolute floors keep tiny baselines -from flagging noise: a change must exceed both the percentage threshold and -the floor to count. Comparisons across materially different environments are -reported as informational and never fail. -""" - -from __future__ import annotations - -import argparse -import dataclasses -import json -import sys -from pathlib import Path -from typing import Any - -DEFAULT_WARN_PCT = 25.0 -DEFAULT_FAIL_PCT = 50.0 -DEFAULT_FLOOR_MS = 5.0 -DEFAULT_FLOOR_BYTES = 1_024 -# Process and heap measurements are inherently noisy; require a much larger -# absolute change than deterministic byte metrics like wire or bundle size. -MEMORY_FLOOR_BYTES = 5_242_880 -MEMORY_METRIC_PREFIXES = ("rss_", "heap_") - -COMPARED_PERCENTILES = ("p50_ms", "p95_ms", "p99_ms") -ENVIRONMENT_KEYS = ("python", "implementation", "operating_system", "machine") - - -@dataclasses.dataclass(frozen=True) -class Finding: - """One compared metric and its regression status.""" - - benchmark: str - metric: str - baseline: float - current: float - change_pct: float - status: str - - def render(self) -> str: - """Format the finding for terminal output. - - Returns: - One aligned report line. - """ - return ( - f"{self.status.upper():<6} {self.benchmark} {self.metric}: " - f"{self.baseline:.2f} -> {self.current:.2f} ({self.change_pct:+.1f}%)" - ) - - -def load_reports(directory: Path) -> dict[str, dict[str, Any]]: - """Index benchmark results from every report file in a directory. - - Args: - directory: Directory containing ``PerformanceReport`` JSON files. - - Returns: - Results keyed by suite, benchmark name, and canonical parameters. - """ - index: dict[str, dict[str, Any]] = {} - for path in sorted(directory.glob("*.json")): - try: - data = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - if not isinstance(data, dict) or "schema_version" not in data: - continue - for result in data.get("results", []): - key = "|".join([ - data.get("suite", path.stem), - result["name"], - json.dumps(result.get("parameters", {}), sort_keys=True), - ]) - index[key] = result | {"environment": data.get("environment", {})} - return index - - -def environments_match( - baseline: dict[str, Any], - current: dict[str, Any], -) -> bool: - """Check whether two results were produced in comparable environments. - - Args: - baseline: Baseline result with attached environment. - current: Current result with attached environment. - - Returns: - Whether the interpreter and platform identifiers all match. - """ - baseline_env = baseline.get("environment", {}) - current_env = current.get("environment", {}) - return all( - baseline_env.get(key) == current_env.get(key) for key in ENVIRONMENT_KEYS - ) - - -def _classify( - baseline: float, - current: float, - floor: float, - warn_pct: float, - fail_pct: float, -) -> tuple[float, str]: - """Classify one metric change; lower values are always better. - - Args: - baseline: Baseline value. - current: Current value. - floor: Minimum absolute increase considered meaningful. - warn_pct: Percentage increase that warns. - fail_pct: Percentage increase that fails. - - Returns: - Percentage change and status (``ok``, ``warn``, or ``fail``). - """ - if baseline <= 0: - return 0.0, "ok" - change_pct = (current - baseline) / baseline * 100 - if current - baseline < floor: - return change_pct, "ok" - if change_pct >= fail_pct: - return change_pct, "fail" - if change_pct >= warn_pct: - return change_pct, "warn" - return change_pct, "ok" - - -def compare( - baseline: dict[str, dict[str, Any]], - current: dict[str, dict[str, Any]], - *, - warn_pct: float = DEFAULT_WARN_PCT, - fail_pct: float = DEFAULT_FAIL_PCT, - floor_ms: float = DEFAULT_FLOOR_MS, - floor_bytes: float = DEFAULT_FLOOR_BYTES, -) -> list[Finding]: - """Compare matched benchmark results between two runs. - - Latency percentiles are compared for results with observations; metrics - named ``*_bytes`` are compared with the byte floor, raised to - ``MEMORY_FLOOR_BYTES`` for noisy process-memory metrics. Counts, ratios, - and unmatched benchmarks are ignored. - - Args: - baseline: Indexed baseline results. - current: Indexed current results. - warn_pct: Percentage increase that warns. - fail_pct: Percentage increase that fails. - floor_ms: Minimum meaningful latency increase in milliseconds. - floor_bytes: Minimum meaningful byte increase. - - Returns: - Findings for every compared metric, in report order. - """ - findings: list[Finding] = [] - for key in sorted(baseline.keys() & current.keys()): - baseline_result, current_result = baseline[key], current[key] - comparisons: list[tuple[str, float, float, float]] = [] - if baseline_result.get("observations_ms") and current_result.get( - "observations_ms" - ): - comparisons.extend( - ( - name, - baseline_result["summary"][name], - current_result["summary"][name], - floor_ms, - ) - for name in COMPARED_PERCENTILES - ) - baseline_metrics = baseline_result.get("metrics", {}) - current_metrics = current_result.get("metrics", {}) - comparisons.extend( - ( - name, - baseline_metrics[name], - current_metrics[name], - max(floor_bytes, MEMORY_FLOOR_BYTES) - if name.startswith(MEMORY_METRIC_PREFIXES) - else floor_bytes, - ) - for name in sorted(baseline_metrics.keys() & current_metrics.keys()) - if name.endswith("_bytes") - ) - for metric, baseline_value, current_value, floor in comparisons: - change_pct, status = _classify( - baseline_value, current_value, floor, warn_pct, fail_pct - ) - findings.append( - Finding(key, metric, baseline_value, current_value, change_pct, status) - ) - return findings - - -def main(argv: list[str] | None = None) -> int: - """Compare two performance result directories. - - Args: - argv: Command-line arguments, defaulting to ``sys.argv``. - - Returns: - Process exit code: 1 when comparable environments show a failure. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("baseline", type=Path, help="baseline results directory") - parser.add_argument("current", type=Path, help="current results directory") - parser.add_argument("--warn-pct", type=float, default=DEFAULT_WARN_PCT) - parser.add_argument("--fail-pct", type=float, default=DEFAULT_FAIL_PCT) - parser.add_argument("--floor-ms", type=float, default=DEFAULT_FLOOR_MS) - parser.add_argument("--floor-bytes", type=float, default=DEFAULT_FLOOR_BYTES) - args = parser.parse_args(argv) - - baseline = load_reports(args.baseline) - current = load_reports(args.current) - if not baseline or not current: - print("No comparable reports found; skipping comparison.") - return 0 - - findings = compare( - baseline, - current, - warn_pct=args.warn_pct, - fail_pct=args.fail_pct, - floor_ms=args.floor_ms, - floor_bytes=args.floor_bytes, - ) - comparable = all( - environments_match(baseline[key], current[key]) - for key in baseline.keys() & current.keys() - ) - for finding in findings: - if finding.status != "ok": - print(finding.render()) - regressions = [finding for finding in findings if finding.status == "fail"] - print( - f"Compared {len(findings)} metrics: " - f"{len(regressions)} fail, " - f"{sum(finding.status == 'warn' for finding in findings)} warn." - ) - if not comparable: - print("Environments differ; comparison is informational only.") - return 0 - return 1 if regressions else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/benchmarks/support/events.py b/tests/benchmarks/support/events.py index 4989024a298..24c8681e8a0 100644 --- a/tests/benchmarks/support/events.py +++ b/tests/benchmarks/support/events.py @@ -12,7 +12,7 @@ from pydantic import BaseModel -class PayloadKind(enum.StrEnum): +class PayloadKind(str, enum.Enum): """Payload shapes exercised by serialization and event benchmarks.""" SCALAR = "scalar" diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py index 4f2b30b49da..95086faeecd 100644 --- a/tests/performance/conftest.py +++ b/tests/performance/conftest.py @@ -68,12 +68,12 @@ def performance_output(request: pytest.FixtureRequest) -> Path: return output -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def performance_load_app(tmp_path_factory) -> Generator[AppHarness, None, None]: - """Build and run the representative production load application once. + """Build and run the representative production load application per module. - Shared session-wide because a production build takes minutes; consumers - isolate through distinct client tokens. + Module scope amortizes the production build within related scenarios while + stopping the in-process servers before unrelated benchmark modules run. Args: tmp_path_factory: Pytest temporary directory factory. diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py index 22b518cd20e..02715a6fce3 100644 --- a/tests/units/benchmarks/test_support.py +++ b/tests/units/benchmarks/test_support.py @@ -5,6 +5,7 @@ import pytest +from tests.benchmarks.support.events import PayloadKind from tests.benchmarks.support.pipeline_trace import PipelineTrace, StageEvent from tests.benchmarks.support.report import ( BenchmarkEnvironment, @@ -13,6 +14,7 @@ percentile, ) from tests.benchmarks.support.socket_client import run_clients +from tests.performance import conftest as performance_conftest def test_percentile_interpolates_and_validates(): @@ -23,6 +25,21 @@ def test_percentile_interpolates_and_validates(): percentile([1], 101) +def test_payload_kind_uses_python_310_compatible_enum_base(): + """Payload enums avoid bases that were added after Python 3.10.""" + assert not any( + base.__module__ == "enum" and base.__name__ == "StrEnum" + for base in PayloadKind.__mro__ + ) + + +def test_production_harness_is_module_scoped(): + """The in-process production harness stops before unrelated modules run.""" + fixture = performance_conftest.performance_load_app + marker = fixture._fixture_function_marker # pyright: ignore[reportPrivateUsage] + assert marker.scope == "module" + + def test_performance_report_round_trip(tmp_path): """Reports retain observations, summaries, metrics, and environment data.""" environment = BenchmarkEnvironment( diff --git a/tests/units/test_compare_performance_results.py b/tests/units/test_compare_performance_results.py deleted file mode 100644 index 93771e5f930..00000000000 --- a/tests/units/test_compare_performance_results.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Tests for the performance report comparison script.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -from scripts.compare_performance_results import ( - compare, - environments_match, - load_reports, - main, -) - -ENVIRONMENT = { - "python": "3.14.0", - "implementation": "CPython", - "operating_system": "Linux", - "machine": "x86_64", -} - - -def _report( - directory: Path, - filename: str, - observations: list[float], - metrics: dict[str, Any] | None = None, - environment: dict[str, Any] | None = None, -) -> None: - """Write a minimal schema-compatible performance report. - - Args: - directory: Destination directory. - filename: Report file name. - observations: Latency observations in milliseconds. - metrics: Optional extra metrics for the single result. - environment: Optional environment override. - """ - ordered = sorted(observations) - summary = { - "count": len(ordered), - "p50_ms": ordered[len(ordered) // 2] if ordered else 0.0, - "p95_ms": ordered[-1] if ordered else 0.0, - "p99_ms": ordered[-1] if ordered else 0.0, - } - directory.joinpath(filename).write_text( - json.dumps({ - "schema_version": 1, - "suite": "demo", - "environment": environment or ENVIRONMENT, - "metadata": {}, - "results": [ - { - "name": "round_trip", - "parameters": {"clients": 1}, - "observations_ms": observations, - "summary": summary, - "metrics": metrics or {}, - } - ], - }) - ) - - -def test_load_reports_ignores_non_reports(tmp_path: Path): - """Files without the report schema are skipped.""" - tmp_path.joinpath("junk.json").write_text('{"unrelated": true}') - tmp_path.joinpath("broken.json").write_text("{") - _report(tmp_path, "real.json", [1.0, 2.0]) - reports = load_reports(tmp_path) - assert len(reports) == 1 - assert next(iter(reports)).startswith("demo|round_trip|") - - -def test_compare_flags_latency_regression(tmp_path: Path): - """A doubled p50 above the floor fails; small drifts stay ok.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [10.0] * 10) - _report(current_dir, "demo.json", [21.0] * 10) - findings = compare(load_reports(baseline_dir), load_reports(current_dir)) - assert {finding.status for finding in findings} == {"fail"} - - _report(current_dir, "demo.json", [10.5] * 10) - findings = compare(load_reports(baseline_dir), load_reports(current_dir)) - assert {finding.status for finding in findings} == {"ok"} - - -def test_compare_respects_absolute_floor(tmp_path: Path): - """A large percentage change under the absolute floor stays ok.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [0.5] * 10) - _report(current_dir, "demo.json", [1.5] * 10) - findings = compare(load_reports(baseline_dir), load_reports(current_dir)) - assert {finding.status for finding in findings} == {"ok"} - - -def test_compare_flags_byte_metrics(tmp_path: Path): - """Metrics named *_bytes are compared with the byte floor.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [], metrics={"wire_bytes": 100_000}) - _report(current_dir, "demo.json", [], metrics={"wire_bytes": 140_000}) - findings = compare(load_reports(baseline_dir), load_reports(current_dir)) - assert [finding.status for finding in findings] == ["warn"] - assert findings[0].metric == "wire_bytes" - - -def test_compare_memory_metrics_use_larger_floor(tmp_path: Path): - """Noisy process-memory metrics need a multi-megabyte absolute change.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [], metrics={"rss_growth_bytes": 1_000_000}) - _report(current_dir, "demo.json", [], metrics={"rss_growth_bytes": 2_000_000}) - findings = compare(load_reports(baseline_dir), load_reports(current_dir)) - assert [finding.status for finding in findings] == ["ok"] - - -def test_environment_mismatch_is_informational(tmp_path: Path, capsys): - """A failing comparison across environments exits zero.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [10.0] * 10) - _report( - current_dir, - "demo.json", - [30.0] * 10, - environment=ENVIRONMENT | {"python": "3.13.0"}, - ) - assert not environments_match( - next(iter(load_reports(baseline_dir).values())), - next(iter(load_reports(current_dir).values())), - ) - assert main([str(baseline_dir), str(current_dir)]) == 0 - assert "informational" in capsys.readouterr().out - - -def test_main_exit_codes(tmp_path: Path): - """Matching environments exit one on failure and zero when clean.""" - baseline_dir, current_dir = tmp_path / "base", tmp_path / "head" - baseline_dir.mkdir() - current_dir.mkdir() - _report(baseline_dir, "demo.json", [10.0] * 10) - _report(current_dir, "demo.json", [30.0] * 10) - assert main([str(baseline_dir), str(current_dir)]) == 1 - - _report(current_dir, "demo.json", [10.0] * 10) - assert main([str(baseline_dir), str(current_dir)]) == 0 - - assert main([str(tmp_path / "missing"), str(current_dir)]) == 0 From 16879803cdb6898a3963aa1d8f8f0bbb857f8378 Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 11:27:27 -0700 Subject: [PATCH 07/13] Fix optional dependency test collection --- tests/units/benchmarks/test_support.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py index 02715a6fce3..852797826b7 100644 --- a/tests/units/benchmarks/test_support.py +++ b/tests/units/benchmarks/test_support.py @@ -5,7 +5,6 @@ import pytest -from tests.benchmarks.support.events import PayloadKind from tests.benchmarks.support.pipeline_trace import PipelineTrace, StageEvent from tests.benchmarks.support.report import ( BenchmarkEnvironment, @@ -14,7 +13,6 @@ percentile, ) from tests.benchmarks.support.socket_client import run_clients -from tests.performance import conftest as performance_conftest def test_percentile_interpolates_and_validates(): @@ -27,6 +25,9 @@ def test_percentile_interpolates_and_validates(): def test_payload_kind_uses_python_310_compatible_enum_base(): """Payload enums avoid bases that were added after Python 3.10.""" + pytest.importorskip("pydantic") + from tests.benchmarks.support.events import PayloadKind + assert not any( base.__module__ == "enum" and base.__name__ == "StrEnum" for base in PayloadKind.__mro__ @@ -35,6 +36,9 @@ def test_payload_kind_uses_python_310_compatible_enum_base(): def test_production_harness_is_module_scoped(): """The in-process production harness stops before unrelated modules run.""" + pytest.importorskip("pydantic") + from tests.performance import conftest as performance_conftest + fixture = performance_conftest.performance_load_app marker = fixture._fixture_function_marker # pyright: ignore[reportPrivateUsage] assert marker.scope == "module" From 29836168eea907f2aa6d5fad1736c09cb7909e84 Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 11:28:13 -0700 Subject: [PATCH 08/13] Remove performance guide --- .github/CODEOWNERS | 1 - PERFORMANCE.md | 67 ---------------------------------------------- 2 files changed, 68 deletions(-) delete mode 100644 PERFORMANCE.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1586fdb1422..313c2072590 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,6 +2,5 @@ /docs/ @Alek99 @reflex-dev/reflex-team /README.md @Alek99 @reflex-dev/reflex-team -/PERFORMANCE.md @Alek99 @reflex-dev/reflex-team /tests/benchmarks/ @Alek99 @reflex-dev/reflex-team /tests/performance/ @Alek99 @reflex-dev/reflex-team diff --git a/PERFORMANCE.md b/PERFORMANCE.md deleted file mode 100644 index 7ae5da08479..00000000000 --- a/PERFORMANCE.md +++ /dev/null @@ -1,67 +0,0 @@ -# Performance benchmark guide - -Reflex uses separate performance tiers because deterministic CPU work, asynchronous wall time, service load, memory, compiler lifecycle, and browser behavior have different noise and cost profiles. - -## Benchmark tiers - -| Tier | Location | Runs | Purpose | -| --- | --- | --- | --- | -| Deterministic CPU | `tests/benchmarks/` | Every pull request | CodSpeed regression detection and flamegraphs | -| Component wall time | `tests/performance/` | Pull-request smoke, manual release scale | Event-loop, task, queue, Redis, and allocation behavior | -| Load and lifecycle | `tests/performance/` | Manual release scale | Tail latency, saturation curve, compiles, hot reload, browser metrics | - -Opt-in coverage beyond timing: - -- `test_wire_size.py` records serialized delta bytes (raw and gzip) per canonical interaction — wire-size regressions are invisible to local-network timing but dominate update latency on real links. -- `test_event_load.py` publishes the latency-throughput curve with a saturation knee, a framework-overhead comparison against a bare Starlette + python-socketio echo server, and a reconnect-storm scenario (simultaneous reconnection after all clients drop). -- `test_session_memory.py` reports marginal backend memory per connected session at small and large per-session state, holding clients in a subprocess so client memory stays out of the measurement. -- `test_browser.py` includes keyed-list DOM operations (create, partial update, select row, swap rows) timed through the following paint. - -Do not add sleep, network, filesystem, browser, or subprocess timings to a CodSpeed instrumentation benchmark. Put those in a wall-time suite and emit a versioned JSON report. - -## Naming and fixture rules - -- Name benchmarks after the user-visible path they protect, not an implementation detail. -- Separate cold and warm paths. -- Keep setup outside the measured call unless setup is the behavior being measured. -- Use `time.perf_counter_ns()` for wall-time observations. -- Use explicit warmup and measurement counts. -- Batch sub-microsecond operations. -- Preserve raw observations in scheduled-suite artifacts. -- Include small representative and stress cases, but reserve full parameter matrices for manual release-scale runs. -- Validate correctness—event ordering, final state, update count, errors—inside every performance scenario. - -Shared fixtures and reporting helpers live in `tests/benchmarks/support/`. Performance reports use schema version 1 and include the commit, branch, Python, OS, CPU, parameters, raw observations, latency summaries, and suite-specific metrics. - -## Commands - -```console -uv run pytest tests/benchmarks --codspeed -uv run pytest tests/performance -m performance --run-performance -``` - -Individual performance suites are controlled by explicit command-line options and environment variables documented in their tests. They are excluded from ordinary unit and integration discovery. - -## Regression policy - -Thresholds become merge-blocking only after at least 20 comparable baseline runs establish their variance and a benchmark owner accepts the budget. Initial review levels are: - -- More than 5% deterministic CodSpeed CPU regression when statistically supported. -- More than 10% stable component wall-time regression. -- More than 15% p95 or p99 load latency regression at equal offered load. -- More than 10% throughput loss at the established latency objective. -- Any event-ordering failure, lost update, orphan task, or monotonic retained-memory growth. - -Use an absolute floor as well as a percentage. Environment mismatches make comparisons informational unless explicitly approved. - -## Adding or changing a benchmark - -1. State the behavior and regression it protects. -2. Choose the correct tier. -3. Add correctness assertions. -4. Record parameters, warmups, iterations, and environment metadata. -5. Run the focused benchmark and its support tests. -6. Review variance before adding a threshold. -7. Update this guide if the suite, schema, or policy changes. - -Benchmark fixtures are reviewed quarterly. Remove tests that no longer represent supported behavior, but preserve historical benchmark names when a time series remains meaningful. From f7d6a863080b7eb454bf7aa6807ff732c51f21ca Mon Sep 17 00:00:00 2001 From: Farhan Date: Tue, 14 Jul 2026 00:06:20 +0500 Subject: [PATCH 09/13] Fix second-round performance benchmark review findings - guard the perf Redis helper: require an explicit db index in REFLEX_PERF_REDIS_URL and refuse to run against a non-empty database; make cleanup exception-safe and flush before close - measure the bundle budget against the shipped client bundle (.web/build/client) instead of all of .web; budget 500KB gzip (~2x a measured 238KB export), overridable via REFLEX_PERFORMANCE_JS_GZIP_BUDGET - bound all event-future waits in the PR-smoke event-loop suite with scale-aware timeouts; add timeout-minutes to the benchmarks CI job - benchmark the real header-decode path via reflex.app._decode_asgi_headers instead of a local copy; re.purge() per round so route-table construction is actually cold; drop asyncio.get_event_loop() from sync benchmarks - move processor construction/lifecycle out of measured regions in the event-loop benchmarks; assert shutdown drain actually drains; raise the drain budget so instrumentation slowdown cannot flip drain to cancel - replace the reconnect-storm settling sleep with token-cleanup polling, scale client timeouts with concurrency, and retry transient storm failures once before asserting - kill and report the session holder on wait timeout; add untraced warmup cycles before the memory leak gate; sweep pre-collected dev-backend children on teardown and retry lost port races - format each async stack frame once in diagnostics; integer Chrome trace tids with thread_name metadata; surface baseline-server thread crashes; use the public api_transformer seam; add tests/units/benchmarks/__init__.py - move the browser smoke into its own CI job and upload performance smoke artifacts --- .github/workflows/performance.yml | 53 +++- reflex/app.py | 20 +- tests/benchmarks/support/apps.py | 7 +- tests/benchmarks/support/baseline_server.py | 24 +- tests/benchmarks/support/diagnostics.py | 8 +- tests/benchmarks/support/pipeline_trace.py | 43 +++- tests/benchmarks/support/redis.py | 47 +++- tests/benchmarks/test_event_loop.py | 256 ++++++++++++++++--- tests/benchmarks/test_event_processing.py | 111 ++++---- tests/benchmarks/test_routes.py | 12 +- tests/benchmarks/test_socket_headers.py | 8 +- tests/performance/test_browser.py | 25 +- tests/performance/test_compiler_lifecycle.py | 109 ++++++-- tests/performance/test_event_load.py | 93 ++++++- tests/performance/test_event_loop.py | 47 +++- tests/performance/test_memory.py | 33 ++- tests/performance/test_session_memory.py | 15 +- tests/units/benchmarks/__init__.py | 1 + tests/units/benchmarks/test_support.py | 203 ++++++++++++++- tests/units/test_app.py | 26 +- 20 files changed, 967 insertions(+), 174 deletions(-) create mode 100644 tests/units/benchmarks/__init__.py diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 84946210b48..c31af0da022 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -21,6 +21,7 @@ jobs: benchmarks: name: Run benchmarks runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -53,6 +54,50 @@ jobs: --performance-output .pytest-tmp/performance -q + - name: Upload performance smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-smoke-benchmarks + path: .pytest-tmp/performance/** + if-no-files-found: ignore + + browser-performance: + name: Run browser performance smoke + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-tags: true + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/setup_build_env + with: + python-version: "3.14" + node-version: "22" + run-uv-sync: true + + - name: Install playwright + run: uv run playwright install chromium --only-shell + + - name: Run browser performance smoke and bundle budget + run: >- + uv run pytest tests/performance/test_browser.py + --run-performance + --performance-scale smoke + --performance-output .pytest-tmp/performance + -q + + - name: Upload performance smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-smoke-browser + path: .pytest-tmp/performance/** + if-no-files-found: ignore + lighthouse: name: Run Lighthouse benchmark runs-on: ubuntu-22.04 @@ -80,14 +125,6 @@ jobs: mkdir -p .pytest-tmp/lighthouse uv run pytest tests/integration/test_lighthouse.py -q -s --tb=no --basetemp=.pytest-tmp/lighthouse - - name: Run browser performance smoke and bundle budget - run: >- - uv run pytest tests/performance/test_browser.py - --run-performance - --performance-scale smoke - --performance-output .pytest-tmp/performance - -q - - name: Upload Lighthouse artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/reflex/app.py b/reflex/app.py index 9364c547436..37dc045eb75 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -20,6 +20,7 @@ Callable, Collection, Coroutine, + Iterable, Mapping, Sequence, ) @@ -1599,6 +1600,18 @@ async def health(_request: Request) -> JSONResponse: return JSONResponse(content=health_status, status_code=status_code) +def _decode_asgi_headers(headers: Iterable[tuple[bytes, bytes]]) -> dict[str, str]: + """Decode raw ASGI scope header pairs into a str-keyed dict. + + Args: + headers: Raw (name, value) byte pairs from the ASGI scope. + + Returns: + A dict mapping decoded header names to decoded values. + """ + return {k.decode("utf-8"): v.decode("utf-8") for (k, v) in headers} + + class EventNamespace(AsyncNamespace): """The event namespace.""" @@ -1770,14 +1783,11 @@ async def on_event(self, sid: str, data: Any): raise RuntimeError(msg) # Get the client headers. - headers = { - k.decode("utf-8"): v.decode("utf-8") - for (k, v) in environ["asgi.scope"]["headers"] - } + headers = _decode_asgi_headers(environ["asgi.scope"]["headers"]) # Get the client IP try: - client_ip = environ["asgi.scope"]["client"][0] + client_ip: str = environ["asgi.scope"]["client"][0] headers["asgi-scope-client"] = client_ip except (KeyError, IndexError): client_ip = environ.get("REMOTE_ADDR", "0.0.0.0") diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py index b7b50c15c67..37c42792c4c 100644 --- a/tests/benchmarks/support/apps.py +++ b/tests/benchmarks/support/apps.py @@ -23,7 +23,9 @@ def lifecycle_app_source( for index in range(pages) ) return f"""import reflex as rx +from starlette.applications import Starlette from starlette.responses import JSONResponse +from starlette.routing import Route RELOAD_VERSION = {reload_version} @@ -46,10 +48,11 @@ def index(label: str = "index"): async def reload_version(_request): return JSONResponse({{"version": RELOAD_VERSION}}) -app = rx.App() +app = rx.App( + api_transformer=Starlette(routes=[Route("/reload-version", reload_version)]), +) app.add_page(index, route="/") {page_registrations} -app._api.add_route("/reload-version", reload_version) """ diff --git a/tests/benchmarks/support/baseline_server.py b/tests/benchmarks/support/baseline_server.py index 554d04a9cd3..dc21ee14f25 100644 --- a/tests/benchmarks/support/baseline_server.py +++ b/tests/benchmarks/support/baseline_server.py @@ -76,9 +76,17 @@ def __init__(self, response: dict[str, Any]): log_level="warning", ) ) - self._thread = threading.Thread(target=self._server.run, daemon=True) + self._thread = threading.Thread(target=self._run_server, daemon=True) + self._exception: BaseException | None = None self.url = "" + def _run_server(self) -> None: + """Run the uvicorn server, capturing a crash for the startup path.""" + try: + self._server.run() + except BaseException as exception: # surfaced by __enter__ + self._exception = exception + def __enter__(self) -> BaselineSocketServer: """Start the server and resolve its ephemeral URL. @@ -86,11 +94,12 @@ def __enter__(self) -> BaselineSocketServer: The running server. Raises: - TimeoutError: If the server does not bind within ten seconds. + TimeoutError: If the server crashes or does not bind within ten + seconds; a crash is chained and quoted in the message. """ self._thread.start() deadline = time.monotonic() + 10 - while time.monotonic() < deadline: + while time.monotonic() < deadline and self._thread.is_alive(): if ( self._server.started and (servers := self._server.servers) @@ -101,8 +110,13 @@ def __enter__(self) -> BaselineSocketServer: return self time.sleep(0.01) self._server.should_exit = True - msg = "Baseline socket server did not start within 10 seconds." - raise TimeoutError(msg) + self._thread.join(timeout=1) + msg = ( + f"Baseline socket server crashed during startup: {self._exception!r}" + if self._exception is not None + else "Baseline socket server did not start within 10 seconds." + ) + raise TimeoutError(msg) from self._exception def __exit__(self, *exc_info) -> None: """Stop the server and join its thread. diff --git a/tests/benchmarks/support/diagnostics.py b/tests/benchmarks/support/diagnostics.py index 8348ff28243..7bfe1bba034 100644 --- a/tests/benchmarks/support/diagnostics.py +++ b/tests/benchmarks/support/diagnostics.py @@ -31,11 +31,9 @@ def capture_async_diagnostics( "name": task.get_name(), "done": task.done(), "cancelled": task.cancelled(), - "stack": [ - line - for frame in task.get_stack() - for line in traceback.format_stack(frame) - ], + "stack": traceback.StackSummary.extract( + (frame, frame.f_lineno) for frame in task.get_stack() + ).format(), }) frames = sys._current_frames() # pyright: ignore [reportPrivateUsage] threads = [] diff --git a/tests/benchmarks/support/pipeline_trace.py b/tests/benchmarks/support/pipeline_trace.py index 2cf31ffaa26..2c78df92e47 100644 --- a/tests/benchmarks/support/pipeline_trace.py +++ b/tests/benchmarks/support/pipeline_trace.py @@ -70,23 +70,38 @@ def durations_ms(self, stages: Sequence[tuple[str, str]]) -> dict[str, list[floa def chrome_trace(self) -> dict[str, Any]: """Convert observations to Chrome instant trace events. + Chrome trace format requires integer pid/tid, so each token maps to a + stable small tid, named via ``thread_name`` metadata events. + Returns: A Chrome trace JSON mapping. """ - return { - "traceEvents": [ - { - "name": event.stage, - "cat": "reflex.event_pipeline", - "ph": "i", - "s": "t", - "pid": 1, - "tid": event.token, - "ts": event.timestamp_ns / 1000, - } - for event in self.events - ] - } + tids: dict[str, int] = {} + for event in self.events: + tids.setdefault(event.token, len(tids) + 1) + thread_names: list[dict[str, Any]] = [ + { + "name": "thread_name", + "ph": "M", + "pid": 1, + "tid": tid, + "args": {"name": token}, + } + for token, tid in tids.items() + ] + instants: list[dict[str, Any]] = [ + { + "name": event.stage, + "cat": "reflex.event_pipeline", + "ph": "i", + "s": "t", + "pid": 1, + "tid": tids[event.token], + "ts": event.timestamp_ns / 1000, + } + for event in self.events + ] + return {"traceEvents": thread_names + instants} def write_chrome_trace(self, path: str | Path) -> Path: """Write Chrome trace JSON. diff --git a/tests/benchmarks/support/redis.py b/tests/benchmarks/support/redis.py index 63fb855533f..cd4dbf7fa69 100644 --- a/tests/benchmarks/support/redis.py +++ b/tests/benchmarks/support/redis.py @@ -5,6 +5,7 @@ import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from urllib.parse import urlparse from redis.asyncio import Redis @@ -17,28 +18,56 @@ def performance_redis_url() -> str: """Return the isolated Redis database used by performance tests. Returns: - Redis connection URL. + Redis connection URL naming an explicit database index. + + Raises: + ValueError: If the configured URL has no explicit database index, which + would silently resolve to the shared default database 0. """ - return os.environ.get(REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + url = os.environ.get(REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + parsed = urlparse(url) + if not parsed.path.strip("/").isdigit() and "db=" not in parsed.query: + msg = ( + f"{REDIS_URL_ENV}={url!r} must name an explicit database index " + "(e.g. redis://host:6379/15); performance tests flush their " + "database and must never target the shared default database." + ) + raise ValueError(msg) + return url @asynccontextmanager async def real_redis_state_manager() -> AsyncIterator[StateManagerRedis]: """Create an isolated state manager backed by a reachable Redis server. + The configured database must be empty and dedicated to performance tests; + it is flushed on exit. + Yields: A state manager whose database is empty at entry and exit. Raises: ConnectionError: If the configured Redis server is unreachable. + RuntimeError: If the configured database already contains keys. """ - redis = Redis.from_url(performance_redis_url()) - await redis.ping() # pyright: ignore [reportGeneralTypeIssues] - await redis.flushdb() - manager = StateManagerRedis(redis=redis) + url = performance_redis_url() + redis = Redis.from_url(url) try: - yield manager + await redis.ping() # pyright: ignore [reportGeneralTypeIssues] + if await redis.dbsize(): + msg = ( + f"Redis database at {url!r} is not empty; performance tests " + "flush it on exit, so point " + f"{REDIS_URL_ENV} at an empty, dedicated database." + ) + raise RuntimeError(msg) + manager = StateManagerRedis(redis=redis) + try: + yield manager + finally: + try: + await redis.flushdb() + finally: + await manager.close() finally: - await manager.close() - await redis.flushdb() await redis.aclose() diff --git a/tests/benchmarks/test_event_loop.py b/tests/benchmarks/test_event_loop.py index 86b37a8e5c6..5e305eab1cb 100644 --- a/tests/benchmarks/test_event_loop.py +++ b/tests/benchmarks/test_event_loop.py @@ -1,19 +1,24 @@ """Deterministic benchmarks for event-processor scheduling and lifecycle paths.""" import asyncio -from collections.abc import Mapping +import contextvars +from collections.abc import Coroutine, Mapping from types import SimpleNamespace from typing import Any, cast from pytest_codspeed import BenchmarkFixture from reflex_base.event.context import EventContext -from reflex_base.event.processor.event_processor import EventProcessor +from reflex_base.event.processor import EventFuture, EventProcessor from reflex_base.registry import RegistrationContext from reflex.app import EventNamespace from reflex.event import Event, EventHandler from reflex.state import StateUpdate +# Drain budget far above any instrumented runtime so measured shutdowns never +# flip from graceful drain to cancellation under benchmark slowdown. +_DRAIN_BUDGET_SECONDS = 60.0 + async def _noop_handler() -> None: """Complete without application work.""" @@ -55,30 +60,90 @@ def _register_handlers() -> RegistrationContext: return context -async def _process(handler: EventHandler, token: str = "token") -> None: - """Process one event through a configured processor. +def _make_processor() -> EventProcessor: + """Construct and configure a processor outside any measured region. + + Returns: + A configured event processor with a generous drain budget. + """ + return EventProcessor(graceful_shutdown_timeout=_DRAIN_BUDGET_SECONDS).configure() + + +def _run_in_context( + loop: asyncio.AbstractEventLoop, + ctx: contextvars.Context, + coro: Coroutine[Any, Any, Any], +) -> Any: + """Run a coroutine on the loop inside a fixed contextvars context. + + Lets ``EventProcessor.start`` and ``stop`` run in separate + ``run_until_complete`` calls while sharing the context that owns the + attached ``EventContext`` token. Args: - handler: Registered handler. - token: Client token. + loop: The event loop to run on. + ctx: The shared context for the wrapping task. + coro: The coroutine to run. + + Returns: + The coroutine's result. """ - processor = EventProcessor(graceful_shutdown_timeout=2) - processor.configure() - async with processor: - future = await processor.enqueue(token, Event.from_event_type(handler())[0]) - await future.wait_all() + return loop.run_until_complete(loop.create_task(coro, context=ctx)) def test_event_queue_dispatch(benchmark: BenchmarkFixture): - """Benchmark enqueue, dispatch, task completion, and clean shutdown. + """Benchmark enqueueing one no-op event and awaiting its completion. + + Processor construction, configuration, startup, and shutdown all happen + in per-round setup and teardown, so only queueing and dispatch are + measured. Args: benchmark: The CodSpeed benchmark fixture. """ loop = asyncio.new_event_loop() registry = _register_handlers() + ctx = contextvars.copy_context() + + async def dispatch(processor: EventProcessor) -> None: + """Enqueue one no-op event and await its completion. + + Args: + processor: The started processor. + """ + future = await processor.enqueue( + "token", Event.from_event_type(NOOP_EVENT())[0] + ) + await future.wait_all() + + def setup(): + """Construct and start a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the started processor. + """ + processor = _make_processor() + _run_in_context(loop, ctx, processor.start()) + return ((processor,), {}) + + def run(processor: EventProcessor) -> None: + """Dispatch one event through the started processor. + + Args: + processor: The started processor. + """ + loop.run_until_complete(dispatch(processor)) + + def teardown(processor: EventProcessor) -> None: + """Stop the measured round's processor. + + Args: + processor: The started processor. + """ + _run_in_context(loop, ctx, processor.stop()) + try: - benchmark(lambda: loop.run_until_complete(_process(NOOP_EVENT))) + benchmark.pedantic(run, setup=setup, teardown=teardown) finally: registry.__exit__(None, None, None) loop.close() @@ -87,13 +152,56 @@ def test_event_queue_dispatch(benchmark: BenchmarkFixture): def test_event_future_tree(benchmark: BenchmarkFixture): """Benchmark a parent event that enqueues and waits for one child. + Processor lifecycle happens in per-round setup and teardown, so only + the parent dispatch, chained child enqueue, and future-tree completion + are measured. + Args: benchmark: The CodSpeed benchmark fixture. """ loop = asyncio.new_event_loop() registry = _register_handlers() + ctx = contextvars.copy_context() + + async def dispatch(processor: EventProcessor) -> None: + """Enqueue one chaining event and await the whole future tree. + + Args: + processor: The started processor. + """ + future = await processor.enqueue( + "token", Event.from_event_type(CHAIN_EVENT())[0] + ) + await future.wait_all() + + def setup(): + """Construct and start a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the started processor. + """ + processor = _make_processor() + _run_in_context(loop, ctx, processor.start()) + return ((processor,), {}) + + def run(processor: EventProcessor) -> None: + """Dispatch the parent event through the started processor. + + Args: + processor: The started processor. + """ + loop.run_until_complete(dispatch(processor)) + + def teardown(processor: EventProcessor) -> None: + """Stop the measured round's processor. + + Args: + processor: The started processor. + """ + _run_in_context(loop, ctx, processor.stop()) + try: - benchmark(lambda: loop.run_until_complete(_process(CHAIN_EVENT))) + benchmark.pedantic(run, setup=setup, teardown=teardown) finally: registry.__exit__(None, None, None) loop.close() @@ -102,22 +210,58 @@ def test_event_future_tree(benchmark: BenchmarkFixture): def test_event_shutdown_drain(benchmark: BenchmarkFixture): """Benchmark graceful draining of ten queued no-op events. + The measured region intentionally covers processor startup, the + ten-event enqueue burst, and the graceful drain performed by shutdown; + construction and configuration happen in per-round setup. Events use + independent tokens because same-token bursts serialize and are partially + cancelled rather than drained at shutdown. + Args: benchmark: The CodSpeed benchmark fixture. """ loop = asyncio.new_event_loop() registry = _register_handlers() + drained: list[EventFuture] = [] + + async def drain(processor: EventProcessor) -> list[EventFuture]: + """Enqueue a burst and let processor shutdown drain it. + + Args: + processor: The configured processor. - async def drain() -> None: - """Enqueue a burst and let processor shutdown drain it.""" - processor = EventProcessor(graceful_shutdown_timeout=2) - processor.configure() + Returns: + The futures for the enqueued burst. + """ async with processor: - for _ in range(10): - await processor.enqueue("token", Event.from_event_type(NOOP_EVENT())[0]) + return [ + await processor.enqueue( + f"token-{index}", Event.from_event_type(NOOP_EVENT())[0] + ) + for index in range(10) + ] + + def setup(): + """Construct a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((_make_processor(),), {}) + + def run(processor: EventProcessor) -> None: + """Run the burst-and-drain round, recording its futures. + + Args: + processor: The configured processor. + """ + drained[:] = loop.run_until_complete(drain(processor)) try: - benchmark(lambda: loop.run_until_complete(drain())) + benchmark.pedantic(run, setup=setup) + assert len(drained) == 10 + assert all(future.done() and not future.cancelled() for future in drained), ( + "shutdown cancelled queued events instead of draining them" + ) finally: registry.__exit__(None, None, None) loop.close() @@ -126,20 +270,25 @@ async def drain() -> None: def test_event_shutdown_cancel(benchmark: BenchmarkFixture): """Benchmark forced cancellation during zero-grace shutdown. + The measured region intentionally covers startup, enqueueing one slow + event, and the zero-budget shutdown that cancels it; construction and + configuration happen in per-round setup. + Args: benchmark: The CodSpeed benchmark fixture. """ loop = asyncio.new_event_loop() registry = _register_handlers() - async def cancel() -> bool: + async def cancel(processor: EventProcessor) -> bool: """Start a slow task and stop without a drain period. + Args: + processor: The configured processor. + Returns: Whether shutdown cancelled its future. """ - processor = EventProcessor(graceful_shutdown_timeout=0) - processor.configure() async with processor: future = await processor.enqueue( "token", Event.from_event_type(SLOW_EVENT())[0] @@ -147,8 +296,27 @@ async def cancel() -> bool: await asyncio.sleep(0) return future.cancelled() + def setup(): + """Construct a zero-grace processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((EventProcessor(graceful_shutdown_timeout=0).configure(),), {}) + + def run(processor: EventProcessor) -> bool: + """Run the cancel round. + + Args: + processor: The configured processor. + + Returns: + Whether shutdown cancelled the slow event's future. + """ + return loop.run_until_complete(cancel(processor)) + try: - assert benchmark(lambda: loop.run_until_complete(cancel())) + assert benchmark.pedantic(run, setup=setup) finally: registry.__exit__(None, None, None) loop.close() @@ -157,20 +325,25 @@ async def cancel() -> bool: def test_yield_intermediate_update(benchmark: BenchmarkFixture): """Benchmark ordered streaming of rapid intermediate deltas. + The measured region intentionally covers startup, streaming both deltas, + and clean shutdown; construction and configuration happen in per-round + setup. + Args: benchmark: The CodSpeed benchmark fixture. """ loop = asyncio.new_event_loop() registry = _register_handlers() - async def collect() -> list[Mapping[str, Any]]: + async def collect(processor: EventProcessor) -> list[Mapping[str, Any]]: """Stream all deltas from the yielding event. + Args: + processor: The configured processor. + Returns: Ordered emitted deltas. """ - processor = EventProcessor(graceful_shutdown_timeout=2) - processor.configure() async with processor: return [ delta @@ -179,8 +352,27 @@ async def collect() -> list[Mapping[str, Any]]: ) ] + def setup(): + """Construct a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((_make_processor(),), {}) + + def run(processor: EventProcessor) -> list[Mapping[str, Any]]: + """Run the streaming round. + + Args: + processor: The configured processor. + + Returns: + Ordered emitted deltas. + """ + return loop.run_until_complete(collect(processor)) + try: - deltas = benchmark(lambda: loop.run_until_complete(collect())) + deltas = benchmark.pedantic(run, setup=setup) assert deltas == [ {"state": {"value": 1}}, {"state": {"value": 2}}, @@ -191,7 +383,11 @@ async def collect() -> list[Mapping[str, Any]]: def test_emit_update(benchmark: BenchmarkFixture): - """Benchmark the real emit-update await and socket-flush scheduling path. + """Benchmark emit_update's socket lookup and task-based flush scheduling. + + The Socket.IO ``emit`` call itself is mocked to a recording no-op, so + only token-to-socket routing and the emit task scheduling are measured, + not any real socket write. Args: benchmark: The CodSpeed benchmark fixture. diff --git a/tests/benchmarks/test_event_processing.py b/tests/benchmarks/test_event_processing.py index fcdfaaa5474..c9993c0a37b 100644 --- a/tests/benchmarks/test_event_processing.py +++ b/tests/benchmarks/test_event_processing.py @@ -12,7 +12,6 @@ from unittest import mock import pytest -import pytest_asyncio from pytest_codspeed import BenchmarkFixture from reflex_base.event import Event from reflex_base.event.context import EventContext @@ -25,8 +24,8 @@ from .fixtures import BenchmarkState -@pytest_asyncio.fixture -async def event_processing_harness(): +@pytest.fixture +def event_processing_harness(): """Set up the full event processing pipeline for benchmarking. Creates a ``BaseStateEventProcessor`` wired to a real @@ -34,8 +33,10 @@ async def event_processing_harness(): enqueued directly and deltas are collected via the emit callback. Yields: - An async event runner and a helper that purges token state between - explicitly cold benchmark rounds. + An async event runner, a helper that purges token state between + explicitly cold benchmark rounds, and the state manager's async + ``close``, which each test must run on its own event loop before + closing it. """ emitted_deltas: list[tuple[str, Mapping[str, Mapping[str, Any]]]] = [] @@ -104,9 +105,7 @@ def purge_tokens(tokens: Sequence[str]) -> None: BaseStateToken(ident=token, cls=BenchmarkState) ) - yield run_events, purge_tokens - - await state_manager.close() + yield run_events, purge_tokens, state_manager.close def test_process_event( @@ -119,17 +118,22 @@ def test_process_event( the existing state (warm path). Only event processing is timed. Args: - event_processing_harness: The event runner and token purge helpers. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, _ = event_processing_harness - loop = asyncio.get_event_loop() + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + + try: + # Each event handler (increment) does a single state mutation with + # no yields, so we expect 1 delta per event = 3 total. + @benchmark + def _(): + loop.run_until_complete(run_events(["benchmark-token"] * 3)) - # Each event handler (increment) does a single state mutation with - # no yields, so we expect 1 delta per event = 3 total. - @benchmark - def _(): - loop.run_until_complete(run_events(["benchmark-token"] * 3)) + finally: + loop.run_until_complete(shutdown()) + loop.close() def test_process_event_cold( @@ -139,11 +143,11 @@ def test_process_event_cold( """Benchmark one event whose token has no existing state. Args: - event_processing_harness: The event runner and token purge helpers. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, purge_tokens = event_processing_harness - loop = asyncio.get_event_loop() + run_events, purge_tokens, shutdown = event_processing_harness + loop = asyncio.new_event_loop() iteration = 0 def setup(): @@ -160,7 +164,11 @@ def teardown(token: str) -> None: """Discard the measured token so cold rounds do not retain state.""" purge_tokens([token]) - benchmark.pedantic(run_cold, setup=setup, teardown=teardown) + try: + benchmark.pedantic(run_cold, setup=setup, teardown=teardown) + finally: + loop.run_until_complete(shutdown()) + loop.close() def test_process_event_warm( @@ -170,18 +178,23 @@ def test_process_event_warm( """Benchmark one event for a token whose state is already initialized. Args: - event_processing_harness: The event runner and token purge helpers. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, _ = event_processing_harness - loop = asyncio.get_event_loop() - token = "warm-token" - loop.run_until_complete(run_events([token])) - - @benchmark - def _(): + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + token = "warm-token" loop.run_until_complete(run_events([token])) + @benchmark + def _(): + loop.run_until_complete(run_events([token])) + + finally: + loop.run_until_complete(shutdown()) + loop.close() + @pytest.mark.parametrize("num_events", [1, 10, 100]) def test_process_event_burst_same_token( @@ -193,17 +206,22 @@ def test_process_event_burst_same_token( Args: num_events: Number of events in the burst. - event_processing_harness: The event runner and token purge helpers. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, _ = event_processing_harness - loop = asyncio.get_event_loop() - tokens = ["burst-token"] * num_events - loop.run_until_complete(run_events([tokens[0]])) + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + tokens = ["burst-token"] * num_events + loop.run_until_complete(run_events([tokens[0]])) - @benchmark - def _(): - loop.run_until_complete(run_events(tokens)) + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) + + finally: + loop.run_until_complete(shutdown()) + loop.close() @pytest.mark.parametrize("num_events", [1, 10, 100]) @@ -216,14 +234,19 @@ def test_process_event_burst_independent_tokens( Args: num_events: Number of independent tokens and events. - event_processing_harness: The event runner and token purge helpers. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, _ = event_processing_harness - loop = asyncio.get_event_loop() - tokens = [f"independent-token-{index}" for index in range(num_events)] - loop.run_until_complete(run_events(tokens)) - - @benchmark - def _(): + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + tokens = [f"independent-token-{index}" for index in range(num_events)] loop.run_until_complete(run_events(tokens)) + + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) + + finally: + loop.run_until_complete(shutdown()) + loop.close() diff --git a/tests/benchmarks/test_routes.py b/tests/benchmarks/test_routes.py index 9585ebb6b74..25fefc67248 100644 --- a/tests/benchmarks/test_routes.py +++ b/tests/benchmarks/test_routes.py @@ -1,5 +1,7 @@ """Benchmarks for route construction and hot-path matching.""" +import re + import pytest from pytest_codspeed import BenchmarkFixture from reflex_base import constants @@ -48,12 +50,20 @@ def match_paths() -> tuple[str | None, ...]: def test_route_table_construction(count: int, benchmark: BenchmarkFixture): """Benchmark cold regex construction and route ordering. + ``re.purge()`` runs in per-round setup so every measured round compiles + the route patterns instead of hitting the ``re`` module's pattern cache. + Args: count: Number of routes. benchmark: The CodSpeed benchmark fixture. """ routes = _routes(count) - router = benchmark(lambda: get_router(routes)) + + def purge_regex_cache(): + """Clear the ``re`` module cache so each round compiles cold.""" + re.purge() + + router = benchmark.pedantic(lambda: get_router(routes), setup=purge_regex_cache) assert router("/posts/reflex") == "posts/[slug]" diff --git a/tests/benchmarks/test_socket_headers.py b/tests/benchmarks/test_socket_headers.py index fb64309be02..3c5b56d6fe4 100644 --- a/tests/benchmarks/test_socket_headers.py +++ b/tests/benchmarks/test_socket_headers.py @@ -3,10 +3,12 @@ import pytest from pytest_codspeed import BenchmarkFixture +from reflex.app import _decode_asgi_headers + @pytest.mark.parametrize("count", [0, 4, 16, 64]) def test_decode_event_headers(count: int, benchmark: BenchmarkFixture): - """Benchmark decoding the ASGI header representation used by socket events. + """Benchmark the header decoding used by ``EventNamespace.on_event``. Args: count: Number of headers. @@ -16,7 +18,5 @@ def test_decode_event_headers(count: int, benchmark: BenchmarkFixture): (f"x-performance-{index}".encode(), f"value-{index}".encode()) for index in range(count) ] - decoded = benchmark( - lambda: {key.decode("utf-8"): value.decode("utf-8") for key, value in headers} - ) + decoded = benchmark(lambda: _decode_asgi_headers(headers)) assert len(decoded) == count diff --git a/tests/performance/test_browser.py b/tests/performance/test_browser.py index 18f5eea6000..fbd8055caef 100644 --- a/tests/performance/test_browser.py +++ b/tests/performance/test_browser.py @@ -3,20 +3,32 @@ from __future__ import annotations import gzip +import os import time from pathlib import Path import pytest from playwright.sync_api import Page, expect +from reflex import constants from reflex.testing import AppHarness from tests.benchmarks.support import BenchmarkResult, PerformanceReport -MAX_JAVASCRIPT_GZIP_BYTES = 14_000_000 +# Gzip budget for the shipped client bundle. A fresh production export of the +# load app measured 238,635 gzip bytes (2026-07-13); the default budget is +# roughly double so only a real bundle regression trips it. CI can tune the +# budget without a code change via the environment variable. +MAX_JAVASCRIPT_GZIP_BYTES = int( + os.environ.get("REFLEX_PERFORMANCE_JS_GZIP_BUDGET", 500_000) +) def _bundle_metrics(root: Path) -> dict[str, int]: - """Calculate bundle module and compressed-size metrics. + """Calculate shipped client-bundle module and compressed-size metrics. + + Only the exported client build (``.web/build/client``) is measured; the + rest of ``.web`` is the build toolchain (``node_modules``, server bundles) + that never reaches a browser. Args: root: Reflex application root. @@ -24,9 +36,10 @@ def _bundle_metrics(root: Path) -> dict[str, int]: Returns: JavaScript module, raw byte, and gzip byte counts. """ + client = root / constants.Dirs.WEB / constants.Dirs.STATIC files = [ path - for path in (root / ".web").rglob("*") + for path in client.rglob("*") if path.is_file() and path.suffix in {".js", ".mjs"} ] return { @@ -140,7 +153,11 @@ def test_browser_report( report.write(performance_output / "browser.json") assert metrics["javascript_modules"] > 0 - assert metrics["javascript_gzip_bytes"] <= MAX_JAVASCRIPT_GZIP_BYTES + assert metrics["javascript_gzip_bytes"] <= MAX_JAVASCRIPT_GZIP_BYTES, ( + f"shipped client bundle is {metrics['javascript_gzip_bytes']} gzip bytes, " + f"over the {MAX_JAVASCRIPT_GZIP_BYTES} budget " + "(override via REFLEX_PERFORMANCE_JS_GZIP_BUDGET)" + ) assert heap_after > 0 diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py index 006207453e2..e065c522297 100644 --- a/tests/performance/test_compiler_lifecycle.py +++ b/tests/performance/test_compiler_lifecycle.py @@ -69,6 +69,10 @@ def _generated_metrics(root: Path) -> dict[str, int]: def _free_port() -> int: """Reserve and release an available local TCP port. + The port must be released before the backend can bind it, so another + process may steal it in between; ``_start_dev_backend_with_retry`` + handles that race by retrying on a fresh port. + Returns: Available port number for the development backend. """ @@ -159,18 +163,41 @@ def _wait_for_reload_version( raise TimeoutError(msg) -def _stop_dev_backend(process: subprocess.Popen[str]) -> None: +def _collect_children( + process: subprocess.Popen[str], + known: dict[int, psutil.Process], +) -> None: + """Snapshot the backend's current child processes for later cleanup. + + Reload workers reparent to init if the backend dies first, so children + must be collected while the parent is still alive to guarantee teardown. + + Args: + process: Development backend process. + known: Accumulated child processes keyed by pid. + """ + with contextlib.suppress(psutil.NoSuchProcess): + for child in psutil.Process(process.pid).children(recursive=True): + known.setdefault(child.pid, child) + + +def _stop_dev_backend( + process: subprocess.Popen[str], + known_children: dict[int, psutil.Process], +) -> None: """Terminate a development backend and all reload workers. Args: process: Development backend process. + known_children: Children collected while the parent was alive, swept + even when the parent has already exited. Stale entries whose pid + was recycled are skipped by psutil's create-time check. """ - try: - parent = psutil.Process(process.pid) - processes = [parent, *parent.children(recursive=True)] - except psutil.NoSuchProcess: - return - for running in reversed(processes): + _collect_children(process, known_children) + processes = list(known_children.values()) + with contextlib.suppress(psutil.NoSuchProcess): + processes.append(psutil.Process(process.pid)) + for running in processes: with contextlib.suppress(psutil.NoSuchProcess): running.terminate() _, alive = psutil.wait_procs(processes, timeout=10) @@ -179,6 +206,56 @@ def _stop_dev_backend(process: subprocess.Popen[str]) -> None: running.kill() +def _start_dev_backend_with_retry( + executable: str, + app_root: Path, + known_children: dict[int, psutil.Process], + attempts: int = 3, +) -> tuple[subprocess.Popen[str], TextIO, Path, int]: + """Start the dev backend, retrying on a lost port-reservation race. + + ``_free_port`` must release its probe socket before the backend can bind, + so another process can grab the port in between; that surfaces as the + backend exiting with "address already in use" and is retried on a fresh + port. + + Args: + executable: Python executable running Reflex. + app_root: Generated application root. + known_children: Accumulated backend children for teardown. + attempts: Maximum start attempts. + + Returns: + Serving backend process, open log stream, log path, and bound port. + + Raises: + RuntimeError: If the backend exits for a reason other than a port + collision, or every attempt loses the port race. + TimeoutError: If a started backend never serves the initial version. + """ + error: RuntimeError | None = None + for _ in range(attempts): + port = _free_port() + process, log_stream, log_path = _start_dev_backend(executable, app_root, port) + try: + _wait_for_reload_version(process, log_path, port, expected=0) + except (RuntimeError, TimeoutError) as exc: + _stop_dev_backend(process, known_children) + log_stream.close() + # Only a lost port race is retried, matching both reflex's + # pre-flight "port ... is already in use" and the server's + # bind-time "address already in use". + if not isinstance(exc, RuntimeError) or ( + "already in use" not in str(exc).lower() + ): + raise + error = exc + else: + return process, log_stream, log_path, port + assert error is not None + raise error + + @pytest.mark.performance def test_compiler_lifecycle_report( tmp_path: Path, @@ -284,19 +361,12 @@ def test_compiler_lifecycle_report( report.add(BenchmarkResult("compile_state_edit", {}, [state_edit_ms], {})) reload_observations = [] - backend_port = _free_port() - backend, backend_log, backend_log_path = _start_dev_backend( - executable, - app_root, - backend_port, + known_children: dict[int, psutil.Process] = {} + backend, backend_log, backend_log_path, backend_port = ( + _start_dev_backend_with_retry(executable, app_root, known_children) ) try: - _wait_for_reload_version( - backend, - backend_log_path, - backend_port, - expected=0, - ) + _collect_children(backend, known_children) for cycle in range(1, reload_cycles + 1): cycle_source = source.replace( "RELOAD_VERSION = 0", @@ -311,8 +381,9 @@ def test_compiler_lifecycle_report( expected=cycle, ) reload_observations.append((time.perf_counter_ns() - started) / 1_000_000) + _collect_children(backend, known_children) finally: - _stop_dev_backend(backend) + _stop_dev_backend(backend, known_children) backend_log.close() report.add( BenchmarkResult( diff --git a/tests/performance/test_event_load.py b/tests/performance/test_event_load.py index 38b633674c1..6d10e042304 100644 --- a/tests/performance/test_event_load.py +++ b/tests/performance/test_event_load.py @@ -28,6 +28,47 @@ KNEE_P99_MARGIN_MS = 25.0 +def _client_timeout(clients: int) -> float: + """Scale the per-client connect/response timeout with concurrency. + + Args: + clients: Number of concurrent clients sharing the backend. + + Returns: + Timeout in seconds. + """ + return max(10.0, clients * 0.5) + + +def _wait_for_token_cleanup( + harness: AppHarness, + token_prefixes: tuple[str, ...], + timeout: float = 10.0, +) -> None: + """Poll until disconnect cleanup has removed the given client tokens. + + Token disconnect cleanup is fire-and-forget on the backend; waiting on the + in-process token map keeps later connections from hitting duplicate-tab + handling without a fixed settling sleep. + + Args: + harness: Running in-process app harness. + token_prefixes: Prefixes of tokens that must disappear. + timeout: Maximum wait in seconds. + """ + app = harness.app_instance + assert app is not None + assert app.event_namespace is not None + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not any( + token.startswith(token_prefixes) + for token in app.event_namespace.token_to_sid + ): + return + time.sleep(0.05) + + def _increment_payload() -> dict[str, Any]: """Build the increment event payload for the running load app. @@ -95,6 +136,7 @@ def test_event_load_report( f"load-token-{clients}-{index}", payload, events_per_client, + timeout=_client_timeout(clients), executor=executor, ), ) @@ -254,19 +296,27 @@ def test_reconnect_storm_report( payload = _increment_payload() tokens = [f"storm-token-{index}" for index in range(clients)] + timeout = _client_timeout(clients) + # Prime each token so the storm exercises state restore, not creation. prime_results = asyncio.run( run_clients( clients, lambda index, executor: run_socket_client( - backend_url, tokens[index], payload, 1, executor=executor + backend_url, + tokens[index], + payload, + 1, + timeout=timeout, + executor=executor, ), ) ) - assert not any(result.errors for result in prime_results) - # Token disconnect cleanup is fire-and-forget; let it finish so the storm - # exercises reconnection instead of duplicate-tab handling. - time.sleep(1) + prime_errors = [error for result in prime_results for error in result.errors] + assert not prime_errors, f"priming clients failed: {prime_errors}" + # Wait for fire-and-forget disconnect cleanup so the storm exercises + # reconnection instead of duplicate-tab handling. + _wait_for_token_cleanup(performance_load_app, ("storm-token-",)) rss_before = current_process_metrics()["rss_bytes"] started = time.perf_counter_ns() @@ -274,13 +324,39 @@ def test_reconnect_storm_report( run_clients( clients, lambda index, executor: run_reconnect_client( - backend_url, tokens[index], payload, executor=executor + backend_url, + tokens[index], + payload, + timeout=timeout, + executor=executor, ), ) ) storm_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 rss_after = current_process_metrics()["rss_bytes"] + # A transient connect failure under the simultaneous storm is retried once + # so only persistent failures trip the assertion; retried clients keep + # their retry measurements but do not affect storm timing. + failed_indexes = [ + index for index, result in enumerate(storm_results) if result.errors + ] + if failed_indexes: + retry_results = asyncio.run( + run_clients( + len(failed_indexes), + lambda index, executor: run_reconnect_client( + backend_url, + tokens[failed_indexes[index]], + payload, + timeout=timeout, + executor=executor, + ), + ) + ) + for index, result in zip(failed_indexes, retry_results, strict=True): + storm_results[index] = result + errors = [error for result in storm_results for error in result.errors] connects = [result.connect_ms for result in storm_results] first_responses = [result.first_response_ms for result in storm_results] @@ -301,9 +377,12 @@ def test_reconnect_storm_report( "reconnects_per_second": clients / storm_seconds, "rss_growth_bytes": rss_after - rss_before, "errors": len(errors), + "retried_clients": len(failed_indexes), }, measurement_iterations=clients, ) ) report.write(performance_output / "reconnect-storm.json") - assert not errors, errors + assert not errors, ( + f"{len(errors)} reconnect client error(s) persisted after retry: {errors}" + ) diff --git a/tests/performance/test_event_loop.py b/tests/performance/test_event_loop.py index f999a55b96e..1e93dae43ec 100644 --- a/tests/performance/test_event_loop.py +++ b/tests/performance/test_event_loop.py @@ -41,6 +41,34 @@ _ORDER_LOG: dict[str, list[int]] = defaultdict(list) _ACTIVE_STATE_TRACE: PipelineTrace | None = None +# Deliberate synchronous block injected by the blocking scenario. Detection is +# asserted at >=2ms of measured loop lag, so 10ms leaves a 5x margin even on a +# loaded CI runner. +_BLOCKING_SLEEP_SECONDS = 0.01 + +# Generous stall guard for event-future waits: a stuck future is the exact +# regression class this suite exists to catch and must fail fast, not hang CI. +_EVENT_WAIT_BASE_TIMEOUT_SECONDS = 60.0 +_EVENT_WAIT_PER_EVENT_SECONDS = 0.1 + + +async def _wait_events_bounded(awaitable: Any, *, events: int, scenario: str) -> None: + """Await event-future completion with a stall-detecting timeout. + + Args: + awaitable: The future wait (or gather of waits) to bound. + events: Number of in-flight events, scaling the allowance. + scenario: Scenario name for the failure message. + """ + timeout = _EVENT_WAIT_BASE_TIMEOUT_SECONDS + events * _EVENT_WAIT_PER_EVENT_SECONDS + try: + await asyncio.wait_for(awaitable, timeout=timeout) + except asyncio.TimeoutError: + pytest.fail( + f"{scenario}: {events} event future(s) still pending after " + f"{timeout:.0f}s; the event pipeline is stuck" + ) + def _record_state_stage(stage: str) -> None: """Record a stage for the active state event. @@ -188,7 +216,7 @@ async def _blocking_handler(sequence: int) -> None: # noqa: RUF029 Args: sequence: Per-token event sequence. """ - time.sleep(0.005) # noqa: ASYNC251 - intentional benchmark scenario + time.sleep(_BLOCKING_SLEEP_SECONDS) # noqa: ASYNC251 - intentional benchmark scenario _ORDER_LOG[EventContext.get().token].append(sequence) @@ -368,7 +396,11 @@ def token_queue_depth() -> int: ) ) futures.append(future) - await asyncio.gather(*(future.wait_all() for future in futures)) + await _wait_events_bounded( + asyncio.gather(*(future.wait_all() for future in futures)), + events=len(futures), + scenario=handler.fn.__name__, + ) await asyncio.sleep(probe_interval * 2) for token, expected in expected_order.items(): @@ -444,7 +476,9 @@ async def emit_delta( StageEvent(future.txid, "received", received), StageEvent(future.txid, "enqueued", time.perf_counter_ns()), ]) - await future.wait_all() + await _wait_events_bounded( + future.wait_all(), events=1, scenario="state_pipeline" + ) finally: await manager.close() _ACTIVE_STATE_TRACE = None @@ -482,7 +516,9 @@ async def _run_failed_state_pipeline() -> None: async with processor: future = await processor.enqueue("failure-token", event) with contextlib.suppress(RuntimeError): - await future.wait_all() + await _wait_events_bounded( + future.wait_all(), events=1, scenario="failed_state_pipeline" + ) finally: await manager.close() _ACTIVE_STATE_TRACE = None @@ -594,6 +630,9 @@ async def test_event_loop_component_report( blocking = next( result for result in report.results if result.name == "blocking_handler" ) + # The probe must detect the deliberately induced block: each blocking event + # stalls the loop for _BLOCKING_SLEEP_SECONDS (10ms), so 2ms of measured + # lag is a 5x-margin detection bound, not a performance threshold. assert blocking.metrics["lag_max_ms"] >= 2 capture_async_diagnostics( performance_output / "event-loop-blocking-diagnostics.json", diff --git a/tests/performance/test_memory.py b/tests/performance/test_memory.py index b723bb661a7..636f8522abd 100644 --- a/tests/performance/test_memory.py +++ b/tests/performance/test_memory.py @@ -40,20 +40,32 @@ def _var_workload() -> None: assert ((left + right) * right - left)._js_expr +# Untraced warmup cycles run before the leak gate observes retention so +# lazily populated caches (compiler memos, Var expression caches) fill first. +WARMUP_CYCLES = 3 + + def _measure_allocations( workload: Callable[[], None], cycles: int, ) -> tuple[list[float], dict[str, float | int]]: """Measure elapsed time, peak allocation, and retained-memory trend. + Runs ``WARMUP_CYCLES`` untraced cycles first so one-time lazy allocations + settle, then traces ``cycles`` measured cycles. ``monotonic_growth`` is + only set when retention grows strictly on every measured cycle after + warmup — the signature of a per-invocation leak, which a warmed cache + cannot produce because it plateaus. + Args: workload: Workload invoked once per cycle. - cycles: Measurement cycles after one warmup. + cycles: Measurement cycles after warmup. Returns: Cycle latencies and allocation metrics. """ - workload() + for _ in range(WARMUP_CYCLES): + workload() gc.collect() tracemalloc.start() baseline, _ = tracemalloc.get_traced_memory() @@ -87,6 +99,11 @@ def _measure_allocations( def test_memory_report(performance_output: Path, performance_scale: str): """Profile allocation and retention across compiler and runtime workloads. + The leak gate fails only when a workload shows strictly monotonic retained + growth on every post-warmup cycle and accumulates more than 1 MB — lazily + populated caches are filled during the untraced warmup cycles, so + sustained growth here indicates a real per-invocation leak. + Args: performance_output: Artifact directory. performance_scale: Selected scenario scale. @@ -105,15 +122,17 @@ def test_memory_report(performance_output: Path, performance_scale: str): parameters={"cycles": cycles}, observations_ms=observations, metrics=metrics, - warmup_iterations=1, + warmup_iterations=WARMUP_CYCLES, measurement_iterations=cycles, ) ) report.write(performance_output / "memory.json") assert all(result.metrics["peak_traced_bytes"] >= 0 for result in report.results) - assert not any( - result.metrics["monotonic_growth"] - and result.metrics["retained_growth_bytes"] > 1_000_000 + leaks = [ + (result.name, result.metrics["retained_growth_bytes"]) for result in report.results - ) + if result.metrics["monotonic_growth"] + and result.metrics["retained_growth_bytes"] > 1_000_000 + ] + assert not leaks, f"sustained post-warmup retained-memory growth: {leaks}" diff --git a/tests/performance/test_session_memory.py b/tests/performance/test_session_memory.py index f052e5c7d22..02b6cb6cb46 100644 --- a/tests/performance/test_session_memory.py +++ b/tests/performance/test_session_memory.py @@ -61,6 +61,10 @@ def _held_sessions( Yields: While the sessions are connected and primed. + + Raises: + RuntimeError: If the holder had to be killed; a holder outliving its + batch would leak client memory into the next RSS measurement. """ process = subprocess.Popen( [ @@ -87,7 +91,16 @@ def _held_sessions( if process.stdin is not None: with contextlib.suppress(OSError): process.stdin.close() - process.wait(timeout=30) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + msg = ( + f"session holder {token_prefix!r} did not exit and was killed; " + "subsequent RSS measurements would have been invalid" + ) + raise RuntimeError(msg) from None def _settled_rss() -> int: diff --git a/tests/units/benchmarks/__init__.py b/tests/units/benchmarks/__init__.py new file mode 100644 index 00000000000..77167d4b4ed --- /dev/null +++ b/tests/units/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Benchmark support tests.""" diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py index 852797826b7..12e54ae9235 100644 --- a/tests/units/benchmarks/test_support.py +++ b/tests/units/benchmarks/test_support.py @@ -2,9 +2,12 @@ import asyncio import json +from types import SimpleNamespace import pytest +from tests.benchmarks.support.apps import lifecycle_app_source +from tests.benchmarks.support.diagnostics import capture_async_diagnostics from tests.benchmarks.support.pipeline_trace import PipelineTrace, StageEvent from tests.benchmarks.support.report import ( BenchmarkEnvironment, @@ -91,10 +94,35 @@ def test_pipeline_trace_durations_and_chrome_output(tmp_path): } path = trace.write_chrome_trace(tmp_path / "trace.json") payload = json.loads(path.read_text()) - assert [event["name"] for event in payload["traceEvents"]] == [ - "enqueued", - "dequeued", - ] + assert [ + event["name"] for event in payload["traceEvents"] if event["ph"] == "i" + ] == ["enqueued", "dequeued"] + + +def test_pipeline_trace_chrome_tids_are_integers(): + """Chrome traces emit integer pids/tids and name tokens via metadata.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token-a", "enqueued", 1_000_000), + StageEvent("token-b", "enqueued", 2_000_000), + StageEvent("token-a", "dequeued", 3_000_000), + ]) + + payload = trace.chrome_trace() + assert all( + isinstance(event["pid"], int) and isinstance(event["tid"], int) + for event in payload["traceEvents"] + ) + instants = [event for event in payload["traceEvents"] if event["ph"] == "i"] + assert instants[0]["tid"] == instants[2]["tid"] + assert instants[0]["tid"] != instants[1]["tid"] + thread_names = { + event["tid"]: event["args"]["name"] + for event in payload["traceEvents"] + if event.get("name") == "thread_name" and event["ph"] == "M" + } + assert thread_names[instants[0]["tid"]] == "token-a" + assert thread_names[instants[1]["tid"]] == "token-b" def test_pipeline_trace_preserves_repeated_stage_pairs(): @@ -124,3 +152,170 @@ async def test_run_clients_leaves_default_executor_usable(): assert results == [0, 1] assert await asyncio.to_thread(lambda: "still-usable") == "still-usable" + + +class _FakeRedis: + """Minimal async Redis stand-in recording lifecycle calls.""" + + def __init__(self, calls: list[str], dbsize: int = 0): + """Initialize the fake. + + Args: + calls: Shared call log appended to by every method. + dbsize: Key count reported by ``dbsize``. + """ + self.calls = calls + self._dbsize = dbsize + + async def ping(self) -> bool: + """Record and acknowledge a ping. + + Returns: + Always True. + """ + self.calls.append("ping") + return True + + async def dbsize(self) -> int: + """Record and report the configured key count. + + Returns: + The configured key count. + """ + self.calls.append("dbsize") + return self._dbsize + + async def flushdb(self) -> None: + """Record a database flush.""" + self.calls.append("flushdb") + + async def aclose(self) -> None: + """Record closing the connection pool.""" + self.calls.append("aclose") + + +def test_performance_redis_url_requires_database_index(monkeypatch): + """URLs without an explicit database index are rejected.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379") + with pytest.raises(ValueError, match="database index"): + redis_support.performance_redis_url() + + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + assert redis_support.performance_redis_url() == "redis://127.0.0.1:6379/15" + + +async def test_real_redis_state_manager_refuses_nonempty_database(monkeypatch): + """A database that already holds keys is never flushed.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + calls: list[str] = [] + fake = _FakeRedis(calls, dbsize=3) + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + monkeypatch.setattr( + redis_support, "Redis", SimpleNamespace(from_url=lambda url: fake) + ) + + with pytest.raises(RuntimeError, match="not empty"): + async with redis_support.real_redis_state_manager(): + pass + + assert "flushdb" not in calls + assert calls[-1] == "aclose" + + +async def test_real_redis_state_manager_cleanup_survives_close_failure(monkeypatch): + """The database is flushed before close and the pool closes regardless.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + calls: list[str] = [] + fake = _FakeRedis(calls) + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + monkeypatch.setattr( + redis_support, "Redis", SimpleNamespace(from_url=lambda url: fake) + ) + + class _FailingManager: + """State-manager stand-in whose close always fails.""" + + def __init__(self, redis): + """Store the redis client. + + Args: + redis: The fake redis client. + """ + self.redis = redis + + async def close(self) -> None: + """Record the close attempt and fail. + + Raises: + RuntimeError: Always, to exercise cleanup ordering. + """ + calls.append("close") + msg = "close failed" + raise RuntimeError(msg) + + monkeypatch.setattr(redis_support, "StateManagerRedis", _FailingManager) + + with pytest.raises(RuntimeError, match="close failed"): + async with redis_support.real_redis_state_manager(): + pass + + assert calls == ["ping", "dbsize", "flushdb", "close", "aclose"] + + +async def test_capture_async_diagnostics_formats_each_frame_once(tmp_path): + """Task stacks include each ancestor frame once instead of once per frame.""" + task = asyncio.current_task() + assert task is not None + task.set_name("diagnostics-under-test") + await asyncio.sleep(0) + + destination = capture_async_diagnostics(tmp_path / "diagnostics.json") + payload = json.loads(destination.read_text()) + stack = "".join( + line + for task_info in payload["tasks"] + if task_info["name"] == "diagnostics-under-test" + for line in task_info["stack"] + ) + assert ", in test_capture_async_diagnostics_formats_each_frame_once" in stack + assert stack.count(", in ") == 1 + + +def test_lifecycle_app_source_uses_public_api_transformer(): + """Generated app sources register extra routes via the public seam.""" + source = lifecycle_app_source(rows=1, pages=1) + assert "_api" not in source + assert "api_transformer" in source + + +def test_baseline_server_startup_failure_surfaces_thread_exception(monkeypatch): + """A server thread crash is reported instead of a bare startup timeout.""" + pytest.importorskip("socketio") + uvicorn = pytest.importorskip("uvicorn") + from tests.benchmarks.support.baseline_server import BaselineSocketServer + + def crash(self) -> None: + """Simulate an immediate server crash. + + Args: + self: The uvicorn server. + + Raises: + RuntimeError: Always. + """ + msg = "boom during startup" + raise RuntimeError(msg) + + monkeypatch.setattr(uvicorn.Server, "run", crash) + with ( + pytest.raises(TimeoutError, match="boom during startup"), + BaselineSocketServer(response={}), + ): + pass diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 270f040f684..611e7ba7b61 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -42,7 +42,12 @@ import reflex as rx from reflex import AdminDash, constants from reflex._upload import upload -from reflex.app import App, ComponentCallable, default_overlay_component +from reflex.app import ( + App, + ComponentCallable, + _decode_asgi_headers, + default_overlay_component, +) from reflex.compiler.compiler import ( _compile_app, _memoize_stateful_app_wraps, @@ -3851,6 +3856,25 @@ def test_call_marks_later_dev_backend_worker_as_hot_reload( assert compile_mock.call_args.kwargs["trigger"] == "hot_reload" +def test_decode_asgi_headers(): + """_decode_asgi_headers decodes raw ASGI header pairs into a str dict.""" + assert _decode_asgi_headers([]) == {} + assert _decode_asgi_headers([ + (b"host", b"example.com"), + (b"x-forwarded-for", b"10.0.0.1, 10.0.0.2"), + ]) == { + "host": "example.com", + "x-forwarded-for": "10.0.0.1, 10.0.0.2", + } + # Later duplicates win, matching dict comprehension semantics. + assert _decode_asgi_headers([ + (b"cookie", b"a=1"), + (b"cookie", b"b=2"), + ]) == {"cookie": "b=2"} + # Names and values are decoded as UTF-8. + assert _decode_asgi_headers([(b"x-name", "café".encode())]) == {"x-name": "café"} + + def test_call_ignores_stale_marker_without_dev_backend_reload( compilable_app: tuple[App, Path], mocker: MockerFixture, From 29890a0e31e836f8fbdfed8088abe15c2ee10cf7 Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 12:20:09 -0700 Subject: [PATCH 10/13] Add performance benchmark news fragment --- news/6751.performance.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/6751.performance.md diff --git a/news/6751.performance.md b/news/6751.performance.md new file mode 100644 index 00000000000..ba04c2e7d48 --- /dev/null +++ b/news/6751.performance.md @@ -0,0 +1 @@ +Add reusable performance benchmarks for backend event-loop responsiveness, event processing, socket headers, compiler lifecycle, memory use, and concurrent browser loads. From 267f35c66067c97035ddd99ef2639c1c7550420b Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 12:30:05 -0700 Subject: [PATCH 11/13] Remove performance benchmark news fragment --- news/6751.performance.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 news/6751.performance.md diff --git a/news/6751.performance.md b/news/6751.performance.md deleted file mode 100644 index ba04c2e7d48..00000000000 --- a/news/6751.performance.md +++ /dev/null @@ -1 +0,0 @@ -Add reusable performance benchmarks for backend event-loop responsiveness, event processing, socket headers, compiler lifecycle, memory use, and concurrent browser loads. From abc199a607b71137bde20f4d3101044a561ee79a Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 15:05:11 -0700 Subject: [PATCH 12/13] Fix benchmark accuracy: measurement-region contamination and vacuous asserts Address review findings that several benchmarks did not measure what they claim: - socket load client: prime + drain the hydrate/on_load deltas a fresh token emits so measured events are 1:1 request/response instead of pipelined two deep, fixing the event-load latency curve and framework-overhead comparison (which could otherwise report negative overhead). - browser hydration: gate on the websocket-driven is_hydrated marker instead of the prerendered #count text, so the timing spans real hydration; add the marker to the load app's index page. - route matching: seed a cached rxconfig so get_config() is a dict lookup as in production, instead of rebuilding Config (sys.path scan + env parse) on every match, which was 55-97% of the measured region. - process_event: purge the shared token per pedantic round so the cold path is actually exercised; CodSpeed's warmup otherwise left every round all-warm. - event-loop leak asserts: capture orphan task/future counts before stop() (which clears them), so the checks and metrics can catch a real leak; wait for the backend-exception task to drain in the failed-pipeline case. - wire size: encode with socket.io's compact separators so wire_bytes equals the real payload size instead of overstating it ~20%. - redis helper: close the manager before flushing (via a fresh client) so oplock lease write-backs land before the wipe and the next run's empty-db guard holds. - compiler lifecycle: keep zipping on the backend export so it measures real work rather than a no-op. --- tests/benchmarks/support/apps.py | 4 ++ tests/benchmarks/support/redis.py | 13 +++++- tests/benchmarks/support/socket_client.py | 43 ++++++++++++++++++++ tests/benchmarks/test_event_processing.py | 21 +++++++--- tests/benchmarks/test_routes.py | 32 +++++++++++++++ tests/performance/test_browser.py | 4 ++ tests/performance/test_compiler_lifecycle.py | 5 ++- tests/performance/test_event_loop.py | 29 ++++++++++--- tests/performance/test_wire_size.py | 8 +++- tests/units/benchmarks/test_support.py | 7 +++- 10 files changed, 149 insertions(+), 17 deletions(-) diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py index 37c42792c4c..dc9386f2546 100644 --- a/tests/benchmarks/support/apps.py +++ b/tests/benchmarks/support/apps.py @@ -91,6 +91,10 @@ def append_large(self): def index(): return rx.vstack( + rx.text( + rx.cond(State.is_hydrated, "hydrated", "loading"), + id="hydrated", + ), rx.text(State.count, id="count"), rx.button("increment", id="increment", on_click=State.increment), rx.button("async", id="async-io", on_click=State.async_io), diff --git a/tests/benchmarks/support/redis.py b/tests/benchmarks/support/redis.py index cd4dbf7fa69..3d0b0f93947 100644 --- a/tests/benchmarks/support/redis.py +++ b/tests/benchmarks/support/redis.py @@ -65,9 +65,18 @@ async def real_redis_state_manager() -> AsyncIterator[StateManagerRedis]: try: yield manager finally: + # Close the manager before flushing: with oplock enabled, close() + # cancels lease-breaker tasks that re-persist cached states, so + # flushing first would leave those write-backs behind and the next + # run's empty-database guard would fail. close() also disposes the + # shared connection pool, so flush through a fresh client. try: - await redis.flushdb() - finally: await manager.close() + finally: + cleanup = Redis.from_url(url) + try: + await cleanup.flushdb() + finally: + await cleanup.aclose() finally: await redis.aclose() diff --git a/tests/benchmarks/support/socket_client.py b/tests/benchmarks/support/socket_client.py index 6f0e497c3c1..e64c782c128 100644 --- a/tests/benchmarks/support/socket_client.py +++ b/tests/benchmarks/support/socket_client.py @@ -15,6 +15,10 @@ import socketio from socketio.exceptions import TimeoutError as SocketTimeoutError +# Silence window that marks the connection quiescent while draining the extra +# deltas the first event on a fresh token emits (hydrate + on_load_internal). +_PRIME_DRAIN_TIMEOUT = 0.5 + @dataclasses.dataclass(frozen=True) class ClientLoadResult: @@ -76,6 +80,44 @@ def _connect( ) +def _drain(client: socketio.SimpleClient, timeout: float) -> None: + """Consume buffered responses until the socket is quiet for ``timeout``. + + Args: + client: Blocking Socket.IO client. + timeout: Silence window that marks the connection quiescent. + """ + while True: + try: + client.receive(timeout=timeout) + except SocketTimeoutError: + return + + +def _prime( + client: socketio.SimpleClient, + event_name: str, + payload: Mapping[str, Any], + timeout: float, +) -> None: + """Hydrate a fresh token so later events observe a clean 1:1 response. + + The first application event on a fresh token triggers the rehydrate path, + which emits extra full-state and ``on_load_internal`` deltas under the same + ``event`` message name. Sending one warmup event and draining every response + keeps the measured loop from reading those deltas as if they answered later + events, which would otherwise report each latency pipelined two events deep. + + Args: + client: Blocking Socket.IO client. + event_name: Socket event name used for requests. + payload: Event payload emitted to trigger hydration. + timeout: Maximum wait bounding the drain silence window. + """ + client.emit(event_name, dict(payload)) + _drain(client, min(timeout, _PRIME_DRAIN_TIMEOUT)) + + async def run_socket_client( url: str, token: str, @@ -153,6 +195,7 @@ def _run_socket_client_sync( try: _connect(client, url, token, namespace, timeout) + _prime(client, event_name, payload, timeout) for _ in range(events): started = time.perf_counter_ns() client.emit(event_name, dict(payload)) diff --git a/tests/benchmarks/test_event_processing.py b/tests/benchmarks/test_event_processing.py index c9993c0a37b..98fc6af2a9c 100644 --- a/tests/benchmarks/test_event_processing.py +++ b/tests/benchmarks/test_event_processing.py @@ -117,20 +117,31 @@ def test_process_event( The first event creates fresh state (cold path), the next two reuse the existing state (warm path). Only event processing is timed. + The token is purged in per-round setup so the cold path is exercised on + every measured round. Without it, CodSpeed's warmup invocation hydrates the + shared token and every subsequent round would measure three warm events, + silently duplicating ``test_process_event_warm``. + Args: event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events, _, shutdown = event_processing_harness + run_events, purge_tokens, shutdown = event_processing_harness loop = asyncio.new_event_loop() - try: + def setup(): + """Return the batch args after purging the token for a cold round.""" + purge_tokens(["benchmark-token"]) # Each event handler (increment) does a single state mutation with # no yields, so we expect 1 delta per event = 3 total. - @benchmark - def _(): - loop.run_until_complete(run_events(["benchmark-token"] * 3)) + return ((["benchmark-token"] * 3,), {}) + def run_batch(tokens): + """Process one cold-then-warm batch for the given tokens.""" + loop.run_until_complete(run_events(tokens)) + + try: + benchmark.pedantic(run_batch, setup=setup) finally: loop.run_until_complete(shutdown()) loop.close() diff --git a/tests/benchmarks/test_routes.py b/tests/benchmarks/test_routes.py index 25fefc67248..15d0fa2e07e 100644 --- a/tests/benchmarks/test_routes.py +++ b/tests/benchmarks/test_routes.py @@ -1,14 +1,45 @@ """Benchmarks for route construction and hot-path matching.""" import re +import sys +import types +from collections.abc import Iterator import pytest from pytest_codspeed import BenchmarkFixture from reflex_base import constants +from reflex_base.config import get_config from reflex.route import get_route_args, get_router +@pytest.fixture +def cached_rxconfig() -> Iterator[None]: + """Seed a cached ``rxconfig`` module so matching hits the config cache. + + In production ``rxconfig`` is imported into ``sys.modules`` and every + ``get_config()`` call (invoked once per ``router(path)`` match) is a dict + lookup. Without a cached module the benchmark environment rebuilds a fresh + ``Config`` on every call — a sys.path scan and full env parse that dominates + the measured region and hides real route-matching regressions. + + Yields: + None, with the cached module installed for the test's duration. + """ + module_name = constants.Config.MODULE + saved = sys.modules.get(module_name) + stub = types.ModuleType(module_name) + stub.config = get_config() # pyright: ignore [reportAttributeAccessIssue] + sys.modules[module_name] = stub + try: + yield + finally: + if saved is not None: + sys.modules[module_name] = saved + else: + sys.modules.pop(module_name, None) + + def _routes(count: int) -> list[str]: """Build representative static and dynamic routes. @@ -24,6 +55,7 @@ def _routes(count: int) -> list[str]: @pytest.mark.parametrize("count", [10, 100, 1000]) +@pytest.mark.usefixtures("cached_rxconfig") def test_route_matching(count: int, benchmark: BenchmarkFixture): """Benchmark repeated matching through representative route tables. diff --git a/tests/performance/test_browser.py b/tests/performance/test_browser.py index fbd8055caef..55d56aea350 100644 --- a/tests/performance/test_browser.py +++ b/tests/performance/test_browser.py @@ -96,6 +96,10 @@ def test_browser_report( ) started = time.perf_counter_ns() page.goto(performance_load_app.frontend_url) + # Gate on the websocket-driven is_hydrated marker, not the prerendered + # #count text (which the static export already ships as "0"), so the timing + # spans real hydration: connect, initial delta, and client state attach. + expect(page.locator("#hydrated")).to_have_text("hydrated") expect(page.locator("#count")).to_have_text("0") hydration_ms = (time.perf_counter_ns() - started) / 1_000_000 heap_before = _browser_heap(page) diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py index e065c522297..fe181642a76 100644 --- a/tests/performance/test_compiler_lifecycle.py +++ b/tests/performance/test_compiler_lifecycle.py @@ -395,6 +395,10 @@ def test_compiler_lifecycle_report( ) ) + # Keep zipping enabled: with --no-zip a backend-only export does no work + # (export() runs frontend build and zip only, both gated off), so the timing + # would measure CLI startup alone. Zipping exercises the real backend export + # path — archiving the app source tree. export_ms = _run( [ executable, @@ -402,7 +406,6 @@ def test_compiler_lifecycle_report( "reflex", "export", "--backend-only", - "--no-zip", ], app_root, ) diff --git a/tests/performance/test_event_loop.py b/tests/performance/test_event_loop.py index 1e93dae43ec..7d274cdaa18 100644 --- a/tests/performance/test_event_loop.py +++ b/tests/performance/test_event_loop.py @@ -402,11 +402,16 @@ def token_queue_depth() -> int: scenario=handler.fn.__name__, ) await asyncio.sleep(probe_interval * 2) + # Capture leak counts while the processor is still running. stop() + # cancels and clears both dicts on block exit, so checking after the + # block would always see zero and never catch an undrained task/future. + orphan_tasks = len(processor._tasks) + orphan_futures = len(processor._futures) for token, expected in expected_order.items(): assert _ORDER_LOG[token] == expected - assert not processor._tasks - assert not processor._futures + assert not orphan_tasks + assert not orphan_futures durations = processor.trace.durations_ms([ ("received", "enqueued"), ("enqueued", "dequeued"), @@ -427,8 +432,8 @@ def token_queue_depth() -> int: "tasks_created": sum( event.stage == "task_created" for event in processor.trace.events ), - "orphan_tasks": len(processor._tasks), - "orphan_futures": len(processor._futures), + "orphan_tasks": orphan_tasks, + "orphan_futures": orphan_futures, } ) return latencies, metrics, processor.trace @@ -512,6 +517,8 @@ async def _run_failed_state_pipeline() -> None: processor.configure(state_manager=manager) event = Event.from_event_type(AttributionState.failure())[0] event = dataclasses.replace(event, router_data={"path": "/", "query": {}}) + orphan_tasks = 0 + orphan_futures = 0 try: async with processor: future = await processor.enqueue("failure-token", event) @@ -519,12 +526,22 @@ async def _run_failed_state_pipeline() -> None: await _wait_events_bounded( future.wait_all(), events=1, scenario="failed_state_pipeline" ) + # The failing handler schedules a backend-exception task after the + # future resolves, so wait for that task to run and its finish + # callback to pop it before capturing counts. Checking after the + # block would only see stop()'s forced teardown, not real cleanup. + for _ in range(200): + await asyncio.sleep(0.01) + if exceptions and not processor._tasks: + break + orphan_tasks = len(processor._tasks) + orphan_futures = len(processor._futures) finally: await manager.close() _ACTIVE_STATE_TRACE = None assert exceptions - assert not processor._tasks - assert not processor._futures + assert not orphan_tasks + assert not orphan_futures @pytest.mark.performance diff --git a/tests/performance/test_wire_size.py b/tests/performance/test_wire_size.py index 27c91143e87..1c738fd7bb3 100644 --- a/tests/performance/test_wire_size.py +++ b/tests/performance/test_wire_size.py @@ -68,7 +68,13 @@ def test_wire_size_report(performance_output: Path, performance_scale: str): for name, interaction in INTERACTIONS.items(): state = initialized_state(STATE_SIZE) interaction(state) - wire = json_dumps(StateUpdate(delta=state.get_delta())).encode() + # Match the compact separators python-socketio uses when it encodes the + # event packet (Packet.encode calls dumps(..., separators=(",", ":"))), + # so wire_bytes equals the real payload size rather than json's spaced + # default, which overstates it by roughly a fifth. + wire = json_dumps( + StateUpdate(delta=state.get_delta()), separators=(",", ":") + ).encode() sizes[name] = len(wire) report.add( BenchmarkResult( diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py index 12e54ae9235..7176c8672e4 100644 --- a/tests/units/benchmarks/test_support.py +++ b/tests/units/benchmarks/test_support.py @@ -228,7 +228,7 @@ async def test_real_redis_state_manager_refuses_nonempty_database(monkeypatch): async def test_real_redis_state_manager_cleanup_survives_close_failure(monkeypatch): - """The database is flushed before close and the pool closes regardless.""" + """The manager closes before the flush, which runs despite a close failure.""" pytest.importorskip("redis") from tests.benchmarks.support import redis as redis_support @@ -266,7 +266,10 @@ async def close(self) -> None: async with redis_support.real_redis_state_manager(): pass - assert calls == ["ping", "dbsize", "flushdb", "close", "aclose"] + # The manager is closed first so any oplock lease write-backs land before + # the flush; the flush then runs on a fresh client even though close() + # raised, and both the fresh client and the original connection are closed. + assert calls == ["ping", "dbsize", "close", "flushdb", "aclose", "aclose"] async def test_capture_async_diagnostics_formats_each_frame_once(tmp_path): From 8745f9eb1c164f60dd4422f71b737e374bc955ad Mon Sep 17 00:00:00 2001 From: Alek Date: Mon, 13 Jul 2026 15:37:08 -0700 Subject: [PATCH 13/13] Fix more benchmark accuracy findings: registry isolation, dispatch path, ordering Second review pass: - state-manager cold get: run the measured construction inside an isolated RegistrationContext registering only PerformanceState, so it instantiates that state's subtree instead of every state in the collected session. Adding or removing an unrelated benchmark state no longer shifts this CodSpeed result. - event queue dispatch / future tree: run the measured dispatch inside the same context where start() attached the EventContext, so enqueue takes the production EventContext.get().fork() path instead of the LookupError fallback; build the event in per-round setup so only queueing and dispatch are measured. - wall-time ordering scenario: track handlers active per token and assert the peak stays 1. Completion-order alone can't catch a regression to concurrent same-token dispatch (equal sleeps still finish in order); the overlap counter can, and it's keyed per token so independent tokens still run concurrently. - compiler lifecycle: disable the live PyPI latest-version check so `init` timing is not contaminated by a network round-trip. --- tests/benchmarks/support/states.py | 20 ++++++++ tests/benchmarks/test_event_loop.py | 50 +++++++++++------- tests/benchmarks/test_state_manager.py | 19 ++++--- tests/performance/test_compiler_lifecycle.py | 9 +++- tests/performance/test_event_loop.py | 54 ++++++++++++++++++-- 5 files changed, 123 insertions(+), 29 deletions(-) diff --git a/tests/benchmarks/support/states.py b/tests/benchmarks/support/states.py index f229871cefa..1e637b971b7 100644 --- a/tests/benchmarks/support/states.py +++ b/tests/benchmarks/support/states.py @@ -52,6 +52,26 @@ def doubled_total(self) -> int: return self.total * 2 +@contextmanager +def isolated_performance_registry() -> Iterator[None]: + """Register only ``PerformanceState`` for deterministic cold construction. + + A state manager builds a token's root state by instantiating the whole + substate tree recorded in the ambient ``RegistrationContext``. Without + isolation that tree includes every module-level state in the collected + benchmark session, so adding or removing a state anywhere shifts a cold + ``get_state`` result on CodSpeed. Entering a fresh context registered with + only ``PerformanceState`` fixes the tree to this state's subtree regardless + of the rest of the session. + + Yields: + None, with the isolated registration context active. + """ + with RegistrationContext(): + RegistrationContext.register_base_state(PerformanceState) + yield + + def initialized_state(size: int) -> PerformanceState: """Create a state with deterministic mutable values. diff --git a/tests/benchmarks/test_event_loop.py b/tests/benchmarks/test_event_loop.py index 5e305eab1cb..509f985518f 100644 --- a/tests/benchmarks/test_event_loop.py +++ b/tests/benchmarks/test_event_loop.py @@ -105,40 +105,47 @@ def test_event_queue_dispatch(benchmark: BenchmarkFixture): registry = _register_handlers() ctx = contextvars.copy_context() - async def dispatch(processor: EventProcessor) -> None: + async def dispatch(processor: EventProcessor, event: Event) -> None: """Enqueue one no-op event and await its completion. Args: processor: The started processor. + event: The pre-built event to enqueue. """ - future = await processor.enqueue( - "token", Event.from_event_type(NOOP_EVENT())[0] - ) + future = await processor.enqueue("token", event) await future.wait_all() def setup(): """Construct and start a processor for the next measured round. Returns: - Pedantic (args, kwargs) holding the started processor. + Pedantic (args, kwargs) holding the started processor and event. """ processor = _make_processor() _run_in_context(loop, ctx, processor.start()) - return ((processor,), {}) + # Build the event in setup so only queueing and dispatch are measured. + event = Event.from_event_type(NOOP_EVENT())[0] + return ((processor, event), {}) - def run(processor: EventProcessor) -> None: + def run(processor: EventProcessor, event: Event) -> None: """Dispatch one event through the started processor. + Runs inside ``ctx`` where ``start`` attached the ``EventContext``, so + enqueue takes the production ``EventContext.get().fork()`` path rather + than the ``LookupError`` fallback a bare loop context would trigger. + Args: processor: The started processor. + event: The pre-built event to enqueue. """ - loop.run_until_complete(dispatch(processor)) + _run_in_context(loop, ctx, dispatch(processor, event)) - def teardown(processor: EventProcessor) -> None: + def teardown(processor: EventProcessor, event: Event) -> None: """Stop the measured round's processor. Args: processor: The started processor. + event: The event dispatched this round (unused at teardown). """ _run_in_context(loop, ctx, processor.stop()) @@ -163,40 +170,47 @@ def test_event_future_tree(benchmark: BenchmarkFixture): registry = _register_handlers() ctx = contextvars.copy_context() - async def dispatch(processor: EventProcessor) -> None: + async def dispatch(processor: EventProcessor, event: Event) -> None: """Enqueue one chaining event and await the whole future tree. Args: processor: The started processor. + event: The pre-built parent event to enqueue. """ - future = await processor.enqueue( - "token", Event.from_event_type(CHAIN_EVENT())[0] - ) + future = await processor.enqueue("token", event) await future.wait_all() def setup(): """Construct and start a processor for the next measured round. Returns: - Pedantic (args, kwargs) holding the started processor. + Pedantic (args, kwargs) holding the started processor and event. """ processor = _make_processor() _run_in_context(loop, ctx, processor.start()) - return ((processor,), {}) + # Build the event in setup so only dispatch and chaining are measured. + event = Event.from_event_type(CHAIN_EVENT())[0] + return ((processor, event), {}) - def run(processor: EventProcessor) -> None: + def run(processor: EventProcessor, event: Event) -> None: """Dispatch the parent event through the started processor. + Runs inside ``ctx`` where ``start`` attached the ``EventContext``, so + enqueue takes the production ``EventContext.get().fork()`` path rather + than the ``LookupError`` fallback a bare loop context would trigger. + Args: processor: The started processor. + event: The pre-built parent event to enqueue. """ - loop.run_until_complete(dispatch(processor)) + _run_in_context(loop, ctx, dispatch(processor, event)) - def teardown(processor: EventProcessor) -> None: + def teardown(processor: EventProcessor, event: Event) -> None: """Stop the measured round's processor. Args: processor: The started processor. + event: The event dispatched this round (unused at teardown). """ _run_in_context(loop, ctx, processor.stop()) diff --git a/tests/benchmarks/test_state_manager.py b/tests/benchmarks/test_state_manager.py index 93ab7d9aaaa..ce91426f737 100644 --- a/tests/benchmarks/test_state_manager.py +++ b/tests/benchmarks/test_state_manager.py @@ -7,7 +7,11 @@ from reflex.istate.manager.memory import StateManagerMemory from reflex.istate.manager.token import BaseStateToken -from .support.states import PerformanceState, get_performance_state +from .support.states import ( + PerformanceState, + get_performance_state, + isolated_performance_registry, +) def test_state_manager_memory_cold_get(benchmark: BenchmarkFixture): @@ -39,11 +43,14 @@ def teardown(token: BaseStateToken) -> None: """Purge the measured state.""" manager._purge_token(token) # pyright: ignore [reportPrivateUsage] - try: - benchmark.pedantic(get_state, setup=setup, teardown=teardown) - finally: - loop.run_until_complete(manager.close()) - loop.close() + # Isolate the registry so the measured cold construction instantiates only + # PerformanceState's subtree, not every state in the collected session. + with isolated_performance_registry(): + try: + benchmark.pedantic(get_state, setup=setup, teardown=teardown) + finally: + loop.run_until_complete(manager.close()) + loop.close() def test_state_manager_memory_warm_get(benchmark: BenchmarkFixture): diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py index fe181642a76..830607690b1 100644 --- a/tests/performance/test_compiler_lifecycle.py +++ b/tests/performance/test_compiler_lifecycle.py @@ -39,7 +39,14 @@ def _run(command: list[str], cwd: Path, timeout: float = 300) -> float: capture_output=True, text=True, timeout=timeout, - env=os.environ | {"REFLEX_TELEMETRY_ENABLED": "false"}, + # Disable the live PyPI latest-version check so `init` timing is not + # contaminated by a network round-trip (and its 2s timeout on a slow or + # offline runner). Telemetry is off for the same reason. + env=os.environ + | { + "REFLEX_TELEMETRY_ENABLED": "false", + "REFLEX_CHECK_LATEST_VERSION": "false", + }, ) return (time.perf_counter_ns() - started) / 1_000_000 diff --git a/tests/performance/test_event_loop.py b/tests/performance/test_event_loop.py index 7d274cdaa18..2f19cc55c83 100644 --- a/tests/performance/test_event_loop.py +++ b/tests/performance/test_event_loop.py @@ -39,8 +39,35 @@ ) _ORDER_LOG: dict[str, list[int]] = defaultdict(list) +# Handlers active per token right now, and the peak seen during a scenario. The +# per-token queue must serialize same-token events, so the peak must stay 1; a +# regression to concurrent same-token dispatch would push it to 2+. +_ACTIVE_PER_TOKEN: dict[str, int] = defaultdict(int) +_PEAK_ACTIVE_PER_TOKEN: dict[str, int] = defaultdict(int) _ACTIVE_STATE_TRACE: PipelineTrace | None = None + +def _record_active(token: str) -> None: + """Mark a handler as active for a token and update the peak. + + Args: + token: The token whose handler is entering. + """ + _ACTIVE_PER_TOKEN[token] += 1 + _PEAK_ACTIVE_PER_TOKEN[token] = max( + _PEAK_ACTIVE_PER_TOKEN[token], _ACTIVE_PER_TOKEN[token] + ) + + +def _clear_active(token: str) -> None: + """Mark a handler as no longer active for a token. + + Args: + token: The token whose handler is exiting. + """ + _ACTIVE_PER_TOKEN[token] -= 1 + + # Deliberate synchronous block injected by the blocking scenario. Detection is # asserted at >=2ms of measured loop lag, so 10ms leaves a 5x margin even on a # loaded CI runner. @@ -206,8 +233,13 @@ async def _cooperative_handler(sequence: int) -> None: Args: sequence: Per-token event sequence. """ - await asyncio.sleep(0.001) - _ORDER_LOG[EventContext.get().token].append(sequence) + token = EventContext.get().token + _record_active(token) + try: + await asyncio.sleep(0.001) + _ORDER_LOG[token].append(sequence) + finally: + _clear_active(token) async def _blocking_handler(sequence: int) -> None: # noqa: RUF029 @@ -216,8 +248,13 @@ async def _blocking_handler(sequence: int) -> None: # noqa: RUF029 Args: sequence: Per-token event sequence. """ - time.sleep(_BLOCKING_SLEEP_SECONDS) # noqa: ASYNC251 - intentional benchmark scenario - _ORDER_LOG[EventContext.get().token].append(sequence) + token = EventContext.get().token + _record_active(token) + try: + time.sleep(_BLOCKING_SLEEP_SECONDS) # noqa: ASYNC251 - intentional benchmark scenario + _ORDER_LOG[token].append(sequence) + finally: + _clear_active(token) COOPERATIVE_EVENT = EventHandler(fn=_cooperative_handler) @@ -341,6 +378,8 @@ async def _run_scenario( Latencies, probe/resource metrics, and pipeline trace. """ _ORDER_LOG.clear() + _ACTIVE_PER_TOKEN.clear() + _PEAK_ACTIVE_PER_TOKEN.clear() processor = TracingEventProcessor(graceful_shutdown_timeout=2) processor.configure() @@ -410,6 +449,13 @@ def token_queue_depth() -> int: for token, expected in expected_order.items(): assert _ORDER_LOG[token] == expected + # Completion order alone cannot catch concurrent same-token dispatch (equal + # sleeps still finish in order), so assert no token ever had two handlers + # active at once — the guarantee the per-token queue exists to provide. + overlapping = { + token: peak for token, peak in _PEAK_ACTIVE_PER_TOKEN.items() if peak > 1 + } + assert not overlapping, f"same-token handlers overlapped: {overlapping}" assert not orphan_tasks assert not orphan_futures durations = processor.trace.durations_ms([