diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 94dc268f2f3..313c2072590 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,5 @@ /docs/ @Alek99 @reflex-dev/reflex-team /README.md @Alek99 @reflex-dev/reflex-team +/tests/benchmarks/ @Alek99 @reflex-dev/reflex-team +/tests/performance/ @Alek99 @reflex-dev/reflex-team diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 571811116ba..c31af0da022 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -21,6 +21,7 @@ jobs: benchmarks: name: Run benchmarks runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -45,6 +46,58 @@ jobs: mode: instrumentation run: uv run pytest -v tests/benchmarks --codspeed + - name: Run event-loop and wire-size smoke + run: >- + uv run pytest tests/performance/test_event_loop.py tests/performance/test_wire_size.py + --run-performance + --performance-scale smoke + --performance-output .pytest-tmp/performance + -q + + - name: Upload performance smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-smoke-benchmarks + path: .pytest-tmp/performance/** + if-no-files-found: ignore + + browser-performance: + name: Run browser performance smoke + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-tags: true + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/setup_build_env + with: + python-version: "3.14" + node-version: "22" + run-uv-sync: true + + - name: Install playwright + run: uv run playwright install chromium --only-shell + + - name: Run browser performance smoke and bundle budget + run: >- + uv run pytest tests/performance/test_browser.py + --run-performance + --performance-scale smoke + --performance-output .pytest-tmp/performance + -q + + - name: Upload performance smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-smoke-browser + path: .pytest-tmp/performance/** + if-no-files-found: ignore + lighthouse: name: Run Lighthouse benchmark runs-on: ubuntu-22.04 diff --git a/pyproject.toml b/pyproject.toml index 437e2fe5f20..099d6108ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -256,6 +256,9 @@ preview = true filterwarnings = "ignore:fields may not start with an underscore:RuntimeWarning" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" +markers = [ + "performance: scheduled wall-time, service, memory, lifecycle, or browser benchmark", +] [tool.codespell] skip = "*.html, examples, *.pyi, */nba.csv, */olympic-winners.json, uv.lock, */bun.lock, node_modules" diff --git a/reflex/app.py b/reflex/app.py index 9364c547436..37dc045eb75 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -20,6 +20,7 @@ Callable, Collection, Coroutine, + Iterable, Mapping, Sequence, ) @@ -1599,6 +1600,18 @@ async def health(_request: Request) -> JSONResponse: return JSONResponse(content=health_status, status_code=status_code) +def _decode_asgi_headers(headers: Iterable[tuple[bytes, bytes]]) -> dict[str, str]: + """Decode raw ASGI scope header pairs into a str-keyed dict. + + Args: + headers: Raw (name, value) byte pairs from the ASGI scope. + + Returns: + A dict mapping decoded header names to decoded values. + """ + return {k.decode("utf-8"): v.decode("utf-8") for (k, v) in headers} + + class EventNamespace(AsyncNamespace): """The event namespace.""" @@ -1770,14 +1783,11 @@ async def on_event(self, sid: str, data: Any): raise RuntimeError(msg) # Get the client headers. - headers = { - k.decode("utf-8"): v.decode("utf-8") - for (k, v) in environ["asgi.scope"]["headers"] - } + headers = _decode_asgi_headers(environ["asgi.scope"]["headers"]) # Get the client IP try: - client_ip = environ["asgi.scope"]["client"][0] + client_ip: str = environ["asgi.scope"]["client"][0] headers["asgi-scope-client"] = client_ip except (KeyError, IndexError): client_ip = environ.get("REMOTE_ADDR", "0.0.0.0") diff --git a/tests/benchmarks/support/__init__.py b/tests/benchmarks/support/__init__.py new file mode 100644 index 00000000000..e3047e6eb64 --- /dev/null +++ b/tests/benchmarks/support/__init__.py @@ -0,0 +1,15 @@ +"""Shared support utilities for Reflex performance benchmarks.""" + +from .diagnostics import capture_async_diagnostics +from .loop_probe import EventLoopProbe +from .pipeline_trace import PipelineTrace +from .report import BenchmarkResult, PerformanceReport, percentile + +__all__ = [ + "BenchmarkResult", + "EventLoopProbe", + "PerformanceReport", + "PipelineTrace", + "capture_async_diagnostics", + "percentile", +] diff --git a/tests/benchmarks/support/apps.py b/tests/benchmarks/support/apps.py new file mode 100644 index 00000000000..dc9386f2546 --- /dev/null +++ b/tests/benchmarks/support/apps.py @@ -0,0 +1,153 @@ +"""Representative Reflex application sources for lifecycle and load tests.""" + +from __future__ import annotations + + +def lifecycle_app_source( + rows: int = 100, + pages: int = 1, + reload_version: int = 0, +) -> str: + """Create a deterministic multi-page application source. + + Args: + rows: Number of stateful rows per page. + pages: Number of routes. + reload_version: Value exposed by the backend reload probe. + + Returns: + Python source for a Reflex application. + """ + page_registrations = "\n".join( + f'app.add_page(lambda route="page-{index}": index(route), route="/page-{index}")' + for index in range(pages) + ) + return f"""import reflex as rx +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route + +RELOAD_VERSION = {reload_version} + +class State(rx.State): + count: int = 0 + + def increment(self): + self.count += 1 + +def row(index: int): + return rx.hstack(rx.text(index), rx.text(State.count)) + +def index(label: str = "index"): + return rx.vstack( + rx.heading(label), + rx.button("increment", on_click=State.increment), + *[row(index) for index in range({rows})], + ) + +async def reload_version(_request): + return JSONResponse({{"version": RELOAD_VERSION}}) + +app = rx.App( + api_transformer=Starlette(routes=[Route("/reload-version", reload_version)]), +) +app.add_page(index, route="/") +{page_registrations} +""" + + +def load_app_source() -> str: + """Create an app covering the load-test handler scenarios. + + Returns: + Python source for a production-mode Reflex load app. + """ + return """import asyncio +import time +import reflex as rx + +class State(rx.State): + count: int = 0 + values: list[int] = [] + + def increment(self): + self.count += 1 + + async def async_io(self): + await asyncio.sleep(0.01) + self.count += 1 + + def blocking(self): + time.sleep(0.02) + self.count += 1 + + def stream(self): + for _ in range(3): + self.count += 1 + yield + + def append_large(self): + self.values.extend(range(1000)) + +def index(): + return rx.vstack( + rx.text( + rx.cond(State.is_hydrated, "hydrated", "loading"), + id="hydrated", + ), + rx.text(State.count, id="count"), + rx.button("increment", id="increment", on_click=State.increment), + rx.button("async", id="async-io", on_click=State.async_io), + rx.button("blocking", id="blocking", on_click=State.blocking), + rx.button("stream", id="stream", on_click=State.stream), + rx.button("large", id="append-large", on_click=State.append_large), + rx.foreach(State.values, lambda value: rx.text(value, class_name="value-row")), + rx.link("second", href="/second", id="second-link"), + ) + +def second(): + return rx.heading("Second page", id="second-heading") + +class RowState(rx.State): + rows: list[str] = [] + selected: int = -1 + + def create_rows(self): + self.rows = [f"row {index}" for index in range(1000)] + + def partial_update(self): + for index in range(0, len(self.rows), 10): + self.rows[index] += " !!!" + + def select_row(self, index: int): + self.selected = index + + def swap_rows(self): + self.rows[1], self.rows[998] = self.rows[998], self.rows[1] + +def rows(): + return rx.vstack( + rx.text( + rx.cond(RowState.is_hydrated, "hydrated", "loading"), + id="rows-hydrated", + ), + rx.text(RowState.selected, id="selected-row"), + rx.button("create", id="create-rows", on_click=RowState.create_rows), + rx.button("partial", id="partial-update", on_click=RowState.partial_update), + rx.button("swap", id="swap-rows", on_click=RowState.swap_rows), + rx.foreach( + RowState.rows, + lambda row, index: rx.text( + row, + id=f"row-{index}", + class_name=rx.cond(index == RowState.selected, "row selected", "row"), + on_click=RowState.select_row(index), + ), + ), + ) + +app = rx.App() +app.add_page(index) +app.add_page(second, route="/second") +app.add_page(rows, route="/rows") +""" diff --git a/tests/benchmarks/support/baseline_server.py b/tests/benchmarks/support/baseline_server.py new file mode 100644 index 00000000000..dc21ee14f25 --- /dev/null +++ b/tests/benchmarks/support/baseline_server.py @@ -0,0 +1,128 @@ +"""Bare Starlette + Socket.IO server mirroring Reflex's websocket substrate. + +Serves the same stack Reflex runs on (uvicorn, Starlette mount at ``/_event``, +python-socketio ``AsyncServer`` on the ``/_event`` namespace) with a handler +that answers every event with a canned state-update payload. Round-trip +latency against this server isolates the substrate cost, so the difference +against a real Reflex app is the framework's own event-pipeline overhead. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any + +import socketio +import uvicorn +from starlette.applications import Starlette +from starlette.routing import Mount + + +class _EchoNamespace(socketio.AsyncNamespace): + """Namespace answering every event with a fixed update payload.""" + + def __init__(self, namespace: str, response: dict[str, Any]): + """Initialize the namespace. + + Args: + namespace: Socket.IO namespace, matching Reflex's event namespace. + response: Payload emitted back for every received event. + """ + super().__init__(namespace) + self._response = response + + async def on_connect(self, sid: str, environ: dict) -> None: + """Accept every connection. + + Args: + sid: Socket.IO session id. + environ: Connection request information. + """ + + async def on_event(self, sid: str, data: Any) -> None: + """Answer an event with the canned update. + + Args: + sid: Socket.IO session id. + data: Received event payload, intentionally unused. + """ + await self.emit("event", self._response, to=sid) + + +class BaselineSocketServer: + """Context manager serving the bare substrate on an ephemeral port.""" + + def __init__(self, response: dict[str, Any]): + """Configure the server without starting it. + + Args: + response: Payload emitted back for every received event. + """ + sio = socketio.AsyncServer( + async_mode="asgi", + cors_allowed_origins="*", + cors_credentials=True, + allow_upgrades=False, + transports=["websocket"], + ) + sio.register_namespace(_EchoNamespace("/_event", response)) + socket_app = socketio.ASGIApp(sio, socketio_path="") + self._server = uvicorn.Server( + uvicorn.Config( + app=Starlette(routes=[Mount("/_event", app=socket_app)]), + host="127.0.0.1", + port=0, + log_level="warning", + ) + ) + self._thread = threading.Thread(target=self._run_server, daemon=True) + self._exception: BaseException | None = None + self.url = "" + + def _run_server(self) -> None: + """Run the uvicorn server, capturing a crash for the startup path.""" + try: + self._server.run() + except BaseException as exception: # surfaced by __enter__ + self._exception = exception + + def __enter__(self) -> BaselineSocketServer: + """Start the server and resolve its ephemeral URL. + + Returns: + The running server. + + Raises: + TimeoutError: If the server crashes or does not bind within ten + seconds; a crash is chained and quoted in the message. + """ + self._thread.start() + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and self._thread.is_alive(): + if ( + self._server.started + and (servers := self._server.servers) + and (sockets := servers[0].sockets) + ): + host, port = sockets[0].getsockname()[:2] + self.url = f"http://{host}:{port}" + return self + time.sleep(0.01) + self._server.should_exit = True + self._thread.join(timeout=1) + msg = ( + f"Baseline socket server crashed during startup: {self._exception!r}" + if self._exception is not None + else "Baseline socket server did not start within 10 seconds." + ) + raise TimeoutError(msg) from self._exception + + def __exit__(self, *exc_info) -> None: + """Stop the server and join its thread. + + Args: + exc_info: Exception information, intentionally unused. + """ + self._server.should_exit = True + self._thread.join(timeout=10) diff --git a/tests/benchmarks/support/diagnostics.py b/tests/benchmarks/support/diagnostics.py new file mode 100644 index 00000000000..7bfe1bba034 --- /dev/null +++ b/tests/benchmarks/support/diagnostics.py @@ -0,0 +1,58 @@ +"""Diagnostic artifacts captured after severe event-loop lag.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import threading +import traceback +from pathlib import Path +from typing import Any + + +def capture_async_diagnostics( + path: str | Path, + *, + metadata: dict[str, Any] | None = None, +) -> Path: + """Capture task and thread stacks in a JSON artifact. + + Args: + path: Destination path. + metadata: Scenario-specific queue, stage, or lag metadata. + + Returns: + Resolved artifact path. + """ + tasks = [] + for task in asyncio.all_tasks(): + tasks.append({ + "name": task.get_name(), + "done": task.done(), + "cancelled": task.cancelled(), + "stack": traceback.StackSummary.extract( + (frame, frame.f_lineno) for frame in task.get_stack() + ).format(), + }) + frames = sys._current_frames() # pyright: ignore [reportPrivateUsage] + threads = [] + for thread in threading.enumerate(): + frame = frames.get(thread.ident) if thread.ident is not None else None + threads.append({ + "name": thread.name, + "ident": thread.ident, + "daemon": thread.daemon, + "stack": traceback.format_stack(frame) if frame is not None else [], + }) + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps( + {"metadata": metadata or {}, "tasks": tasks, "threads": threads}, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return destination diff --git a/tests/benchmarks/support/events.py b/tests/benchmarks/support/events.py new file mode 100644 index 00000000000..24c8681e8a0 --- /dev/null +++ b/tests/benchmarks/support/events.py @@ -0,0 +1,91 @@ +"""Representative event payloads for performance benchmarks.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import uuid +from typing import Any + +from pydantic import BaseModel + + +class PayloadKind(str, enum.Enum): + """Payload shapes exercised by serialization and event benchmarks.""" + + SCALAR = "scalar" + LIST = "list" + MAPPING = "mapping" + DATACLASS = "dataclass" + MODEL = "model" + + +@dataclasses.dataclass(frozen=True) +class PayloadRow: + """Nested dataclass payload row.""" + + identifier: int + label: str + values: tuple[int, ...] + + +class PayloadModel(BaseModel): + """Nested Pydantic payload model.""" + + identifier: int + label: str + values: list[int] + + +def make_payload(kind: PayloadKind, rows: int) -> Any: + """Build a deterministic payload with the requested shape. + + Args: + kind: Payload representation. + rows: Number of repeated rows or scalar characters. + + Returns: + A representative payload. + """ + if kind is PayloadKind.SCALAR: + return "x" * rows + if kind is PayloadKind.LIST: + return list(range(rows)) + if kind is PayloadKind.MAPPING: + return {f"key_{index}": index for index in range(rows)} + if kind is PayloadKind.DATACLASS: + return [ + PayloadRow(index, f"row-{index}", (index, index + 1)) + for index in range(rows) + ] + if kind is PayloadKind.MODEL: + return [ + PayloadModel( + identifier=index, + label=f"row-{index}", + values=[index, index + 1], + ) + for index in range(rows) + ] + msg = f"Unhandled payload kind: {kind}" + raise AssertionError(msg) + + +def wire_edge_cases() -> dict[str, Any]: + """Return values that exercise the complete socket serialization contract. + + Returns: + A mapping containing special numeric and structured values. + """ + return { + "unicode": "Reflex ⚡", + "date": datetime.date(2026, 1, 2), + "datetime": datetime.datetime(2026, 1, 2, 3, 4, 5), + "decimal": decimal.Decimal("123.45"), + "uuid": uuid.UUID("12345678-1234-5678-1234-567812345678"), + "set": {1, 2, 3}, + "large_integer": 2**80, + "dataclass": PayloadRow(1, "row", (1, 2)), + } diff --git a/tests/benchmarks/support/idle_clients.py b/tests/benchmarks/support/idle_clients.py new file mode 100644 index 00000000000..f21321d13b8 --- /dev/null +++ b/tests/benchmarks/support/idle_clients.py @@ -0,0 +1,48 @@ +"""Hold idle Socket.IO sessions from a separate process. + +Memory-per-session benchmarks run the backend inside the test process, so +client-side allocations would pollute resident-memory measurements. This +module connects a number of sessions, primes each with one event so the +backend materializes its state, reports readiness on stdout, and holds the +connections until stdin closes. +""" + +from __future__ import annotations + +import json +import sys + +import socketio + +from tests.benchmarks.support.socket_client import _connect + + +def main() -> int: + """Connect, prime, and hold the requested sessions. + + Returns: + Process exit code. + """ + url, token_prefix, count, payload_json = sys.argv[1:5] + payload = json.loads(payload_json) + clients: list[socketio.SimpleClient] = [] + try: + for index in range(int(count)): + client = socketio.SimpleClient(logger=False) + _connect(client, url, f"{token_prefix}-{index}", "/_event", timeout=10) + client.emit("event", payload) + response = client.receive(timeout=10) + while response and response[0] != "event": + response = client.receive(timeout=10) + clients.append(client) + print("ready", flush=True) + sys.stdin.read() + finally: + for client in clients: + if client.connected: + client.disconnect() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/benchmarks/support/loop_probe.py b/tests/benchmarks/support/loop_probe.py new file mode 100644 index 00000000000..366a65bcf9c --- /dev/null +++ b/tests/benchmarks/support/loop_probe.py @@ -0,0 +1,122 @@ +"""Wall-time probes for asyncio performance benchmarks.""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from types import TracebackType + +from .report import percentile + + +@dataclass +class EventLoopProbe: + """Sample event-loop lag, tasks, and benchmark-specific gauges.""" + + interval: float = 0.01 + gauges: Mapping[str, Callable[[], int]] = field(default_factory=dict) + lag_samples: list[float] = field(default_factory=list, init=False) + task_samples: list[int] = field(default_factory=list, init=False) + gauge_samples: dict[str, list[int]] = field(default_factory=dict, init=False) + _task: asyncio.Task[None] | None = field(default=None, init=False, repr=False) + _deadline: float | None = field(default=None, init=False, repr=False) + + @property + def peak_tasks(self) -> int: + """Return the largest observed live-task count. + + Returns: + Peak live tasks. + """ + return max(self.task_samples, default=0) + + async def __aenter__(self) -> EventLoopProbe: + """Start sampling and allow the sampler task to initialize. + + Returns: + The running probe. + """ + if self.interval <= 0: + msg = "Event-loop probe interval must be greater than zero." + raise ValueError(msg) + self.lag_samples.clear() + self.task_samples.clear() + self.gauge_samples = {name: [] for name in self.gauges} + self._deadline = None + self._task = asyncio.create_task( + self._sample(), + name="reflex_benchmark_event_loop_probe", + ) + await asyncio.sleep(0) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Stop the sampler. + + Args: + exc_type: Exception type raised by the measured workload, if any. + exc_value: Exception raised by the measured workload, if any. + traceback: Traceback raised by the measured workload, if any. + """ + if self._task is None: + return + loop = asyncio.get_running_loop() + if self._deadline is not None and loop.time() >= self._deadline: + self._record_sample(loop, self._deadline) + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _sample(self) -> None: + """Record scheduling delay, tasks, and custom gauges.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + self.interval + while True: + self._deadline = deadline + await asyncio.sleep(max(0.0, deadline - loop.time())) + now = self._record_sample(loop, deadline) + deadline = max(deadline + self.interval, now + self.interval) + + def _record_sample(self, loop: asyncio.AbstractEventLoop, deadline: float) -> float: + """Record one probe sample and return its observation time. + + Returns: + Event-loop clock time used for the sample. + """ + now = loop.time() + self.lag_samples.append(max(0.0, now - deadline)) + tasks = asyncio.all_tasks(loop) + if self._task is not None: + tasks.discard(self._task) + self.task_samples.append(len(tasks)) + for name, gauge in self.gauges.items(): + self.gauge_samples[name].append(gauge()) + return now + + def summary(self) -> dict[str, float | int]: + """Return stable summary fields for benchmark JSON output. + + Returns: + Sample count, peak tasks, custom gauge peaks, and lag percentiles. + """ + summary: dict[str, float | int] = { + "sample_count": len(self.lag_samples), + "peak_tasks": self.peak_tasks, + "lag_p50_ms": percentile(self.lag_samples, 50) * 1000, + "lag_p95_ms": percentile(self.lag_samples, 95) * 1000, + "lag_p99_ms": percentile(self.lag_samples, 99) * 1000, + "lag_max_ms": max(self.lag_samples, default=0.0) * 1000, + } + summary.update({ + f"peak_{name}": max(samples, default=0) + for name, samples in self.gauge_samples.items() + }) + return summary diff --git a/tests/benchmarks/support/pipeline_trace.py b/tests/benchmarks/support/pipeline_trace.py new file mode 100644 index 00000000000..2c78df92e47 --- /dev/null +++ b/tests/benchmarks/support/pipeline_trace.py @@ -0,0 +1,129 @@ +"""Benchmark-only event pipeline tracing.""" + +from __future__ import annotations + +import dataclasses +import json +import time +from collections import defaultdict, deque +from collections.abc import Iterable, Sequence +from pathlib import Path +from typing import Any + + +@dataclasses.dataclass(frozen=True) +class StageEvent: + """One monotonic pipeline stage observation.""" + + token: str + stage: str + timestamp_ns: int + + +@dataclasses.dataclass +class PipelineTrace: + """Collect per-token pipeline stages without changing runtime code.""" + + events: list[StageEvent] = dataclasses.field(default_factory=list) + + def record(self, token: str, stage: str) -> None: + """Record a pipeline stage using a monotonic clock. + + Args: + token: Event token or transaction identifier. + stage: Stable stage name. + """ + self.events.append( + StageEvent(token=token, stage=stage, timestamp_ns=time.perf_counter_ns()) + ) + + def durations_ms(self, stages: Sequence[tuple[str, str]]) -> dict[str, list[float]]: + """Calculate named adjacent or non-adjacent stage durations. + + Args: + stages: Start/end stage pairs. + + Returns: + Durations grouped under ``start_to_end`` keys. + """ + grouped: dict[str, list[StageEvent]] = defaultdict(list) + for event in self.events: + grouped[event.token].append(event) + + durations: dict[str, list[float]] = { + f"{start}_to_{end}": [] for start, end in stages + } + for token_events in grouped.values(): + ordered_events = sorted(token_events, key=lambda event: event.timestamp_ns) + for start, end in stages: + pending_starts: deque[int] = deque() + for event in ordered_events: + if event.stage == start: + pending_starts.append(event.timestamp_ns) + elif event.stage == end and pending_starts: + started_at = pending_starts.popleft() + durations[f"{start}_to_{end}"].append( + (event.timestamp_ns - started_at) / 1_000_000 + ) + return durations + + def chrome_trace(self) -> dict[str, Any]: + """Convert observations to Chrome instant trace events. + + Chrome trace format requires integer pid/tid, so each token maps to a + stable small tid, named via ``thread_name`` metadata events. + + Returns: + A Chrome trace JSON mapping. + """ + tids: dict[str, int] = {} + for event in self.events: + tids.setdefault(event.token, len(tids) + 1) + thread_names: list[dict[str, Any]] = [ + { + "name": "thread_name", + "ph": "M", + "pid": 1, + "tid": tid, + "args": {"name": token}, + } + for token, tid in tids.items() + ] + instants: list[dict[str, Any]] = [ + { + "name": event.stage, + "cat": "reflex.event_pipeline", + "ph": "i", + "s": "t", + "pid": 1, + "tid": tids[event.token], + "ts": event.timestamp_ns / 1000, + } + for event in self.events + ] + return {"traceEvents": thread_names + instants} + + def write_chrome_trace(self, path: str | Path) -> Path: + """Write Chrome trace JSON. + + Args: + path: Destination path. + + Returns: + The resolved destination path. + """ + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(self.chrome_trace(), indent=2) + "\n", + encoding="utf-8", + ) + return destination + + def extend(self, events: Iterable[StageEvent]) -> None: + """Append existing observations. + + Args: + events: Observations to append. + """ + self.events.extend(events) diff --git a/tests/benchmarks/support/redis.py b/tests/benchmarks/support/redis.py new file mode 100644 index 00000000000..3d0b0f93947 --- /dev/null +++ b/tests/benchmarks/support/redis.py @@ -0,0 +1,82 @@ +"""Real-Redis helpers for scheduled performance suites.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from urllib.parse import urlparse + +from redis.asyncio import Redis + +from reflex.istate.manager.redis import StateManagerRedis + +REDIS_URL_ENV = "REFLEX_PERF_REDIS_URL" + + +def performance_redis_url() -> str: + """Return the isolated Redis database used by performance tests. + + Returns: + Redis connection URL naming an explicit database index. + + Raises: + ValueError: If the configured URL has no explicit database index, which + would silently resolve to the shared default database 0. + """ + url = os.environ.get(REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + parsed = urlparse(url) + if not parsed.path.strip("/").isdigit() and "db=" not in parsed.query: + msg = ( + f"{REDIS_URL_ENV}={url!r} must name an explicit database index " + "(e.g. redis://host:6379/15); performance tests flush their " + "database and must never target the shared default database." + ) + raise ValueError(msg) + return url + + +@asynccontextmanager +async def real_redis_state_manager() -> AsyncIterator[StateManagerRedis]: + """Create an isolated state manager backed by a reachable Redis server. + + The configured database must be empty and dedicated to performance tests; + it is flushed on exit. + + Yields: + A state manager whose database is empty at entry and exit. + + Raises: + ConnectionError: If the configured Redis server is unreachable. + RuntimeError: If the configured database already contains keys. + """ + url = performance_redis_url() + redis = Redis.from_url(url) + try: + await redis.ping() # pyright: ignore [reportGeneralTypeIssues] + if await redis.dbsize(): + msg = ( + f"Redis database at {url!r} is not empty; performance tests " + "flush it on exit, so point " + f"{REDIS_URL_ENV} at an empty, dedicated database." + ) + raise RuntimeError(msg) + manager = StateManagerRedis(redis=redis) + try: + yield manager + finally: + # Close the manager before flushing: with oplock enabled, close() + # cancels lease-breaker tasks that re-persist cached states, so + # flushing first would leave those write-backs behind and the next + # run's empty-database guard would fail. close() also disposes the + # shared connection pool, so flush through a fresh client. + try: + await manager.close() + finally: + cleanup = Redis.from_url(url) + try: + await cleanup.flushdb() + finally: + await cleanup.aclose() + finally: + await redis.aclose() diff --git a/tests/benchmarks/support/report.py b/tests/benchmarks/support/report.py new file mode 100644 index 00000000000..2cd8c105386 --- /dev/null +++ b/tests/benchmarks/support/report.py @@ -0,0 +1,238 @@ +"""Versioned result schema for wall-time and load benchmarks.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import platform +import subprocess +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 + + +def percentile(values: Sequence[float], value: float) -> float: + """Return a linearly interpolated percentile. + + Args: + values: Numeric observations. + value: Percentile in the inclusive range 0 through 100. + + Returns: + The interpolated percentile, or zero for an empty sequence. + + Raises: + ValueError: If the requested percentile is outside the valid range. + """ + if not 0 <= value <= 100: + msg = "Percentile must be between 0 and 100." + raise ValueError(msg) + if not values: + return 0.0 + ordered = sorted(values) + position = (len(ordered) - 1) * value / 100 + lower_index = int(position) + upper_index = min(lower_index + 1, len(ordered) - 1) + fraction = position - lower_index + return ( + ordered[lower_index] + (ordered[upper_index] - ordered[lower_index]) * fraction + ) + + +def _git_value(*args: str) -> str | None: + """Read a value from the current Git checkout when available. + + Args: + args: Arguments following ``git``. + + Returns: + Stripped command output, or ``None`` outside a Git checkout. + """ + try: + result = subprocess.run( + ["git", *args], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout.strip() or None + + +@dataclasses.dataclass(frozen=True) +class BenchmarkEnvironment: + """Runtime and repository metadata attached to a performance report.""" + + commit: str | None + branch: str | None + python: str + implementation: str + operating_system: str + machine: str + processor: str + cpu_count: int | None + ci: bool + + @classmethod + def detect(cls) -> BenchmarkEnvironment: + """Detect the current performance environment. + + Returns: + Environment metadata suitable for cross-run validation. + """ + return cls( + commit=os.environ.get("GITHUB_SHA") or _git_value("rev-parse", "HEAD"), + branch=os.environ.get("GITHUB_HEAD_REF") + or os.environ.get("GITHUB_REF_NAME") + or _git_value("branch", "--show-current"), + python=platform.python_version(), + implementation=platform.python_implementation(), + operating_system=platform.platform(), + machine=platform.machine(), + processor=platform.processor(), + cpu_count=os.cpu_count(), + ci=os.environ.get("CI", "").lower() == "true", + ) + + +@dataclasses.dataclass(frozen=True) +class BenchmarkResult: + """One parameterized performance benchmark result.""" + + name: str + parameters: Mapping[str, Any] + observations_ms: Sequence[float] + metrics: Mapping[str, float | int] + warmup_iterations: int = 0 + measurement_iterations: int = 1 + + def summary(self) -> dict[str, float | int]: + """Summarize the benchmark's latency observations. + + Returns: + Count and common latency percentiles in milliseconds. + """ + values = self.observations_ms + return { + "count": len(values), + "mean_ms": sum(values) / len(values) if values else 0.0, + "p50_ms": percentile(values, 50), + "p90_ms": percentile(values, 90), + "p95_ms": percentile(values, 95), + "p99_ms": percentile(values, 99), + "max_ms": max(values, default=0.0), + } + + def as_dict(self) -> dict[str, Any]: + """Serialize this result. + + Returns: + A JSON-compatible result mapping. + """ + return { + "name": self.name, + "parameters": dict(self.parameters), + "warmup_iterations": self.warmup_iterations, + "measurement_iterations": self.measurement_iterations, + "observations_ms": list(self.observations_ms), + "summary": self.summary(), + "metrics": dict(self.metrics), + } + + +@dataclasses.dataclass +class PerformanceReport: + """Collection of results produced in one benchmark environment.""" + + suite: str + results: list[BenchmarkResult] = dataclasses.field(default_factory=list) + environment: BenchmarkEnvironment = dataclasses.field( + default_factory=BenchmarkEnvironment.detect + ) + metadata: dict[str, Any] = dataclasses.field(default_factory=dict) + + def add(self, result: BenchmarkResult) -> None: + """Append a benchmark result. + + Args: + result: Result to append. + """ + self.results.append(result) + + def as_dict(self) -> dict[str, Any]: + """Serialize the report. + + Returns: + A JSON-compatible report mapping. + """ + return { + "schema_version": SCHEMA_VERSION, + "suite": self.suite, + "environment": dataclasses.asdict(self.environment), + "metadata": self.metadata, + "results": [result.as_dict() for result in self.results], + } + + def write(self, path: str | Path) -> Path: + """Write the report as deterministic formatted JSON. + + Args: + path: Destination path. + + Returns: + The resolved destination path. + """ + destination = Path(path).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(self.as_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return destination + + +def current_process_metrics() -> dict[str, int]: + """Return portable process resource metrics. + + Returns: + Resident memory, thread count, and process ID. + """ + import psutil + + process = psutil.Process() + return { + "pid": process.pid, + "rss_bytes": process.memory_info().rss, + "threads": process.num_threads(), + } + + +def runtime_metadata() -> dict[str, str]: + """Return additional runtime identifiers when installed. + + Returns: + Node and Redis version strings when their commands are available. + """ + metadata = {"python_executable": sys.executable} + for name, command in { + "node": ["node", "--version"], + "redis": ["redis-server", "--version"], + }.items(): + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + continue + metadata[name] = (result.stdout or result.stderr).strip() + return metadata diff --git a/tests/benchmarks/support/socket_client.py b/tests/benchmarks/support/socket_client.py new file mode 100644 index 00000000000..e64c782c128 --- /dev/null +++ b/tests/benchmarks/support/socket_client.py @@ -0,0 +1,334 @@ +"""Protocol-aware Socket.IO client used by scheduled load tests.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import functools +import json +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.parse import quote + +import socketio +from socketio.exceptions import TimeoutError as SocketTimeoutError + +# Silence window that marks the connection quiescent while draining the extra +# deltas the first event on a fresh token emits (hydrate + on_load_internal). +_PRIME_DRAIN_TIMEOUT = 0.5 + + +@dataclasses.dataclass(frozen=True) +class ClientLoadResult: + """Latency observations and failures for one load client.""" + + token: str + latencies_ms: tuple[float, ...] + errors: tuple[str, ...] + payload_bytes: tuple[int, ...] = () + + +@dataclasses.dataclass(frozen=True) +class ReconnectResult: + """Connection and first-response observations for one reconnect client.""" + + token: str + connect_ms: float + first_response_ms: float + errors: tuple[str, ...] + + +def _payload_size(response: Any) -> int: + """Approximate the wire size of a received update payload. + + Args: + response: Decoded ``[event, payload]`` message from the client. + + Returns: + Compact JSON byte length of the payload, excluding Socket.IO framing. + """ + payload = response[1] if len(response) > 1 else None + return len(json.dumps(payload, separators=(",", ":"), default=str)) + + +def _connect( + client: socketio.SimpleClient, + url: str, + token: str, + namespace: str, + timeout: float, +) -> None: + """Connect a client the way the Reflex frontend does. + + Args: + client: Blocking Socket.IO client. + url: Backend Socket.IO URL. + token: Reflex client token. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection wait. + """ + separator = "&" if "?" in url else "?" + client.connect( + f"{url}{separator}token={quote(token)}", + transports=["websocket"], + socketio_path="_event", + headers={"Origin": url}, + namespace=namespace, + wait_timeout=max(1, int(timeout)), + ) + + +def _drain(client: socketio.SimpleClient, timeout: float) -> None: + """Consume buffered responses until the socket is quiet for ``timeout``. + + Args: + client: Blocking Socket.IO client. + timeout: Silence window that marks the connection quiescent. + """ + while True: + try: + client.receive(timeout=timeout) + except SocketTimeoutError: + return + + +def _prime( + client: socketio.SimpleClient, + event_name: str, + payload: Mapping[str, Any], + timeout: float, +) -> None: + """Hydrate a fresh token so later events observe a clean 1:1 response. + + The first application event on a fresh token triggers the rehydrate path, + which emits extra full-state and ``on_load_internal`` deltas under the same + ``event`` message name. Sending one warmup event and draining every response + keeps the measured loop from reading those deltas as if they answered later + events, which would otherwise report each latency pipelined two events deep. + + Args: + client: Blocking Socket.IO client. + event_name: Socket event name used for requests. + payload: Event payload emitted to trigger hydration. + timeout: Maximum wait bounding the drain silence window. + """ + client.emit(event_name, dict(payload)) + _drain(client, min(timeout, _PRIME_DRAIN_TIMEOUT)) + + +async def run_socket_client( + url: str, + token: str, + payload: Mapping[str, Any], + events: int, + *, + event_name: str = "event", + response_name: str = "event", + namespace: str = "/_event", + timeout: float = 10, + executor: concurrent.futures.Executor | None = None, +) -> ClientLoadResult: + """Send events sequentially and measure matching socket responses. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted for each operation. + events: Number of operations. + event_name: Socket event name used for requests. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum response wait per operation. + executor: Optional executor that owns the blocking socket client. + + Returns: + Per-operation latency and error observations. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + executor, + functools.partial( + _run_socket_client_sync, + url, + token, + payload, + events, + event_name, + response_name, + namespace, + timeout, + ), + ) + + +def _run_socket_client_sync( + url: str, + token: str, + payload: Mapping[str, Any], + events: int, + event_name: str, + response_name: str, + namespace: str, + timeout: float, +) -> ClientLoadResult: + """Run a load client with the websocket-client transport. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted for each operation. + events: Number of operations. + event_name: Socket event name used for requests. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum response wait per operation. + + Returns: + Per-operation latency and error observations. + """ + client = socketio.SimpleClient(logger=False) + latencies: list[float] = [] + payload_sizes: list[int] = [] + errors: list[str] = [] + + try: + _connect(client, url, token, namespace, timeout) + _prime(client, event_name, payload, timeout) + for _ in range(events): + started = time.perf_counter_ns() + client.emit(event_name, dict(payload)) + try: + response = client.receive(timeout=timeout) + while response and response[0] != response_name: + response = client.receive(timeout=timeout) + except SocketTimeoutError: + errors.append("response_timeout") + continue + latencies.append((time.perf_counter_ns() - started) / 1_000_000) + payload_sizes.append(_payload_size(response)) + except Exception as err: + errors.append(f"{type(err).__name__}: {err}") + finally: + if client.connected: + client.disconnect() + + return ClientLoadResult( + token, tuple(latencies), tuple(errors), tuple(payload_sizes) + ) + + +async def run_reconnect_client( + url: str, + token: str, + payload: Mapping[str, Any], + *, + event_name: str = "event", + response_name: str = "event", + namespace: str = "/_event", + timeout: float = 10, + executor: concurrent.futures.Executor | None = None, +) -> ReconnectResult: + """Connect, send one event, and measure connect and first-response time. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted after connecting. + event_name: Socket event name used for the request. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection and response wait. + executor: Optional executor that owns the blocking socket client. + + Returns: + Connect and first-response observations. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + executor, + functools.partial( + _run_reconnect_client_sync, + url, + token, + payload, + event_name, + response_name, + namespace, + timeout, + ), + ) + + +def _run_reconnect_client_sync( + url: str, + token: str, + payload: Mapping[str, Any], + event_name: str, + response_name: str, + namespace: str, + timeout: float, +) -> ReconnectResult: + """Run one reconnect client with the websocket-client transport. + + Args: + url: Backend Socket.IO URL. + token: Reflex client token. + payload: Event payload emitted after connecting. + event_name: Socket event name used for the request. + response_name: Socket event name carrying state updates. + namespace: Reflex Socket.IO namespace. + timeout: Maximum connection and response wait. + + Returns: + Connect and first-response observations. + """ + client = socketio.SimpleClient(logger=False) + connect_ms = 0.0 + first_response_ms = 0.0 + errors: list[str] = [] + + try: + started = time.perf_counter_ns() + _connect(client, url, token, namespace, timeout) + connect_ms = (time.perf_counter_ns() - started) / 1_000_000 + started = time.perf_counter_ns() + client.emit(event_name, dict(payload)) + response = client.receive(timeout=timeout) + while response and response[0] != response_name: + response = client.receive(timeout=timeout) + first_response_ms = (time.perf_counter_ns() - started) / 1_000_000 + except SocketTimeoutError: + errors.append("response_timeout") + except Exception as err: + errors.append(f"{type(err).__name__}: {err}") + finally: + if client.connected: + client.disconnect() + + return ReconnectResult(token, connect_ms, first_response_ms, tuple(errors)) + + +async def run_clients( + clients: int, + factory: Callable[[int, concurrent.futures.Executor], Any], +) -> list[Any]: + """Run a parameterized set of clients concurrently. + + Args: + clients: Number of clients. + factory: Callable returning an awaitable for a client index and executor. + + Returns: + Client results in index order. + """ + executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(1, clients)) + try: + return list( + await asyncio.gather( + *(factory(index, executor) for index in range(clients)) + ) + ) + finally: + executor.shutdown(wait=True) diff --git a/tests/benchmarks/support/states.py b/tests/benchmarks/support/states.py new file mode 100644 index 00000000000..1e637b971b7 --- /dev/null +++ b/tests/benchmarks/support/states.py @@ -0,0 +1,199 @@ +"""Scalable state fixtures shared by performance suites.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import cast + +from reflex_base.registry import RegistrationContext + +import reflex as rx +from reflex.state import BaseState + + +class PerformanceState(rx.State): + """Representative state with mutable and computed values.""" + + counter: rx.Field[int] = rx.field(0) + numbers: rx.Field[list[int]] = rx.field(default_factory=list) + mapping: rx.Field[dict[str, int]] = rx.field(default_factory=dict) + + @rx.event + def increment(self): + """Increment the scalar counter.""" + self.counter += 1 + + @rx.event + def append_number(self, value: int): + """Append a number. + + Args: + value: Value to append. + """ + self.numbers.append(value) + + @rx.var(cache=True) + def total(self) -> int: + """Return the sum of numbers and the counter. + + Returns: + Current aggregate. + """ + return self.counter + sum(self.numbers) + + @rx.var(cache=True) + def doubled_total(self) -> int: + """Return a computed value depending on another computed value. + + Returns: + Twice the current aggregate. + """ + return self.total * 2 + + +@contextmanager +def isolated_performance_registry() -> Iterator[None]: + """Register only ``PerformanceState`` for deterministic cold construction. + + A state manager builds a token's root state by instantiating the whole + substate tree recorded in the ambient ``RegistrationContext``. Without + isolation that tree includes every module-level state in the collected + benchmark session, so adding or removing a state anywhere shifts a cold + ``get_state`` result on CodSpeed. Entering a fresh context registered with + only ``PerformanceState`` fixes the tree to this state's subtree regardless + of the rest of the session. + + Yields: + None, with the isolated registration context active. + """ + with RegistrationContext(): + RegistrationContext.register_base_state(PerformanceState) + yield + + +def initialized_state(size: int) -> PerformanceState: + """Create a state with deterministic mutable values. + + Args: + size: Number of list and mapping elements. + + Returns: + Initialized performance state. + """ + state = PerformanceState( + _reflex_internal_init=True # pyright: ignore [reportCallIssue] + ) + state.numbers = list(range(size)) + state.mapping = {f"key_{index}": index for index in range(size)} + state._clean() + return state + + +def get_performance_state(root_state: BaseState) -> PerformanceState: + """Return the performance state from a managed root hierarchy. + + Args: + root_state: Root state returned by a state manager. + + Returns: + Performance substate used by benchmark workloads. + """ + return cast( + PerformanceState, + root_state.get_substate(PerformanceState.get_full_name().split(".")), + ) + + +@contextmanager +def computed_fanout_state(fanout: int) -> Iterator[tuple[BaseState, tuple[str, ...]]]: + """Create a state whose scalar value feeds a requested number of vars. + + Args: + fanout: Number of cached computed vars depending on ``counter``. + + Yields: + State instance and the generated computed-var names. + """ + with RegistrationContext(): + namespace: dict[str, object] = { + "__module__": __name__, + "__annotations__": {"counter": rx.Field[int]}, + "counter": rx.field(0), + } + names = tuple(f"derived_{index}" for index in range(fanout)) + for index, name in enumerate(names): + + def derived(self: BaseState, offset: int = index) -> int: + """Return the source counter offset by a stable value.""" + return cast(PerformanceState, self).counter + offset + + derived.__name__ = name + namespace[name] = rx.var(cache=True)(derived) + state_type = cast( + type[BaseState], + type(f"PerformanceFanout{fanout}", (rx.State,), namespace), + ) + yield ( + state_type( # pyright: ignore [reportCallIssue] + _reflex_internal_init=True + ), + names, + ) + + +@contextmanager +def state_tree(shape: str) -> Iterator[BaseState]: + """Build an isolated registered state hierarchy. + + Args: + shape: ``depth_10``, ``width_10``, or ``three_by_three``. + + Yields: + Root state containing the requested substate hierarchy. + + Raises: + ValueError: If the shape is unknown. + """ + with RegistrationContext(): + RegistrationContext.register_base_state(PerformanceState) + if shape == "depth_10": + parent: type[BaseState] = PerformanceState + for index in range(10): + parent = cast( + type[BaseState], + type( + f"PerformanceDepth{index}", + (parent,), + {"__module__": __name__}, + ), + ) + elif shape == "width_10": + for index in range(10): + type( + f"PerformanceWide{index}", + (PerformanceState,), + {"__module__": __name__}, + ) + elif shape == "three_by_three": + for parent_index in range(3): + parent = cast( + type[BaseState], + type( + f"PerformanceBranch{parent_index}", + (PerformanceState,), + {"__module__": __name__}, + ), + ) + for child_index in range(3): + type( + f"PerformanceBranch{parent_index}Child{child_index}", + (parent,), + {"__module__": __name__}, + ) + else: + msg = f"Unknown performance state-tree shape: {shape}" + raise ValueError(msg) + yield PerformanceState.get_root_state()( # pyright: ignore [reportCallIssue] + _reflex_internal_init=True + ) diff --git a/tests/benchmarks/test_event_loop.py b/tests/benchmarks/test_event_loop.py new file mode 100644 index 00000000000..509f985518f --- /dev/null +++ b/tests/benchmarks/test_event_loop.py @@ -0,0 +1,443 @@ +"""Deterministic benchmarks for event-processor scheduling and lifecycle paths.""" + +import asyncio +import contextvars +from collections.abc import Coroutine, Mapping +from types import SimpleNamespace +from typing import Any, cast + +from pytest_codspeed import BenchmarkFixture +from reflex_base.event.context import EventContext +from reflex_base.event.processor import EventFuture, EventProcessor +from reflex_base.registry import RegistrationContext + +from reflex.app import EventNamespace +from reflex.event import Event, EventHandler +from reflex.state import StateUpdate + +# Drain budget far above any instrumented runtime so measured shutdowns never +# flip from graceful drain to cancellation under benchmark slowdown. +_DRAIN_BUDGET_SECONDS = 60.0 + + +async def _noop_handler() -> None: + """Complete without application work.""" + + +async def _chain_handler() -> None: + """Enqueue a child event from the active event context.""" + await EventContext.get().enqueue(Event.from_event_type(NOOP_EVENT())[0]) + + +async def _slow_handler() -> None: + """Remain pending until shutdown cancellation.""" + await asyncio.sleep(60) + + +async def _yield_handler() -> None: + """Emit rapid intermediate deltas in order.""" + context = EventContext.get() + await context.emit_delta({"state": {"value": 1}}) + await context.emit_delta({"state": {"value": 2}}) + + +NOOP_EVENT = EventHandler(fn=_noop_handler) +CHAIN_EVENT = EventHandler(fn=_chain_handler) +SLOW_EVENT = EventHandler(fn=_slow_handler) +YIELD_EVENT = EventHandler(fn=_yield_handler) + + +def _register_handlers() -> RegistrationContext: + """Create an attached registry containing benchmark handlers. + + Returns: + Attached registration context. + """ + context = RegistrationContext() + context.__enter__() + for handler in (NOOP_EVENT, CHAIN_EVENT, SLOW_EVENT, YIELD_EVENT): + RegistrationContext.register_event_handler(handler) + return context + + +def _make_processor() -> EventProcessor: + """Construct and configure a processor outside any measured region. + + Returns: + A configured event processor with a generous drain budget. + """ + return EventProcessor(graceful_shutdown_timeout=_DRAIN_BUDGET_SECONDS).configure() + + +def _run_in_context( + loop: asyncio.AbstractEventLoop, + ctx: contextvars.Context, + coro: Coroutine[Any, Any, Any], +) -> Any: + """Run a coroutine on the loop inside a fixed contextvars context. + + Lets ``EventProcessor.start`` and ``stop`` run in separate + ``run_until_complete`` calls while sharing the context that owns the + attached ``EventContext`` token. + + Args: + loop: The event loop to run on. + ctx: The shared context for the wrapping task. + coro: The coroutine to run. + + Returns: + The coroutine's result. + """ + return loop.run_until_complete(loop.create_task(coro, context=ctx)) + + +def test_event_queue_dispatch(benchmark: BenchmarkFixture): + """Benchmark enqueueing one no-op event and awaiting its completion. + + Processor construction, configuration, startup, and shutdown all happen + in per-round setup and teardown, so only queueing and dispatch are + measured. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + ctx = contextvars.copy_context() + + async def dispatch(processor: EventProcessor, event: Event) -> None: + """Enqueue one no-op event and await its completion. + + Args: + processor: The started processor. + event: The pre-built event to enqueue. + """ + future = await processor.enqueue("token", event) + await future.wait_all() + + def setup(): + """Construct and start a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the started processor and event. + """ + processor = _make_processor() + _run_in_context(loop, ctx, processor.start()) + # Build the event in setup so only queueing and dispatch are measured. + event = Event.from_event_type(NOOP_EVENT())[0] + return ((processor, event), {}) + + def run(processor: EventProcessor, event: Event) -> None: + """Dispatch one event through the started processor. + + Runs inside ``ctx`` where ``start`` attached the ``EventContext``, so + enqueue takes the production ``EventContext.get().fork()`` path rather + than the ``LookupError`` fallback a bare loop context would trigger. + + Args: + processor: The started processor. + event: The pre-built event to enqueue. + """ + _run_in_context(loop, ctx, dispatch(processor, event)) + + def teardown(processor: EventProcessor, event: Event) -> None: + """Stop the measured round's processor. + + Args: + processor: The started processor. + event: The event dispatched this round (unused at teardown). + """ + _run_in_context(loop, ctx, processor.stop()) + + try: + benchmark.pedantic(run, setup=setup, teardown=teardown) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_future_tree(benchmark: BenchmarkFixture): + """Benchmark a parent event that enqueues and waits for one child. + + Processor lifecycle happens in per-round setup and teardown, so only + the parent dispatch, chained child enqueue, and future-tree completion + are measured. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + ctx = contextvars.copy_context() + + async def dispatch(processor: EventProcessor, event: Event) -> None: + """Enqueue one chaining event and await the whole future tree. + + Args: + processor: The started processor. + event: The pre-built parent event to enqueue. + """ + future = await processor.enqueue("token", event) + await future.wait_all() + + def setup(): + """Construct and start a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the started processor and event. + """ + processor = _make_processor() + _run_in_context(loop, ctx, processor.start()) + # Build the event in setup so only dispatch and chaining are measured. + event = Event.from_event_type(CHAIN_EVENT())[0] + return ((processor, event), {}) + + def run(processor: EventProcessor, event: Event) -> None: + """Dispatch the parent event through the started processor. + + Runs inside ``ctx`` where ``start`` attached the ``EventContext``, so + enqueue takes the production ``EventContext.get().fork()`` path rather + than the ``LookupError`` fallback a bare loop context would trigger. + + Args: + processor: The started processor. + event: The pre-built parent event to enqueue. + """ + _run_in_context(loop, ctx, dispatch(processor, event)) + + def teardown(processor: EventProcessor, event: Event) -> None: + """Stop the measured round's processor. + + Args: + processor: The started processor. + event: The event dispatched this round (unused at teardown). + """ + _run_in_context(loop, ctx, processor.stop()) + + try: + benchmark.pedantic(run, setup=setup, teardown=teardown) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_shutdown_drain(benchmark: BenchmarkFixture): + """Benchmark graceful draining of ten queued no-op events. + + The measured region intentionally covers processor startup, the + ten-event enqueue burst, and the graceful drain performed by shutdown; + construction and configuration happen in per-round setup. Events use + independent tokens because same-token bursts serialize and are partially + cancelled rather than drained at shutdown. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + drained: list[EventFuture] = [] + + async def drain(processor: EventProcessor) -> list[EventFuture]: + """Enqueue a burst and let processor shutdown drain it. + + Args: + processor: The configured processor. + + Returns: + The futures for the enqueued burst. + """ + async with processor: + return [ + await processor.enqueue( + f"token-{index}", Event.from_event_type(NOOP_EVENT())[0] + ) + for index in range(10) + ] + + def setup(): + """Construct a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((_make_processor(),), {}) + + def run(processor: EventProcessor) -> None: + """Run the burst-and-drain round, recording its futures. + + Args: + processor: The configured processor. + """ + drained[:] = loop.run_until_complete(drain(processor)) + + try: + benchmark.pedantic(run, setup=setup) + assert len(drained) == 10 + assert all(future.done() and not future.cancelled() for future in drained), ( + "shutdown cancelled queued events instead of draining them" + ) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_event_shutdown_cancel(benchmark: BenchmarkFixture): + """Benchmark forced cancellation during zero-grace shutdown. + + The measured region intentionally covers startup, enqueueing one slow + event, and the zero-budget shutdown that cancels it; construction and + configuration happen in per-round setup. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + + async def cancel(processor: EventProcessor) -> bool: + """Start a slow task and stop without a drain period. + + Args: + processor: The configured processor. + + Returns: + Whether shutdown cancelled its future. + """ + async with processor: + future = await processor.enqueue( + "token", Event.from_event_type(SLOW_EVENT())[0] + ) + await asyncio.sleep(0) + return future.cancelled() + + def setup(): + """Construct a zero-grace processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((EventProcessor(graceful_shutdown_timeout=0).configure(),), {}) + + def run(processor: EventProcessor) -> bool: + """Run the cancel round. + + Args: + processor: The configured processor. + + Returns: + Whether shutdown cancelled the slow event's future. + """ + return loop.run_until_complete(cancel(processor)) + + try: + assert benchmark.pedantic(run, setup=setup) + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_yield_intermediate_update(benchmark: BenchmarkFixture): + """Benchmark ordered streaming of rapid intermediate deltas. + + The measured region intentionally covers startup, streaming both deltas, + and clean shutdown; construction and configuration happen in per-round + setup. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + registry = _register_handlers() + + async def collect(processor: EventProcessor) -> list[Mapping[str, Any]]: + """Stream all deltas from the yielding event. + + Args: + processor: The configured processor. + + Returns: + Ordered emitted deltas. + """ + async with processor: + return [ + delta + async for delta in processor.enqueue_stream_delta( + "token", Event.from_event_type(YIELD_EVENT())[0] + ) + ] + + def setup(): + """Construct a processor for the next measured round. + + Returns: + Pedantic (args, kwargs) holding the configured processor. + """ + return ((_make_processor(),), {}) + + def run(processor: EventProcessor) -> list[Mapping[str, Any]]: + """Run the streaming round. + + Args: + processor: The configured processor. + + Returns: + Ordered emitted deltas. + """ + return loop.run_until_complete(collect(processor)) + + try: + deltas = benchmark.pedantic(run, setup=setup) + assert deltas == [ + {"state": {"value": 1}}, + {"state": {"value": 2}}, + ] + finally: + registry.__exit__(None, None, None) + loop.close() + + +def test_emit_update(benchmark: BenchmarkFixture): + """Benchmark emit_update's socket lookup and task-based flush scheduling. + + The Socket.IO ``emit`` call itself is mocked to a recording no-op, so + only token-to-socket routing and the emit task scheduling are measured, + not any real socket write. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + loop = asyncio.new_event_loop() + emitted: list[StateUpdate] = [] + + async def emit( # noqa: RUF029 + _event: str, update: StateUpdate, *, to: str + ) -> None: + """Record a stand-in Socket.IO write. + + Args: + _event: Socket event name. + update: State update. + to: Socket ID. + """ + assert to == "sid" + emitted.append(update) + + token_manager = SimpleNamespace( + instance_id="instance", + token_to_socket={"token": SimpleNamespace(instance_id="instance", sid="sid")}, + ) + namespace = cast( + EventNamespace, + SimpleNamespace(_token_manager=token_manager, emit=emit), + ) + update = StateUpdate(delta={"state": {"value": 1}}) + + try: + benchmark( + lambda: loop.run_until_complete( + EventNamespace.emit_update(namespace, update, "token") + ) + ) + assert emitted + finally: + loop.close() diff --git a/tests/benchmarks/test_event_payload.py b/tests/benchmarks/test_event_payload.py new file mode 100644 index 00000000000..f250010e076 --- /dev/null +++ b/tests/benchmarks/test_event_payload.py @@ -0,0 +1,43 @@ +"""Benchmarks for chained-event payload snapshotting.""" + +from typing import Any + +from pytest_codspeed import BenchmarkFixture + +from reflex.event import Event, EventHandler + +from .support.states import initialized_state + + +def _receive_payload(payload: Any) -> None: + """Define a handler accepting arbitrary representative payloads. + + Args: + payload: Event payload. + """ + + +PAYLOAD_EVENT = EventHandler(fn=_receive_payload) + + +def test_event_payload_plain_collection(benchmark: BenchmarkFixture): + """Benchmark snapshotting a large proxy-free event payload. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + payload = [{"index": index, "label": f"row-{index}"} for index in range(10_000)] + events = benchmark(lambda: Event.from_event_type(PAYLOAD_EVENT(payload))) + assert len(events[0].payload["payload"]) == len(payload) + + +def test_event_payload_mutable_proxy(benchmark: BenchmarkFixture): + """Benchmark the required detachment of a state-bound mutable proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + state = initialized_state(10_000) + payload = state.numbers + events = benchmark(lambda: Event.from_event_type(PAYLOAD_EVENT(payload))) + assert events[0].payload["payload"] == list(range(10_000)) diff --git a/tests/benchmarks/test_event_processing.py b/tests/benchmarks/test_event_processing.py index 15acf8094d4..98fc6af2a9c 100644 --- a/tests/benchmarks/test_event_processing.py +++ b/tests/benchmarks/test_event_processing.py @@ -7,12 +7,11 @@ import asyncio import traceback -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any from unittest import mock import pytest -import pytest_asyncio from pytest_codspeed import BenchmarkFixture from reflex_base.event import Event from reflex_base.event.context import EventContext @@ -20,12 +19,13 @@ from reflex_base.utils.format import format_event_handler from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import BaseStateToken from .fixtures import BenchmarkState -@pytest_asyncio.fixture -async def event_processing_harness(): +@pytest.fixture +def event_processing_harness(): """Set up the full event processing pipeline for benchmarking. Creates a ``BaseStateEventProcessor`` wired to a real @@ -33,8 +33,10 @@ async def event_processing_harness(): enqueued directly and deltas are collected via the emit callback. Yields: - An async callable that enqueues the given number of events - and waits for all expected deltas. + An async event runner, a helper that purges token state between + explicitly cold benchmark rounds, and the state manager's async + ``close``, which each test must run on its own event loop before + closing it. """ emitted_deltas: list[tuple[str, Mapping[str, Mapping[str, Any]]]] = [] @@ -67,7 +69,6 @@ def handle_backend_exception(ex: Exception) -> None: ) processor._root_context = root_context - token = "benchmark-token" handler_name = format_event_handler(BenchmarkState.event_handlers["increment"]) event = Event( name=handler_name, @@ -78,25 +79,33 @@ def handle_backend_exception(ex: Exception) -> None: payload={}, ) - async def run_events(num_events: int, num_expected_deltas: int) -> None: - """Enqueue events and wait for all deltas to be emitted. + async def run_events(tokens: Sequence[str]) -> None: + """Enqueue events for the given tokens and wait for their deltas. Args: - num_events: Number of increment events to enqueue. - num_expected_deltas: How many deltas to wait for. + tokens: Client tokens to enqueue one increment event for. """ emitted_deltas.clear() async with processor as p: async for _ in asyncio.as_completed([ - await p.enqueue(token, event) for _ in range(num_events) + await p.enqueue(token, event) for token in tokens ]): pass - assert len(emitted_deltas) == num_expected_deltas + assert len(emitted_deltas) == len(tokens) - yield run_events + def purge_tokens(tokens: Sequence[str]) -> None: + """Remove benchmark tokens from the in-memory state manager. - await state_manager.close() + Args: + tokens: Client tokens whose state and lock bookkeeping should be removed. + """ + for token in tokens: + state_manager._purge_token( # pyright: ignore [reportPrivateUsage] + BaseStateToken(ident=token, cls=BenchmarkState) + ) + + yield run_events, purge_tokens, state_manager.close def test_process_event( @@ -108,15 +117,147 @@ def test_process_event( The first event creates fresh state (cold path), the next two reuse the existing state (warm path). Only event processing is timed. + The token is purged in per-round setup so the cold path is exercised on + every measured round. Without it, CodSpeed's warmup invocation hydrates the + shared token and every subsequent round would measure three warm events, + silently duplicating ``test_process_event_warm``. + Args: - event_processing_harness: The run_events async callable. + event_processing_harness: The event runner, token purge, and shutdown helpers. benchmark: The codspeed benchmark fixture. """ - run_events = event_processing_harness - loop = asyncio.get_event_loop() - - # Each event handler (increment) does a single state mutation with - # no yields, so we expect 1 delta per event = 3 total. - @benchmark - def _(): - loop.run_until_complete(run_events(num_events=3, num_expected_deltas=3)) + run_events, purge_tokens, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + + def setup(): + """Return the batch args after purging the token for a cold round.""" + purge_tokens(["benchmark-token"]) + # Each event handler (increment) does a single state mutation with + # no yields, so we expect 1 delta per event = 3 total. + return ((["benchmark-token"] * 3,), {}) + + def run_batch(tokens): + """Process one cold-then-warm batch for the given tokens.""" + loop.run_until_complete(run_events(tokens)) + + try: + benchmark.pedantic(run_batch, setup=setup) + finally: + loop.run_until_complete(shutdown()) + loop.close() + + +def test_process_event_cold( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark one event whose token has no existing state. + + Args: + event_processing_harness: The event runner, token purge, and shutdown helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, purge_tokens, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + iteration = 0 + + def setup(): + """Return a unique cold token for the next measured round.""" + nonlocal iteration + iteration += 1 + return ((f"cold-token-{iteration}",), {}) + + def run_cold(token: str) -> None: + """Process one event with no state cached for its token.""" + loop.run_until_complete(run_events([token])) + + def teardown(token: str) -> None: + """Discard the measured token so cold rounds do not retain state.""" + purge_tokens([token]) + + try: + benchmark.pedantic(run_cold, setup=setup, teardown=teardown) + finally: + loop.run_until_complete(shutdown()) + loop.close() + + +def test_process_event_warm( + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark one event for a token whose state is already initialized. + + Args: + event_processing_harness: The event runner, token purge, and shutdown helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + token = "warm-token" + loop.run_until_complete(run_events([token])) + + @benchmark + def _(): + loop.run_until_complete(run_events([token])) + + finally: + loop.run_until_complete(shutdown()) + loop.close() + + +@pytest.mark.parametrize("num_events", [1, 10, 100]) +def test_process_event_burst_same_token( + num_events: int, + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark queued events serialized through one token. + + Args: + num_events: Number of events in the burst. + event_processing_harness: The event runner, token purge, and shutdown helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + tokens = ["burst-token"] * num_events + loop.run_until_complete(run_events([tokens[0]])) + + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) + + finally: + loop.run_until_complete(shutdown()) + loop.close() + + +@pytest.mark.parametrize("num_events", [1, 10, 100]) +def test_process_event_burst_independent_tokens( + num_events: int, + event_processing_harness, + benchmark: BenchmarkFixture, +): + """Benchmark queued events distributed across independent tokens. + + Args: + num_events: Number of independent tokens and events. + event_processing_harness: The event runner, token purge, and shutdown helpers. + benchmark: The codspeed benchmark fixture. + """ + run_events, _, shutdown = event_processing_harness + loop = asyncio.new_event_loop() + try: + tokens = [f"independent-token-{index}" for index in range(num_events)] + loop.run_until_complete(run_events(tokens)) + + @benchmark + def _(): + loop.run_until_complete(run_events(tokens)) + + finally: + loop.run_until_complete(shutdown()) + loop.close() diff --git a/tests/benchmarks/test_routes.py b/tests/benchmarks/test_routes.py new file mode 100644 index 00000000000..15d0fa2e07e --- /dev/null +++ b/tests/benchmarks/test_routes.py @@ -0,0 +1,114 @@ +"""Benchmarks for route construction and hot-path matching.""" + +import re +import sys +import types +from collections.abc import Iterator + +import pytest +from pytest_codspeed import BenchmarkFixture +from reflex_base import constants +from reflex_base.config import get_config + +from reflex.route import get_route_args, get_router + + +@pytest.fixture +def cached_rxconfig() -> Iterator[None]: + """Seed a cached ``rxconfig`` module so matching hits the config cache. + + In production ``rxconfig`` is imported into ``sys.modules`` and every + ``get_config()`` call (invoked once per ``router(path)`` match) is a dict + lookup. Without a cached module the benchmark environment rebuilds a fresh + ``Config`` on every call — a sys.path scan and full env parse that dominates + the measured region and hides real route-matching regressions. + + Yields: + None, with the cached module installed for the test's duration. + """ + module_name = constants.Config.MODULE + saved = sys.modules.get(module_name) + stub = types.ModuleType(module_name) + stub.config = get_config() # pyright: ignore [reportAttributeAccessIssue] + sys.modules[module_name] = stub + try: + yield + finally: + if saved is not None: + sys.modules[module_name] = saved + else: + sys.modules.pop(module_name, None) + + +def _routes(count: int) -> list[str]: + """Build representative static and dynamic routes. + + Args: + count: Approximate route count. + + Returns: + Route templates. + """ + routes = ["index", "posts/[slug]", "files/[[...splat]]"] + routes.extend(f"section-{index}/item-{index}" for index in range(count - 3)) + return routes + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +@pytest.mark.usefixtures("cached_rxconfig") +def test_route_matching(count: int, benchmark: BenchmarkFixture): + """Benchmark repeated matching through representative route tables. + + Args: + count: Number of routes. + benchmark: The CodSpeed benchmark fixture. + """ + router = get_router(_routes(count)) + paths = ["/", "/posts/reflex", f"/section-{count - 4}/item-{count - 4}"] + + def match_paths() -> tuple[str | None, ...]: + """Match the fixed path set. + + Returns: + Matched route templates. + """ + return tuple(router(path) for path in paths) + + result = benchmark(match_paths) + assert result == ("index", "posts/[slug]", paths[-1].removeprefix("/")) + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_route_table_construction(count: int, benchmark: BenchmarkFixture): + """Benchmark cold regex construction and route ordering. + + ``re.purge()`` runs in per-round setup so every measured round compiles + the route patterns instead of hitting the ``re`` module's pattern cache. + + Args: + count: Number of routes. + benchmark: The CodSpeed benchmark fixture. + """ + routes = _routes(count) + + def purge_regex_cache(): + """Clear the ``re`` module cache so each round compiles cold.""" + re.purge() + + router = benchmark.pedantic(lambda: get_router(routes), setup=purge_regex_cache) + assert router("/posts/reflex") == "posts/[slug]" + + +def test_route_argument_extraction(benchmark: BenchmarkFixture): + """Benchmark parsing dynamic and catch-all route arguments. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + route = "/org/[org_id]/project/[project_id]/files/[[...splat]]" + result = benchmark(lambda: get_route_args(route)) + assert result == { + "org_id": constants.RouteArgType.SINGLE, + "project_id": constants.RouteArgType.SINGLE, + "splat": constants.RouteArgType.LIST, + } diff --git a/tests/benchmarks/test_serialization.py b/tests/benchmarks/test_serialization.py new file mode 100644 index 00000000000..de613f00cea --- /dev/null +++ b/tests/benchmarks/test_serialization.py @@ -0,0 +1,61 @@ +"""Benchmarks for state-update and wire serialization.""" + +import pytest +from pytest_codspeed import BenchmarkFixture +from reflex_base.utils.format import json_dumps + +from reflex.state import StateUpdate, serialize_state_update + +from .support.events import PayloadKind, make_payload, wire_edge_cases + + +@pytest.mark.parametrize( + ("kind", "rows"), + [ + pytest.param(PayloadKind.SCALAR, 100, id="scalar_100b"), + pytest.param(PayloadKind.SCALAR, 10_000, id="scalar_10kb"), + pytest.param(PayloadKind.SCALAR, 1_000_000, id="scalar_1mb"), + pytest.param(PayloadKind.MAPPING, 100, id="mapping_100"), + pytest.param(PayloadKind.DATACLASS, 1000, id="dataclass_1000"), + pytest.param(PayloadKind.MODEL, 1000, id="model_1000"), + ], +) +def test_state_update_wire_serialization( + kind: PayloadKind, + rows: int, + benchmark: BenchmarkFixture, +): + """Benchmark encoding representative state updates to socket JSON. + + Args: + kind: Payload shape. + rows: Payload size parameter. + benchmark: The CodSpeed benchmark fixture. + """ + update = StateUpdate( + delta={"performance_state": {"payload": make_payload(kind, rows)}} + ) + encoded = benchmark(lambda: json_dumps(update)) + assert encoded + + +def test_state_update_dataclass_serialization(benchmark: BenchmarkFixture): + """Benchmark conversion of a state update into its transport dictionary. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + update = StateUpdate(delta={"performance_state": {"payload": list(range(10_000))}}) + result = benchmark(lambda: serialize_state_update(update)) + assert result["delta"] + + +def test_wire_edge_case_serialization(benchmark: BenchmarkFixture): + """Benchmark the complete set of supported special wire values. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + payload = wire_edge_cases() + encoded = benchmark(lambda: json_dumps(payload)) + assert "Reflex" in encoded diff --git a/tests/benchmarks/test_socket_headers.py b/tests/benchmarks/test_socket_headers.py new file mode 100644 index 00000000000..3c5b56d6fe4 --- /dev/null +++ b/tests/benchmarks/test_socket_headers.py @@ -0,0 +1,22 @@ +"""Benchmarks for per-event ASGI header decoding.""" + +import pytest +from pytest_codspeed import BenchmarkFixture + +from reflex.app import _decode_asgi_headers + + +@pytest.mark.parametrize("count", [0, 4, 16, 64]) +def test_decode_event_headers(count: int, benchmark: BenchmarkFixture): + """Benchmark the header decoding used by ``EventNamespace.on_event``. + + Args: + count: Number of headers. + benchmark: The CodSpeed benchmark fixture. + """ + headers = [ + (f"x-performance-{index}".encode(), f"value-{index}".encode()) + for index in range(count) + ] + decoded = benchmark(lambda: _decode_asgi_headers(headers)) + assert len(decoded) == count diff --git a/tests/benchmarks/test_state_delta.py b/tests/benchmarks/test_state_delta.py new file mode 100644 index 00000000000..513344f368b --- /dev/null +++ b/tests/benchmarks/test_state_delta.py @@ -0,0 +1,214 @@ +"""Benchmarks for dirty propagation and state delta generation.""" + +import pytest +from pytest_codspeed import BenchmarkFixture + +from .support.states import ( + PerformanceState, + computed_fanout_state, + initialized_state, + state_tree, +) + + +@pytest.mark.parametrize("size", [10, 100, 1000, 10_000]) +def test_state_delta_scalar_mutation(size: int, benchmark: BenchmarkFixture): + """Benchmark a scalar mutation and delta extraction across state sizes. + + Args: + size: Mutable collection size carried by the state. + benchmark: The CodSpeed benchmark fixture. + """ + state = initialized_state(size) + + def mutate_and_get_delta(): + """Mutate one field, calculate its delta, and reset dirtiness. + + Returns: + State delta. + """ + state.counter += 1 + delta = state.get_delta() + state._clean() + return delta + + delta = benchmark(mutate_and_get_delta) + assert delta + + +@pytest.mark.parametrize("appends", [1, 10, 100]) +def test_dirty_computed_var_propagation( + appends: int, + benchmark: BenchmarkFixture, +): + """Benchmark repeated mutable updates with computed dependencies. + + Args: + appends: Number of mutations in one measured operation. + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a stable state for one measured round. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = initialized_state(10) + _ = state.doubled_total + state._clean() + return ((state,), {}) + + def append_and_recompute(state: PerformanceState) -> int: + """Append values and resolve the dependent computed var. + + Args: + state: Fresh state for this measured round. + + Returns: + Recomputed aggregate. + """ + for value in range(appends): + state.numbers.append(value) + result = state.doubled_total + state._clean() + return result + + assert benchmark.pedantic(append_and_recompute, setup=setup) >= 0 + + +@pytest.mark.parametrize("fanout", [0, 2, 10, 50]) +def test_computed_dependency_fanout(fanout: int, benchmark: BenchmarkFixture): + """Benchmark dirty propagation across a controlled dependency fan-out. + + Args: + fanout: Number of cached vars directly depending on one source field. + benchmark: The CodSpeed benchmark fixture. + """ + + def mutate_and_delta(state): + """Mutate the shared dependency and return its resulting delta. + + Returns: + The updated source value. + """ + state.counter += 1 + state.get_delta() + state._clean() + return state.counter + + with computed_fanout_state(fanout) as (state, names): + for name in names: + getattr(state, name) + state._clean() + assert benchmark(mutate_and_delta, state) >= 1 + + +def test_resolved_delta(benchmark: BenchmarkFixture): + """Benchmark the async resolved-delta path used by event processing. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + import asyncio + + state = initialized_state(1000) + loop = asyncio.new_event_loop() + + def resolve_delta(): + """Mutate and resolve the full delta. + + Returns: + Resolved state delta. + """ + state.counter += 1 + delta = loop.run_until_complete(state._get_resolved_delta()) + state._clean() + return delta + + try: + assert benchmark(resolve_delta) + finally: + loop.close() + + +@pytest.mark.parametrize("shape", ["depth_10", "width_10", "three_by_three"]) +def test_state_tree_delta(shape: str, benchmark: BenchmarkFixture): + """Benchmark delta traversal through representative substate shapes. + + Args: + shape: Registered hierarchy shape. + benchmark: The CodSpeed benchmark fixture. + """ + contexts = [] + + def setup(): + """Create an isolated hierarchy for one measured round. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + context = state_tree(shape) + root = context.__enter__() + contexts.append(context) + return ((root,), {}) + + def mutate_and_delta(root): + """Mutate every performance substate and calculate one root delta. + + Args: + root: Root state. + + Returns: + Root delta. + """ + stack = [root] + while stack: + state = stack.pop() + stack.extend(state.substates.values()) + if hasattr(state, "counter"): + state.counter += 1 + return root.get_delta() + + def teardown(_root) -> None: + """Detach the isolated registration context.""" + contexts.pop().__exit__(None, None, None) + + assert benchmark.pedantic( + mutate_and_delta, + setup=setup, + teardown=teardown, + ) + + +@pytest.mark.parametrize("size", [10, 1000, 100_000]) +def test_state_collection_assignment(size: int, benchmark: BenchmarkFixture): + """Benchmark state assignment and its type-validation hot path. + + Args: + size: Assigned list size. + benchmark: The CodSpeed benchmark fixture. + """ + values = list(range(size)) + + def setup(): + """Create a fresh state for one assignment. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + return ((initialized_state(0),), {}) + + def assign(state: PerformanceState) -> int: + """Assign the representative collection. + + Args: + state: Fresh state. + + Returns: + Assigned collection size. + """ + state.numbers = values + return len(state.numbers) + + assert benchmark.pedantic(assign, setup=setup) == size diff --git a/tests/benchmarks/test_state_manager.py b/tests/benchmarks/test_state_manager.py new file mode 100644 index 00000000000..ce91426f737 --- /dev/null +++ b/tests/benchmarks/test_state_manager.py @@ -0,0 +1,131 @@ +"""Deterministic benchmarks for in-memory state-manager operations.""" + +import asyncio + +from pytest_codspeed import BenchmarkFixture + +from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import BaseStateToken + +from .support.states import ( + PerformanceState, + get_performance_state, + isolated_performance_registry, +) + + +def test_state_manager_memory_cold_get(benchmark: BenchmarkFixture): + """Benchmark state construction for an uncached token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + iteration = 0 + + def setup(): + """Return a unique token for one cold measurement.""" + nonlocal iteration + iteration += 1 + token = BaseStateToken(ident=f"cold-{iteration}", cls=PerformanceState) + return ((token,), {}) + + def get_state(token: BaseStateToken) -> PerformanceState: + """Fetch a state through the async manager API. + + Returns: + Managed state. + """ + return get_performance_state(loop.run_until_complete(manager.get_state(token))) + + def teardown(token: BaseStateToken) -> None: + """Purge the measured state.""" + manager._purge_token(token) # pyright: ignore [reportPrivateUsage] + + # Isolate the registry so the measured cold construction instantiates only + # PerformanceState's subtree, not every state in the collected session. + with isolated_performance_registry(): + try: + benchmark.pedantic(get_state, setup=setup, teardown=teardown) + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_warm_get(benchmark: BenchmarkFixture): + """Benchmark state lookup for a cached token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = BaseStateToken(ident="warm", cls=PerformanceState) + loop.run_until_complete(manager.get_state(token)) + + try: + state = benchmark( + lambda: get_performance_state( + loop.run_until_complete(manager.get_state(token)) + ) + ) + assert isinstance(state, PerformanceState) + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_modify(benchmark: BenchmarkFixture): + """Benchmark lock acquisition, mutation, and release for one token. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = BaseStateToken(ident="modify", cls=PerformanceState) + + async def modify() -> int: + """Increment one state under the manager lock. + + Returns: + Updated counter. + """ + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) + state.counter += 1 + return state.counter + + try: + assert benchmark(lambda: loop.run_until_complete(modify())) > 0 + finally: + loop.run_until_complete(manager.close()) + loop.close() + + +def test_state_manager_memory_read_only_modify(benchmark: BenchmarkFixture): + """Benchmark a read-only lock context that should not produce a write. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + manager = StateManagerMemory() + loop = asyncio.new_event_loop() + token = BaseStateToken(ident="read-only", cls=PerformanceState) + + async def read_only() -> int: + """Read one field under the manager lock. + + Returns: + Current counter. + """ + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) + return state.counter + + try: + assert benchmark(lambda: loop.run_until_complete(read_only())) == 0 + finally: + loop.run_until_complete(manager.close()) + loop.close() diff --git a/tests/benchmarks/test_state_proxy.py b/tests/benchmarks/test_state_proxy.py index 455cb8376ab..859a2e0a5fe 100644 --- a/tests/benchmarks/test_state_proxy.py +++ b/tests/benchmarks/test_state_proxy.py @@ -93,3 +93,67 @@ def test_var_access(access_fn, benchmark: BenchmarkFixture): """ state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] benchmark(lambda: access_fn(state)) + + +def test_proxy_list_mutation(benchmark: BenchmarkFixture): + """Benchmark a batch of writes through a mutable list proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a fresh state for one measured write batch. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] + return ((state,), {}) + + def mutate(state: ProxyBenchmarkState) -> int: + """Append one hundred values through the proxy. + + Args: + state: Fresh benchmark state. + + Returns: + Final list length. + """ + for value in range(100): + state.numbers.append(value) + return len(state.numbers) + + assert benchmark.pedantic(mutate, setup=setup) == N + 100 + + +def test_proxy_mapping_mutation(benchmark: BenchmarkFixture): + """Benchmark a batch of writes through a mutable mapping proxy. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + + def setup(): + """Create a fresh state for one measured write batch. + + Returns: + CodSpeed pedantic positional and keyword arguments. + """ + state = ProxyBenchmarkState() # pyright: ignore [reportCallIssue] + return ((state,), {}) + + def mutate(state: ProxyBenchmarkState) -> int: + """Update one hundred values through the proxy. + + Args: + state: Fresh benchmark state. + + Returns: + Final mapping length. + """ + for value in range(100): + state.mapping[value] = value + return len(state.mapping) + + assert benchmark.pedantic(mutate, setup=setup) == N diff --git a/tests/benchmarks/test_var_operations.py b/tests/benchmarks/test_var_operations.py new file mode 100644 index 00000000000..aa8edaa4e89 --- /dev/null +++ b/tests/benchmarks/test_var_operations.py @@ -0,0 +1,39 @@ +"""Benchmarks for Var construction and type dispatch.""" + +from pytest_codspeed import BenchmarkFixture + +from reflex.vars import Var + + +def test_var_arithmetic_chain(benchmark: BenchmarkFixture): + """Benchmark construction of a representative arithmetic expression. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + left = Var.create(1) + right = Var.create(2) + result = benchmark(lambda: ((left + right) * right - left) / right) + assert result._js_expr + + +def test_var_to_dispatch(benchmark: BenchmarkFixture): + """Benchmark conversion through the Var subclass registry. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + value = Var(_js_expr="value", _var_type=int) + result = benchmark(lambda: value.to(float)) + assert result._var_type is float + + +def test_var_guess_type_dispatch(benchmark: BenchmarkFixture): + """Benchmark inferred dispatch through the Var subclass registry. + + Args: + benchmark: The CodSpeed benchmark fixture. + """ + value = Var(_js_expr="value", _var_type=list[int]) + result = benchmark(value.guess_type) + assert result._var_type == list[int] diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 00000000000..c4e2b92c0cc --- /dev/null +++ b/tests/performance/__init__.py @@ -0,0 +1 @@ +"""Scheduled performance suites excluded from ordinary test runs.""" diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py new file mode 100644 index 00000000000..95086faeecd --- /dev/null +++ b/tests/performance/conftest.py @@ -0,0 +1,102 @@ +"""Configuration for scheduled performance suites.""" + +from collections.abc import Generator +from pathlib import Path + +import pytest + +from reflex.testing import AppHarness, AppHarnessProd +from tests.benchmarks.support.apps import load_app_source + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Register explicit performance-suite options. + + Args: + parser: Pytest command-line parser. + """ + group = parser.getgroup("performance") + group.addoption( + "--run-performance", + action="store_true", + default=False, + help="run scheduled performance tests", + ) + group.addoption( + "--performance-output", + default=".pytest-tmp/performance", + help="directory for performance JSON, traces, and profiles", + ) + group.addoption( + "--performance-scale", + choices=("smoke", "release"), + default="smoke", + help="performance scenario scale", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, + items: list[pytest.Item], +) -> None: + """Skip scheduled performance tests unless explicitly enabled. + + Args: + config: Active Pytest configuration. + items: Collected tests. + """ + if config.getoption("--run-performance"): + return + skip = pytest.mark.skip(reason="pass --run-performance to execute this suite") + for item in items: + if "performance" in item.keywords: + item.add_marker(skip) + + +@pytest.fixture +def performance_output(request: pytest.FixtureRequest) -> Path: + """Return and create the performance artifact directory. + + Args: + request: Pytest fixture request. + + Returns: + Artifact directory. + """ + output = Path(request.config.getoption("--performance-output")).resolve() + output.mkdir(parents=True, exist_ok=True) + return output + + +@pytest.fixture(scope="module") +def performance_load_app(tmp_path_factory) -> Generator[AppHarness, None, None]: + """Build and run the representative production load application per module. + + Module scope amortizes the production build within related scenarios while + stopping the in-process servers before unrelated benchmark modules run. + + Args: + tmp_path_factory: Pytest temporary directory factory. + + Yields: + Running production app harness. + """ + with AppHarnessProd.create( + root=tmp_path_factory.mktemp("performance_load"), + app_source=load_app_source(), + app_name="performance_load", + ) as harness: + yield harness + + +@pytest.fixture +def performance_scale(request: pytest.FixtureRequest) -> str: + """Return the selected scenario scale. + + Args: + request: Pytest fixture request. + + Returns: + ``smoke`` or ``release``. + """ + return request.config.getoption("--performance-scale") diff --git a/tests/performance/test_browser.py b/tests/performance/test_browser.py new file mode 100644 index 00000000000..55d56aea350 --- /dev/null +++ b/tests/performance/test_browser.py @@ -0,0 +1,253 @@ +"""Production browser, bundle, hydration, and DOM-update benchmarks.""" + +from __future__ import annotations + +import gzip +import os +import time +from pathlib import Path + +import pytest +from playwright.sync_api import Page, expect + +from reflex import constants +from reflex.testing import AppHarness +from tests.benchmarks.support import BenchmarkResult, PerformanceReport + +# Gzip budget for the shipped client bundle. A fresh production export of the +# load app measured 238,635 gzip bytes (2026-07-13); the default budget is +# roughly double so only a real bundle regression trips it. CI can tune the +# budget without a code change via the environment variable. +MAX_JAVASCRIPT_GZIP_BYTES = int( + os.environ.get("REFLEX_PERFORMANCE_JS_GZIP_BUDGET", 500_000) +) + + +def _bundle_metrics(root: Path) -> dict[str, int]: + """Calculate shipped client-bundle module and compressed-size metrics. + + Only the exported client build (``.web/build/client``) is measured; the + rest of ``.web`` is the build toolchain (``node_modules``, server bundles) + that never reaches a browser. + + Args: + root: Reflex application root. + + Returns: + JavaScript module, raw byte, and gzip byte counts. + """ + client = root / constants.Dirs.WEB / constants.Dirs.STATIC + files = [ + path + for path in client.rglob("*") + if path.is_file() and path.suffix in {".js", ".mjs"} + ] + return { + "javascript_modules": len(files), + "javascript_bytes": sum(path.stat().st_size for path in files), + "javascript_gzip_bytes": sum( + len(gzip.compress(path.read_bytes())) for path in files + ), + } + + +def _browser_heap(page: Page) -> int: + """Read Chromium's JavaScript heap metric. + + Args: + page: Playwright page. + + Returns: + Used JavaScript heap bytes, or zero if unavailable. + """ + session = page.context.new_cdp_session(page) + try: + session.send("Performance.enable") + metrics = session.send("Performance.getMetrics")["metrics"] + finally: + session.detach() + values = {metric["name"]: metric["value"] for metric in metrics} + return int(values.get("JSHeapUsedSize", 0)) + + +@pytest.mark.performance +def test_browser_report( + performance_load_app: AppHarness, + page: Page, + performance_output: Path, + performance_scale: str, +): + """Measure hydration, state-to-DOM updates, large lists, navigation, and heap. + + Args: + performance_load_app: Running production app. + page: Playwright browser page. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + assert performance_load_app.frontend_url is not None + page.add_init_script( + """ + window.__reflexLongTasks = []; + new PerformanceObserver((list) => { + window.__reflexLongTasks.push(...list.getEntries().map((entry) => entry.duration)); + }).observe({entryTypes: ['longtask']}); + """ + ) + started = time.perf_counter_ns() + page.goto(performance_load_app.frontend_url) + # Gate on the websocket-driven is_hydrated marker, not the prerendered + # #count text (which the static export already ships as "0"), so the timing + # spans real hydration: connect, initial delta, and client state attach. + expect(page.locator("#hydrated")).to_have_text("hydrated") + expect(page.locator("#count")).to_have_text("0") + hydration_ms = (time.perf_counter_ns() - started) / 1_000_000 + heap_before = _browser_heap(page) + + updates = {"smoke": 10, "release": 1000}[performance_scale] + started = time.perf_counter_ns() + for _ in range(updates): + page.locator("#increment").click() + expect(page.locator("#count")).to_have_text(str(updates)) + update_ms = (time.perf_counter_ns() - started) / 1_000_000 + + started = time.perf_counter_ns() + page.locator("#append-large").click() + expect(page.locator(".value-row")).to_have_count(1000) + large_list_ms = (time.perf_counter_ns() - started) / 1_000_000 + + started = time.perf_counter_ns() + page.locator("#second-link").click() + expect(page.locator("#second-heading")).to_have_text("Second page") + navigation_ms = (time.perf_counter_ns() - started) / 1_000_000 + heap_after = _browser_heap(page) + long_tasks = page.evaluate("window.__reflexLongTasks || []") + assert isinstance(long_tasks, list) + + metrics: dict[str, float | int] = dict( + _bundle_metrics(performance_load_app.app_path) + ) + metrics.update({ + "heap_before_bytes": heap_before, + "heap_after_bytes": heap_after, + "heap_growth_bytes": heap_after - heap_before, + "long_task_count": len(long_tasks), + "long_task_total_ms": sum(float(value) for value in long_tasks), + "javascript_gzip_budget_bytes": MAX_JAVASCRIPT_GZIP_BYTES, + }) + report = PerformanceReport( + "browser", + metadata={"scale": performance_scale}, + ) + for name, observations, parameters in [ + ("hydration", [hydration_ms], {}), + ("state_update_to_dom", [update_ms / updates], {"updates": updates}), + ("render_1000_rows", [large_list_ms], {"rows": 1000}), + ("route_navigation", [navigation_ms], {}), + ]: + report.add( + BenchmarkResult( + name, + parameters, + observations, + metrics, + measurement_iterations=len(observations), + ) + ) + report.write(performance_output / "browser.json") + + assert metrics["javascript_modules"] > 0 + assert metrics["javascript_gzip_bytes"] <= MAX_JAVASCRIPT_GZIP_BYTES, ( + f"shipped client bundle is {metrics['javascript_gzip_bytes']} gzip bytes, " + f"over the {MAX_JAVASCRIPT_GZIP_BYTES} budget " + "(override via REFLEX_PERFORMANCE_JS_GZIP_BUDGET)" + ) + assert heap_after > 0 + + +def _timed_through_paint(page: Page, action, expectation) -> float: + """Time an interaction from click through the following paint. + + Args: + page: Playwright browser page. + action: Callable triggering the interaction. + expectation: Callable asserting the resulting DOM change. + + Returns: + Elapsed milliseconds including a double requestAnimationFrame flush. + """ + started = time.perf_counter_ns() + action() + expectation() + page.evaluate( + "() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))" + ) + return (time.perf_counter_ns() - started) / 1_000_000 + + +@pytest.mark.performance +def test_browser_rows_report( + performance_load_app: AppHarness, + page: Page, + performance_output: Path, + performance_scale: str, +): + """Measure keyed-list DOM operations: create, partial update, select, swap. + + The discriminating js-framework-benchmark operations for a keyed list + rendered through ``rx.foreach``: full create, every-tenth-row update, a + selection that must not re-render other rows, and a two-row swap that + stresses keyed reconciliation. Durations include the following paint. + + Args: + performance_load_app: Running production app. + page: Playwright browser page. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + assert performance_load_app.frontend_url is not None + page.goto(performance_load_app.frontend_url.rstrip("/") + "/rows") + expect(page.locator("#rows-hydrated")).to_have_text("hydrated") + expect(page.locator("#selected-row")).to_have_text("-1") + + create_ms = _timed_through_paint( + page, + lambda: page.locator("#create-rows").click(), + lambda: expect(page.locator(".row")).to_have_count(1000), + ) + partial_ms = _timed_through_paint( + page, + lambda: page.locator("#partial-update").click(), + lambda: expect(page.locator("#row-990")).to_have_text("row 990 !!!"), + ) + select_ms = _timed_through_paint( + page, + lambda: page.locator("#row-500").click(), + lambda: expect(page.locator("#selected-row")).to_have_text("500"), + ) + swap_ms = _timed_through_paint( + page, + lambda: page.locator("#swap-rows").click(), + lambda: expect(page.locator("#row-1")).to_have_text("row 998"), + ) + + report = PerformanceReport( + "browser-rows", + metadata={"scale": performance_scale, "rows": 1000}, + ) + for name, duration in [ + ("create_1000_rows", create_ms), + ("partial_update_every_10th", partial_ms), + ("select_row", select_ms), + ("swap_rows", swap_ms), + ]: + report.add( + BenchmarkResult( + name, + {"rows": 1000}, + [duration], + {}, + measurement_iterations=1, + ) + ) + report.write(performance_output / "browser-rows.json") diff --git a/tests/performance/test_compiler_lifecycle.py b/tests/performance/test_compiler_lifecycle.py new file mode 100644 index 00000000000..830607690b1 --- /dev/null +++ b/tests/performance/test_compiler_lifecycle.py @@ -0,0 +1,423 @@ +"""Developer lifecycle benchmarks for init, compile, edits, and export.""" + +from __future__ import annotations + +import contextlib +import json +import os +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import TextIO +from urllib.request import urlopen + +import psutil +import pytest + +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.apps import lifecycle_app_source + + +def _run(command: list[str], cwd: Path, timeout: float = 300) -> float: + """Run a lifecycle command and return elapsed milliseconds. + + Args: + command: Command and arguments. + cwd: Working directory. + timeout: Command timeout. + + Returns: + Elapsed wall time in milliseconds. + """ + started = time.perf_counter_ns() + subprocess.run( + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=timeout, + # Disable the live PyPI latest-version check so `init` timing is not + # contaminated by a network round-trip (and its 2s timeout on a slow or + # offline runner). Telemetry is off for the same reason. + env=os.environ + | { + "REFLEX_TELEMETRY_ENABLED": "false", + "REFLEX_CHECK_LATEST_VERSION": "false", + }, + ) + return (time.perf_counter_ns() - started) / 1_000_000 + + +def _generated_metrics(root: Path) -> dict[str, int]: + """Count generated frontend files and bytes. + + Args: + root: Application root. + + Returns: + Generated file, JavaScript module, and byte counts. + """ + web = root / ".web" + files = [path for path in web.rglob("*") if path.is_file()] if web.exists() else [] + javascript = [ + path for path in files if path.suffix in {".js", ".jsx", ".ts", ".tsx"} + ] + return { + "generated_files": len(files), + "javascript_modules": len(javascript), + "generated_bytes": sum(path.stat().st_size for path in files), + "javascript_bytes": sum(path.stat().st_size for path in javascript), + } + + +def _free_port() -> int: + """Reserve and release an available local TCP port. + + The port must be released before the backend can bind it, so another + process may steal it in between; ``_start_dev_backend_with_retry`` + handles that race by retrying on a fresh port. + + Returns: + Available port number for the development backend. + """ + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def _start_dev_backend( + executable: str, + app_root: Path, + port: int, +) -> tuple[subprocess.Popen[str], TextIO, Path]: + """Start a persistent reload-capable development backend. + + Args: + executable: Python executable running Reflex. + app_root: Generated application root. + port: Backend port. + + Returns: + Backend process, open log stream, and log path. + """ + log_path = app_root / "hot-reload.log" + log_stream = log_path.open("w+", encoding="utf-8") + try: + process = subprocess.Popen( + [ + executable, + "-m", + "reflex", + "run", + "--backend-only", + "--backend-port", + str(port), + "--loglevel", + "error", + ], + cwd=app_root, + env=os.environ | {"REFLEX_TELEMETRY_ENABLED": "false"}, + stdout=log_stream, + stderr=subprocess.STDOUT, + text=True, + ) + except OSError: + log_stream.close() + raise + return process, log_stream, log_path + + +def _wait_for_reload_version( + process: subprocess.Popen[str], + log_path: Path, + port: int, + expected: int, + timeout: float = 120, +) -> None: + """Wait until the live backend exposes an expected source version. + + Args: + process: Development backend process. + log_path: Backend log used for failure context. + port: Backend port. + expected: Source version expected after reload. + timeout: Maximum wait in seconds. + + Raises: + RuntimeError: If the backend exits before serving the version. + TimeoutError: If the reload does not complete in time. + """ + deadline = time.monotonic() + timeout + url = f"http://127.0.0.1:{port}/reload-version" + while time.monotonic() < deadline: + if process.poll() is not None: + log = log_path.read_text(encoding="utf-8")[-4000:] + msg = f"Development backend exited during reload:\n{log}" + raise RuntimeError(msg) + try: + with urlopen(url, timeout=0.5) as response: + payload = json.load(response) + if payload.get("version") == expected: + return + except (OSError, ValueError): + pass + time.sleep(0.05) + log = log_path.read_text(encoding="utf-8")[-4000:] + msg = f"Timed out waiting for reload version {expected}:\n{log}" + raise TimeoutError(msg) + + +def _collect_children( + process: subprocess.Popen[str], + known: dict[int, psutil.Process], +) -> None: + """Snapshot the backend's current child processes for later cleanup. + + Reload workers reparent to init if the backend dies first, so children + must be collected while the parent is still alive to guarantee teardown. + + Args: + process: Development backend process. + known: Accumulated child processes keyed by pid. + """ + with contextlib.suppress(psutil.NoSuchProcess): + for child in psutil.Process(process.pid).children(recursive=True): + known.setdefault(child.pid, child) + + +def _stop_dev_backend( + process: subprocess.Popen[str], + known_children: dict[int, psutil.Process], +) -> None: + """Terminate a development backend and all reload workers. + + Args: + process: Development backend process. + known_children: Children collected while the parent was alive, swept + even when the parent has already exited. Stale entries whose pid + was recycled are skipped by psutil's create-time check. + """ + _collect_children(process, known_children) + processes = list(known_children.values()) + with contextlib.suppress(psutil.NoSuchProcess): + processes.append(psutil.Process(process.pid)) + for running in processes: + with contextlib.suppress(psutil.NoSuchProcess): + running.terminate() + _, alive = psutil.wait_procs(processes, timeout=10) + for running in alive: + with contextlib.suppress(psutil.NoSuchProcess): + running.kill() + + +def _start_dev_backend_with_retry( + executable: str, + app_root: Path, + known_children: dict[int, psutil.Process], + attempts: int = 3, +) -> tuple[subprocess.Popen[str], TextIO, Path, int]: + """Start the dev backend, retrying on a lost port-reservation race. + + ``_free_port`` must release its probe socket before the backend can bind, + so another process can grab the port in between; that surfaces as the + backend exiting with "address already in use" and is retried on a fresh + port. + + Args: + executable: Python executable running Reflex. + app_root: Generated application root. + known_children: Accumulated backend children for teardown. + attempts: Maximum start attempts. + + Returns: + Serving backend process, open log stream, log path, and bound port. + + Raises: + RuntimeError: If the backend exits for a reason other than a port + collision, or every attempt loses the port race. + TimeoutError: If a started backend never serves the initial version. + """ + error: RuntimeError | None = None + for _ in range(attempts): + port = _free_port() + process, log_stream, log_path = _start_dev_backend(executable, app_root, port) + try: + _wait_for_reload_version(process, log_path, port, expected=0) + except (RuntimeError, TimeoutError) as exc: + _stop_dev_backend(process, known_children) + log_stream.close() + # Only a lost port race is retried, matching both reflex's + # pre-flight "port ... is already in use" and the server's + # bind-time "address already in use". + if not isinstance(exc, RuntimeError) or ( + "already in use" not in str(exc).lower() + ): + raise + error = exc + else: + return process, log_stream, log_path, port + assert error is not None + raise error + + +@pytest.mark.performance +def test_compiler_lifecycle_report( + tmp_path: Path, + performance_output: Path, + performance_scale: str, +): + """Measure import, init, cold/warm compiles, edits, reload cycles, and export. + + Args: + tmp_path: Temporary application parent. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + sizes = { + "smoke": (10, 1, 2), + "release": (1000, 25, 100), + } + rows, pages, reload_cycles = sizes[performance_scale] + app_root = tmp_path / "lifecycle_app" + app_root.mkdir() + executable = sys.executable + report = PerformanceReport( + "compiler-lifecycle", + metadata={"scale": performance_scale, "rows": rows, "pages": pages}, + ) + + import_ms = _run([executable, "-c", "import reflex"], app_root) + report.add( + BenchmarkResult( + "import_reflex", + {}, + [import_ms], + {}, + measurement_iterations=1, + ) + ) + + init_ms = _run( + [ + executable, + "-m", + "reflex", + "init", + "--name", + "lifecycle_app", + "--template", + "blank", + "--no-agents", + ], + app_root, + ) + app_source = app_root / "lifecycle_app" / "lifecycle_app.py" + app_source.write_text(lifecycle_app_source(rows, pages), encoding="utf-8") + report.add(BenchmarkResult("init", {}, [init_ms], {}, measurement_iterations=1)) + + compile_command = [executable, "-m", "reflex", "compile", "--no-rich"] + cold_ms = _run(compile_command, app_root) + warm_ms = _run(compile_command, app_root) + report.add( + BenchmarkResult( + "compile_cold", + {"rows": rows, "pages": pages}, + [cold_ms], + _generated_metrics(app_root), + ) + ) + report.add( + BenchmarkResult( + "compile_warm_no_change", + {"rows": rows, "pages": pages}, + [warm_ms], + _generated_metrics(app_root), + ) + ) + + source = app_source.read_text() + page_source = source.replace( + "rx.heading(label)", + 'rx.heading(f"edited {label}")', + ) + app_source.write_text(page_source, encoding="utf-8") + page_edit_ms = _run(compile_command, app_root) + report.add(BenchmarkResult("compile_page_edit", {}, [page_edit_ms], {})) + + shared_source = page_source.replace( + "rx.text(index)", + 'rx.text(index, class_name="shared-row-label")', + ) + app_source.write_text(shared_source, encoding="utf-8") + shared_edit_ms = _run(compile_command, app_root) + report.add( + BenchmarkResult( + "compile_shared_component_edit", + {}, + [shared_edit_ms], + {}, + ) + ) + + source = shared_source.replace("count: int = 0", "count: int = 1") + app_source.write_text(source, encoding="utf-8") + state_edit_ms = _run(compile_command, app_root) + report.add(BenchmarkResult("compile_state_edit", {}, [state_edit_ms], {})) + + reload_observations = [] + known_children: dict[int, psutil.Process] = {} + backend, backend_log, backend_log_path, backend_port = ( + _start_dev_backend_with_retry(executable, app_root, known_children) + ) + try: + _collect_children(backend, known_children) + for cycle in range(1, reload_cycles + 1): + cycle_source = source.replace( + "RELOAD_VERSION = 0", + f"RELOAD_VERSION = {cycle}", + ) + started = time.perf_counter_ns() + app_source.write_text(cycle_source, encoding="utf-8") + _wait_for_reload_version( + backend, + backend_log_path, + backend_port, + expected=cycle, + ) + reload_observations.append((time.perf_counter_ns() - started) / 1_000_000) + _collect_children(backend, known_children) + finally: + _stop_dev_backend(backend, known_children) + backend_log.close() + report.add( + BenchmarkResult( + "backend_hot_reload", + {"cycles": reload_cycles, "watcher": "reflex run"}, + reload_observations, + _generated_metrics(app_root), + measurement_iterations=reload_cycles, + ) + ) + + # Keep zipping enabled: with --no-zip a backend-only export does no work + # (export() runs frontend build and zip only, both gated off), so the timing + # would measure CLI startup alone. Zipping exercises the real backend export + # path — archiving the app source tree. + export_ms = _run( + [ + executable, + "-m", + "reflex", + "export", + "--backend-only", + ], + app_root, + ) + report.add(BenchmarkResult("export_backend", {}, [export_ms], {})) + report.write(performance_output / "compiler-lifecycle.json") + + assert report.results + assert _generated_metrics(app_root)["generated_files"] > 0 diff --git a/tests/performance/test_event_load.py b/tests/performance/test_event_load.py new file mode 100644 index 00000000000..6d10e042304 --- /dev/null +++ b/tests/performance/test_event_load.py @@ -0,0 +1,388 @@ +"""Production Socket.IO concurrency, throughput, and tail-latency benchmarks.""" + +from __future__ import annotations + +import asyncio +import json +import time +from pathlib import Path +from typing import Any + +import pytest +from reflex_base.config import get_config +from reflex_base.registry import RegistrationContext + +from reflex.testing import AppHarness +from tests.benchmarks.support import BenchmarkResult, PerformanceReport, percentile +from tests.benchmarks.support.baseline_server import BaselineSocketServer +from tests.benchmarks.support.report import current_process_metrics +from tests.benchmarks.support.socket_client import ( + run_clients, + run_reconnect_client, + run_socket_client, +) + +# A load level saturates once p99 exceeds the single-client baseline by this +# factor and by this absolute margin, so tiny baselines do not flag noise. +KNEE_P99_FACTOR = 3.0 +KNEE_P99_MARGIN_MS = 25.0 + + +def _client_timeout(clients: int) -> float: + """Scale the per-client connect/response timeout with concurrency. + + Args: + clients: Number of concurrent clients sharing the backend. + + Returns: + Timeout in seconds. + """ + return max(10.0, clients * 0.5) + + +def _wait_for_token_cleanup( + harness: AppHarness, + token_prefixes: tuple[str, ...], + timeout: float = 10.0, +) -> None: + """Poll until disconnect cleanup has removed the given client tokens. + + Token disconnect cleanup is fire-and-forget on the backend; waiting on the + in-process token map keeps later connections from hitting duplicate-tab + handling without a fixed settling sleep. + + Args: + harness: Running in-process app harness. + token_prefixes: Prefixes of tokens that must disappear. + timeout: Maximum wait in seconds. + """ + app = harness.app_instance + assert app is not None + assert app.event_namespace is not None + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not any( + token.startswith(token_prefixes) + for token in app.event_namespace.token_to_sid + ): + return + time.sleep(0.05) + + +def _increment_payload() -> dict[str, Any]: + """Build the increment event payload for the running load app. + + Returns: + Socket.IO event payload targeting the app's increment handler. + """ + handler_name = next( + name + for name in RegistrationContext.get().event_handlers + if name.endswith(".increment") and "performance_load" in name + ) + return { + "name": handler_name, + "payload": {}, + "router_data": {"path": "/", "pathname": "/", "query": {}}, + } + + +def _backend_url() -> str: + """Return the running backend's Socket.IO URL. + + Returns: + Backend URL without a trailing slash. + """ + return get_config().api_url.rstrip("/") + + +@pytest.mark.performance +def test_event_load_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure production Socket.IO latency and throughput across client counts. + + Reports the full latency-throughput curve and the saturation knee: the + first load level whose p99 exceeds the single-client baseline by both + ``KNEE_P99_FACTOR`` and ``KNEE_P99_MARGIN_MS``. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + scales = { + "smoke": ([1, 5], 3), + "release": ([1, 10, 50, 200], 50), + } + client_counts, events_per_client = scales[performance_scale] + backend_url = _backend_url() + payload = _increment_payload() + report = PerformanceReport( + "event-load", + metadata={"scale": performance_scale, "backend_url": backend_url}, + ) + + curve: list[dict[str, float | int]] = [] + for clients in client_counts: + started = time.perf_counter_ns() + results = asyncio.run( + run_clients( + clients, + lambda index, executor, clients=clients: run_socket_client( + backend_url, + f"load-token-{clients}-{index}", + payload, + events_per_client, + timeout=_client_timeout(clients), + executor=executor, + ), + ) + ) + elapsed_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 + latencies = [latency for result in results for latency in result.latencies_ms] + payload_sizes = [size for result in results for size in result.payload_bytes] + errors = [error for result in results for error in result.errors] + per_client_max = [max(result.latencies_ms, default=0) for result in results] + expected = clients * events_per_client + throughput = len(latencies) / elapsed_seconds + p99 = percentile(latencies, 99) + curve.append({ + "clients": clients, + "throughput_events_per_second": throughput, + "p50_ms": percentile(latencies, 50), + "p99_ms": p99, + }) + report.add( + BenchmarkResult( + name=f"unique_tokens_{clients}_clients", + parameters={ + "clients": clients, + "events_per_client": events_per_client, + }, + observations_ms=latencies, + metrics={ + "throughput_events_per_second": throughput, + "errors": len(errors), + "expected_updates": expected, + "received_updates": len(latencies), + "fairness_spread_ms": max(per_client_max, default=0) + - min(per_client_max, default=0), + "payload_bytes_mean": sum(payload_sizes) / len(payload_sizes) + if payload_sizes + else 0, + "payload_bytes_max": max(payload_sizes, default=0), + }, + measurement_iterations=len(latencies), + ) + ) + assert not errors, errors + assert len(latencies) == expected + + baseline_p99 = curve[0]["p99_ms"] + knee_threshold = max( + baseline_p99 * KNEE_P99_FACTOR, baseline_p99 + KNEE_P99_MARGIN_MS + ) + knee_clients = next( + (level["clients"] for level in curve[1:] if level["p99_ms"] > knee_threshold), + 0, + ) + report.metadata["latency_curve"] = curve + report.metadata["knee_threshold_ms"] = knee_threshold + report.metadata["knee_clients"] = knee_clients + + report.write(performance_output / "event-load.json") + + +@pytest.mark.performance +def test_framework_overhead_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Compare Reflex round-trips against the bare Starlette+Socket.IO substrate. + + Runs the same client, event payload, and response size against a bare + ``uvicorn`` + Starlette + python-socketio echo server, isolating how much + of the round-trip is Reflex's event pipeline versus the substrate. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + events, warmup = {"smoke": (25, 5), "release": (500, 20)}[performance_scale] + payload = _increment_payload() + + reflex_result = asyncio.run( + run_socket_client(_backend_url(), "overhead-reflex", payload, events + warmup) + ) + assert not reflex_result.errors, reflex_result.errors + + # Pad the canned response so both servers answer with equal payload bytes. + response_bytes = max(reflex_result.payload_bytes, default=0) + canned: dict[str, Any] = {"delta": {"state": {"count": 1}}, "final": True} + base_bytes = len(json.dumps(canned | {"pad": ""}, separators=(",", ":"))) + canned["pad"] = "x" * max(0, response_bytes - base_bytes) + with BaselineSocketServer(canned) as baseline: + baseline_result = asyncio.run( + run_socket_client( + baseline.url, "overhead-baseline", payload, events + warmup + ) + ) + assert not baseline_result.errors, baseline_result.errors + + report = PerformanceReport( + "framework-overhead", + metadata={"scale": performance_scale, "response_bytes": response_bytes}, + ) + reflex_latencies = reflex_result.latencies_ms[warmup:] + baseline_latencies = baseline_result.latencies_ms[warmup:] + for name, latencies in [ + ("reflex_round_trip", reflex_latencies), + ("bare_starlette_round_trip", baseline_latencies), + ]: + report.add( + BenchmarkResult( + name=name, + parameters={"events": events}, + observations_ms=list(latencies), + metrics={}, + warmup_iterations=warmup, + measurement_iterations=len(latencies), + ) + ) + reflex_p50 = percentile(reflex_latencies, 50) + baseline_p50 = percentile(baseline_latencies, 50) + report.add( + BenchmarkResult( + name="framework_overhead", + parameters={"events": events}, + observations_ms=[], + metrics={ + "overhead_p50_ms": reflex_p50 - baseline_p50, + "overhead_p99_ms": percentile(reflex_latencies, 99) + - percentile(baseline_latencies, 99), + "overhead_ratio_p50": reflex_p50 / baseline_p50 if baseline_p50 else 0, + }, + ) + ) + report.write(performance_output / "framework-overhead.json") + assert reflex_p50 > 0 + assert baseline_p50 > 0 + + +@pytest.mark.performance +def test_reconnect_storm_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure simultaneous reconnection after all clients drop. + + Primes per-token state, then reconnects every client at once — the + deploy-restart storm — measuring connect and first-update latency + percentiles and resident-memory growth. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + clients = {"smoke": 5, "release": 100}[performance_scale] + backend_url = _backend_url() + payload = _increment_payload() + tokens = [f"storm-token-{index}" for index in range(clients)] + + timeout = _client_timeout(clients) + + # Prime each token so the storm exercises state restore, not creation. + prime_results = asyncio.run( + run_clients( + clients, + lambda index, executor: run_socket_client( + backend_url, + tokens[index], + payload, + 1, + timeout=timeout, + executor=executor, + ), + ) + ) + prime_errors = [error for result in prime_results for error in result.errors] + assert not prime_errors, f"priming clients failed: {prime_errors}" + # Wait for fire-and-forget disconnect cleanup so the storm exercises + # reconnection instead of duplicate-tab handling. + _wait_for_token_cleanup(performance_load_app, ("storm-token-",)) + + rss_before = current_process_metrics()["rss_bytes"] + started = time.perf_counter_ns() + storm_results = asyncio.run( + run_clients( + clients, + lambda index, executor: run_reconnect_client( + backend_url, + tokens[index], + payload, + timeout=timeout, + executor=executor, + ), + ) + ) + storm_seconds = (time.perf_counter_ns() - started) / 1_000_000_000 + rss_after = current_process_metrics()["rss_bytes"] + + # A transient connect failure under the simultaneous storm is retried once + # so only persistent failures trip the assertion; retried clients keep + # their retry measurements but do not affect storm timing. + failed_indexes = [ + index for index, result in enumerate(storm_results) if result.errors + ] + if failed_indexes: + retry_results = asyncio.run( + run_clients( + len(failed_indexes), + lambda index, executor: run_reconnect_client( + backend_url, + tokens[failed_indexes[index]], + payload, + timeout=timeout, + executor=executor, + ), + ) + ) + for index, result in zip(failed_indexes, retry_results, strict=True): + storm_results[index] = result + + errors = [error for result in storm_results for error in result.errors] + connects = [result.connect_ms for result in storm_results] + first_responses = [result.first_response_ms for result in storm_results] + report = PerformanceReport( + "reconnect-storm", + metadata={"scale": performance_scale, "backend_url": backend_url}, + ) + report.add( + BenchmarkResult( + name=f"reconnect_storm_{clients}_clients", + parameters={"clients": clients}, + observations_ms=first_responses, + metrics={ + "connect_p50_ms": percentile(connects, 50), + "connect_p99_ms": percentile(connects, 99), + "connect_max_ms": max(connects, default=0), + "storm_seconds": storm_seconds, + "reconnects_per_second": clients / storm_seconds, + "rss_growth_bytes": rss_after - rss_before, + "errors": len(errors), + "retried_clients": len(failed_indexes), + }, + measurement_iterations=clients, + ) + ) + report.write(performance_output / "reconnect-storm.json") + assert not errors, ( + f"{len(errors)} reconnect client error(s) persisted after retry: {errors}" + ) diff --git a/tests/performance/test_event_loop.py b/tests/performance/test_event_loop.py new file mode 100644 index 00000000000..2f19cc55c83 --- /dev/null +++ b/tests/performance/test_event_loop.py @@ -0,0 +1,709 @@ +"""Wall-time event-loop, queueing, ordering, and blocking benchmarks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import time +from collections import defaultdict +from collections.abc import AsyncIterator, Mapping +from pathlib import Path +from typing import Any + +import pytest +from reflex_base.event.context import EventContext +from reflex_base.event.processor import BaseStateEventProcessor +from reflex_base.event.processor.event_processor import EventProcessor, EventQueueEntry +from reflex_base.registry import RegisteredEventHandler, RegistrationContext +from typing_extensions import Unpack, override + +import reflex as rx +from reflex.event import Event, EventHandler +from reflex.istate.manager import StateModificationContext +from reflex.istate.manager.memory import StateManagerMemory +from reflex.istate.manager.token import TOKEN_TYPE, StateToken +from reflex.state import BaseState +from tests.benchmarks.support import ( + BenchmarkResult, + EventLoopProbe, + PerformanceReport, + PipelineTrace, + capture_async_diagnostics, +) +from tests.benchmarks.support.pipeline_trace import StageEvent +from tests.benchmarks.support.report import ( + current_process_metrics, + percentile, + runtime_metadata, +) + +_ORDER_LOG: dict[str, list[int]] = defaultdict(list) +# Handlers active per token right now, and the peak seen during a scenario. The +# per-token queue must serialize same-token events, so the peak must stay 1; a +# regression to concurrent same-token dispatch would push it to 2+. +_ACTIVE_PER_TOKEN: dict[str, int] = defaultdict(int) +_PEAK_ACTIVE_PER_TOKEN: dict[str, int] = defaultdict(int) +_ACTIVE_STATE_TRACE: PipelineTrace | None = None + + +def _record_active(token: str) -> None: + """Mark a handler as active for a token and update the peak. + + Args: + token: The token whose handler is entering. + """ + _ACTIVE_PER_TOKEN[token] += 1 + _PEAK_ACTIVE_PER_TOKEN[token] = max( + _PEAK_ACTIVE_PER_TOKEN[token], _ACTIVE_PER_TOKEN[token] + ) + + +def _clear_active(token: str) -> None: + """Mark a handler as no longer active for a token. + + Args: + token: The token whose handler is exiting. + """ + _ACTIVE_PER_TOKEN[token] -= 1 + + +# Deliberate synchronous block injected by the blocking scenario. Detection is +# asserted at >=2ms of measured loop lag, so 10ms leaves a 5x margin even on a +# loaded CI runner. +_BLOCKING_SLEEP_SECONDS = 0.01 + +# Generous stall guard for event-future waits: a stuck future is the exact +# regression class this suite exists to catch and must fail fast, not hang CI. +_EVENT_WAIT_BASE_TIMEOUT_SECONDS = 60.0 +_EVENT_WAIT_PER_EVENT_SECONDS = 0.1 + + +async def _wait_events_bounded(awaitable: Any, *, events: int, scenario: str) -> None: + """Await event-future completion with a stall-detecting timeout. + + Args: + awaitable: The future wait (or gather of waits) to bound. + events: Number of in-flight events, scaling the allowance. + scenario: Scenario name for the failure message. + """ + timeout = _EVENT_WAIT_BASE_TIMEOUT_SECONDS + events * _EVENT_WAIT_PER_EVENT_SECONDS + try: + await asyncio.wait_for(awaitable, timeout=timeout) + except asyncio.TimeoutError: + pytest.fail( + f"{scenario}: {events} event future(s) still pending after " + f"{timeout:.0f}s; the event pipeline is stuck" + ) + + +def _record_state_stage(stage: str) -> None: + """Record a stage for the active state event. + + Args: + stage: Pipeline stage name. + """ + trace = _ACTIVE_STATE_TRACE + assert trace is not None + trace.record(EventContext.get().txid, stage) + + +class AttributionState(rx.State): + """State whose handler exposes precise benchmark-only boundaries.""" + + value: rx.Field[int] = rx.field(0) + + @rx.event + def increment(self): + """Record handler and pre-delta boundaries around a mutation.""" + _record_state_stage("handler_started") + self.value += 1 + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def cpu_heavy(self): + """Perform synchronous CPU work on the event loop.""" + _record_state_stage("handler_started") + self.value += sum(range(100_000)) + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def blocking(self): + """Perform a deliberately blocking synchronous wait.""" + _record_state_stage("handler_started") + time.sleep(0.005) + self.value += 1 + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + async def thread_offload(self): + """Move CPU work off the event-loop thread.""" + _record_state_stage("handler_started") + self.value += await asyncio.to_thread(sum, range(100_000)) + _record_state_stage("handler_finished") + _record_state_stage("delta_started") + + @rx.event + def sync_generator(self): + """Yield three synchronous intermediate state updates.""" + _record_state_stage("handler_started") + for _ in range(3): + self.value += 1 + _record_state_stage("delta_started") + yield + _record_state_stage("handler_finished") + + @rx.event + async def async_generator(self): + """Yield three cooperative asynchronous state updates.""" + _record_state_stage("handler_started") + for _ in range(3): + await asyncio.sleep(0) + self.value += 1 + _record_state_stage("delta_started") + yield + _record_state_stage("handler_finished") + + @rx.event + def failure(self): + """Raise an exception after recording handler entry.""" + _record_state_stage("handler_started") + msg = "intentional performance scenario failure" + raise RuntimeError(msg) + + +class TracingStateManagerMemory(StateManagerMemory): + """Memory manager recording lock and state-flush boundaries.""" + + def __init__(self, trace: PipelineTrace): + """Initialize the manager. + + Args: + trace: Pipeline trace receiving observations. + """ + super().__init__() + self.trace = trace + + @override + @contextlib.asynccontextmanager + async def modify_state( + self, + token: StateToken[TOKEN_TYPE], + **context: Unpack[StateModificationContext], + ) -> AsyncIterator[TOKEN_TYPE]: + """Record lock wait, hold, and flush stages. + + Args: + token: Managed state token. + context: State modification context. + + Yields: + Managed state. + """ + transaction = EventContext.get().txid + self.trace.record(transaction, "lock_wait_started") + async with super().modify_state(token, **context) as state: + self.trace.record(transaction, "lock_acquired") + try: + yield state + finally: + self.trace.record(transaction, "state_flush_started") + self.trace.record(transaction, "state_flush_finished") + self.trace.record(transaction, "lock_released") + + +class AttributionEventProcessor(BaseStateEventProcessor): + """State processor that omits first-connect hydration from attribution.""" + + @override + async def _rehydrate(self, root_state: BaseState) -> None: + """Skip hydration so the scenario isolates one application event. + + Args: + root_state: Root state, intentionally unused. + """ + + +async def _cooperative_handler(sequence: int) -> None: + """Yield to the loop before recording event completion. + + Args: + sequence: Per-token event sequence. + """ + token = EventContext.get().token + _record_active(token) + try: + await asyncio.sleep(0.001) + _ORDER_LOG[token].append(sequence) + finally: + _clear_active(token) + + +async def _blocking_handler(sequence: int) -> None: # noqa: RUF029 + """Deliberately block the loop before recording completion. + + Args: + sequence: Per-token event sequence. + """ + token = EventContext.get().token + _record_active(token) + try: + time.sleep(_BLOCKING_SLEEP_SECONDS) # noqa: ASYNC251 - intentional benchmark scenario + _ORDER_LOG[token].append(sequence) + finally: + _clear_active(token) + + +COOPERATIVE_EVENT = EventHandler(fn=_cooperative_handler) +BLOCKING_EVENT = EventHandler(fn=_blocking_handler) + + +class TracingEventProcessor(EventProcessor): + """Event processor that records benchmark-only task pipeline stages.""" + + def __init__(self, graceful_shutdown_timeout: float | None = None): + """Initialize the processor and its isolated trace. + + Args: + graceful_shutdown_timeout: Maximum graceful drain time. + """ + super().__init__(graceful_shutdown_timeout=graceful_shutdown_timeout) + self.trace = PipelineTrace() + + async def enqueue(self, token: str, event: Event, ev_ctx=None): + """Record ingress and queue completion around normal enqueue. + + Args: + token: Client token. + event: Event to enqueue. + ev_ctx: Optional existing event context. + + Returns: + Tracked event future. + """ + started = time.perf_counter_ns() + future = await super().enqueue(token, event, ev_ctx) + self.trace.extend([ + _stage(future.txid, "received", started), + _stage(future.txid, "enqueued", time.perf_counter_ns()), + ]) + return future + + def _enqueue_for_token( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> None: + """Record shared-queue dequeue and per-token queue entry.""" + self.trace.record(entry.ctx.txid, "dequeued") + self.trace.record(entry.ctx.txid, "token_queued") + super()._enqueue_for_token( + entry=entry, + registered_handler=registered_handler, + ) + + def _create_event_task( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> asyncio.Task: + """Record when per-token ordering permits task creation. + + Returns: + Created event task. + """ + self.trace.record(entry.ctx.txid, "token_ready") + self.trace.record(entry.ctx.txid, "task_created") + return super()._create_event_task( + entry=entry, + registered_handler=registered_handler, + ) + + async def _process_event_queue_entry( + self, + *, + entry: EventQueueEntry, + registered_handler: RegisteredEventHandler, + ) -> None: + """Record scheduling, handler, and completion boundaries.""" + self.trace.record(entry.ctx.txid, "task_started") + self.trace.record(entry.ctx.txid, "handler_started") + try: + await super()._process_event_queue_entry( + entry=entry, + registered_handler=registered_handler, + ) + finally: + self.trace.record(entry.ctx.txid, "handler_finished") + self.trace.record(entry.ctx.txid, "completed") + + +def _stage(token: str, stage: str, timestamp_ns: int) -> StageEvent: + """Create a pipeline stage without another clock read. + + Args: + token: Transaction identifier. + stage: Stage name. + timestamp_ns: Existing monotonic timestamp. + + Returns: + Stage event. + """ + return StageEvent(token, stage, timestamp_ns) + + +async def _run_scenario( + handler: EventHandler, + *, + tokens: int, + events_per_token: int, + probe_interval: float, + hot_token_fraction: float = 0, +) -> tuple[list[float], dict[str, float | int], PipelineTrace]: + """Run one event-loop scenario and collect latency and resource metrics. + + Args: + handler: Registered workload handler. + tokens: Number of independent client tokens. + events_per_token: Sequential events per token. + probe_interval: Loop-lag sample interval. + hot_token_fraction: Fraction of all work assigned to token zero. + + Returns: + Latencies, probe/resource metrics, and pipeline trace. + """ + _ORDER_LOG.clear() + _ACTIVE_PER_TOKEN.clear() + _PEAK_ACTIVE_PER_TOKEN.clear() + processor = TracingEventProcessor(graceful_shutdown_timeout=2) + processor.configure() + + def queue_depth() -> int: + """Return shared queue depth.""" + return processor._queue.qsize() if processor._queue is not None else 0 + + def token_queue_depth() -> int: + """Return total queued per-token work.""" + return sum(len(queue) for queue in processor._token_queues.values()) + + probe = EventLoopProbe( + interval=probe_interval, + gauges={ + "queue_depth": queue_depth, + "token_queue_depth": token_queue_depth, + "processor_tasks": lambda: len(processor._tasks), + }, + ) + latencies: list[float] = [] + started: dict[str, int] = {} + expected_order: dict[str, list[int]] = defaultdict(list) + if hot_token_fraction: + total_events = tokens * events_per_token + hot_events = int(total_events * hot_token_fraction) + token_events = [("token-0", sequence) for sequence in range(hot_events)] + for offset in range(total_events - hot_events): + token_index = 1 + offset % max(1, tokens - 1) + token = f"token-{token_index}" + token_events.append((token, len(expected_order[token]))) + expected_order[token].append(len(expected_order[token])) + expected_order["token-0"] = list(range(hot_events)) + else: + token_events = [ + (f"token-{token_index}", sequence) + for token_index in range(tokens) + for sequence in range(events_per_token) + ] + for token_index in range(tokens): + expected_order[f"token-{token_index}"] = list(range(events_per_token)) + + async with probe, processor: + futures = [] + for token, sequence in token_events: + future = await processor.enqueue( + token, + Event.from_event_type(handler(sequence))[0], + ) + started[future.txid] = time.perf_counter_ns() + future.add_done_callback( + lambda completed, txid=future.txid: latencies.append( + (time.perf_counter_ns() - started[txid]) / 1_000_000 + ) + ) + futures.append(future) + await _wait_events_bounded( + asyncio.gather(*(future.wait_all() for future in futures)), + events=len(futures), + scenario=handler.fn.__name__, + ) + await asyncio.sleep(probe_interval * 2) + # Capture leak counts while the processor is still running. stop() + # cancels and clears both dicts on block exit, so checking after the + # block would always see zero and never catch an undrained task/future. + orphan_tasks = len(processor._tasks) + orphan_futures = len(processor._futures) + + for token, expected in expected_order.items(): + assert _ORDER_LOG[token] == expected + # Completion order alone cannot catch concurrent same-token dispatch (equal + # sleeps still finish in order), so assert no token ever had two handlers + # active at once — the guarantee the per-token queue exists to provide. + overlapping = { + token: peak for token, peak in _PEAK_ACTIVE_PER_TOKEN.items() if peak > 1 + } + assert not overlapping, f"same-token handlers overlapped: {overlapping}" + assert not orphan_tasks + assert not orphan_futures + durations = processor.trace.durations_ms([ + ("received", "enqueued"), + ("enqueued", "dequeued"), + ("dequeued", "task_started"), + ("token_queued", "token_ready"), + ("task_created", "task_started"), + ("handler_started", "handler_finished"), + ]) + stage_metrics = { + f"{name}_p95_ms": percentile(observations, 95) + for name, observations in durations.items() + } + metrics = ( + probe.summary() + | current_process_metrics() + | stage_metrics + | { + "tasks_created": sum( + event.stage == "task_created" for event in processor.trace.events + ), + "orphan_tasks": orphan_tasks, + "orphan_futures": orphan_futures, + } + ) + return latencies, metrics, processor.trace + + +async def _run_state_pipeline( + handler: Any, +) -> tuple[float, dict[str, float | int], PipelineTrace]: + """Run a full state mutation, delta, flush, and emit pipeline. + + Returns: + End-to-end latency, stage metrics, and trace. + """ + global _ACTIVE_STATE_TRACE + trace = PipelineTrace() + _ACTIVE_STATE_TRACE = trace + manager = TracingStateManagerMemory(trace) + processor = AttributionEventProcessor(graceful_shutdown_timeout=2) + processor.configure(state_manager=manager) + assert processor._root_context is not None # pyright: ignore [reportPrivateUsage] + + async def emit_delta( + _token: str, + _delta: Mapping[str, Mapping[str, Any]], + ) -> None: + """Record delta completion and simulated socket flush boundaries.""" + transaction = EventContext.get().txid + trace.record(transaction, "delta_finished") + trace.record(transaction, "emit_started") + await asyncio.sleep(0) + trace.record(transaction, "emit_finished") + + processor._root_context = dataclasses.replace( # pyright: ignore [reportPrivateUsage] + processor._root_context, # pyright: ignore [reportPrivateUsage] + emit_delta_impl=emit_delta, + ) + event = Event.from_event_type(handler())[0] + event = dataclasses.replace(event, router_data={"path": "/", "query": {}}) + started = time.perf_counter_ns() + try: + async with processor: + received = time.perf_counter_ns() + future = await processor.enqueue("attribution-token", event) + trace.extend([ + StageEvent(future.txid, "received", received), + StageEvent(future.txid, "enqueued", time.perf_counter_ns()), + ]) + await _wait_events_bounded( + future.wait_all(), events=1, scenario="state_pipeline" + ) + finally: + await manager.close() + _ACTIVE_STATE_TRACE = None + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + durations = trace.durations_ms([ + ("received", "enqueued"), + ("lock_wait_started", "lock_acquired"), + ("handler_started", "handler_finished"), + ("delta_started", "delta_finished"), + ("state_flush_started", "state_flush_finished"), + ("emit_started", "emit_finished"), + ("lock_acquired", "lock_released"), + ]) + metrics: dict[str, float | int] = {} + for name, observations in durations.items(): + metrics[f"{name}_p95_ms"] = percentile(observations, 95) + return elapsed, metrics, trace + + +async def _run_failed_state_pipeline() -> None: + """Verify exception cleanup leaves no processor tasks or futures.""" + global _ACTIVE_STATE_TRACE + trace = PipelineTrace() + _ACTIVE_STATE_TRACE = trace + manager = TracingStateManagerMemory(trace) + exceptions: list[Exception] = [] + processor = AttributionEventProcessor( + graceful_shutdown_timeout=2, + backend_exception_handler=exceptions.append, + ) + processor.configure(state_manager=manager) + event = Event.from_event_type(AttributionState.failure())[0] + event = dataclasses.replace(event, router_data={"path": "/", "query": {}}) + orphan_tasks = 0 + orphan_futures = 0 + try: + async with processor: + future = await processor.enqueue("failure-token", event) + with contextlib.suppress(RuntimeError): + await _wait_events_bounded( + future.wait_all(), events=1, scenario="failed_state_pipeline" + ) + # The failing handler schedules a backend-exception task after the + # future resolves, so wait for that task to run and its finish + # callback to pop it before capturing counts. Checking after the + # block would only see stop()'s forced teardown, not real cleanup. + for _ in range(200): + await asyncio.sleep(0.01) + if exceptions and not processor._tasks: + break + orphan_tasks = len(processor._tasks) + orphan_futures = len(processor._futures) + finally: + await manager.close() + _ACTIVE_STATE_TRACE = None + assert exceptions + assert not orphan_tasks + assert not orphan_futures + + +@pytest.mark.performance +async def test_event_loop_component_report( + performance_output: Path, + performance_scale: str, +): + """Measure cooperative, contended, concurrent, and blocking event workloads. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + scales = { + "smoke": (5, 4, 10), + "release": (200, 50, 1000), + } + independent_tokens, independent_events, same_token_events = scales[ + performance_scale + ] + scenarios = [ + ("same_token", COOPERATIVE_EVENT, 1, same_token_events, 0), + ( + "independent_tokens", + COOPERATIVE_EVENT, + independent_tokens, + independent_events, + 0, + ), + ( + "hot_token_80_20", + COOPERATIVE_EVENT, + max(5, independent_tokens), + independent_events, + 0.8, + ), + ("blocking_handler", BLOCKING_EVENT, 2, 5, 0), + ] + registry = RegistrationContext() + with registry: + for handler in (COOPERATIVE_EVENT, BLOCKING_EVENT): + RegistrationContext.register_event_handler(handler) + RegistrationContext.register_base_state(AttributionState) + + report = PerformanceReport( + "event-loop", + metadata=runtime_metadata() | {"scale": performance_scale}, + ) + traces: list[PipelineTrace] = [] + for name, handler, tokens, events_per_token, hot_fraction in scenarios: + latencies, metrics, trace = await _run_scenario( + handler, + tokens=tokens, + events_per_token=events_per_token, + probe_interval=0.001, + hot_token_fraction=hot_fraction, + ) + report.add( + BenchmarkResult( + name=name, + parameters={ + "tokens": tokens, + "events_per_token": events_per_token, + "hot_token_fraction": hot_fraction, + }, + observations_ms=latencies, + metrics=metrics, + measurement_iterations=len(latencies), + ) + ) + traces.append(trace) + + for state_name, state_handler in [ + ("state_pipeline_attribution", AttributionState.increment), + ("cpu_heavy_handler", AttributionState.cpu_heavy), + ("blocking_state_handler", AttributionState.blocking), + ("thread_offload_handler", AttributionState.thread_offload), + ("sync_generator_handler", AttributionState.sync_generator), + ("async_generator_handler", AttributionState.async_generator), + ]: + state_latency, state_metrics, state_trace = await _run_state_pipeline( + state_handler + ) + report.add( + BenchmarkResult( + name=state_name, + parameters={"state_manager": "memory"}, + observations_ms=[state_latency], + metrics=state_metrics, + measurement_iterations=1, + ) + ) + traces.append(state_trace) + await _run_failed_state_pipeline() + + report.write(performance_output / "event-loop.json") + combined_trace = PipelineTrace() + for trace in traces: + combined_trace.extend(trace.events) + combined_trace.write_chrome_trace(performance_output / "event-loop.trace.json") + + assert report.results + blocking = next( + result for result in report.results if result.name == "blocking_handler" + ) + # The probe must detect the deliberately induced block: each blocking event + # stalls the loop for _BLOCKING_SLEEP_SECONDS (10ms), so 2ms of measured + # lag is a 5x-margin detection bound, not a performance threshold. + assert blocking.metrics["lag_max_ms"] >= 2 + capture_async_diagnostics( + performance_output / "event-loop-blocking-diagnostics.json", + metadata={ + "scenario": blocking.name, + "metrics": dict(blocking.metrics), + "last_stages": { + event.token: event.stage for event in combined_trace.events + }, + }, + ) diff --git a/tests/performance/test_memory.py b/tests/performance/test_memory.py new file mode 100644 index 00000000000..636f8522abd --- /dev/null +++ b/tests/performance/test_memory.py @@ -0,0 +1,138 @@ +"""Allocation and retained-memory performance benchmarks.""" + +from __future__ import annotations + +import gc +import itertools +import time +import tracemalloc +from collections.abc import Callable +from pathlib import Path + +import pytest +from reflex_base.utils.format import json_dumps + +from reflex.compiler import compiler +from reflex.state import StateUpdate +from reflex.vars import Var +from tests.benchmarks.fixtures import _complicated_page +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.states import initialized_state + + +def _compile_workload() -> None: + """Build and compile a representative component tree.""" + assert compiler._compile_page(_complicated_page()) + + +def _state_delta_workload() -> None: + """Mutate and serialize a state carrying large mutable values.""" + state = initialized_state(1000) + state.counter += 1 + assert json_dumps(StateUpdate(delta=state.get_delta())) + + +def _var_workload() -> None: + """Construct a batch of Var operations and release them.""" + left = Var.create(1) + right = Var.create(2) + for _ in range(1000): + assert ((left + right) * right - left)._js_expr + + +# Untraced warmup cycles run before the leak gate observes retention so +# lazily populated caches (compiler memos, Var expression caches) fill first. +WARMUP_CYCLES = 3 + + +def _measure_allocations( + workload: Callable[[], None], + cycles: int, +) -> tuple[list[float], dict[str, float | int]]: + """Measure elapsed time, peak allocation, and retained-memory trend. + + Runs ``WARMUP_CYCLES`` untraced cycles first so one-time lazy allocations + settle, then traces ``cycles`` measured cycles. ``monotonic_growth`` is + only set when retention grows strictly on every measured cycle after + warmup — the signature of a per-invocation leak, which a warmed cache + cannot produce because it plateaus. + + Args: + workload: Workload invoked once per cycle. + cycles: Measurement cycles after warmup. + + Returns: + Cycle latencies and allocation metrics. + """ + for _ in range(WARMUP_CYCLES): + workload() + gc.collect() + tracemalloc.start() + baseline, _ = tracemalloc.get_traced_memory() + retained: list[int] = [] + latencies: list[float] = [] + peak = baseline + try: + for _ in range(cycles): + started = time.perf_counter_ns() + workload() + latencies.append((time.perf_counter_ns() - started) / 1_000_000) + gc.collect() + current, current_peak = tracemalloc.get_traced_memory() + retained.append(current - baseline) + peak = max(peak, current_peak) + finally: + tracemalloc.stop() + + monotonic_growth = all( + current > previous for previous, current in itertools.pairwise(retained) + ) + return latencies, { + "peak_traced_bytes": peak - baseline, + "retained_bytes": retained[-1] if retained else 0, + "retained_growth_bytes": retained[-1] - retained[0] if len(retained) > 1 else 0, + "monotonic_growth": int(monotonic_growth), + } + + +@pytest.mark.performance +def test_memory_report(performance_output: Path, performance_scale: str): + """Profile allocation and retention across compiler and runtime workloads. + + The leak gate fails only when a workload shows strictly monotonic retained + growth on every post-warmup cycle and accumulates more than 1 MB — lazily + populated caches are filled during the untraced warmup cycles, so + sustained growth here indicates a real per-invocation leak. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + cycles = {"smoke": 3, "release": 100}[performance_scale] + report = PerformanceReport("memory", metadata={"scale": performance_scale}) + for name, workload in { + "component_compile": _compile_workload, + "state_delta_serialization": _state_delta_workload, + "var_operations": _var_workload, + }.items(): + observations, metrics = _measure_allocations(workload, cycles) + report.add( + BenchmarkResult( + name=name, + parameters={"cycles": cycles}, + observations_ms=observations, + metrics=metrics, + warmup_iterations=WARMUP_CYCLES, + measurement_iterations=cycles, + ) + ) + + report.write(performance_output / "memory.json") + assert all(result.metrics["peak_traced_bytes"] >= 0 for result in report.results) + leaks = [ + (result.name, result.metrics["retained_growth_bytes"]) + for result in report.results + if result.metrics["monotonic_growth"] + and result.metrics["retained_growth_bytes"] > 1_000_000 + ] + assert not leaks, f"sustained post-warmup retained-memory growth: {leaks}" diff --git a/tests/performance/test_redis_latency.py b/tests/performance/test_redis_latency.py new file mode 100644 index 00000000000..a5054398466 --- /dev/null +++ b/tests/performance/test_redis_latency.py @@ -0,0 +1,253 @@ +"""Real-Redis state-manager latency and command benchmarks.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Any + +import pytest + +from reflex.istate.manager.token import BaseStateToken +from tests.benchmarks.support import BenchmarkResult, EventLoopProbe, PerformanceReport +from tests.benchmarks.support.redis import real_redis_state_manager +from tests.benchmarks.support.states import PerformanceState, get_performance_state + + +async def _measure( + operation: Callable[[], Awaitable[Any]], + iterations: int, +) -> list[float]: + """Measure sequential async operations. + + Args: + operation: Async operation factory. + iterations: Number of calls. + + Returns: + Per-call latencies in milliseconds. + """ + observations = [] + for _ in range(iterations): + started = time.perf_counter_ns() + await operation() + observations.append((time.perf_counter_ns() - started) / 1_000_000) + return observations + + +def _command_calls(info: dict[str, Any]) -> int: + """Sum Redis command calls from ``INFO commandstats``. + + Args: + info: Redis commandstats mapping. + + Returns: + Total command invocations. + """ + return sum( + int(value.get("calls", 0)) + for key, value in info.items() + if key.startswith("cmdstat_") and isinstance(value, dict) + ) + + +def _command_counts(info: dict[str, Any]) -> dict[str, int]: + """Extract per-command call counts. + + Args: + info: Redis commandstats mapping. + + Returns: + Command name to call count. + """ + return { + key.removeprefix("cmdstat_"): int(value.get("calls", 0)) + for key, value in info.items() + if key.startswith("cmdstat_") and isinstance(value, dict) + } + + +@pytest.mark.performance +async def test_redis_state_manager_report( + performance_output: Path, + performance_scale: str, +): + """Measure cold/warm state access, writes, contention, and large payloads. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + iterations = {"smoke": 5, "release": 200}[performance_scale] + concurrency = {"smoke": 5, "release": 200}[performance_scale] + report = PerformanceReport("redis", metadata={"scale": performance_scale}) + + async with real_redis_state_manager() as manager: + before_info = await manager.redis.info("commandstats") + cold_iteration = 0 + + async def cold_get() -> PerformanceState: + """Fetch a unique uncached state. + + Returns: + Newly created state. + """ + nonlocal cold_iteration + cold_iteration += 1 + token = BaseStateToken( + ident=f"cold-{cold_iteration}", + cls=PerformanceState, + ) + return get_performance_state(await manager.get_state(token)) + + cold = await _measure(cold_get, iterations) + warm_token = BaseStateToken(ident="warm", cls=PerformanceState) + warm_root = await manager.get_state(warm_token) + warm_state = get_performance_state(warm_root) + warm_state.counter += 1 + await manager.set_state(warm_token, warm_root) + warm_key = str(warm_token.with_cls(type(warm_state))) + assert await manager.redis.exists(warm_key) + + async def warm_get() -> PerformanceState: + """Fetch and resolve an existing persisted state. + + Returns: + Persisted performance state. + """ + return get_performance_state(await manager.get_state(warm_token)) + + warm = await _measure(warm_get, iterations) + + async def modify(token: BaseStateToken) -> int: + """Modify one token under its Redis lock. + + Args: + token: State token. + + Returns: + Updated counter. + """ + async with manager.modify_state_with_links(token) as root_state: + state = get_performance_state(root_state) + state.counter += 1 + return state.counter + + async def diagnose_modify() -> dict[str, float | int]: + """Attribute lock/read, serialization, and flush/write costs. + + Returns: + Stage timings and serialized byte count. + """ + context = manager.modify_state_with_links(warm_token) + lock_started = time.perf_counter_ns() + root_state = await context.__aenter__() + lock_read_ms = (time.perf_counter_ns() - lock_started) / 1_000_000 + state = get_performance_state(root_state) + state.counter += 1 + serialize_started = time.perf_counter_ns() + serialized = BaseStateToken.serialize(state) + serialize_ms = (time.perf_counter_ns() - serialize_started) / 1_000_000 + flush_started = time.perf_counter_ns() + await context.__aexit__(None, None, None) + flush_write_ms = (time.perf_counter_ns() - flush_started) / 1_000_000 + return { + "lock_and_read_ms": lock_read_ms, + "state_serialize_ms": serialize_ms, + "serialized_bytes": len(serialized), + "flush_and_write_ms": flush_write_ms, + } + + stage_metrics = await diagnose_modify() + writes = await _measure(lambda: modify(warm_token), iterations) + + async def concurrent(tokens: list[BaseStateToken]) -> list[float]: + """Measure a concurrent batch. + + Args: + tokens: Token for each concurrent modification. + + Returns: + Per-operation completion latencies. + """ + started = time.perf_counter_ns() + + async def timed(token: BaseStateToken) -> float: + """Measure one concurrent modification. + + Args: + token: State token. + + Returns: + Completion latency from batch start. + """ + await modify(token) + return (time.perf_counter_ns() - started) / 1_000_000 + + return list(await asyncio.gather(*(timed(token) for token in tokens))) + + probe = EventLoopProbe(interval=0.001) + async with probe: + same_token = await concurrent([warm_token] * concurrency) + independent_tokens = await concurrent([ + BaseStateToken(ident=f"independent-{index}", cls=PerformanceState) + for index in range(concurrency) + ]) + + large_token = BaseStateToken(ident="large", cls=PerformanceState) + + async def large_write() -> int: + """Write a state large enough to exercise serialization offloading. + + Returns: + Number of stored elements. + """ + async with manager.modify_state_with_links(large_token) as root_state: + state = get_performance_state(root_state) + state.numbers = list(range(100_000)) + return len(state.numbers) + + large = await _measure(large_write, 1 if performance_scale == "smoke" else 3) + after_info = await manager.redis.info("commandstats") + + command_delta = _command_calls(after_info) - _command_calls(before_info) + before_commands = _command_counts(before_info) + after_commands = _command_counts(after_info) + command_metrics = { + f"redis_command_{command}": count - before_commands.get(command, 0) + for command, count in after_commands.items() + if count - before_commands.get(command, 0) > 0 + } + common_metrics = ( + probe.summary() + | stage_metrics + | command_metrics + | {"redis_commands": command_delta} + ) + for name, observations, parameters in [ + ("cold_get", cold, {"iterations": iterations}), + ("warm_get", warm, {"iterations": iterations}), + ("modify", writes, {"iterations": iterations}), + ("same_token_contention", same_token, {"concurrency": concurrency}), + ( + "independent_token_concurrency", + independent_tokens, + {"concurrency": concurrency}, + ), + ("large_state_write", large, {"elements": 100_000}), + ]: + report.add( + BenchmarkResult( + name, + parameters, + observations, + common_metrics, + measurement_iterations=len(observations), + ) + ) + report.write(performance_output / "redis.json") + + assert command_delta > 0 + assert len(same_token) == len(independent_tokens) == concurrency diff --git a/tests/performance/test_session_memory.py b/tests/performance/test_session_memory.py new file mode 100644 index 00000000000..02b6cb6cb46 --- /dev/null +++ b/tests/performance/test_session_memory.py @@ -0,0 +1,168 @@ +"""Backend memory cost per connected session as state size grows.""" + +from __future__ import annotations + +import contextlib +import gc +import json +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from reflex_base.config import get_config +from reflex_base.registry import RegistrationContext + +from reflex.testing import AppHarness +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.report import current_process_metrics + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _handler_payload(suffix: str) -> dict[str, Any]: + """Build an event payload for a load-app handler. + + Args: + suffix: Handler method name. + + Returns: + Socket.IO event payload targeting the handler. + """ + handler_name = next( + name + for name in RegistrationContext.get().event_handlers + if name.endswith(f".{suffix}") and "performance_load" in name + ) + return { + "name": handler_name, + "payload": {}, + "router_data": {"path": "/", "pathname": "/", "query": {}}, + } + + +@contextlib.contextmanager +def _held_sessions( + url: str, + token_prefix: str, + count: int, + payload: dict[str, Any], +) -> Iterator[None]: + """Hold primed sessions from a subprocess so client memory stays external. + + Args: + url: Backend Socket.IO URL. + token_prefix: Unique token prefix for this batch. + count: Number of sessions to hold. + payload: Priming event payload sent once per session. + + Yields: + While the sessions are connected and primed. + + Raises: + RuntimeError: If the holder had to be killed; a holder outliving its + batch would leak client memory into the next RSS measurement. + """ + process = subprocess.Popen( + [ + sys.executable, + "-m", + "tests.benchmarks.support.idle_clients", + url, + token_prefix, + str(count), + json.dumps(payload), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + cwd=REPO_ROOT, + ) + try: + assert process.stdout is not None + assert process.stdin is not None + ready = process.stdout.readline().strip() + assert ready == "ready", f"session holder failed: {ready!r}" + yield + finally: + if process.stdin is not None: + with contextlib.suppress(OSError): + process.stdin.close() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + msg = ( + f"session holder {token_prefix!r} did not exit and was killed; " + "subsequent RSS measurements would have been invalid" + ) + raise RuntimeError(msg) from None + + +def _settled_rss() -> int: + """Collect garbage, let the loop settle, and read resident memory. + + Returns: + Resident set size in bytes. + """ + gc.collect() + time.sleep(0.2) + return current_process_metrics()["rss_bytes"] + + +@pytest.mark.performance +def test_session_memory_report( + performance_load_app: AppHarness, + performance_output: Path, + performance_scale: str, +): + """Measure marginal backend memory per connected, primed session. + + The capacity-planning number: bytes of backend memory per session at a + small and a large per-session state, measured between two held batches so + interpreter warmup does not inflate the marginal cost. + + Args: + performance_load_app: Running production app. + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + batch = {"smoke": 5, "release": 50}[performance_scale] + backend_url = get_config().api_url.rstrip("/") + report = PerformanceReport( + "session-memory", + metadata={"scale": performance_scale, "batch": batch}, + ) + + for variant, handler in [ + ("small_state", "increment"), + ("large_state", "append_large"), + ]: + payload = _handler_payload(handler) + rss_start = _settled_rss() + with _held_sessions(backend_url, f"mem-{variant}-a", batch, payload): + rss_first = _settled_rss() + with _held_sessions(backend_url, f"mem-{variant}-b", batch, payload): + rss_second = _settled_rss() + report.add( + BenchmarkResult( + name=f"session_memory_{variant}", + parameters={"sessions_per_batch": batch, "handler": handler}, + observations_ms=[], + metrics={ + "rss_start_bytes": rss_start, + "rss_first_batch_bytes": rss_first, + "rss_second_batch_bytes": rss_second, + "first_batch_bytes_per_session": (rss_first - rss_start) / batch, + "marginal_bytes_per_session": (rss_second - rss_first) / batch, + }, + measurement_iterations=batch, + ) + ) + + report.write(performance_output / "session-memory.json") + assert len(report.results) == 2 diff --git a/tests/performance/test_wire_size.py b/tests/performance/test_wire_size.py new file mode 100644 index 00000000000..1c738fd7bb3 --- /dev/null +++ b/tests/performance/test_wire_size.py @@ -0,0 +1,94 @@ +"""Wire-size benchmarks: serialized delta bytes per canonical interaction.""" + +from __future__ import annotations + +import gzip +from collections.abc import Callable +from pathlib import Path + +import pytest +from reflex_base.utils.format import json_dumps + +from reflex.state import StateUpdate +from tests.benchmarks.support import BenchmarkResult, PerformanceReport +from tests.benchmarks.support.states import PerformanceState, initialized_state + +STATE_SIZE = 1000 + + +def _scalar_change(state: PerformanceState) -> None: + """Increment the scalar counter.""" + state.counter += 1 + + +def _list_append(state: PerformanceState) -> None: + """Append one element to the list value.""" + state.numbers.append(STATE_SIZE) + + +def _partial_row_update(state: PerformanceState) -> None: + """Mutate a single element in the middle of the list value.""" + state.numbers[STATE_SIZE // 2] += 1 + + +def _mapping_update(state: PerformanceState) -> None: + """Update one existing mapping entry.""" + state.mapping["key_0"] += 1 + + +def _large_append(state: PerformanceState) -> None: + """Extend the list value by a thousand elements.""" + state.numbers.extend(range(STATE_SIZE)) + + +INTERACTIONS: dict[str, Callable[[PerformanceState], None]] = { + "scalar_change": _scalar_change, + "list_append": _list_append, + "partial_row_update": _partial_row_update, + "mapping_update": _mapping_update, + "large_append": _large_append, +} + + +@pytest.mark.performance +def test_wire_size_report(performance_output: Path, performance_scale: str): + """Record serialized state-update bytes for canonical interactions. + + Wire bytes regress silently: they are invisible to timing benchmarks on a + local network but dominate update latency on real links. Each interaction + mutates a warm state and serializes the resulting update exactly as the + event pipeline would. + + Args: + performance_output: Artifact directory. + performance_scale: Selected scenario scale. + """ + report = PerformanceReport("wire-size", metadata={"scale": performance_scale}) + sizes: dict[str, int] = {} + for name, interaction in INTERACTIONS.items(): + state = initialized_state(STATE_SIZE) + interaction(state) + # Match the compact separators python-socketio uses when it encodes the + # event packet (Packet.encode calls dumps(..., separators=(",", ":"))), + # so wire_bytes equals the real payload size rather than json's spaced + # default, which overstates it by roughly a fifth. + wire = json_dumps( + StateUpdate(delta=state.get_delta()), separators=(",", ":") + ).encode() + sizes[name] = len(wire) + report.add( + BenchmarkResult( + name=name, + parameters={"state_size": STATE_SIZE}, + observations_ms=[], + metrics={ + "wire_bytes": len(wire), + "wire_gzip_bytes": len(gzip.compress(wire)), + }, + ) + ) + + report.write(performance_output / "wire-size.json") + assert all(size > 0 for size in sizes.values()) + # A scalar change must stay far below a whole-collection resend. + assert sizes["scalar_change"] < sizes["large_append"] diff --git a/tests/units/benchmarks/__init__.py b/tests/units/benchmarks/__init__.py new file mode 100644 index 00000000000..77167d4b4ed --- /dev/null +++ b/tests/units/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Benchmark support tests.""" diff --git a/tests/units/benchmarks/test_loop_probe.py b/tests/units/benchmarks/test_loop_probe.py new file mode 100644 index 00000000000..6c9ad5057c5 --- /dev/null +++ b/tests/units/benchmarks/test_loop_probe.py @@ -0,0 +1,81 @@ +"""Tests for event-loop benchmark probes.""" + +import asyncio +import time + +import pytest + +from tests.benchmarks.support.loop_probe import EventLoopProbe + + +@pytest.mark.asyncio +async def test_event_loop_probe_detects_blocking_work(): + """A synchronous pause is reported as event-loop scheduling lag.""" + interval = 0.001 + async with EventLoopProbe(interval=interval) as probe: + await asyncio.sleep(interval * 2) + time.sleep(0.02) # noqa: ASYNC251 - intentional event-loop blocking + await asyncio.sleep(interval * 2) + + summary = probe.summary() + assert summary["sample_count"] >= 2 + assert summary["peak_tasks"] >= 1 + assert summary["lag_max_ms"] >= 10 + + +@pytest.mark.asyncio +async def test_event_loop_probe_detects_terminal_blocking_work(): + """A block immediately before context exit still produces a lag sample.""" + async with EventLoopProbe(interval=0.001) as probe: + time.sleep(0.02) # noqa: ASYNC251 - intentional event-loop blocking + + assert probe.summary()["lag_max_ms"] >= 10 + + +@pytest.mark.asyncio +async def test_event_loop_probe_excludes_sampler_task(): + """Task metrics exclude the probe's own sampler task.""" + async with EventLoopProbe(interval=0.001) as probe: + await asyncio.sleep(0.003) + + assert probe.peak_tasks == 1 + + +@pytest.mark.asyncio +async def test_event_loop_probe_resets_between_runs(): + """Reusing a probe does not mix observations from separate workloads.""" + probe = EventLoopProbe(interval=0.001) + async with probe: + await asyncio.sleep(0.005) + first_sample_count = len(probe.lag_samples) + probe.lag_samples.append(-1) + + async with probe: + await asyncio.sleep(0.003) + + assert first_sample_count > 0 + assert probe.lag_samples + assert -1 not in probe.lag_samples + + +def test_event_loop_probe_summary_percentiles(): + """Summary values use linearly interpolated lag percentiles.""" + probe = EventLoopProbe() + probe.lag_samples.extend([0.001, 0.002, 0.003, 0.004, 0.005]) + + assert probe.summary() == { + "sample_count": 5, + "peak_tasks": 0, + "lag_p50_ms": 3.0, + "lag_p95_ms": 4.8, + "lag_p99_ms": 4.96, + "lag_max_ms": 5.0, + } + + +@pytest.mark.asyncio +async def test_event_loop_probe_rejects_non_positive_interval(): + """Sampling intervals must advance the event-loop clock.""" + with pytest.raises(ValueError, match="greater than zero"): + async with EventLoopProbe(interval=0): + pass diff --git a/tests/units/benchmarks/test_support.py b/tests/units/benchmarks/test_support.py new file mode 100644 index 00000000000..7176c8672e4 --- /dev/null +++ b/tests/units/benchmarks/test_support.py @@ -0,0 +1,324 @@ +"""Tests for shared performance benchmark support.""" + +import asyncio +import json +from types import SimpleNamespace + +import pytest + +from tests.benchmarks.support.apps import lifecycle_app_source +from tests.benchmarks.support.diagnostics import capture_async_diagnostics +from tests.benchmarks.support.pipeline_trace import PipelineTrace, StageEvent +from tests.benchmarks.support.report import ( + BenchmarkEnvironment, + BenchmarkResult, + PerformanceReport, + percentile, +) +from tests.benchmarks.support.socket_client import run_clients + + +def test_percentile_interpolates_and_validates(): + """Percentiles interpolate and reject values outside the valid range.""" + assert percentile([], 50) == 0 + assert percentile([1, 2, 3, 4, 5], 95) == pytest.approx(4.8) + with pytest.raises(ValueError, match="between 0 and 100"): + percentile([1], 101) + + +def test_payload_kind_uses_python_310_compatible_enum_base(): + """Payload enums avoid bases that were added after Python 3.10.""" + pytest.importorskip("pydantic") + from tests.benchmarks.support.events import PayloadKind + + assert not any( + base.__module__ == "enum" and base.__name__ == "StrEnum" + for base in PayloadKind.__mro__ + ) + + +def test_production_harness_is_module_scoped(): + """The in-process production harness stops before unrelated modules run.""" + pytest.importorskip("pydantic") + from tests.performance import conftest as performance_conftest + + fixture = performance_conftest.performance_load_app + marker = fixture._fixture_function_marker # pyright: ignore[reportPrivateUsage] + assert marker.scope == "module" + + +def test_performance_report_round_trip(tmp_path): + """Reports retain observations, summaries, metrics, and environment data.""" + environment = BenchmarkEnvironment( + commit="abc", + branch="branch", + python="3.14", + implementation="CPython", + operating_system="test", + machine="machine", + processor="processor", + cpu_count=4, + ci=True, + ) + report = PerformanceReport("event-loop", environment=environment) + report.add( + BenchmarkResult( + name="warm-event", + parameters={"tokens": 1}, + observations_ms=[1, 2, 3], + metrics={"peak_tasks": 4}, + warmup_iterations=1, + measurement_iterations=3, + ) + ) + + output = report.write(tmp_path / "report.json") + payload = json.loads(output.read_text()) + + assert payload["schema_version"] == 1 + assert payload["environment"]["commit"] == "abc" + assert payload["results"][0]["summary"]["p50_ms"] == 2 + assert payload["results"][0]["metrics"]["peak_tasks"] == 4 + + +def test_pipeline_trace_durations_and_chrome_output(tmp_path): + """Pipeline traces calculate durations and produce Chrome trace events.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token", "enqueued", 1_000_000), + StageEvent("token", "dequeued", 2_500_000), + ]) + + assert trace.durations_ms([("enqueued", "dequeued")]) == { + "enqueued_to_dequeued": [1.5] + } + path = trace.write_chrome_trace(tmp_path / "trace.json") + payload = json.loads(path.read_text()) + assert [ + event["name"] for event in payload["traceEvents"] if event["ph"] == "i" + ] == ["enqueued", "dequeued"] + + +def test_pipeline_trace_chrome_tids_are_integers(): + """Chrome traces emit integer pids/tids and name tokens via metadata.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token-a", "enqueued", 1_000_000), + StageEvent("token-b", "enqueued", 2_000_000), + StageEvent("token-a", "dequeued", 3_000_000), + ]) + + payload = trace.chrome_trace() + assert all( + isinstance(event["pid"], int) and isinstance(event["tid"], int) + for event in payload["traceEvents"] + ) + instants = [event for event in payload["traceEvents"] if event["ph"] == "i"] + assert instants[0]["tid"] == instants[2]["tid"] + assert instants[0]["tid"] != instants[1]["tid"] + thread_names = { + event["tid"]: event["args"]["name"] + for event in payload["traceEvents"] + if event.get("name") == "thread_name" and event["ph"] == "M" + } + assert thread_names[instants[0]["tid"]] == "token-a" + assert thread_names[instants[1]["tid"]] == "token-b" + + +def test_pipeline_trace_preserves_repeated_stage_pairs(): + """Repeated streaming stages produce one duration per occurrence.""" + trace = PipelineTrace() + trace.extend([ + StageEvent("token", "delta_started", 1_000_000), + StageEvent("token", "delta_finished", 2_000_000), + StageEvent("token", "delta_started", 3_000_000), + StageEvent("token", "delta_finished", 5_000_000), + ]) + + assert trace.durations_ms([("delta_started", "delta_finished")]) == { + "delta_started_to_delta_finished": [1.0, 2.0] + } + + +@pytest.mark.asyncio +async def test_run_clients_leaves_default_executor_usable(): + """The load helper does not close the event loop's default executor.""" + results = await run_clients( + 2, + lambda index, executor: asyncio.get_running_loop().run_in_executor( + executor, lambda: index + ), + ) + + assert results == [0, 1] + assert await asyncio.to_thread(lambda: "still-usable") == "still-usable" + + +class _FakeRedis: + """Minimal async Redis stand-in recording lifecycle calls.""" + + def __init__(self, calls: list[str], dbsize: int = 0): + """Initialize the fake. + + Args: + calls: Shared call log appended to by every method. + dbsize: Key count reported by ``dbsize``. + """ + self.calls = calls + self._dbsize = dbsize + + async def ping(self) -> bool: + """Record and acknowledge a ping. + + Returns: + Always True. + """ + self.calls.append("ping") + return True + + async def dbsize(self) -> int: + """Record and report the configured key count. + + Returns: + The configured key count. + """ + self.calls.append("dbsize") + return self._dbsize + + async def flushdb(self) -> None: + """Record a database flush.""" + self.calls.append("flushdb") + + async def aclose(self) -> None: + """Record closing the connection pool.""" + self.calls.append("aclose") + + +def test_performance_redis_url_requires_database_index(monkeypatch): + """URLs without an explicit database index are rejected.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379") + with pytest.raises(ValueError, match="database index"): + redis_support.performance_redis_url() + + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + assert redis_support.performance_redis_url() == "redis://127.0.0.1:6379/15" + + +async def test_real_redis_state_manager_refuses_nonempty_database(monkeypatch): + """A database that already holds keys is never flushed.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + calls: list[str] = [] + fake = _FakeRedis(calls, dbsize=3) + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + monkeypatch.setattr( + redis_support, "Redis", SimpleNamespace(from_url=lambda url: fake) + ) + + with pytest.raises(RuntimeError, match="not empty"): + async with redis_support.real_redis_state_manager(): + pass + + assert "flushdb" not in calls + assert calls[-1] == "aclose" + + +async def test_real_redis_state_manager_cleanup_survives_close_failure(monkeypatch): + """The manager closes before the flush, which runs despite a close failure.""" + pytest.importorskip("redis") + from tests.benchmarks.support import redis as redis_support + + calls: list[str] = [] + fake = _FakeRedis(calls) + monkeypatch.setenv(redis_support.REDIS_URL_ENV, "redis://127.0.0.1:6379/15") + monkeypatch.setattr( + redis_support, "Redis", SimpleNamespace(from_url=lambda url: fake) + ) + + class _FailingManager: + """State-manager stand-in whose close always fails.""" + + def __init__(self, redis): + """Store the redis client. + + Args: + redis: The fake redis client. + """ + self.redis = redis + + async def close(self) -> None: + """Record the close attempt and fail. + + Raises: + RuntimeError: Always, to exercise cleanup ordering. + """ + calls.append("close") + msg = "close failed" + raise RuntimeError(msg) + + monkeypatch.setattr(redis_support, "StateManagerRedis", _FailingManager) + + with pytest.raises(RuntimeError, match="close failed"): + async with redis_support.real_redis_state_manager(): + pass + + # The manager is closed first so any oplock lease write-backs land before + # the flush; the flush then runs on a fresh client even though close() + # raised, and both the fresh client and the original connection are closed. + assert calls == ["ping", "dbsize", "close", "flushdb", "aclose", "aclose"] + + +async def test_capture_async_diagnostics_formats_each_frame_once(tmp_path): + """Task stacks include each ancestor frame once instead of once per frame.""" + task = asyncio.current_task() + assert task is not None + task.set_name("diagnostics-under-test") + await asyncio.sleep(0) + + destination = capture_async_diagnostics(tmp_path / "diagnostics.json") + payload = json.loads(destination.read_text()) + stack = "".join( + line + for task_info in payload["tasks"] + if task_info["name"] == "diagnostics-under-test" + for line in task_info["stack"] + ) + assert ", in test_capture_async_diagnostics_formats_each_frame_once" in stack + assert stack.count(", in ") == 1 + + +def test_lifecycle_app_source_uses_public_api_transformer(): + """Generated app sources register extra routes via the public seam.""" + source = lifecycle_app_source(rows=1, pages=1) + assert "_api" not in source + assert "api_transformer" in source + + +def test_baseline_server_startup_failure_surfaces_thread_exception(monkeypatch): + """A server thread crash is reported instead of a bare startup timeout.""" + pytest.importorskip("socketio") + uvicorn = pytest.importorskip("uvicorn") + from tests.benchmarks.support.baseline_server import BaselineSocketServer + + def crash(self) -> None: + """Simulate an immediate server crash. + + Args: + self: The uvicorn server. + + Raises: + RuntimeError: Always. + """ + msg = "boom during startup" + raise RuntimeError(msg) + + monkeypatch.setattr(uvicorn.Server, "run", crash) + with ( + pytest.raises(TimeoutError, match="boom during startup"), + BaselineSocketServer(response={}), + ): + pass diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 270f040f684..611e7ba7b61 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -42,7 +42,12 @@ import reflex as rx from reflex import AdminDash, constants from reflex._upload import upload -from reflex.app import App, ComponentCallable, default_overlay_component +from reflex.app import ( + App, + ComponentCallable, + _decode_asgi_headers, + default_overlay_component, +) from reflex.compiler.compiler import ( _compile_app, _memoize_stateful_app_wraps, @@ -3851,6 +3856,25 @@ def test_call_marks_later_dev_backend_worker_as_hot_reload( assert compile_mock.call_args.kwargs["trigger"] == "hot_reload" +def test_decode_asgi_headers(): + """_decode_asgi_headers decodes raw ASGI header pairs into a str dict.""" + assert _decode_asgi_headers([]) == {} + assert _decode_asgi_headers([ + (b"host", b"example.com"), + (b"x-forwarded-for", b"10.0.0.1, 10.0.0.2"), + ]) == { + "host": "example.com", + "x-forwarded-for": "10.0.0.1, 10.0.0.2", + } + # Later duplicates win, matching dict comprehension semantics. + assert _decode_asgi_headers([ + (b"cookie", b"a=1"), + (b"cookie", b"b=2"), + ]) == {"cookie": "b=2"} + # Names and values are decoded as UTF-8. + assert _decode_asgi_headers([(b"x-name", "café".encode())]) == {"x-name": "café"} + + def test_call_ignores_stale_marker_without_dev_backend_reload( compilable_app: tuple[App, Path], mocker: MockerFixture,