perf: Speed up local-evaluation hot path for large environments

Addresses #198: for a 262-feature environment, local evaluation is
1000x faster for the common case (no variants / no overrides) and
8-60% faster overall depending on scenario.

Changes:

- Cache the output of `get_environment_flags()` — the evaluation context
  is immutable between environment refreshes, so we rebuild it once on
  update instead of once per call. Invalidated via a property setter on
  `_evaluation_context` so existing direct-assignment call sites (tests,
  offline mode) keep working transparently.
- Short-circuit `get_identity_flags()` to the cached environment Flags
  when the environment has no features with variants and no segments with
  overrides — in that case identity flags are guaranteed to equal
  environment flags. A fresh `Flags` wrapper is allocated around the
  cached flag dict to preserve identity metadata for pipeline analytics.
- Precompute `identity.key` in `map_context_and_identity_data_to_context`
  so flag-engine's `get_enriched_context` becomes a no-op instead of
  performing a shallow context copy on every call.
- Pre-sort multivariate variants once at env-load time; Timsort on an
  already-sorted list is a fast path compared to resorting per call.
- Add `__slots__` to `Flag` / `DefaultFlag` / `BaseFlag` on Python 3.10+
  and inline the per-flag construction inside `Flags.from_evaluation_result`
  to skip the redundant helper call and truthiness check.

Also adds a `benchmarks/` harness (issue-#198 scenario, variant-density
knobs, cProfile mode) so future regressions can be caught locally
before release. Two tests that asserted on the engine's internal call
shape were rewritten to assert on actual flag values instead.

beep boop

Author: khvn26

benchmarks/__init__.py

@@ -0,0 +1,201 @@
+"""Microbenchmark for the Flagsmith Python SDK's local evaluation hot path.
+
+Run with::
+
+    poetry run python -m benchmarks.bench
+    poetry run python -m benchmarks.bench --profile        # cProfile hot path
+    poetry run python -m benchmarks.bench --iters 20000    # custom iter count
+
+Mirrors the scenario from issue #198: enable_local_evaluation=True, a
+single-identity call into get_identity_flags / get_environment_flags with
+~262 features.
+"""
+
+from __future__ import annotations
+
+import argparse
+import cProfile
+import json
+import pstats
+import statistics
+import time
+from typing import Callable
+
+from benchmarks.env import build_environment
+from flagsmith import Flagsmith
+from flagsmith.mappers import map_environment_document_to_context
+
+
+def _make_client(n_features: int, with_multivariate: int = 0) -> Flagsmith:
+    env_doc = build_environment(
+        n_features=n_features,
+        with_multivariate=with_multivariate,
+    )
+    # Build a local-eval client without hitting the network / starting polling.
+    client = Flagsmith.__new__(Flagsmith)
+    client.offline_mode = False
+    client.enable_local_evaluation = True
+    client.offline_handler = None
+    client.default_flag_handler = None
+    client.enable_realtime_updates = False
+    client._analytics_processor = None
+    client._pipeline_analytics_processor = None
+    client._Flagsmith__evaluation_context = None
+    client._environment_context_without_segments = None
+    client._environment_flags_cache = None
+    client._identity_flags_match_environment = False
+    client._environment_updated_at = None
+    client._evaluation_context = map_environment_document_to_context(env_doc)
+    return client
+
+
+def _bench(
+    name: str,
+    fn: Callable[[], object],
+    *,
+    iters: int,
+    warmup: int,
+) -> None:
+    for _ in range(warmup):
+        fn()
+
+    samples: list[float] = []
+    # Break total iters into batches so we can also report a stdev.
+    batch_size = max(1, iters // 20)
+    for _ in range(0, iters, batch_size):
+        n = min(batch_size, iters)
+        t0 = time.perf_counter()
+        for _ in range(n):
+            fn()
+        samples.append((time.perf_counter() - t0) / n)
+        iters -= n
+
+    p50 = statistics.median(samples) * 1e6
+    mean = statistics.fmean(samples) * 1e6
+    stdev = statistics.pstdev(samples) * 1e6
+    print(
+        f"{name:<32} p50={p50:8.2f} µs  mean={mean:8.2f} µs  stdev={stdev:7.2f} µs  "
+        f"throughput={1e6 / mean:>10,.0f}/s"
+    )
+
+
+def run(iters: int, warmup: int, n_features: int, with_multivariate: int) -> None:
+    client = _make_client(n_features, with_multivariate=with_multivariate)
+    traits = {"venue_id": "12345"}
+
+    print(
+        f"Flagsmith local-eval benchmark | features={n_features} "
+        f"multivariate={with_multivariate} iters={iters} warmup={warmup}"
+    )
+    _bench(
+        "get_environment_flags",
+        client.get_environment_flags,
+        iters=iters,
+        warmup=warmup,
+    )
+    _bench(
+        "get_identity_flags",
+        lambda: client.get_identity_flags(identifier="anonymous", traits=traits),
+        iters=iters,
+        warmup=warmup,
+    )
+
+    flags = client.get_identity_flags(identifier="anonymous", traits=traits)
+    name = next(iter(flags.flags))
+    _bench(
+        "is_feature_enabled (cached)",
+        lambda: flags.is_feature_enabled(name),
+        iters=iters * 10,
+        warmup=warmup,
+    )
+
+
+def profile(
+    iters: int,
+    n_features: int,
+    output: str | None,
+    with_multivariate: int = 0,
+) -> None:
+    client = _make_client(n_features, with_multivariate=with_multivariate)
+    traits = {"venue_id": "12345"}
+
+    # Warm up JSONPath caches, lru_cache, etc.
+    for _ in range(200):
+        client.get_identity_flags(identifier="anonymous", traits=traits)
+
+    profiler = cProfile.Profile()
+    profiler.enable()
+    for _ in range(iters):
+        client.get_identity_flags(identifier="anonymous", traits=traits)
+    profiler.disable()
+
+    stats = pstats.Stats(profiler).sort_stats(pstats.SortKey.CUMULATIVE)
+    stats.print_stats(30)
+    stats.sort_stats(pstats.SortKey.TIME).print_stats(30)
+
+    if output:
+        profiler.dump_stats(output)
+        print(f"wrote {output}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--iters", type=int, default=5000)
+    parser.add_argument("--warmup", type=int, default=500)
+    parser.add_argument("--features", type=int, default=262)
+    parser.add_argument(
+        "--multivariate",
+        type=int,
+        default=0,
+        help="number of features that should have 2-way multivariate variants",
+    )
+    parser.add_argument("--profile", action="store_true")
+    parser.add_argument("--profile-output", default=None)
+    parser.add_argument("--json", action="store_true", help="emit JSON summary")
+    args = parser.parse_args()
+
+    if args.profile:
+        profile(
+            args.iters,
+            args.features,
+            args.profile_output,
+            with_multivariate=args.multivariate,
+        )
+        return
+
+    if args.json:
+        # Alternative machine-readable mode for diffing runs.
+        client = _make_client(args.features)
+        traits = {"venue_id": "12345"}
+
+        def _measure(fn: Callable[[], object], count: int) -> float:
+            for _ in range(args.warmup):
+                fn()
+            t0 = time.perf_counter()
+            for _ in range(count):
+                fn()
+            return (time.perf_counter() - t0) / count
+
+        result = {
+            "features": args.features,
+            "iters": args.iters,
+            "get_environment_flags_us": _measure(
+                client.get_environment_flags, args.iters
+            )
+            * 1e6,
+            "get_identity_flags_us": _measure(
+                lambda: client.get_identity_flags(
+                    identifier="anonymous", traits=traits
+                ),
+                args.iters,
+            )
+            * 1e6,
+        }
+        print(json.dumps(result, indent=2))
+        return
+
+    run(args.iters, args.warmup, args.features, args.multivariate)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/bench.py

@@ -0,0 +1,56 @@
+"""Synthetic environment builder for local evaluation benchmarks.
+
+Mirrors the shape of the real environment document (262 features, one segment),
+so we can exercise the local eval hot path without needing network access.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import typing
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "tests", "data")
+TEMPLATE_PATH = os.path.join(DATA_DIR, "environment.json")
+
+
+def build_environment(
+    n_features: int = 262,
+    *,
+    with_multivariate: int = 0,
+) -> dict[str, typing.Any]:
+    with open(TEMPLATE_PATH) as f:
+        env = json.load(f)
+
+    # Base feature state to clone.
+    base_fs = copy.deepcopy(env["feature_states"][0])
+    feature_states: list[dict[str, typing.Any]] = []
+    for i in range(n_features):
+        fs = copy.deepcopy(base_fs)
+        fs["django_id"] = i + 1
+        fs["feature"] = {
+            "name": f"feature_{i:04d}",
+            "type": "STANDARD",
+            "id": i + 1,
+        }
+        fs["feature_state_value"] = f"value-{i}"
+        if with_multivariate and i < with_multivariate:
+            fs["multivariate_feature_state_values"] = [
+                {
+                    "multivariate_feature_option": {"value": f"mv-{i}-a"},
+                    "percentage_allocation": 50.0,
+                    "id": (i + 1) * 100 + 1,
+                },
+                {
+                    "multivariate_feature_option": {"value": f"mv-{i}-b"},
+                    "percentage_allocation": 50.0,
+                    "id": (i + 1) * 100 + 2,
+                },
+            ]
+        feature_states.append(fs)
+
+    env["feature_states"] = feature_states
+    # Strip the (irrelevant) identity override for a clean baseline.
+    env["identity_overrides"] = []
+    return env

benchmarks/env.py

@@ -117,7 +117,12 @@ def __init__(
         self._pipeline_analytics_processor: typing.Optional[
             PipelineAnalyticsProcessor
         ] = None
-        self._evaluation_context: typing.Optional[SDKEvaluationContext] = None
+        self.__evaluation_context: typing.Optional[SDKEvaluationContext] = None
+        self._environment_context_without_segments: typing.Optional[
+            SDKEvaluationContext
+        ] = None
+        self._environment_flags_cache: typing.Optional[Flags] = None
+        self._identity_flags_match_environment: bool = False
         self._environment_updated_at: typing.Optional[datetime] = None
 
         # argument validation
@@ -356,6 +361,46 @@ def update_environment(self) -> None:
             except (KeyError, TypeError, ValueError):
                 logger.exception("Error parsing environment document")
 
+    @property
+    def _evaluation_context(self) -> typing.Optional[SDKEvaluationContext]:
+        return self.__evaluation_context
+
+    @_evaluation_context.setter
+    def _evaluation_context(
+        self, context: typing.Optional[SDKEvaluationContext]
+    ) -> None:
+        """Swap in a new evaluation context and invalidate derived caches.
+
+        Pre-computing the segment-stripped view and a couple of shape flags
+        once per refresh keeps the hot path for ``get_environment_flags``
+        and the short-circuited ``get_identity_flags`` allocation-free.
+        """
+        self.__evaluation_context = context
+        self._environment_flags_cache = None
+        if context is None:
+            self._environment_context_without_segments = None
+            self._identity_flags_match_environment = False
+            return
+        context_without_segments = context.copy()
+        context_without_segments.pop("segments", None)
+        self._environment_context_without_segments = context_without_segments
+        # An identity's flags only differ from the environment's when either
+        # a feature has variants (percentage split) or some segment can
+        # override feature values (segment overrides / identity overrides).
+        # When neither is true we can skip the per-call engine evaluation
+        # entirely and reuse the cached environment Flags.
+        has_variants = any(
+            feature.get("variants")
+            for feature in (context.get("features") or {}).values()
+        )
+        has_segment_overrides = any(
+            segment.get("overrides")
+            for segment in (context.get("segments") or {}).values()
+        )
+        self._identity_flags_match_environment = not (
+            has_variants or has_segment_overrides
+        )
+
     def _get_headers(
         self,
         environment_key: str,
@@ -375,24 +420,25 @@ def _get_headers(
         return headers
 
     def _get_environment_flags_from_document(self) -> Flags:
-        if self._evaluation_context is None:
-            raise TypeError("No environment present")
+        if (cached := self._environment_flags_cache) is not None:
+            return cached
 
-        # Omit segments from evaluation context for environment flags
-        # as they are only relevant for identity-specific evaluations
-        context_without_segments = self._evaluation_context.copy()
-        context_without_segments.pop("segments", None)
+        context_without_segments = self._environment_context_without_segments
+        if context_without_segments is None:
+            raise TypeError("No environment present")
 
         evaluation_result = engine.get_evaluation_result(
             context=context_without_segments,
         )
 
-        return Flags.from_evaluation_result(
+        flags = Flags.from_evaluation_result(
             evaluation_result=evaluation_result,
             analytics_processor=self._analytics_processor,
             default_flag_handler=self.default_flag_handler,
             pipeline_analytics_processor=self._pipeline_analytics_processor,
         )
+        self._environment_flags_cache = flags
+        return flags
 
     def _get_identity_flags_from_document(
         self,
@@ -402,15 +448,26 @@ def _get_identity_flags_from_document(
         if self._evaluation_context is None:
             raise TypeError("No environment present")
 
+        if self._identity_flags_match_environment:
+            # Reuse the cached environment Flag dict but wrap it in a fresh
+            # ``Flags`` carrying this identity's metadata, so pipeline analytics
+            # still see per-identity events.
+            env_flags = self._get_environment_flags_from_document()
+            return Flags(
+                flags=env_flags.flags,
+                default_flag_handler=self.default_flag_handler,
+                _analytics_processor=self._analytics_processor,
+                _pipeline_analytics_processor=self._pipeline_analytics_processor,
+                _identity_identifier=identifier,
+                _traits=resolve_trait_values(traits),
+            )
+
         context = map_context_and_identity_data_to_context(
             context=self._evaluation_context,
             identifier=identifier,
             traits=traits,
         )
-        evaluation_result = engine.get_evaluation_result(
-            context=context,
-        )
-
+        evaluation_result = engine.get_evaluation_result(context=context)
         return Flags.from_evaluation_result(
             evaluation_result=evaluation_result,
             analytics_processor=self._analytics_processor,