From 840560595439945b464d085cd41843acf17b5576 Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 10:38:34 -0700 Subject: [PATCH 1/6] wip --- src/Stache/Indexes/Index.php | 7 +++++ src/Stache/Indexes/Terms/Associations.php | 38 ++++++++++++++++++++--- src/Stache/Stache.php | 3 +- src/Stache/Stores/AggregateStore.php | 10 ++++++ src/Stache/Stores/Store.php | 32 ++++++++++++++++--- 5 files changed, 81 insertions(+), 9 deletions(-) diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 61182a2e005..00126931048 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -59,6 +59,13 @@ public function push($value) $this->items[] = $value; } + public function setItems(array $items): static + { + $this->items = $items; + + return $this; + } + public function load() { if ($this->loaded) { diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 1ddef3781e3..1737d5d2e01 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -2,6 +2,7 @@ namespace Statamic\Stache\Indexes\Terms; +use Statamic\Facades\Stache; use Statamic\Facades\Taxonomy; use Statamic\Stache\Indexes\Index; use Statamic\Support\Str; @@ -13,6 +14,35 @@ public function getItems() return Taxonomy::findByHandle($handle = $this->store->childKey()) ->collections() ->flatMap(function ($collection) use ($handle) { + $entriesStore = Stache::store('entries')->store($collection->handle()); + $collectionHandle = $collection->handle(); + + // Fast path: warmValueIndexes() already built entries' category index in Redis. + $storeKey = $entriesStore->key(); + $cacheKey = "stache::indexes::{$storeKey}::{$handle}"; + $taxData = Stache::cacheStore()->get($cacheKey); + + if ($taxData !== null) { + $taxValues = collect($taxData)->filter(fn ($v) => ! empty($v)); + $siteData = Stache::cacheStore()->get("stache::indexes::{$storeKey}::site"); + $sites = $siteData !== null ? collect($siteData) : null; + + return $taxValues->flatMap(function ($value, $entryId) use ($collectionHandle, $entriesStore, $sites) { + $site = $sites !== null + ? $sites->get($entryId) + : $entriesStore->getItem($entryId)?->locale(); + + return collect((array) $value)->map(fn ($v) => [ + 'value' => $v, + 'slug' => Str::slug($v), + 'entry' => $entryId, + 'collection' => $collectionHandle, + 'site' => $site, + ]); + }); + } + + // Cold path fallback (fires outside of a 2-pass warm, e.g. in tests or direct calls). return $collection->queryEntries() ->where($handle, '<>', null) ->get() @@ -20,11 +50,11 @@ public function getItems() return collect($entry->value($handle)) ->map(function ($value) use ($entry) { return [ - 'value' => $value, - 'slug' => Str::slug($value), - 'entry' => $entry->id(), + 'value' => $value, + 'slug' => Str::slug($value), + 'entry' => $entry->id(), 'collection' => $entry->collectionHandle(), - 'site' => $entry->locale(), + 'site' => $entry->locale(), ]; }); })->all(); diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 406980b04c7..4705f5aeaa0 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -132,7 +132,8 @@ public function warm() if ($this->shouldUseParallelWarming($stores)) { $this->warmInParallel($stores); } else { - $stores->each->warm(); + $stores->each->warmValueIndexes(); + $stores->each->warmOtherIndexes(); } $this->stopTimer(); diff --git a/src/Stache/Stores/AggregateStore.php b/src/Stache/Stores/AggregateStore.php index 334c0b63835..255aad54599 100644 --- a/src/Stache/Stores/AggregateStore.php +++ b/src/Stache/Stores/AggregateStore.php @@ -87,6 +87,16 @@ public function warm() $this->discoverStores()->each->warm(); } + public function warmValueIndexes() + { + $this->discoverStores()->each->warmValueIndexes(); + } + + public function warmOtherIndexes() + { + $this->discoverStores()->each->warmOtherIndexes(); + } + public function paths() { return $this->discoverStores()->flatMap(function ($store) { diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index c220b679e1b..cebc1d82f9f 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -406,12 +406,36 @@ public function clear() public function warm() { - $this->shouldCacheFileItems = true; + $this->warmValueIndexes(); + $this->warmOtherIndexes(); + } + + public function warmValueIndexes() + { + $valueIndexes = $this->resolveIndexes()->filter( + fn ($index) => method_exists($index, 'getItemValue') + ); - $this->resolveIndexes()->each->update(); + $accumulated = $valueIndexes->map(fn () => [])->all(); - $this->shouldCacheFileItems = false; - $this->fileItems = null; + foreach ($this->paths()->keys() as $key) { + $item = $this->getItem($key); + + foreach ($valueIndexes as $name => $index) { + $accumulated[$name][$key] = $index->getItemValue($item); + } + } + + $valueIndexes->each(function ($index, $name) use ($accumulated) { + $index->setItems($accumulated[$name])->cache(); + }); + } + + public function warmOtherIndexes() + { + $this->resolveIndexes() + ->filter(fn ($index) => ! method_exists($index, 'getItemValue')) + ->each->update(); } public function keys() From 65fda10ae2ef88fc3612fe5656e53a317aa7af56 Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 10:41:24 -0700 Subject: [PATCH 2/6] comments --- src/Stache/Indexes/Index.php | 4 ++++ src/Stache/Indexes/Terms/Associations.php | 21 +++++++++++++++++---- src/Stache/Stache.php | 4 ++++ src/Stache/Stores/AggregateStore.php | 2 ++ src/Stache/Stores/Store.php | 11 +++++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 00126931048..dcbe5101c93 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -59,6 +59,10 @@ public function push($value) $this->items[] = $value; } + /** + * Bulk-sets the index items without going through getItems(). Used by + * Store::warmValueIndexes() to write pre-accumulated values in one shot. + */ public function setItems(array $items): static { $this->items = $items; diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 1737d5d2e01..bb04132995a 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -9,6 +9,20 @@ class Associations extends Index { + /** + * Builds the term→entry association map for a taxonomy. + * + * This index is the reason warm() runs in two passes. Associations needs to know + * which entries reference each term, but the only way to find that (without loading + * every Entry from disk) is to read the entries' already-warmed taxonomy index from + * Redis. The 2-pass warm guarantees that index exists before this method is called. + * + * Fast path (used during stache:warm): reads the flat `[entryId => termValue]` and + * `[entryId => site]` arrays directly from Redis — no Entry objects are constructed. + * + * Cold path (used outside of stache:warm, e.g. on-demand index builds): queries + * entries via Eloquent as before. Slower but always correct. + */ public function getItems() { return Taxonomy::findByHandle($handle = $this->store->childKey()) @@ -17,12 +31,11 @@ public function getItems() $entriesStore = Stache::store('entries')->store($collection->handle()); $collectionHandle = $collection->handle(); - // Fast path: warmValueIndexes() already built entries' category index in Redis. $storeKey = $entriesStore->key(); - $cacheKey = "stache::indexes::{$storeKey}::{$handle}"; - $taxData = Stache::cacheStore()->get($cacheKey); + $taxData = Stache::cacheStore()->get("stache::indexes::{$storeKey}::{$handle}"); if ($taxData !== null) { + // Fast path: entries' value indexes are already in Redis (Pass 1 ran first). $taxValues = collect($taxData)->filter(fn ($v) => ! empty($v)); $siteData = Stache::cacheStore()->get("stache::indexes::{$storeKey}::site"); $sites = $siteData !== null ? collect($siteData) : null; @@ -42,7 +55,7 @@ public function getItems() }); } - // Cold path fallback (fires outside of a 2-pass warm, e.g. in tests or direct calls). + // Cold path: Redis miss — fall back to querying entries directly. return $collection->queryEntries() ->where($handle, '<>', null) ->get() diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 4705f5aeaa0..59eb0ccdc17 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -132,6 +132,10 @@ public function warm() if ($this->shouldUseParallelWarming($stores)) { $this->warmInParallel($stores); } else { + // Two-pass warm: Pass 1 writes all Value indexes (including entries' taxonomy + // indexes) to Redis across every store before Pass 2 runs. This lets + // Terms\Associations read from Redis in Pass 2 instead of loading all Entry + // objects from disk, which was the main source of slow warm times. $stores->each->warmValueIndexes(); $stores->each->warmOtherIndexes(); } diff --git a/src/Stache/Stores/AggregateStore.php b/src/Stache/Stores/AggregateStore.php index 255aad54599..5d537e919ec 100644 --- a/src/Stache/Stores/AggregateStore.php +++ b/src/Stache/Stores/AggregateStore.php @@ -87,11 +87,13 @@ public function warm() $this->discoverStores()->each->warm(); } + /** @see Store::warmValueIndexes() */ public function warmValueIndexes() { $this->discoverStores()->each->warmValueIndexes(); } + /** @see Store::warmOtherIndexes() */ public function warmOtherIndexes() { $this->discoverStores()->each->warmOtherIndexes(); diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index cebc1d82f9f..dc2092df109 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -410,6 +410,12 @@ public function warm() $this->warmOtherIndexes(); } + /** + * Pass 1 of the 2-pass warm. Loads every file once and accumulates values for + * all Value-based indexes in a single loop, then writes each index to Redis in + * one shot. This ensures entries' taxonomy indexes (e.g. `categories`) are in + * Redis before Pass 2 runs, so Terms\Associations can use the fast path. + */ public function warmValueIndexes() { $valueIndexes = $this->resolveIndexes()->filter( @@ -431,6 +437,11 @@ public function warmValueIndexes() }); } + /** + * Pass 2 of the 2-pass warm. Runs after all stores have completed Pass 1, so + * non-Value indexes (e.g. Terms\Associations) can read from Redis instead of + * loading Entry objects from disk. + */ public function warmOtherIndexes() { $this->resolveIndexes() From 666238e9824947e3c5b544b0b97134161fd647d5 Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 11:07:18 -0700 Subject: [PATCH 3/6] pint --- src/Stache/Indexes/Terms/Associations.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index bb04132995a..4bb5e1eba0f 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -46,11 +46,11 @@ public function getItems() : $entriesStore->getItem($entryId)?->locale(); return collect((array) $value)->map(fn ($v) => [ - 'value' => $v, - 'slug' => Str::slug($v), - 'entry' => $entryId, + 'value' => $v, + 'slug' => Str::slug($v), + 'entry' => $entryId, 'collection' => $collectionHandle, - 'site' => $site, + 'site' => $site, ]); }); } @@ -63,11 +63,11 @@ public function getItems() return collect($entry->value($handle)) ->map(function ($value) use ($entry) { return [ - 'value' => $value, - 'slug' => Str::slug($value), - 'entry' => $entry->id(), + 'value' => $value, + 'slug' => Str::slug($value), + 'entry' => $entry->id(), 'collection' => $entry->collectionHandle(), - 'site' => $entry->locale(), + 'site' => $entry->locale(), ]; }); })->all(); From 03c02774aea25c2933dd2085bd271b95dc428246 Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 12:26:10 -0700 Subject: [PATCH 4/6] tidy --- src/Stache/Indexes/Terms/Associations.php | 42 ++++++++++++----------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 4bb5e1eba0f..f3cb999fee3 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -15,10 +15,10 @@ class Associations extends Index * This index is the reason warm() runs in two passes. Associations needs to know * which entries reference each term, but the only way to find that (without loading * every Entry from disk) is to read the entries' already-warmed taxonomy index from - * Redis. The 2-pass warm guarantees that index exists before this method is called. + * the cache. The 2-pass warm guarantees that index exists before this method is called. * * Fast path (used during stache:warm): reads the flat `[entryId => termValue]` and - * `[entryId => site]` arrays directly from Redis — no Entry objects are constructed. + * `[entryId => site]` arrays directly from the cache — no Entry objects are constructed. * * Cold path (used outside of stache:warm, e.g. on-demand index builds): queries * entries via Eloquent as before. Slower but always correct. @@ -32,27 +32,29 @@ public function getItems() $collectionHandle = $collection->handle(); $storeKey = $entriesStore->key(); - $taxData = Stache::cacheStore()->get("stache::indexes::{$storeKey}::{$handle}"); - if ($taxData !== null) { - // Fast path: entries' value indexes are already in Redis (Pass 1 ran first). - $taxValues = collect($taxData)->filter(fn ($v) => ! empty($v)); - $siteData = Stache::cacheStore()->get("stache::indexes::{$storeKey}::site"); - $sites = $siteData !== null ? collect($siteData) : null; + // array of [entry_id => [term]] + $taxonomyData = collect(Stache::cacheStore()->get("stache::indexes::{$storeKey}::{$handle}")); - return $taxValues->flatMap(function ($value, $entryId) use ($collectionHandle, $entriesStore, $sites) { - $site = $sites !== null - ? $sites->get($entryId) - : $entriesStore->getItem($entryId)?->locale(); + if (! is_null($taxonomyData)) { + // Fast path: entries' value indexes are already in the cache (Pass 1 ran first). + $sites = collect(Stache::cacheStore()->get("stache::indexes::{$storeKey}::site")); - return collect((array) $value)->map(fn ($v) => [ - 'value' => $v, - 'slug' => Str::slug($v), - 'entry' => $entryId, - 'collection' => $collectionHandle, - 'site' => $site, - ]); - }); + return $taxonomyData + ->filter(fn (?array $entryTerms) => ! empty($entryTerms)) + ->flatMap(function ($value, $entryId) use ($collectionHandle, $entriesStore, $sites) { + $site = $sites->isNotEmpty() + ? $sites->get($entryId) + : $entriesStore->getItem($entryId)?->locale(); + + return collect((array) $value)->map(fn ($v) => [ + 'value' => $v, + 'slug' => Str::slug($v), + 'entry' => $entryId, + 'collection' => $collectionHandle, + 'site' => $site, + ]); + }); } // Cold path: Redis miss — fall back to querying entries directly. From 7990850f954be2d578008fa8dc0b08d5be238cec Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 12:45:10 -0700 Subject: [PATCH 5/6] more tidy --- src/Stache/Indexes/Terms/Associations.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index f3cb999fee3..8c23142828d 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -36,20 +36,20 @@ public function getItems() // array of [entry_id => [term]] $taxonomyData = collect(Stache::cacheStore()->get("stache::indexes::{$storeKey}::{$handle}")); - if (! is_null($taxonomyData)) { + if ($taxonomyData->isNotEmpty()) { // Fast path: entries' value indexes are already in the cache (Pass 1 ran first). $sites = collect(Stache::cacheStore()->get("stache::indexes::{$storeKey}::site")); return $taxonomyData - ->filter(fn (?array $entryTerms) => ! empty($entryTerms)) - ->flatMap(function ($value, $entryId) use ($collectionHandle, $entriesStore, $sites) { + ->filter(fn (?array $entryTerms): bool => ! empty($entryTerms)) + ->flatMap(function (array $terms, string|int $entryId) use ($collectionHandle, $entriesStore, $sites) { $site = $sites->isNotEmpty() ? $sites->get($entryId) : $entriesStore->getItem($entryId)?->locale(); - return collect((array) $value)->map(fn ($v) => [ - 'value' => $v, - 'slug' => Str::slug($v), + return collect($terms)->map(fn (string $term) => [ + 'value' => $term, + 'slug' => Str::slug($term), 'entry' => $entryId, 'collection' => $collectionHandle, 'site' => $site, From 9f70dd63b9c1be78422b19272e412784ac4b8631 Mon Sep 17 00:00:00 2001 From: edalzell Date: Sat, 13 Jun 2026 12:49:56 -0700 Subject: [PATCH 6/6] tidy & comment --- src/Stache/Stores/Store.php | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index dc2092df109..cc28a1abdf1 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -418,10 +418,18 @@ public function warm() */ public function warmValueIndexes() { - $valueIndexes = $this->resolveIndexes()->filter( - fn ($index) => method_exists($index, 'getItemValue') - ); - + $valueIndexes = $this + ->resolveIndexes() + ->filter( + fn ($index) => method_exists($index, 'getItemValue') + ); + + /* + This sets up a structure like ['fieldName' => [], 'anotherField' => []] before the loop below + it iterates over every item in the store. Each inner array then gets populated with $key => $value + pairs as items are processed, so all items are batched by index rather than writing to cache one at a time. + It's a performance optimization — collect everything first, then flush each index to cache in one shot on line 449. + */ $accumulated = $valueIndexes->map(fn () => [])->all(); foreach ($this->paths()->keys() as $key) {