Randomize cache index for singleton allocations

src/alloc.c

@@ -196,8 +196,38 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z
   }
 }
 
+// Large singleton pages start at slice boundaries. Returning that address
+// unchanged for a run of equally-sized allocations can map every object to
+// the same cache sets. Use some of the size-class slack to vary the cache
+// index without reserving any additional memory.
+static void* mi_heap_malloc_cache_index_randomize(mi_heap_t* heap, void* p, size_t size) {
+  #if MI_TRACK_ENABLED
+  MI_UNUSED(heap); MI_UNUSED(size);
+  return p;
+  #else
+  if (p == NULL) return NULL;
+  mi_page_t* const page = _mi_ptr_page(p);
+  if (page->reserved != 1 || mi_page_has_aligned(page)) return p;
+
+  const size_t bsize = mi_page_usable_block_size(page);
+  const size_t slack = (bsize >= size ? bsize - size : 0);
+  const size_t cache_line = 64;
+  const size_t max_cache_offset = 63 * cache_line;
+  if (slack < cache_line) return p;
+
+  const size_t offset_count = 1 + ((slack < max_cache_offset ? slack : max_cache_offset) / cache_line);
+  const size_t offset = (_mi_heap_random_next(heap) % offset_count) * cache_line;
+  if (offset == 0) return p;
+
+  mi_page_set_has_aligned(page, true);
+  _mi_padding_shrink(page, (mi_block_t*)p, offset + size);
+  return (uint8_t*)p + offset;
+  #endif
+}
+
 extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
-  return _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL);
+  void* const p = _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL);
+  return mi_heap_malloc_cache_index_randomize(heap, p, size);
 }
 
 mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {

src/free.c

@@ -504,6 +504,14 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si
   mi_track_mem_defined(padding,sizeof(mi_padding_t));
   padding->delta = (uint32_t)new_delta;
   mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
+  #if MI_PADDING_CHECK
+  // The usable prefix grew, so move the overflow-detection bytes with it.
+  uint8_t* fill = (uint8_t*)block + bsize - new_delta;
+  const size_t maxpad = (new_delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : new_delta);
+  mi_track_mem_defined(fill, maxpad);
+  for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
+  mi_track_mem_noaccess(fill, maxpad);
+  #endif
 }
 #else
 static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {

test/test-api.c

@@ -102,6 +102,26 @@ int main(void) {
     void* p = mi_malloc(67108872);
     mi_free(p);
   };
+  CHECK_BODY("malloc-large-cache-index") { // see issue #1121
+    #if (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)
+    result = true; // cache-index randomization is disabled with external trackers
+    #else
+    const size_t count = 64;
+    const size_t size = 64 * 1024 * sizeof(float) + sizeof(float);
+    void* blocks[64];
+    uintptr_t first_index = 0;
+    bool varied = false;
+    for (size_t i = 0; i < count; i++) {
+      blocks[i] = mi_malloc(size);
+      assert(blocks[i] != NULL);
+      const uintptr_t index = (uintptr_t)blocks[i] % 4096;
+      if (i == 0) { first_index = index; }
+      else if (index != first_index) { varied = true; }
+    }
+    for (size_t i = 0; i < count; i++) { mi_free(blocks[i]); }
+    result = varied;
+    #endif
+  };
 
   // ---------------------------------------------------
   // Extended