diff --git a/src/alloc.c b/src/alloc.c index 313efa5a1..a31bd9c57 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -196,8 +196,38 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z } } +// Large singleton pages start at slice boundaries. Returning that address +// unchanged for a run of equally-sized allocations can map every object to +// the same cache sets. Use some of the size-class slack to vary the cache +// index without reserving any additional memory. +static void* mi_heap_malloc_cache_index_randomize(mi_heap_t* heap, void* p, size_t size) { + #if MI_TRACK_ENABLED + MI_UNUSED(heap); MI_UNUSED(size); + return p; + #else + if (p == NULL) return NULL; + mi_page_t* const page = _mi_ptr_page(p); + if (page->reserved != 1 || mi_page_has_aligned(page)) return p; + + const size_t bsize = mi_page_usable_block_size(page); + const size_t slack = (bsize >= size ? bsize - size : 0); + const size_t cache_line = 64; + const size_t max_cache_offset = 63 * cache_line; + if (slack < cache_line) return p; + + const size_t offset_count = 1 + ((slack < max_cache_offset ? slack : max_cache_offset) / cache_line); + const size_t offset = (_mi_heap_random_next(heap) % offset_count) * cache_line; + if (offset == 0) return p; + + mi_page_set_has_aligned(page, true); + _mi_padding_shrink(page, (mi_block_t*)p, offset + size); + return (uint8_t*)p + offset; + #endif +} + extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { - return _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL); + void* const p = _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL); + return mi_heap_malloc_cache_index_randomize(heap, p, size); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { diff --git a/src/free.c b/src/free.c index 8762ae1e8..deee8db9d 100644 --- a/src/free.c +++ b/src/free.c @@ -504,6 +504,14 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si mi_track_mem_defined(padding,sizeof(mi_padding_t)); padding->delta = (uint32_t)new_delta; mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); + #if MI_PADDING_CHECK + // The usable prefix grew, so move the overflow-detection bytes with it. + uint8_t* fill = (uint8_t*)block + bsize - new_delta; + const size_t maxpad = (new_delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : new_delta); + mi_track_mem_defined(fill, maxpad); + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + mi_track_mem_noaccess(fill, maxpad); + #endif } #else static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) { diff --git a/test/test-api.c b/test/test-api.c index 763e4cef2..c183045af 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -102,6 +102,26 @@ int main(void) { void* p = mi_malloc(67108872); mi_free(p); }; + CHECK_BODY("malloc-large-cache-index") { // see issue #1121 + #if (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW) + result = true; // cache-index randomization is disabled with external trackers + #else + const size_t count = 64; + const size_t size = 64 * 1024 * sizeof(float) + sizeof(float); + void* blocks[64]; + uintptr_t first_index = 0; + bool varied = false; + for (size_t i = 0; i < count; i++) { + blocks[i] = mi_malloc(size); + assert(blocks[i] != NULL); + const uintptr_t index = (uintptr_t)blocks[i] % 4096; + if (i == 0) { first_index = index; } + else if (index != first_index) { varied = true; } + } + for (size_t i = 0; i < count; i++) { mi_free(blocks[i]); } + result = varied; + #endif + }; // --------------------------------------------------- // Extended