diff --git a/src/runtime/internal/region_allocator.h b/src/runtime/internal/region_allocator.h index 8c04116a3a65..b8dccc377e60 100644 --- a/src/runtime/internal/region_allocator.h +++ b/src/runtime/internal/region_allocator.h @@ -94,6 +94,9 @@ class RegionAllocator { // Invokes the deallocation callback to free memory for the block region int free_block_region(void *user_context, BlockRegion *region); + // Deallocate only if the region is unused. + int deallocate_block_region_if_unused(void *user_context, BlockRegion *region); + // Returns true if the given block region is the last region in the list bool is_last_block_region(void *user_context, const BlockRegion *region) const; @@ -389,24 +392,11 @@ bool RegionAllocator::can_coalesce(const BlockRegion *block_region) const { } BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) { - - if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) { -#ifdef DEBUG_RUNTIME_INTERNAL - debug(user_context) << "RegionAllocator: Freeing unused region to coalesce (" - << "block_ptr=" << (void *)block_region->block_ptr << " " - << "block_region=" << (void *)block_region << " " - << "memory_size=" << (uint32_t)(block_region->memory.allocation.size) << " " - << "block_reserved=" << (uint32_t)block->reserved << " " - << ")"; -#endif - halide_abort_if_false(user_context, allocators.region.deallocate != nullptr); - MemoryRegion *memory_region = &(block_region->memory); - allocators.region.deallocate(user_context, memory_region); - block_region->memory.handle = nullptr; - } + deallocate_block_region_if_unused(user_context, block_region); BlockRegion *prev_region = block_region->prev_ptr; if (is_available(prev_region) && (prev_region != block_region)) { + deallocate_block_region_if_unused(user_context, prev_region); #ifdef DEBUG_RUNTIME_INTERNAL debug(user_context) << "RegionAllocator: Coalescing " @@ -448,29 +438,18 @@ bool RegionAllocator::can_split(const BlockRegion *block_region, const MemoryReq } BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, const MemoryRequest &request) { + const size_t original_size = block_region->memory.allocation.size; + const size_t original_offset = block_region->memory.allocation.offset; - if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) { -#ifdef DEBUG_RUNTIME_INTERNAL - debug(user_context) << "RegionAllocator: Split deallocate region (" - << "block_ptr=" << (void *)block_region->block_ptr << " " - << "block_region=" << (void *)block_region << " " - << "memory_size=" << (uint32_t)(block_region->memory.allocation.size) << " " - << "block_reserved=" << (uint32_t)block_region->block_ptr->reserved << " " - << ")"; -#endif - halide_abort_if_false(user_context, allocators.region.deallocate != nullptr); - MemoryRegion *memory_region = &(block_region->memory); - allocators.region.deallocate(user_context, memory_region); - block_region->memory.handle = nullptr; - } + deallocate_block_region_if_unused(user_context, block_region); MemoryRequest split_request = request; - split_request.size = block_region->memory.allocation.size - request.size; - split_request.offset = block_region->memory.allocation.offset + request.size; + split_request.size = original_size - request.size; + split_request.offset = original_offset + request.size; #ifdef DEBUG_RUNTIME_INTERNAL debug(user_context) << "RegionAllocator: Splitting " - << "current region (offset=" << (int32_t)block_region->memory.allocation.offset << " size=" << (int32_t)(block_region->memory.allocation.size) << " bytes) " + << "current region (offset=" << (int32_t)original_offset << " size=" << (int32_t)(original_size) << " bytes) " << "to create empty region (offset=" << (int32_t)split_request.offset << " size=" << (int32_t)(split_request.size) << " bytes)"; #endif BlockRegion *next_region = block_region->next_ptr; @@ -483,7 +462,13 @@ BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion } empty_region->prev_ptr = block_region; block_region->next_ptr = empty_region; - block_region->memory.allocation.size -= empty_region->memory.allocation.size; + + // Derive the allocated region's size from the empty region's conformed offset; + // subtracting the empty region's size would shrink this region below the request + // whenever conforming the empty region adjusted its offset or size. + block_region->memory.allocation.size = empty_region->memory.allocation.offset - original_offset; + halide_abort_if_false(user_context, block_region->memory.allocation.size >= request.size); + halide_abort_if_false(user_context, empty_region->memory.allocation.offset + empty_region->memory.allocation.size <= original_offset + original_size); return empty_region; } @@ -651,27 +636,32 @@ int RegionAllocator::free_block_region(void *user_context, BlockRegion *block_re << "usage_count=" << (uint32_t)block_region->usage_count << " " << "block_reserved=" << (uint32_t)block->reserved << ")"; #endif - int error_code = 0; - if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) { -#ifdef DEBUG_RUNTIME_INTERNAL - debug(user_context) << " deallocating region (" - << "block_ptr=" << (void *)block_region->block_ptr << " " - << "block_region=" << (void *)block_region << " " - << "memory_size=" << (uint32_t)(block_region->memory.allocation.size) << " " - << "block_reserved=" << (uint32_t)block->reserved << " " - << ")"; -#endif - // NOTE: Deallocate but leave memory size as is, so that coalesce can compute region merging sizes - halide_abort_if_false(user_context, allocators.region.deallocate != nullptr); - MemoryRegion *memory_region = &(block_region->memory); - error_code = allocators.region.deallocate(user_context, memory_region); - block_region->memory.handle = nullptr; - } + int error_code = deallocate_block_region_if_unused(user_context, block_region); block_region->usage_count = 0; block_region->status = AllocationStatus::Available; return error_code; } +int RegionAllocator::deallocate_block_region_if_unused(void *user_context, BlockRegion *region) { + if ((region->usage_count != 0) || (region->memory.handle == nullptr)) { + return 0; + } + +#ifdef DEBUG_RUNTIME_INTERNAL + debug(user_context) << "RegionAllocator: deallocating unused region (" + << "block_ptr=" << (void *)region->block_ptr << " " + << "block_region=" << (void *)region << " " + << "memory_size=" << (uint32_t)(region->memory.allocation.size) << " " + << "block_reserved=" << (uint32_t)block->reserved << " " + << ")"; +#endif + halide_abort_if_false(user_context, allocators.region.deallocate != nullptr); + MemoryRegion *memory_region = &(region->memory); + int error_code = allocators.region.deallocate(user_context, memory_region); + region->memory.handle = nullptr; + return error_code; +} + int RegionAllocator::release(void *user_context) { #ifdef DEBUG_RUNTIME_INTERNAL debug(user_context) << "RegionAllocator: Releasing all regions (" diff --git a/test/runtime/block_allocator.cpp b/test/runtime/block_allocator.cpp index 7bc3d5ebd92f..80e53e79b0f9 100644 --- a/test/runtime/block_allocator.cpp +++ b/test/runtime/block_allocator.cpp @@ -43,8 +43,19 @@ int deallocate_block(void *user_context, MemoryBlock *block) { return halide_error_code_success; } +void *encode_region_handle(size_t size) { + return (void *)(size + 1); +} + +void check_region(void *user_context, MemoryRegion *region) { + HALIDE_CHECK(user_context, region != nullptr); + HALIDE_CHECK(user_context, region->handle == encode_region_handle(region->allocation.size)); +} + int allocate_region(void *user_context, MemoryRegion *region) { - region->handle = (void *)1; + HALIDE_CHECK(user_context, region->handle == nullptr); + + region->handle = encode_region_handle(region->allocation.size); allocated_region_memory += region->allocation.size; debug(user_context) << "Test : allocate_region (" @@ -57,7 +68,8 @@ int allocate_region(void *user_context, MemoryRegion *region) { } int deallocate_region(void *user_context, MemoryRegion *region) { - region->handle = (void *)0; + HALIDE_CHECK(user_context, region->handle == encode_region_handle(region->allocation.size)); + region->handle = nullptr; allocated_region_memory -= region->allocation.size; debug(user_context) << "Test : deallocate_region (" @@ -120,12 +132,12 @@ int main(int argc, char **argv) { request.properties.usage = MemoryUsage::DefaultUsage; MemoryRegion *r1 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r1 != nullptr); + check_region(user_context, r1); HALIDE_CHECK(user_context, allocated_block_memory == block_size); HALIDE_CHECK(user_context, allocated_region_memory == request.size); MemoryRegion *r2 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r2 != nullptr); + check_region(user_context, r2); HALIDE_CHECK(user_context, allocated_block_memory == block_size); HALIDE_CHECK(user_context, allocated_region_memory == (2 * request.size)); @@ -133,7 +145,7 @@ int main(int argc, char **argv) { HALIDE_CHECK(user_context, allocated_region_memory == (1 * request.size)); MemoryRegion *r3 = instance->reserve(user_context, request); - halide_abort_if_false(user_context, r3 != nullptr); + check_region(user_context, r3); halide_abort_if_false(user_context, allocated_block_memory == block_size); halide_abort_if_false(user_context, allocated_region_memory == (2 * request.size)); instance->retain(user_context, r3); @@ -152,9 +164,9 @@ int main(int argc, char **argv) { request.size = block_size / 2; // request two half-size regions MemoryRegion *r4 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r4 != nullptr); + check_region(user_context, r4); MemoryRegion *r5 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r5 != nullptr); + check_region(user_context, r5); HALIDE_CHECK(user_context, nullptr == instance->reserve(user_context, request)); // requesting a third should fail HALIDE_CHECK(user_context, allocated_block_memory == block_size); @@ -167,7 +179,7 @@ int main(int argc, char **argv) { request.size = block_size; MemoryRegion *r6 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r6 != nullptr); + check_region(user_context, r6); instance->destroy(user_context); deallocate_block(user_context, memory_block); @@ -308,12 +320,12 @@ int main(int argc, char **argv) { request.properties.usage = MemoryUsage::DefaultUsage; MemoryRegion *r1 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r1 != nullptr); + check_region(user_context, r1); HALIDE_CHECK(user_context, allocated_block_memory == block_size); HALIDE_CHECK(user_context, allocated_region_memory == padded_size); MemoryRegion *r2 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r2 != nullptr); + check_region(user_context, r2); HALIDE_CHECK(user_context, allocated_block_memory == block_size); HALIDE_CHECK(user_context, allocated_region_memory == (2 * padded_size)); @@ -324,9 +336,9 @@ int main(int argc, char **argv) { request.size = block_size / 2; // request two half-size regions MemoryRegion *r4 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r4 != nullptr); + check_region(user_context, r4); MemoryRegion *r5 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r5 != nullptr); + check_region(user_context, r5); HALIDE_CHECK(user_context, nullptr == instance->reserve(user_context, request)); // requesting a third should fail HALIDE_CHECK(user_context, allocated_block_memory == block_size); @@ -339,7 +351,7 @@ int main(int argc, char **argv) { request.size = block_size; MemoryRegion *r6 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r6 != nullptr); + check_region(user_context, r6); instance->destroy(user_context); deallocate_block(user_context, memory_block); @@ -380,12 +392,12 @@ int main(int argc, char **argv) { request.properties.usage = MemoryUsage::DefaultUsage; MemoryRegion *r1 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r1 != nullptr); + check_region(user_context, r1); HALIDE_CHECK(user_context, allocated_block_memory == config.minimum_block_size); HALIDE_CHECK(user_context, allocated_region_memory == request.size); MemoryRegion *r2 = instance->reserve(user_context, request); - HALIDE_CHECK(user_context, r2 != nullptr); + check_region(user_context, r2); HALIDE_CHECK(user_context, allocated_block_memory == config.minimum_block_size); HALIDE_CHECK(user_context, allocated_region_memory == (2 * request.size)); @@ -393,7 +405,7 @@ int main(int argc, char **argv) { HALIDE_CHECK(user_context, allocated_region_memory == (1 * request.size)); MemoryRegion *r3 = instance->reserve(user_context, request); - halide_abort_if_false(user_context, r3 != nullptr); + check_region(user_context, r3); halide_abort_if_false(user_context, allocated_block_memory == config.minimum_block_size); halide_abort_if_false(user_context, allocated_region_memory == (2 * request.size)); instance->retain(user_context, r3); @@ -489,6 +501,7 @@ int main(int argc, char **argv) { count = count > 1 ? count : 1; request.size = count * sizeof(int); MemoryRegion *region = instance->reserve(user_context, request); + check_region(user_context, region); pointers.append(user_context, region); } @@ -533,6 +546,7 @@ int main(int argc, char **argv) { request.size = count * sizeof(int); total_allocation_size += request.size; MemoryRegion *region = instance->reserve(user_context, request); + check_region(user_context, region); pointers.append(user_context, region); } @@ -549,6 +563,7 @@ int main(int argc, char **argv) { count = count > 1 ? count : 1; request.size = count * sizeof(int); MemoryRegion *region = instance->reserve(user_context, request); + check_region(user_context, region); pointers.append(user_context, region); }