From 10ea5780cf072482013161efd2467138b283fc26 Mon Sep 17 00:00:00 2001 From: Patrick LoPresti Date: Wed, 5 Aug 2026 10:32:38 -0700 Subject: [PATCH] nv-vm: skip unallocated pages during sysmem OOM cleanup nv_alloc_system_pages() sets pages to UC only after the whole array is allocated. On an out-of-memory partial allocation it jumps to failed: -> nv_free_system_pages() before that, which then calls nv_set_memory_type(at, NV_MEMORY_WRITEBACK) based on the requested at->cache_type. nv_set_memory_type() iterates over all at->num_pages with no unallocated-slot guard, unlike the two loops in nv_free_system_pages() that break at page_ptr->virt_addr == 0. For a large descriptor that failed early, most slots are unallocated (phys_addr == 0), so NV_GET_PAGE_STRUCT(0) yields PFN 0 and set_memory_wb()/ set_memory_array_wb() is invoked on [0x0-0xfff] once per slot. This floods the kernel log with "x86/PAT: freeing invalid memtype [mem 0x0-0xfff]" and spins in an O(num_pages) TLB-flushing loop while holding the RM API write lock (rmapiLockAcquire in serverAllocResource), deadlocking all other threads until reboot. Bound the cache-type reset to actually-allocated pages by breaking at the first virt_addr == 0, mirroring the existing guards in nv_free_system_pages(). Fully-allocated frees are unaffected; an OOM now returns NV_ERR_NO_MEMORY cleanly instead of wedging the system. Fixes #1277 Signed-off-by: Patrick LoPresti --- kernel-open/nvidia/nv-vm.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/kernel-open/nvidia/nv-vm.c b/kernel-open/nvidia/nv-vm.c index e36654744..ee38cd908 100644 --- a/kernel-open/nvidia/nv-vm.c +++ b/kernel-open/nvidia/nv-vm.c @@ -168,7 +168,9 @@ static inline void nv_set_memory_type(nv_alloc_t *at, NvU32 type) if (at->flags.contig) { - nv_set_contig_memory_type(&at->page_table[0], at->num_pages, type); + // Skip if the (single) allocation never happened (e.g. OOM cleanup). + if (at->page_table[0].virt_addr != 0) + nv_set_contig_memory_type(&at->page_table[0], at->num_pages, type); return; } @@ -197,6 +199,12 @@ static inline void nv_set_memory_type(nv_alloc_t *at, NvU32 type) for (i = 0; i < at->num_pages; i++) { page_ptr = &at->page_table[i]; + // Stop at the first unallocated slot. On a partial-allocation + // failure the tail of page_table is zeroed; touching those slots + // would call set_memory_wb() on PFN 0 ("freeing invalid memtype + // [mem 0x0-0xfff]") once per slot -- a huge loop under the RM lock. + if (page_ptr->virt_addr == 0) + break; page = NV_GET_PAGE_STRUCT(page_ptr->phys_addr); #if defined(NV_SET_MEMORY_ARRAY_UC_PRESENT) pages[i] = (unsigned long)page_address(page); @@ -204,11 +212,14 @@ static inline void nv_set_memory_type(nv_alloc_t *at, NvU32 type) pages[i] = page; #endif } + if (i > 0) + { #if defined(NV_SET_MEMORY_ARRAY_UC_PRESENT) - nv_set_memory_array_type(pages, at->num_pages, type); + nv_set_memory_array_type(pages, i, type); #elif defined(NV_SET_PAGES_ARRAY_UC_PRESENT) - nv_set_pages_array_type(pages, at->num_pages, type); + nv_set_pages_array_type(pages, i, type); #endif + } os_free_mem(pages); } @@ -220,7 +231,12 @@ static inline void nv_set_memory_type(nv_alloc_t *at, NvU32 type) else { for (i = 0; i < at->num_pages; i++) + { + // Stop at the first unallocated slot (see comment above). + if (at->page_table[i].virt_addr == 0) + break; nv_set_contig_memory_type(&at->page_table[i], 1, type); + } } }