From 2f8f802a0578c51646f34fdaf2b8f3e624931313 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 25 Mar 2026 15:56:51 +0100 Subject: [PATCH 1/4] fast-get: fix userspace thread detection Checking number of memory domain partitions in thread's domain to determine whether it's a userspace thread isn't reliable and is a layering violation. Instead check the K_USER flag which indicates exactly that. Also add a missing check when adding a partition for the newly allocated entry. Suggested-by: Adrian Warecki Signed-off-by: Guennadi Liakhovetski --- posix/include/rtos/kernel.h | 8 ++++++++ zephyr/include/rtos/kernel.h | 7 +++++++ zephyr/lib/fast-get.c | 8 +++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/posix/include/rtos/kernel.h b/posix/include/rtos/kernel.h index 714fc6672a1d..9fa3962a3589 100644 --- a/posix/include/rtos/kernel.h +++ b/posix/include/rtos/kernel.h @@ -10,6 +10,7 @@ #include +#include #include #ifdef __ZEPHYR__ @@ -28,6 +29,13 @@ typedef struct { #define Z_TIMEOUT_MS(t) ((k_timeout_t) { .ticks = clock_ms_to_ticks(PLATFORM_DEFAULT_CLOCK, t) }) +struct k_thread; +static inline bool thread_is_userspace(struct k_thread *thread) +{ + (void)thread; + return false; +} + static inline void k_sleep(k_timeout_t timeout) { wait_delay(timeout.ticks); diff --git a/zephyr/include/rtos/kernel.h b/zephyr/include/rtos/kernel.h index 17d388cbf8e8..328fc96d2eaa 100644 --- a/zephyr/include/rtos/kernel.h +++ b/zephyr/include/rtos/kernel.h @@ -8,4 +8,11 @@ #include +#include + +static inline bool thread_is_userspace(struct k_thread *thread) +{ + return !!(thread->base.user_options & K_USER); +} + #endif /* __ZEPHYR_RTOS_KERNEL_H__ */ diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index c681252ae27a..d3c25d87420c 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -132,6 +132,9 @@ static int fast_get_access_grant(struct k_mem_domain *mdom, void *addr, size_t s const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) { +#if CONFIG_USERSPACE + bool current_is_userspace = thread_is_userspace(k_current_get()); +#endif struct sof_fast_get_data *data = &fast_get_data; uint32_t alloc_flags = SOF_MEM_FLAG_USER; struct sof_fast_get_entry *entry; @@ -189,8 +192,7 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) * We only get there for large buffers, since small buffers with * enabled userspace don't create fast-get entries */ - if (mdom->num_partitions > 1) { - /* A userspace thread makes the request */ + if (current_is_userspace) { if (mdom != entry->mdom && !fast_get_partition_exists(k_current_get(), ret, ALIGN_UP(size, CONFIG_MM_DRV_PAGE_SIZE))) { @@ -235,7 +237,7 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) #if CONFIG_USERSPACE entry->mdom = k_current_get()->mem_domain_info.mem_domain; - if (size > FAST_GET_MAX_COPY_SIZE) { + if (size > FAST_GET_MAX_COPY_SIZE && current_is_userspace) { /* Otherwise we've allocated on thread's heap, so it already has access */ int err = fast_get_access_grant(entry->mdom, ret, size); From ea733432faa421ddd24747f95ebf815b43dee117 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 25 Mar 2026 16:01:43 +0100 Subject: [PATCH 2/4] fast-get: fix heap when freeing the buffer Use the same heap to free the buffer as the one, used to allocate it. Signed-off-by: Guennadi Liakhovetski --- zephyr/lib/fast-get.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index d3c25d87420c..7bb161dbb775 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -243,7 +243,7 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) if (err < 0) { LOG_ERR("failed to grant access err=%d", err); - sof_heap_free(NULL, ret); + sof_heap_free(heap, ret); ret = NULL; goto out; } From 7a8a8a6ba4b4c990de67952dafe7daf84d2413c9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 25 Mar 2026 16:03:46 +0100 Subject: [PATCH 3/4] fast-get: remove mdom from the entry structure The mdom member of struct sof_fast_get_entry is redundant, remove it. Signed-off-by: Guennadi Liakhovetski --- zephyr/lib/fast-get.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 7bb161dbb775..ea7157e78804 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -28,9 +28,6 @@ struct sof_fast_get_entry { const void *dram_ptr; void *sram_ptr; -#if CONFIG_USERSPACE - struct k_mem_domain *mdom; -#endif size_t size; unsigned int refcount; }; @@ -103,10 +100,8 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data * #endif #if CONFIG_USERSPACE -static bool fast_get_partition_exists(struct k_thread *thread, void *start, size_t size) +static bool fast_get_partition_exists(struct k_mem_domain *domain, void *start, size_t size) { - struct k_mem_domain *domain = thread->mem_domain_info.mem_domain; - for (unsigned int i = 0; i < domain->num_partitions; i++) { struct k_mem_partition *dpart = &domain->partitions[i]; @@ -170,8 +165,8 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) } while (!entry); #if CONFIG_USERSPACE - LOG_DBG("userspace %u part %#zx bytes alloc %p entry %p DRAM %p", - k_current_get()->mem_domain_info.mem_domain->num_partitions, size, + LOG_DBG("%s: %#zx bytes alloc %p entry %p DRAM %p", + current_is_userspace ? "userspace" : "kernel", size, alloc_ptr, entry->sram_ptr, dram_ptr); #endif @@ -193,11 +188,9 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) * enabled userspace don't create fast-get entries */ if (current_is_userspace) { - if (mdom != entry->mdom && - !fast_get_partition_exists(k_current_get(), ret, + if (!fast_get_partition_exists(mdom, ret, ALIGN_UP(size, CONFIG_MM_DRV_PAGE_SIZE))) { - LOG_DBG("grant access to domain %p first was %p", mdom, - entry->mdom); + LOG_DBG("grant access to domain %p", mdom); int err = fast_get_access_grant(mdom, ret, size); @@ -236,10 +229,10 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) dcache_writeback_region((__sparse_force void __sparse_cache *)entry->sram_ptr, size); #if CONFIG_USERSPACE - entry->mdom = k_current_get()->mem_domain_info.mem_domain; if (size > FAST_GET_MAX_COPY_SIZE && current_is_userspace) { /* Otherwise we've allocated on thread's heap, so it already has access */ - int err = fast_get_access_grant(entry->mdom, ret, size); + int err = fast_get_access_grant(k_current_get()->mem_domain_info.mem_domain, + ret, size); if (err < 0) { LOG_ERR("failed to grant access err=%d", err); @@ -298,11 +291,8 @@ void fast_put(struct k_heap *heap, struct k_mem_domain *mdom, const void *sram_p * For large buffers, each thread that called fast_get() has a partition * in its memory domain. Each thread must remove its own partition here * to prevent partition leaks. - * - * Order matters: free buffer first (needs partition for cache access), - * then remove partition. */ - if (entry->size > FAST_GET_MAX_COPY_SIZE && entry->mdom && mdom) { + if (entry->size > FAST_GET_MAX_COPY_SIZE && mdom) { struct k_mem_partition part = { .start = (uintptr_t)sram_ptr, .size = ALIGN_UP(entry->size, CONFIG_MM_DRV_PAGE_SIZE), From 24bb8dd0a77a1a31b85b8ac51c6c87b670ac8eb8 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 17 Apr 2026 17:18:12 +0200 Subject: [PATCH 4/4] fast-get: don't corrupt entries when failing When fast_get() fails it shouldn't leave partially overwritten entries behind. Also fix the call to memcpy_s() to use the proper destination size. Signed-off-by: Guennadi Liakhovetski --- zephyr/lib/fast-get.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index ea7157e78804..edb490df0e06 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -223,10 +223,6 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) ret = sof_heap_alloc(heap, alloc_flags, alloc_size, alloc_align); if (!ret) goto out; - entry->size = size; - entry->sram_ptr = ret; - memcpy_s(entry->sram_ptr, entry->size, dram_ptr, size); - dcache_writeback_region((__sparse_force void __sparse_cache *)entry->sram_ptr, size); #if CONFIG_USERSPACE if (size > FAST_GET_MAX_COPY_SIZE && current_is_userspace) { @@ -244,6 +240,10 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) #endif /* CONFIG_USERSPACE */ entry->dram_ptr = dram_ptr; + entry->size = size; + entry->sram_ptr = ret; + memcpy_s(ret, alloc_size, dram_ptr, size); + dcache_writeback_region((__sparse_force void __sparse_cache *)ret, size); entry->refcount = 1; out: k_spin_unlock(&data->lock, key);