From d9c0a949e6fc859bdc84de86f7d1a4d5e3e185d9 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Thu, 6 Aug 2026 17:53:23 +0000 Subject: [PATCH 1/5] Memoize the driver extension list `properties(::ZeKernel)` consults `extension_properties` to decide whether to link the max-group-size extension struct, and the KernelAbstractions backend reaches it through `launch_configuration` on every dispatch of a kernel with a dynamic workgroupsize. Rebuilding the answer each time takes two driver round-trips plus a `String` allocation and a `Dict` insertion per extension, so the query dominates the host cost of launch-bound workloads. A driver's extension list is fixed for its lifetime, and Level Zero has no driver-destroy entry point, so the result can be cached per driver. `ZeDriver` compares and hashes by handle, keying the cache by value, and the cache is emptied in `__init__` so no entry can be inherited from a precompiled image. --- lib/level-zero/driver.jl | 27 +++++++++++++++++++++++++++ lib/level-zero/module.jl | 1 - lib/level-zero/oneL0.jl | 4 ++++ test/level-zero.jl | 5 +++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/level-zero/driver.jl b/lib/level-zero/driver.jl index 0444e049..61cf3f3a 100644 --- a/lib/level-zero/driver.jl +++ b/lib/level-zero/driver.jl @@ -100,7 +100,34 @@ function ipc_properties(drv::ZeDriver) ) end +# Memoized, because this lands on the per-launch path: `properties(::ZeKernel)` consults it +# to decide whether to link the max-group-size extension struct, and KernelAbstractions +# reaches that from `launch_configuration` on every dispatch of a kernel whose +# workgroupsize is dynamic. Answering it is not cheap -- two driver round-trips, a copy of +# the property vector, and a `String` allocation plus a `Dict` insertion per extension -- +# and on a Data Center GPU Max 1550, whose driver advertises many extensions, it measured +# ~90 us per call and accounted for 90% of the host time of a launch-bound GPU callback. +# +# Safe to remember: a driver reports a fixed set of extensions, and Level Zero has no +# driver-destroy entry point, so a `ZeDriver` handle stays valid and keeps its meaning for +# the lifetime of the process. The cache is emptied in `__init__` so that entries can never +# be inherited from the process that generated a precompiled image. +# +# The returned `Dict` is the cached object itself; callers must treat it as read-only. +const extension_properties_cache = Dict{ZeDriver,Dict{String,VersionNumber}}() +const extension_properties_lock = ReentrantLock() + function extension_properties(drv::ZeDriver) + Base.@lock extension_properties_lock begin + cached = get(extension_properties_cache, drv, nothing) + cached === nothing || return cached + extensions = _extension_properties(drv) + extension_properties_cache[drv] = extensions + return extensions + end +end + +function _extension_properties(drv::ZeDriver) count_ref = Ref{UInt32}(0) zeDriverGetExtensionProperties(drv, count_ref, C_NULL) diff --git a/lib/level-zero/module.jl b/lib/level-zero/module.jl index 2af65b23..4fcce7a3 100644 --- a/lib/level-zero/module.jl +++ b/lib/level-zero/module.jl @@ -238,7 +238,6 @@ function properties(kernel::ZeKernel) link_extensions(props_ref, preferred_group_size_props_ref) if haskey(oneL0.extension_properties(kernel.mod.context.driver), "ZE_extension_kernel_max_group_size_properties") - # TODO: memoize max_group_size_props_ref = Ref(ze_kernel_max_group_size_properties_ext_t()) link_extensions(preferred_group_size_props_ref, max_group_size_props_ref) else diff --git a/lib/level-zero/oneL0.jl b/lib/level-zero/oneL0.jl index 948a213a..d24ba412 100644 --- a/lib/level-zero/oneL0.jl +++ b/lib/level-zero/oneL0.jl @@ -166,6 +166,10 @@ function __init__() precompiling = ccall(:jl_generating_output, Cint, ()) != 0 precompiling && return + # Driver handles are process-local, so nothing keyed on one may survive into another + # process through a precompiled image. + empty!(extension_properties_cache) + # Resolve the LTS master switch up front, before the driver-availability early # returns below: it gates codegen and behavior and must be set even on hosts # without a functional GPU. Default off (rolling stack); an LTS deployment such as diff --git a/test/level-zero.jl b/test/level-zero.jl index bbf8f465..a7d19807 100644 --- a/test/level-zero.jl +++ b/test/level-zero.jl @@ -22,6 +22,11 @@ properties(drv) ipc_properties(drv) extension_properties(drv) +# the extension list is memoized: it is answered from the cache, and that answer is the +# one the driver gives +@test extension_properties(drv) === extension_properties(drv) +@test extension_properties(drv) == oneL0._extension_properties(drv) + end drv = first(drivers()) From 6892d590ec15ba9703f2b8147bd766283aaf2663 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Thu, 6 Aug 2026 17:53:35 +0000 Subject: [PATCH 2/5] Fill arrays with a kernel instead of the memory-fill command The `Base.fill!` specialization drove Level Zero's memory-fill command, which requires the fill pattern to live in USM memory: a host allocation, a residency call and a free around every call, plus a full queue synchronize to keep the pattern alive until the asynchronous fill has read it. The repeated host allocations also pushed `zeMemAllocHost` into `retry_reclaim`'s garbage collections. GPUArrays' generic definition lowers to a single fill kernel and does none of this, so drop the specialization and let it apply. --- src/array.jl | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/array.jl b/src/array.jl index 3fd539b1..6fdeaef6 100644 --- a/src/array.jl +++ b/src/array.jl @@ -544,23 +544,11 @@ ones(dims...) = ones(Float64, dims...) fill(v, dims...) = fill!(oneArray{typeof(v)}(undef, dims...), v) fill(v, dims::Dims) = fill!(oneArray{typeof(v)}(undef, dims...), v) -function Base.fill!(A::oneDenseArray{T}, val) where T - length(A) == 0 && return A - val = convert(T, val) - sizeof(T) == 0 && return A - - # execute! is async, so we need to allocate the pattern in USM memory and keep it alive - # until the operation completes. The fill reads this host buffer on the GPU, so it must - # be made resident on the device like any other USM a kernel reads (see - # `allocate(::Type{oneL0.HostBuffer}, ...)`). - buf = oneL0.host_alloc(context(A), sizeof(T), Base.datatype_alignment(T)) - oneL0.make_resident(context(A), device(), buf) - unsafe_store!(convert(Ptr{T}, buf), val) - unsafe_fill!(context(A), device(), pointer(A), convert(ZePtr{T}, buf), length(A)) - synchronize(global_queue(context(A), device())) - oneL0.free(buf) - A -end +# NOTE: `Base.fill!` is deliberately not specialized here. GPUArrays' generic definition +# lowers to a single fill kernel, whereas the Level Zero memory-fill command requires the +# pattern to live in USM memory: a host allocation, a residency call and a free around +# every call, plus a full queue synchronize to keep the pattern alive until the +# asynchronous fill has read it. ## derived arrays From e1f4cc6b2ece28078e89dc6d25f6b71b1f3dde57 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Thu, 6 Aug 2026 17:54:09 +0000 Subject: [PATCH 3/5] Account for spill memory when picking a group size `launch_configuration` returned the kernel's `maxGroupSize`, which Level Zero reports without regard to spill. The driver allocates `spillMemSize * group_size` bytes of scratch per work-group, so a heavily spilling kernel is reported as launchable at group sizes with a very large scratch demand. Cap the group size so a spilling kernel's per-group scratch stays within a fixed budget. Level Zero exposes no scratch-space query, so the budget is a conservative constant; CUDA.jl gets the equivalent from an occupancy API that accounts for register pressure. The inner division is clamped so a kernel spilling more than the whole budget still gets a group size of one. --- src/compiler/execution.jl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/compiler/execution.jl b/src/compiler/execution.jl index cc8d3310..327cf8c7 100644 --- a/src/compiler/execution.jl +++ b/src/compiler/execution.jl @@ -209,6 +209,18 @@ struct HostKernel{F,TT} <: AbstractKernel{F,TT} fun::ZeKernel end +# Upper bound on the spill (scratch) memory a single work-group may require, in bytes. +# +# The driver allocates `spillMemSize * group_size` of scratch per work-group. `maxGroupSize` +# does not account for spill, so a heavily spilling kernel is reported as launchable at a +# group size whose scratch demand is enormous: on a Data Center GPU Max 1550, a kernel +# spilling 3648 B/thread is reported launchable at 1024 items/group, i.e. ~3.7 MB of scratch +# for one work-group. +# +# Level Zero exposes no scratch-space query, so the budget is a conservative constant rather +# than a derived one. +const MAX_GROUP_SCRATCH = 1024 * 1024 + function launch_configuration(kernel::HostKernel{F,TT}) where {F,TT} # Level Zero's zeKernelSuggestGroupSize provides a launch configuration # that exactly cover the input size. This can result in very awkward @@ -230,6 +242,16 @@ function launch_configuration(kernel::HostKernel{F,TT}) where {F,TT} group_size = max_size ÷ 2 end + # keep a spilling kernel's per-group scratch within budget. CUDA.jl gets this for free + # from an occupancy API that is register-pressure aware; Level Zero reports the spill + # size but does not fold it into `maxGroupSize`, so account for it here. Rounded down to + # a power of two, both because group sizes want to be anyway and to stay clear of the + # limit rather than right at it. + spill = kernel_props.spillMemSize + if spill > 0 && group_size * spill > MAX_GROUP_SCRATCH + group_size = max(1, prevpow(2, max(1, MAX_GROUP_SCRATCH ÷ spill))) + end + # TODO: align the group size based on preferredGroupSize return group_size From 06070946739a2068419aa65eacb1a80609b54545 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Fri, 7 Aug 2026 13:36:28 +0000 Subject: [PATCH 4/5] Hold the kernel-property extension structs across the query `link_extensions` chains the extension descriptors together with raw interior pointers stored into each other's `pNext` fields, which the GC cannot see: `zeKernelGetProperties` receives only `props_ref` and reaches the other structs through the chain, so nothing in the call itself keeps them rooted. Preserve all three refs across the query, as `device_alloc`, `ZeModule` and `ZeKernel` already do for the buffers their descriptors point at. Defensive; not a fix for an observed failure. --- lib/level-zero/module.jl | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/level-zero/module.jl b/lib/level-zero/module.jl index 4fcce7a3..f143f8fa 100644 --- a/lib/level-zero/module.jl +++ b/lib/level-zero/module.jl @@ -235,15 +235,25 @@ export properties function properties(kernel::ZeKernel) props_ref = Ref(ze_kernel_properties_t()) preferred_group_size_props_ref = Ref(ze_kernel_preferred_group_size_properties_t()) - link_extensions(props_ref, preferred_group_size_props_ref) - if haskey(oneL0.extension_properties(kernel.mod.context.driver), - "ZE_extension_kernel_max_group_size_properties") - max_group_size_props_ref = Ref(ze_kernel_max_group_size_properties_ext_t()) - link_extensions(preferred_group_size_props_ref, max_group_size_props_ref) - else - max_group_size_props_ref = nothing + max_group_size_props_ref = + if haskey(oneL0.extension_properties(kernel.mod.context.driver), + "ZE_extension_kernel_max_group_size_properties") + Ref(ze_kernel_max_group_size_properties_ext_t()) + else + nothing + end + + # `link_extensions` chains these together with raw interior pointers stored into each + # other's `pNext` field, which the GC cannot see: only `props_ref` is passed to the + # query, and the driver reaches the rest by following those pointers. Hold them across + # the call, as `device_alloc` does for its relaxed-allocation extension. + GC.@preserve props_ref preferred_group_size_props_ref max_group_size_props_ref begin + link_extensions(props_ref, preferred_group_size_props_ref) + if max_group_size_props_ref !== nothing + link_extensions(preferred_group_size_props_ref, max_group_size_props_ref) + end + zeKernelGetProperties(kernel, props_ref) end - zeKernelGetProperties(kernel, props_ref) props = props_ref[] return ( From 0690b28a2850e109d03729fb3148175004af5b55 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Fri, 7 Aug 2026 15:26:11 +0000 Subject: [PATCH 5/5] Apply Runic formatting --- lib/level-zero/driver.jl | 2 +- lib/level-zero/module.jl | 14 ++++++++------ test/level-zero.jl | 8 ++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/level-zero/driver.jl b/lib/level-zero/driver.jl index 61cf3f3a..9a5b59ee 100644 --- a/lib/level-zero/driver.jl +++ b/lib/level-zero/driver.jl @@ -114,7 +114,7 @@ end # be inherited from the process that generated a precompiled image. # # The returned `Dict` is the cached object itself; callers must treat it as read-only. -const extension_properties_cache = Dict{ZeDriver,Dict{String,VersionNumber}}() +const extension_properties_cache = Dict{ZeDriver, Dict{String, VersionNumber}}() const extension_properties_lock = ReentrantLock() function extension_properties(drv::ZeDriver) diff --git a/lib/level-zero/module.jl b/lib/level-zero/module.jl index f143f8fa..0c9ddbb1 100644 --- a/lib/level-zero/module.jl +++ b/lib/level-zero/module.jl @@ -236,12 +236,14 @@ function properties(kernel::ZeKernel) props_ref = Ref(ze_kernel_properties_t()) preferred_group_size_props_ref = Ref(ze_kernel_preferred_group_size_properties_t()) max_group_size_props_ref = - if haskey(oneL0.extension_properties(kernel.mod.context.driver), - "ZE_extension_kernel_max_group_size_properties") - Ref(ze_kernel_max_group_size_properties_ext_t()) - else - nothing - end + if haskey( + oneL0.extension_properties(kernel.mod.context.driver), + "ZE_extension_kernel_max_group_size_properties" + ) + Ref(ze_kernel_max_group_size_properties_ext_t()) + else + nothing + end # `link_extensions` chains these together with raw interior pointers stored into each # other's `pNext` field, which the GC cannot see: only `props_ref` is passed to the diff --git a/test/level-zero.jl b/test/level-zero.jl index a7d19807..0fbc87db 100644 --- a/test/level-zero.jl +++ b/test/level-zero.jl @@ -22,10 +22,10 @@ properties(drv) ipc_properties(drv) extension_properties(drv) -# the extension list is memoized: it is answered from the cache, and that answer is the -# one the driver gives -@test extension_properties(drv) === extension_properties(drv) -@test extension_properties(drv) == oneL0._extension_properties(drv) + # the extension list is memoized: it is answered from the cache, and that answer is the + # one the driver gives + @test extension_properties(drv) === extension_properties(drv) + @test extension_properties(drv) == oneL0._extension_properties(drv) end