From e4d47143b64054d3a5c2b56724e0eb4d47576215 Mon Sep 17 00:00:00 2001 From: Chris Rackauckas Date: Sat, 8 Aug 2026 04:50:12 -0400 Subject: [PATCH] Read backward reductions freshest-last; bump to 0.2.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backward kernels' nmuladd reduction read the already-solved trailing columns with ascending nk, so its first loads hit the block the previous kernel call had just stored, stalling on stores still draining. Walking nk descending reads the oldest columns first and the freshest block last — the mirror of the forward kernels' ascending reduction, which also ends on its most recently written block. The down-counting loop also compiles to the same shared-scaled-index address form as the forward loop (two address updates per trip instead of six). Measured (EPYC 7502/AVX2, Float64, nrhs=8, min times), left-upper ldiv! vs left-lower at equal UNIT flag, after the chain fix in #44: n=48 1.15x -> 1.10x, n=64 1.11x -> 1.06x, n=128 1.06x -> 1.04x, n=256 1.05x -> 1.02x, n=1000 1.02x -> 1.00x. Vs OpenBLAS trsm the upper leg is now 2.0-2.9x. Forward kernels untouched. 0.2.2 was registered from a0118b0 (pre-#44), so the #44 and this-PR improvements ship as 0.2.3. Co-Authored-By: Chris Rackauckas --- Project.toml | 2 +- src/TriangularSolve.jl | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Project.toml b/Project.toml index d7b2fef..bb3b149 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TriangularSolve" uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" authors = ["chriselrod and contributors"] -version = "0.2.2" +version = "0.2.3" [deps] CloseOpenIntervals = "fb6a15b2-703c-40df-9091-08a04967cfa9" diff --git a/src/TriangularSolve.jl b/src/TriangularSolve.jl index 354062e..153f940 100644 --- a/src/TriangularSolve.jl +++ b/src/TriangularSolve.jl @@ -1505,7 +1505,12 @@ end # kernels' `SafeCloseOpen(n)` loops spck = gesp(spc, ($z, n + $W)) splk = gesp(spl, (n + $W, n)) - for nk ∈ SafeCloseOpen(nend - n - $W) # nmuladd + # nk descends so the most recently solved (just stored) columns are read + # last, giving their stores time to drain — the mirror of the forward + # kernels' ascending reduction, which also ends on the freshest block + nk = nend - n - $W + while nk > 0 + nk -= 1 A11 = vload(spck, $(Unroll{1,W,U,1,W,zero(UInt),1})(($(StaticInt(0)), nk))) Base.Cartesian.@nexprs $W c -> C11_c = vfnmadd_fast(A11, vload(splk, (nk, $z + (c - 1))), C11_c) @@ -1544,7 +1549,9 @@ end Base.Cartesian.@nexprs $W c -> C11_c = C11[c] spck = gesp(spc, ($z, n + $W)) splk = gesp(spl, (n + $W, n)) - for nk ∈ SafeCloseOpen(nend - n - $W) # nmuladd + nk = nend - n - $W + while nk > 0 + nk -= 1 A11 = vload(spck, ($(MM{W}(z)), nk), mask) Base.Cartesian.@nexprs $W c -> C11_c = vfnmadd_fast(A11, vload(splk, (nk, $z + (c - 1))), C11_c) @@ -1834,7 +1841,9 @@ end # kernels' `SafeCloseOpen(n)` loops spck = gesp(spc, ($z, n + $(W * U))) splk = gesp(spl, (n + $(W * U), n)) - for nk ∈ SafeCloseOpen(nend - n - $(W * U)) # nmuladd + nk = nend - n - $(W * U) + while nk > 0 + nk -= 1 L_ki = vload(splk, $(Unroll{2,W,U,2,W,zero(UInt),1})((nk, $z))) Base.Cartesian.@nexprs $W c -> A11_c = vfnmadd_fast(L_ki, vload(spck, (static(c - 1), nk)), A11_c) @@ -1933,7 +1942,9 @@ end Base.Cartesian.@nexprs $W c -> A11_c = getfield(A11, c) spck = gesp(spc, ($z, n + $W)) splk = gesp(spl, (n + $W, n)) - for nk ∈ SafeCloseOpen(nend - n - $W) # nmuladd + nk = nend - n - $W + while nk > 0 + nk -= 1 L_ki = vload(splk, (nk, $(MM{W}(z)))) Base.Cartesian.@nexprs $W c -> A11_c = vfnmadd_fast(L_ki, vload(spck, (static(c - 1), nk)), A11_c) @@ -1966,7 +1977,9 @@ end Base.Cartesian.@nexprs $R r -> A11_r = getfield(A11, r) spck = gesp(spc, ($z, n + $W)) splk = gesp(spl, (n + $W, n)) - for nk ∈ SafeCloseOpen(nend - n - $W) # nmuladd + nk = nend - n - $W + while nk > 0 + nk -= 1 L_ki = vload(splk, (nk, $(MM{W}(z)))) Base.Cartesian.@nexprs $R r -> A11_r = vfnmadd_fast(L_ki, vload(spck, (static(r - 1), nk)), A11_r)