From 5a1e81e8708f043e03eb2a8a85b07cd0cc67ecb7 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Thu, 20 Aug 2026 15:18:48 -0400 Subject: [PATCH 1/3] fix: clamp clip_skip to 2 for SDXL to avoid blank images SDXL's CLIP-G (OpenCLIP ViT-bigG) is only ever trained against its penultimate layer output. An explicit clip_skip of 1 (or any value below 2) runs its untrained final layer, whose activations overflow fp16 on some backends (observed on ROCm/gfx1100) and propagate through the UNet and VAE decode into a blank white image. The existing clip_skip<=0 default already resolved to 2 for SDXL, so this only affects requests that explicitly pass a value of 0 < n < 2. --- src/conditioning/conditioner.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 3d7ff0397..9ef88c9b9 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -425,6 +425,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { if (clip_skip <= 0) { clip_skip = (sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1; } + if (sd_version_is_sdxl(version) && clip_skip < 2) { + LOG_WARN("invalid clip_skip=%d for SDXL, using 2", clip_skip); + clip_skip = 2; + } size_t chunk_len = 77; size_t chunk_count = tokens.size() / chunk_len; From acf737ed65e423dd714ecb2d6e34e083b81a986f Mon Sep 17 00:00:00 2001 From: Rojikku Date: Fri, 21 Aug 2026 00:13:10 -0400 Subject: [PATCH 2/3] Revert "fix: clamp clip_skip to 2 for SDXL to avoid blank images" This reverts commit 5a1e81e8708f043e03eb2a8a85b07cd0cc67ecb7. --- src/conditioning/conditioner.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 9ef88c9b9..3d7ff0397 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -425,10 +425,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { if (clip_skip <= 0) { clip_skip = (sd_version_is_sd2(version) || sd_version_is_sdxl(version)) ? 2 : 1; } - if (sd_version_is_sdxl(version) && clip_skip < 2) { - LOG_WARN("invalid clip_skip=%d for SDXL, using 2", clip_skip); - clip_skip = 2; - } size_t chunk_len = 77; size_t chunk_count = tokens.size() / chunk_len; From 324655137d952086dbf26b91b68a53f5968cc1a0 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Fri, 21 Aug 2026 01:01:57 -0400 Subject: [PATCH 3/3] fix: detect and route around corrupted CLIP-L final layer on SDXL Many distributed SDXL checkpoints (verified: waiIllustriousSDXL, cyberrealisticXL, both independently) carry NaN weights across every weight matrix (q/k/v/out_proj, mlp.fc1/fc2) in CLIP-L's final transformer layer. This is not a precision or backend issue - verified by reading the raw safetensors bytes directly: the layer is NaN in the file itself, on both CPU and GPU, regardless of backend. SDXL's UNet was only ever trained against CLIP-L's penultimate layer, so this last layer is architecturally unused, and it appears the common conversion/merge pipeline these checkpoints share never populated it with valid data. Requesting a clip_skip low enough to reach that layer (clip_skip=1) therefore produces a NaN encoder output, which propagates through generation and comes out as a blank white image. Rather than unconditionally clamping clip_skip for all SDXL requests (which would needlessly break a checkpoint using a valid standalone CLIP-L, e.g. via --clip-l), detect the corruption from the actual computed output: if requesting clip_skip<2 on SDXL yields a NaN result, warn and recompute with clip_skip=2. Costs one extra (cheap, CLIP-L-only) compute, only when explicitly requesting the affected range on an affected checkpoint. CLIP-G's own final layer was checked and confirmed valid in both tested checkpoints, so this only touches the CLIP-L path. --- src/conditioning/conditioner.hpp | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 3d7ff0397..2ef457d03 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -465,6 +465,40 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { true, true); GGML_ASSERT(!chunk_hidden_states.empty()); + if (sd_version_is_sdxl(version) && clip_skip < 2) { + // Many distributed SDXL checkpoints carry NaN weights in CLIP-L's + // final transformer layer, since SDXL's UNet was only ever trained + // against the penultimate layer and that last layer is otherwise + // unused. Detect it from the actual output rather than assuming + // every checkpoint is affected, so a valid override (e.g. --clip-l) + // still works at clip_skip=1. + bool has_nan = false; + for (float v : chunk_hidden_states.values()) { + if (std::isnan(v)) { + has_nan = true; + break; + } + } + if (has_nan) { + LOG_WARN( + "clip_skip=%d produced invalid output from this checkpoint's CLIP-L " + "text encoder (a common defect in merged SDXL checkpoints' unused " + "final layer); using clip_skip=2 instead", + clip_skip); + clip_skip = 2; + chunk_hidden_states = text_model->compute(n_threads, + input_ids, + num_custom_embeddings, + token_embed_custom.data(), + max_token_idx, + false, + clip_skip, + false, + true, + true); + GGML_ASSERT(!chunk_hidden_states.empty()); + } + } if (sd_version_is_sdxl(version)) { auto chunk_hidden_states2 = text_model2->compute(n_threads, input_ids2,