From b77cda03392eda8961cdd62033a5db9baf2fda79 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Thu, 14 May 2026 16:20:28 -0700 Subject: [PATCH] layer: Force bypass when requested format is unsupported on fallback When canBypassXWayland returns false at swapchain creation and the requested format isn't on the fallback surface, refusing the swapchain leaves the app with no recovery path. Instead, prefer bypass over INITIALIZATION_FAILED. Star Citizen reproduces this during the game's launch process: it recreates its swapchain exactly when its X11 toplevel and inner window are mid-resize, so canBypass returns false (geometry mismatch) and the create lands on the XCB fallback surface. The XCB fallback only advertises B8G8R8A8 family formats, which means the recreate is refused for any other format. Without HDR exposed SC asks for VK_FORMAT_R8G8B8A8_UNORM, and with HDR exposed it instead tries to use VK_FORMAT_R16G16B16A16_SFLOAT. Either way, the game stalls on an infinite black screen after gamescope refuses the swapchain recreate. Track the bypass override on the swapchain so QueuePresentKHR doesn't ask the app to recreate when the live canBypass disagrees. --- layer/VkLayer_FROG_gamescope_wsi.cpp | 86 +++++++++++++++++----------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/layer/VkLayer_FROG_gamescope_wsi.cpp b/layer/VkLayer_FROG_gamescope_wsi.cpp index af0c200993..8f706eed12 100644 --- a/layer/VkLayer_FROG_gamescope_wsi.cpp +++ b/layer/VkLayer_FROG_gamescope_wsi.cpp @@ -500,6 +500,7 @@ namespace GamescopeWSILayer { VkSurfaceKHR surface; // Always the Gamescope Surface surface -- so the Wayland one. bool isWayland; bool isBypassingXWayland; + bool forcedBypass; bool forceFifo; VkPresentModeKHR presentMode; VkExtent2D extent; @@ -1111,20 +1112,10 @@ namespace GamescopeWSILayer { return pDispatch->CreateSwapchainKHR(device, pCreateInfo, pAllocator, pSwapchain); } - const bool canBypass = gamescopeSurface->canBypassXWayland(); + bool canBypass = gamescopeSurface->canBypassXWayland(); VkSwapchainCreateInfoKHR swapchainInfo = *pCreateInfo; - if (pCreateInfo->oldSwapchain) { - if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) { - gamescopeSwapchain->retired = true; - // If we are going to/from being able to bypass XWayland, make sure - // we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types. - if (gamescopeSwapchain->isBypassingXWayland != canBypass) - swapchainInfo.oldSwapchain = VK_NULL_HANDLE; - } - } - if (gamescopeSurface->flags & GamescopeLayerClient::Flag::ForceSwapchainExtent) { if (!gamescopeSurface->isWayland()) { auto rect = xcb::getWindowRect(gamescopeSurface->connection, gamescopeSurface->window); @@ -1135,6 +1126,42 @@ namespace GamescopeWSILayer { } } + auto isFormatSupportedOnSurface = [&](VkSurfaceKHR vkSurface, VkFormat format) { + std::vector formats; + vkroots::helpers::enumerate( + pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR, + formats, + pDispatch->PhysicalDevice, + vkSurface); + return std::ranges::any_of( + formats, + std::bind_front(std::equal_to{}, format), + &VkSurfaceFormatKHR::format); + }; + + // If bypass supports the format but the XCB fallback doesn't, force bypass. + // Refusing here would leave the app with no recovery path. + bool forcedBypass = false; + if (!canBypass + && !isFormatSupportedOnSurface(gamescopeSurface->fallbackSurface, pCreateInfo->imageFormat) + && isFormatSupportedOnSurface(pCreateInfo->surface, pCreateInfo->imageFormat)) { + fprintf(stderr, "[Gamescope WSI] Forcing bypass (format unsupported on fallback) for xid: 0x%0x - format: %s\n", + gamescopeSurface->window, + vkroots::helpers::enumString(pCreateInfo->imageFormat)); + canBypass = true; + forcedBypass = true; + } + + if (pCreateInfo->oldSwapchain) { + if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) { + gamescopeSwapchain->retired = true; + // If we are going to/from being able to bypass XWayland, make sure + // we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types. + if (gamescopeSwapchain->isBypassingXWayland != canBypass) + swapchainInfo.oldSwapchain = VK_NULL_HANDLE; + } + } + // If we can't flip, fallback to the regular XCB surface on the XCB window. if (!canBypass) swapchainInfo.surface = gamescopeSurface->fallbackSurface; @@ -1175,28 +1202,14 @@ namespace GamescopeWSILayer { // Check for VkFormat support and return VK_ERROR_INITIALIZATION_FAILED // if that VkFormat is unsupported for the underlying surface. - { - std::vector supportedSurfaceFormats; - vkroots::helpers::enumerate( - pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR, - supportedSurfaceFormats, - pDispatch->PhysicalDevice, - swapchainInfo.surface); - - bool supportedSwapchainFormat = std::ranges::any_of( - supportedSurfaceFormats, - std::bind_front(std::equal_to{}, swapchainInfo.imageFormat), - &VkSurfaceFormatKHR::format) ; - - if (!supportedSwapchainFormat) { - fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n", - gamescopeSurface->window, - vkroots::helpers::enumString(pCreateInfo->imageFormat), - vkroots::helpers::enumString(pCreateInfo->imageColorSpace), - canBypass ? "true" : "false"); - - return VK_ERROR_INITIALIZATION_FAILED; - } + if (!isFormatSupportedOnSurface(swapchainInfo.surface, swapchainInfo.imageFormat)) { + fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n", + gamescopeSurface->window, + vkroots::helpers::enumString(pCreateInfo->imageFormat), + vkroots::helpers::enumString(pCreateInfo->imageColorSpace), + canBypass ? "true" : "false"); + + return VK_ERROR_INITIALIZATION_FAILED; } uint32_t serverId = ~0u; @@ -1232,6 +1245,7 @@ namespace GamescopeWSILayer { .surface = pCreateInfo->surface, // Always the Wayland side surface. .isWayland = gamescopeSurface->isWayland(), .isBypassingXWayland = canBypass, + .forcedBypass = forcedBypass, .forceFifo = gamescopeIsForcingFifo(), // Were we forcing fifo when this swapchain was made? .presentMode = pCreateInfo->presentMode, // The new present mode. .extent = pCreateInfo->imageExtent, @@ -1431,8 +1445,10 @@ namespace GamescopeWSILayer { if (canBypass) { if (!(gamescopeSurface->flags & GamescopeLayerClient::Flag::NoSuboptimal)) UpdateSwapchainResult(VK_SUBOPTIMAL_KHR); - } else { - UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR); + } else if (!gamescopeSwapchain->forcedBypass) { + // A forced-bypass swapchain has no fallback path; recreating + // would re-force bypass and loop. + UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR); } }