diff --git a/README.md b/README.md index f3c13255..7f486278 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ Untold Engine is well-suited for: - [Geometry Streaming System](docs/API/UsingGeometryStreamingSystem.md) - [LOD-Batching-Streaming](docs/API/UsingLOD-Batching-Streaming.md) - [Spatial Input](docs/API/UsingSpatialInput.md) +- [XR Immersion Modes](docs/API/UsingXRImmersionMode.md) - [Gaussian System](docs/API/UsingGaussianSystem.md) - [Spatial Debugger](docs/API/SpatialDebugger.md) - [Profiler](docs/API/UsingProfiler.md) diff --git a/Sources/UntoldEngine/Shaders/OutputTransformShader.metal b/Sources/UntoldEngine/Shaders/OutputTransformShader.metal index e8206243..c19b3216 100644 --- a/Sources/UntoldEngine/Shaders/OutputTransformShader.metal +++ b/Sources/UntoldEngine/Shaders/OutputTransformShader.metal @@ -51,6 +51,11 @@ fragment OutputTransformFragmentOut fragmentOutputTransformShader( OutputTransformFragmentOut out; out.color = c; - out.depth = depth; + // Reverse-Z: sky pixels carry the clear value 0.0 = infinite distance. The + // visionOS compositor reprojects using view distance (near/depth), so exactly + // 0.0 divides to infinity and those pixels are discarded (rendered black). + // Clamp to a finite far depth (~1km) that is visually indistinguishable from + // infinity but valid for the compositor's reprojection. + out.depth = max(depth, 1e-4); return out; } diff --git a/Sources/UntoldEngine/Shaders/preCompShader.metal b/Sources/UntoldEngine/Shaders/preCompShader.metal index 2f2e97d8..11d6f634 100644 --- a/Sources/UntoldEngine/Shaders/preCompShader.metal +++ b/Sources/UntoldEngine/Shaders/preCompShader.metal @@ -54,6 +54,14 @@ fragment float4 fragmentPreCompositeShader(VertexCompositeOutput vertexOut [[sta baseColor.rgb = gaussianColor.rgb + baseColor.rgb * (1.0 - gaussianColor.a); baseColor.a = gaussianColor.a + baseColor.a * (1.0 - gaussianColor.a); + // Non-passthrough modes (full immersion, macOS, AR) present an opaque layer: + // the environment already fills every background pixel, so alpha must be 1.0 + // regardless of what upstream content (HDR alpha, material alpha, Gaussians) + // produced. Only mixed passthrough is allowed to punch through via alpha. + if (isPassthrough == false) { + baseColor.a = 1.0; + } + // Sample gizmo float3 gizmoColor = gizmoTexture.sample(s, vertexOut.uvCoords).rgb; float gizmoLumen = getLuminance(gizmoColor); diff --git a/Sources/UntoldEngine/Systems/RenderingSystem.swift b/Sources/UntoldEngine/Systems/RenderingSystem.swift index 86677bd2..d920687e 100644 --- a/Sources/UntoldEngine/Systems/RenderingSystem.swift +++ b/Sources/UntoldEngine/Systems/RenderingSystem.swift @@ -1330,7 +1330,8 @@ public let outputTransformRenderPass: RenderPasses.RenderPassExecution = { comma renderPassDescriptor.colorAttachments[0].loadAction = .clear renderPassDescriptor.colorAttachments[0].storeAction = .store - renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0.0, 0.0, 0.0, 0.0) + // Do not override clearColor here: the XR layer sets it per immersion mode + // (UntoldEngineXR.swift), and the fullscreen quad overwrites every pixel anyway. guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { handleError(.renderPassCreationFailed, "Output Transform Pass: encoder creation failed") diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air index f6c49147..cf9f0159 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib index b1083f1d..ec8101e2 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air index 8ccfa38d..7448af4f 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib index 3d3c5793..0b3ed6a1 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air index 3fdef239..b332ad16 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib index 547e84dc..0d2bde07 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air index 617629c6..c6fc7190 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib index 73bd135f..295d0551 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air index 8e85815a..3de2c98b 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib index 7f65e70f..8f8093b5 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib index 4f9558b0..ab18ecf9 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib differ diff --git a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift index c2f98d94..8a9696b9 100644 --- a/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift +++ b/Sources/UntoldEngine/Utils/EngineSettingsAPI.swift @@ -77,6 +77,14 @@ public enum RenderingEnvironmentProperty: Sendable { case visible(Bool) case lightingMode(RuntimeEnvironmentLightingMode) case realWorldLightingContribution(Float) + /// Loads an HDR environment asset and regenerates the IBL textures + /// (mipmaps + pre-filtered irradiance/specular). + /// + /// When `directory` is provided, `name` (including extension) is resolved + /// against it. Otherwise the engine's standard asset search is used — + /// notably `assetBasePath` (GameData) and its `HDR/` subfolder — trying + /// the `.hdr`, `.exr`, and `.png` extensions when `name` has none. + case asset(String, directory: URL? = nil) } public enum WireframeProperty: Sendable { @@ -144,9 +152,41 @@ private func applyRenderingEnvironmentProperty(_ property: RenderingEnvironmentP RuntimeEnvironmentLightingStore.shared.mode = value case let .realWorldLightingContribution(value): RuntimeEnvironmentLightingStore.shared.realWorldLightingContribution = value + case let .asset(name, directory): + applyEnvironmentAsset(name, directory: directory) } } +private func applyEnvironmentAsset(_ name: String, directory: URL?) { + // Explicit directory: preserve generateHDR's legacy behavior (name must + // include the file extension). + if let directory { + generateHDR(name, from: directory) + return + } + + // No directory: resolve through the engine's standard asset search + // (assetBasePath/GameData incl. the HDR/ subfolder, app bundle, engine bundle). + let nsName = name as NSString + let providedExtension = nsName.pathExtension + let candidates: [(base: String, ext: String)] = providedExtension.isEmpty + ? ["hdr", "exr", "png"].map { (name, $0) } + : [(nsName.deletingPathExtension, providedExtension)] + + for candidate in candidates { + if let url = LoadingSystem.shared.resourceURL( + forResource: candidate.base, withExtension: candidate.ext, subResource: nil + ) { + generateHDR(url.lastPathComponent, from: url.deletingLastPathComponent()) + return + } + } + + // Fall back to generateHDR's own resolution (engine bundle) so behavior + // degrades to the legacy path and its error reporting. + generateHDR(name, from: nil) +} + private func applyWireframeProperty(_ property: WireframeProperty) { wireframeRenderStateLock.lock() let current = wireframeRenderState diff --git a/Sources/UntoldEngine/Utils/FuncUtils.swift b/Sources/UntoldEngine/Utils/FuncUtils.swift index 5c4c9453..5449ddfa 100644 --- a/Sources/UntoldEngine/Utils/FuncUtils.swift +++ b/Sources/UntoldEngine/Utils/FuncUtils.swift @@ -105,6 +105,23 @@ public func loadHDR(_ textureName: String, from directory: URL? = nil) throws -> bitmapContext.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height)) + // Sanitize the half-float pixels. Radiance RGBE can encode values beyond + // Float16 range (e.g. the sun disk in "puresky" HDRIs decodes to Inf). + // A single Inf/NaN poisons mip generation and the IBL pre-filter integral, + // which turns every IBL-lit object black. Inf -> max finite half (65504), + // NaN and negatives -> 0. + let componentCount = cgImage.width * cgImage.height * 4 + let halfBits = bitmapContext.data!.bindMemory(to: UInt16.self, capacity: componentCount) + let maxFiniteHalf: UInt16 = 0x7BFF // 65504.0 + for i in 0 ..< componentCount { + let bits = halfBits[i] + if bits & 0x8000 != 0 { + halfBits[i] = 0 // negative radiance is invalid + } else if bits & 0x7C00 == 0x7C00 { + halfBits[i] = (bits & 0x03FF) == 0 ? maxFiniteHalf : 0 // Inf : NaN + } + } + let descriptor = MTLTextureDescriptor() descriptor.pixelFormat = .rgba16Float descriptor.width = cgImage.width diff --git a/docs/API/UsingXRImmersionMode.md b/docs/API/UsingXRImmersionMode.md new file mode 100644 index 00000000..dedb028c --- /dev/null +++ b/docs/API/UsingXRImmersionMode.md @@ -0,0 +1,183 @@ +# Using XR Immersion Modes + +This guide shows how to run your Vision Pro game in **mixed** (passthrough) or +**full** immersion mode, how to enable the environment (skybox), and how to +load a custom IBL/HDR environment. + +--- + +## Choosing an Immersion Mode + +Two settings control the immersion mode, and **they must match**: + +1. The SwiftUI `ImmersiveSpace` style in your app entry point. +2. The engine's immersion mode, set with `setImmersionMode`. + +If they disagree — for example, the space is `.mixed` but the engine is set to +`.full` — you will see camera passthrough bleeding through your scene or a +black background where content should be. + +### Full Immersion + +Your rendered scene replaces the real world entirely. The environment map is +used as the background. + +```swift +@main +struct MyGameApp: App { + + @State private var immersionStyle: ImmersionStyle = .full + + var body: some Scene { + ImmersiveSpace(id: "ImmersiveSpace") { + CompositorLayer(configuration: UntoldEngineConfiguration(), renderer: { layerRenderer in + if let xr = UntoldEngineXR(layerRenderer: layerRenderer) { + XRHolder.shared.xr = xr + + // Must match the ImmersiveSpace style above + xr.setImmersionMode(xrImmersionMode: .full) + + // ... game scene + callbacks + render thread setup + } + }) + } + .immersionStyle(selection: $immersionStyle, in: .full) + } +} +``` + +### Mixed Immersion (Passthrough) + +Your 3D content is composited over the real world. The environment map is not +shown — the camera passthrough is your background. + +```swift +@State private var immersionStyle: ImmersionStyle = .mixed +``` + +```swift +xr.setImmersionMode(xrImmersionMode: .mixed) +``` + +```swift +.immersionStyle(selection: $immersionStyle, in: .mixed) +``` + +### Supporting Both Modes + +Allow both styles in the space and keep the engine in sync whenever the +selection changes: + +```swift +.immersionStyle(selection: $immersionStyle, in: .mixed, .full) +``` + +```swift +// Whenever you switch immersionStyle at runtime: +XRHolder.shared.xr?.setImmersionMode( + xrImmersionMode: immersionStyle is FullImmersionStyle ? .full : .mixed +) +``` + +--- + +## Enabling the Environment + +Enable environment rendering from your `GameScene` setup: + +```swift +setRendering(.environment(.visible(true))) +``` + +- In **full immersion**, the environment map fills the background behind your + scene. +- In **mixed immersion**, the environment is not composited — passthrough is + the background — but the environment still drives image-based lighting (IBL) + on your objects. + +To control whether the environment contributes IBL lighting to your materials: + +```swift +setRendering(.environment(.ibl(true))) // objects are lit by the environment +setRendering(.environment(.ibl(false))) // disable environment lighting +``` + +### Lighting Mode + +Choose where environment lighting comes from: + +```swift +// Use only your authored HDR environment for lighting +setRendering(.environment(.lightingMode(.authoredOnly))) + +// Blend in the real-world lighting estimate (mixed immersion) +setRendering(.environment(.lightingMode(.realWorldEstimate))) +setRendering(.environment(.realWorldLightingContribution(1.0))) +``` + +--- + +## Setting a Custom IBL Environment + +Load your own HDR environment with the `.asset` property. The engine loads the +image, generates mipmaps, and rebuilds the IBL textures (irradiance, specular, +BRDF) in one call: + +```swift +setRendering(.environment(.asset("suburban_garden_2k.hdr"))) +``` + +### Where to Put the File + +Place the HDR inside your project's `GameData/HDR` folder: + +``` +MyGame/ +└── Sources/ + └── MyGame/ + └── GameData/ + └── HDR/ + └── suburban_garden_2k.hdr +``` + +The name is resolved through the engine's standard asset search +(`assetBasePath`, which the generated project points at `GameData`). The +extension is optional — when omitted, the engine tries `.hdr`, `.exr`, and +`.png` in that order: + +```swift +setRendering(.environment(.asset("suburban_garden_2k"))) +``` + +### Loading From a Custom Location + +Pass an explicit directory (the name must include the extension), or use an +absolute path: + +```swift +setRendering(.environment(.asset("studio.hdr", directory: myAssetsURL))) + +setRendering(.environment(.asset("/Users/me/HDRIs/studio.hdr"))) +``` + +### HDR File Guidelines + +- Use an **equirectangular** image with a **2:1 aspect ratio** (e.g. + 2048×1024). +- **2K is the recommended resolution** — it keeps memory low (~20 MB on the + GPU) and looks sharp as a background. IBL lighting quality does not improve + beyond 2K. +- Load the environment during scene setup (for example in your `GameScene` + init), not every frame — the call blocks briefly while the IBL textures are + generated. + +### Complete Example + +```swift +private func configureEngineSystems() { + setRendering(.environment(.visible(true))) + setRendering(.environment(.ibl(true))) + setRendering(.environment(.lightingMode(.authoredOnly))) + setRendering(.environment(.asset("suburban_garden_2k.hdr"))) +} +``` diff --git a/mkdocs.yml b/mkdocs.yml index 7c145abc..0cb60631 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,6 +55,7 @@ nav: - Spatial Input: API/UsingSpatialInput.md - Lighting System: API/UsingLightingSystem.md - XR Lighting: API/UsingXRLighting.md + - XR Immersion Modes: API/UsingXRImmersionMode.md - Scene Channels: API/UsingSceneChannels.md - Light Portals: API/UsingLightPortals.md - Materials: API/UsingMaterials.md