[OpenXR] Adds a minimal API to request and control Passthrough

sources/Directory.Packages.props

@@ -23,6 +23,7 @@
     <PackageVersion Include="Silk.NET.OpenGLES" Version="2.20.0" />
     <PackageVersion Include="Silk.NET.OpenGLES.Extensions.EXT" Version="2.20.0" />
     <PackageVersion Include="Silk.NET.OpenXR" Version="2.20.0" />
+    <PackageVersion Include="Silk.NET.OpenXR.Extensions.FB" Version="2.20.0" />
     <PackageVersion Include="Silk.NET.Sdl" Version="2.20.0" />
     <PackageVersion Include="Silk.NET.Windowing.Sdl" Version="2.19.0" />
     <PackageVersion Include="Stride.SharpFont" Version="1.0.0" />

sources/engine/Stride.Engine/Rendering/Compositing/ForwardRenderer.cs

@@ -185,6 +185,7 @@ protected override void InitializeCore()
                     vrSystem.RequireMirror = VRSettings.CopyMirror;
                     vrSystem.MirrorWidth = GraphicsDevice.Presenter.BackBuffer.Width;
                     vrSystem.MirrorHeight = GraphicsDevice.Presenter.BackBuffer.Height;
+                    vrSystem.RequestPassthrough = VRSettings.RequestPassthrough;
 
                     vrSystem.Enabled = true; //careful this will trigger the whole chain of initialization!
                     vrSystem.Visible = true;

sources/engine/Stride.Engine/Rendering/Compositing/VRRendererSettings.cs

@@ -41,6 +41,9 @@ public class VRRendererSettings
         [DataMember(40)]
         public List<VROverlayRenderer> Overlays { get; } = new List<VROverlayRenderer>();
 
+        [DataMember(50)]
+        public bool RequestPassthrough { get; set; }
+
         [DataMemberIgnore]
         public RenderView[] RenderViews = { new RenderView(), new RenderView() };
 

sources/engine/Stride.VirtualReality/OpenXR/OpenXRExt_FB_passthrough.cs

@@ -0,0 +1,107 @@
+using Silk.NET.Core;
+using Silk.NET.Core.Contexts;
+using Silk.NET.OpenXR;
+using Silk.NET.OpenXR.Extensions.FB;
+using System;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Stride.VirtualReality
+{
+    internal unsafe class OpenXRExt_FB_Passthrough
+    {
+        private readonly Session session;
+        private readonly FBPassthrough api;
+        private readonly PassthroughFB handle;
+        private readonly CompositionLayerPassthroughFB* compositionlayer;
+
+        private PassthroughLayerFB passthrough_Layer;
+
+        public OpenXRExt_FB_Passthrough(XR xr, Session session, Instance instance)
+        {
+            this.session = session;
+
+            api = new FBPassthrough(new LamdaNativeContext(TryGetProcAddress));
+
+            var passthroughCreateInfo = new PassthroughCreateInfoFB
+            {
+                Next = null,
+                Flags = 0,
+                Type = StructureType.PassthroughCreateInfoFB
+            };
+            api.CreatePassthroughFB(session, passthroughCreateInfo, ref handle).CheckResult();
+
+            this.compositionlayer = (CompositionLayerPassthroughFB*)Marshal.AllocHGlobal(sizeof(CompositionLayerPassthroughFB));
+            Unsafe.InitBlockUnaligned((byte*)this.compositionlayer, 0, (uint)sizeof(CompositionLayerPassthroughFB));
+
+            bool TryGetProcAddress(string n, out nint fptr)
+            {
+                PfnVoidFunction function;
+                var result = xr.GetInstanceProcAddr(instance, n, ref function);
+                if (result.Success())
+                {
+                    fptr = function;
+                    return true;
+                }
+                else
+                {
+                    fptr = default;
+                    return false;
+                }
+            }
+        }
+
+        public bool Enabled
+        {
+            get => passthrough_Layer.Handle != default;
+            set 
+            {
+                if (value != Enabled) 
+                { 
+                    if (value)
+                    {
+                        //start the extension
+                        api.PassthroughStartFB(handle).CheckResult();
+
+                        //create the layer
+                        var passthroughLayerCreateInfo = new PassthroughLayerCreateInfoFB
+                        {
+                            Next = null,
+                            Flags = PassthroughFlagsFB.IsRunningATCreationBitFB,
+                            Passthrough = handle,
+                            Purpose = PassthroughLayerPurposeFB.ReconstructionFB,
+                            Type = StructureType.PassthroughLayerCreateInfoFB
+                        };
+
+                        api.CreatePassthroughLayerFB(session, in passthroughLayerCreateInfo, ref passthrough_Layer).CheckResult();
+                    }
+                    else
+                    {
+                        api.DestroyPassthroughLayerFB(passthrough_Layer);
+                        passthrough_Layer = default;
+
+                        api.PassthroughPauseFB(handle);
+                    }
+                }
+            }
+        }
+
+        internal unsafe CompositionLayerPassthroughFB* GetCompositionLayer()
+        {
+            compositionlayer->Next = null;
+            compositionlayer->Flags = CompositionLayerFlags.BlendTextureSourceAlphaBit;
+            compositionlayer->LayerHandle = passthrough_Layer;
+            compositionlayer->Type = StructureType.CompositionLayerPassthroughFB;
+            return this.compositionlayer;
+        }
+
+        internal unsafe void Destroy()
+        {
+            Enabled = false;
+            api.DestroyPassthroughFB(handle);
+            Marshal.FreeHGlobal(new nint(compositionlayer));
+            api.Dispose();
+        }
+    }
+}

sources/engine/Stride.VirtualReality/OpenXR/OpenXRHmd.cs

@@ -11,6 +11,7 @@
 using System.Diagnostics;
 using Silk.NET.Core.Native;
 using System.Runtime.CompilerServices;
+using Stride.Core;
 
 namespace Stride.VirtualReality
 {
@@ -19,7 +20,7 @@ public class OpenXRHmd : VRDevice
         // Public static variable to add extensions to the initialization of the openXR session
         public static List<string> extensions = new List<string>();
 
-        public static OpenXRHmd? New()
+        public static OpenXRHmd? New(bool requestPassthrough)
         {
             // Create our API object for OpenXR.
             var xr = XR.GetApi();
@@ -30,7 +31,7 @@ public class OpenXRHmd : VRDevice
             }
 
             // Takes ownership on API object
-            return new OpenXRHmd(xr);
+            return new OpenXRHmd(xr, requestPassthrough);
         }
 
         // API Objects for accessing OpenXR
@@ -59,6 +60,9 @@ public class OpenXRHmd : VRDevice
         private bool sessionRunning = false;
         private SessionState state = SessionState.Unknown;
 
+        // Passthrough
+        private OpenXRExt_FB_Passthrough? passthroughExt;
+
         private GraphicsDevice? baseDevice;
 
         private Size2 renderSize;
@@ -75,6 +79,7 @@ public class OpenXRHmd : VRDevice
         // array of view_count containers for submitting swapchains with rendered VR frames
         private CompositionLayerProjectionView[]? projection_views;
         private View[]? views;
+        private readonly unsafe List<IntPtr> compositionLayers = new();
 
         public override Size2 ActualRenderFrameSize
         {
@@ -124,12 +129,18 @@ public override bool CanInitialize
 
         public override TrackedItem[]? TrackedItems => null;
 
-        private OpenXRHmd(XR xr)
+        private OpenXRHmd(XR xr, bool requestPassthrough)
         {
             Xr = xr;
             VRApi = VRApi.OpenXR;
-            Instance = OpenXRUtils.CreateRuntime(xr, extensions, Logger);
+
+            var requestedExtensions = new List<string>(extensions);
+            if (requestPassthrough)
+                requestedExtensions.Add(OpenXRUtils.XR_FB_PASSTHROUGH_EXTENSION_NAME);
+
+            Instance = OpenXRUtils.CreateRuntime(xr, requestedExtensions, Logger);
             SystemId = Instance.Handle != 0 ? OpenXRUtils.GetSystem(xr, Instance, Logger) : default;
+            SupportsPassthrough = requestPassthrough && Xr.IsInstanceExtensionPresent(null, OpenXRUtils.XR_FB_PASSTHROUGH_EXTENSION_NAME);
         }
 
         public override unsafe void Enable(GraphicsDevice device, GraphicsDeviceManager graphicsDeviceManager, bool requireMirror, int mirrorWidth, int mirrorHeight)
@@ -348,6 +359,22 @@ public override unsafe void Enable(GraphicsDevice device, GraphicsDeviceManager
             Xr.StringToPath(Instance, "/user/hand/left", ref leftHandPath);
         }
 
+        public override IDisposable StartPassthrough()
+        {
+            if (!SupportsPassthrough)
+                throw new NotSupportedException();
+
+            if (passthroughExt is null)
+                passthroughExt = new OpenXRExt_FB_Passthrough(Xr, globalSession, Instance);
+
+            if (passthroughExt.Enabled)
+                throw new InvalidOperationException("Passthrough already started");
+
+            passthroughExt.Enabled = true;
+
+            return new AnonymousDisposable(() => passthroughExt.Enabled = false);
+        }
+
         private void EndNullFrame()
         {
             FrameEndInfo frame_end_info = new FrameEndInfo()
@@ -492,7 +519,7 @@ public override void Commit(CommandList commandList, Texture renderFrame)
             #if STRIDE_GRAPHICS_API_DIRECT3D11
             if (render_targets is null)
                 return;
-            #endif
+#endif
 
             // if we didn't wait a frame, don't commit
             if (begunFrame == false)
@@ -503,18 +530,19 @@ public override void Commit(CommandList commandList, Texture renderFrame)
             if (swapImageCollected)
             {
 #if STRIDE_GRAPHICS_API_DIRECT3D11
-            Debug.Assert(commandList.NativeDeviceContext == baseDevice.NativeDeviceContext);
-            // Logger.Warning("Blit render target");
-            baseDevice.NativeDeviceContext.CopyResource(renderFrame.NativeRenderTargetView.Resource, render_targets[swapchainPointer].Resource);
+                Debug.Assert(commandList.NativeDeviceContext == baseDevice.NativeDeviceContext);
+                // Logger.Warning("Blit render target");
+                baseDevice.NativeDeviceContext.CopyResource(renderFrame.NativeRenderTargetView.Resource, render_targets[swapchainPointer].Resource);
 #endif
 
-            // Release the swapchain image
-            // Logger.Warning("ReleaseSwapchainImage");
-            var releaseInfo = new SwapchainImageReleaseInfo() { 
-                Type = StructureType.SwapchainImageReleaseInfo,
-                Next = null,
-            };
-            Xr.ReleaseSwapchainImage(globalSwapchain, in releaseInfo).CheckResult();
+                // Release the swapchain image
+                // Logger.Warning("ReleaseSwapchainImage");
+                var releaseInfo = new SwapchainImageReleaseInfo()
+                {
+                    Type = StructureType.SwapchainImageReleaseInfo,
+                    Next = null,
+                };
+                Xr.ReleaseSwapchainImage(globalSwapchain, in releaseInfo).CheckResult();
 
                 for (var eye = 0; eye < 2; eye++)
                 {
@@ -524,28 +552,40 @@ public override void Commit(CommandList commandList, Texture renderFrame)
 
                 unsafe
                 {
-                fixed (CompositionLayerProjectionView* projection_views_ptr = &projection_views[0])
-                {
-                    var projectionLayer = new CompositionLayerProjection
-                    (
-                        viewCount: (uint)projection_views.Length,
-                        views: projection_views_ptr,
-                        space: globalPlaySpace
-                    );
-
-                    var layerPointer = (CompositionLayerBaseHeader*)&projectionLayer;
-                    var frameEndInfo = new FrameEndInfo()
+                    // Add composition layers from extensions
+                    compositionLayers.Clear();
+                    if (passthroughExt != null && passthroughExt.Enabled)
                     {
-                        Type = StructureType.FrameEndInfo,
-                        DisplayTime = globalFrameState.PredictedDisplayTime,
-                        EnvironmentBlendMode = EnvironmentBlendMode.Opaque,
-                        LayerCount = 1,
-                        Layers = &layerPointer,
-                        Next = null,
-                    };
-
-                    //Logger.Warning("EndFrame");
-                    Xr.EndFrame(globalSession, in frameEndInfo).CheckResult();
+                        var layer = passthroughExt.GetCompositionLayer();
+                        compositionLayers.Add(new IntPtr(layer));
+                    }
+
+                    fixed (CompositionLayerProjectionView* projection_views_ptr = &projection_views[0])
+                    {
+                        var projectionLayer = new CompositionLayerProjection
+                        (
+                            viewCount: (uint)projection_views.Length,
+                            views: projection_views_ptr,
+                            space: globalPlaySpace,
+                            layerFlags: compositionLayers.Count > 0 ? CompositionLayerFlags.BlendTextureSourceAlphaBit : 0
+                        );
+
+                        compositionLayers.Add(new IntPtr(&projectionLayer));
+                        fixed (nint* layersPtr = CollectionsMarshal.AsSpan(compositionLayers))
+                        {
+                            var frameEndInfo = new FrameEndInfo()
+                            {
+                                Type = StructureType.FrameEndInfo,
+                                DisplayTime = globalFrameState.PredictedDisplayTime,
+                                EnvironmentBlendMode = EnvironmentBlendMode.Opaque,
+                                LayerCount = (uint)compositionLayers.Count,
+                                Layers = (CompositionLayerBaseHeader**)layersPtr,
+                                Next = null,
+                            };
+
+                            //Logger.Warning("EndFrame");
+                            Xr.EndFrame(globalSession, in frameEndInfo).CheckResult();
+                        }
                     }
                 }
             }
@@ -843,6 +883,8 @@ public override void Dispose()
             if (globalSwapchain.Handle != 0)
                 Xr.DestroySwapchain(globalSwapchain).CheckResult();
 
+            passthroughExt?.Destroy();
+
             if (globalSession.Handle != 0)
                 Xr.DestroySession(globalSession).CheckResult();
 

sources/engine/Stride.VirtualReality/OpenXR/OpenXRUtils.cs

@@ -18,6 +18,7 @@ namespace Stride.VirtualReality
     internal static unsafe class OpenXRUtils
     {
         public const string XR_KHR_D3D11_ENABLE_EXTENSION_NAME = "XR_KHR_D3D11_enable";
+        public const string XR_FB_PASSTHROUGH_EXTENSION_NAME = "XR_FB_passthrough";
 
         public static bool Success(this Result result) => result >= 0;
 

sources/engine/Stride.VirtualReality/Stride.VirtualReality.csproj

@@ -53,6 +53,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Silk.NET.OpenXR" />
+    <PackageReference Include="Silk.NET.OpenXR.Extensions.FB" />
   </ItemGroup>
   <Import Project="$(StrideSdkTargets)" />
 </Project>

sources/engine/Stride.VirtualReality/VRDevice.cs

@@ -54,6 +54,13 @@ protected VRDevice()
 
         public bool SupportsOverlays { get; protected set; } = false;
 
+        /// <summary>
+        /// Whether or not passthrough is supported by the device. 
+        /// It needs to be requested on device creation by enabling <see cref="VRDeviceSystem.RequestPassthrough"/>.
+        /// If supported, passthrough can be started (and stopped) with the <see cref="StartPassthrough"/> method.
+        /// </summary>
+        public bool SupportsPassthrough { get; protected set; } = false;
+
         public virtual VROverlay CreateOverlay(int width, int height, int mipLevels, int sampleCount)
         {
             return null;
@@ -63,6 +70,16 @@ public virtual void ReleaseOverlay(VROverlay overlay)
         {         
         }
 
+        /// <summary>
+        /// Starts a passthrough. When enabled the scene is rendered on top of the camera image of the device.
+        /// </summary>
+        /// <returns>A disposable which will stop the passthrough on dispose.</returns>
+        /// <exception cref="NotSupportedException">Thrown if passthrough is not supported by the device.</exception>
+        public virtual IDisposable StartPassthrough()
+        {
+            throw new NotSupportedException();
+        }
+
         public abstract void Enable(GraphicsDevice device, GraphicsDeviceManager graphicsDeviceManager, bool requireMirror, int mirrorWidth, int mirrorHeight);
 
         public virtual void Recenter()

sources/engine/Stride.VirtualReality/VRDeviceSystem.cs

@@ -33,6 +33,8 @@ public VRDeviceSystem(IServiceRegistry registry) : base(registry)
 
         public int MirrorHeight;
 
+        public bool RequestPassthrough;
+
         public bool PreviousUseCustomProjectionMatrix;
 
         public bool PreviousUseCustomViewMatrix;
@@ -83,7 +85,7 @@ private void OnEnabledChanged(object sender, EventArgs eventArgs)
                         case VRApi.OpenXR:
                             {
 #if STRIDE_GRAPHICS_API_DIRECT3D11
-                                Device = OpenXRHmd.New();
+                                Device = OpenXRHmd.New(RequestPassthrough);
 #endif
                                 break;
                             }