diff --git a/sources/Directory.Packages.props b/sources/Directory.Packages.props index c7acf09e24..03d61bae8b 100644 --- a/sources/Directory.Packages.props +++ b/sources/Directory.Packages.props @@ -23,6 +23,7 @@ + diff --git a/sources/engine/Stride.Engine/Rendering/Compositing/ForwardRenderer.cs b/sources/engine/Stride.Engine/Rendering/Compositing/ForwardRenderer.cs index cd0e9b18f7..9ab70f1993 100644 --- a/sources/engine/Stride.Engine/Rendering/Compositing/ForwardRenderer.cs +++ b/sources/engine/Stride.Engine/Rendering/Compositing/ForwardRenderer.cs @@ -185,6 +185,7 @@ protected override void InitializeCore() vrSystem.RequireMirror = VRSettings.CopyMirror; vrSystem.MirrorWidth = GraphicsDevice.Presenter.BackBuffer.Width; vrSystem.MirrorHeight = GraphicsDevice.Presenter.BackBuffer.Height; + vrSystem.RequestPassthrough = VRSettings.RequestPassthrough; vrSystem.Enabled = true; //careful this will trigger the whole chain of initialization! vrSystem.Visible = true; diff --git a/sources/engine/Stride.Engine/Rendering/Compositing/VRRendererSettings.cs b/sources/engine/Stride.Engine/Rendering/Compositing/VRRendererSettings.cs index 2ddced95d5..65584881ee 100644 --- a/sources/engine/Stride.Engine/Rendering/Compositing/VRRendererSettings.cs +++ b/sources/engine/Stride.Engine/Rendering/Compositing/VRRendererSettings.cs @@ -41,6 +41,9 @@ public class VRRendererSettings [DataMember(40)] public List Overlays { get; } = new List(); + [DataMember(50)] + public bool RequestPassthrough { get; set; } + [DataMemberIgnore] public RenderView[] RenderViews = { new RenderView(), new RenderView() }; diff --git a/sources/engine/Stride.VirtualReality/OpenXR/OpenXRExt_FB_passthrough.cs b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRExt_FB_passthrough.cs new file mode 100644 index 0000000000..ebdf6ca8f8 --- /dev/null +++ b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRExt_FB_passthrough.cs @@ -0,0 +1,107 @@ +using Silk.NET.Core; +using Silk.NET.Core.Contexts; +using Silk.NET.OpenXR; +using Silk.NET.OpenXR.Extensions.FB; +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Stride.VirtualReality +{ + internal unsafe class OpenXRExt_FB_Passthrough + { + private readonly Session session; + private readonly FBPassthrough api; + private readonly PassthroughFB handle; + private readonly CompositionLayerPassthroughFB* compositionlayer; + + private PassthroughLayerFB passthrough_Layer; + + public OpenXRExt_FB_Passthrough(XR xr, Session session, Instance instance) + { + this.session = session; + + api = new FBPassthrough(new LamdaNativeContext(TryGetProcAddress)); + + var passthroughCreateInfo = new PassthroughCreateInfoFB + { + Next = null, + Flags = 0, + Type = StructureType.PassthroughCreateInfoFB + }; + api.CreatePassthroughFB(session, passthroughCreateInfo, ref handle).CheckResult(); + + this.compositionlayer = (CompositionLayerPassthroughFB*)Marshal.AllocHGlobal(sizeof(CompositionLayerPassthroughFB)); + Unsafe.InitBlockUnaligned((byte*)this.compositionlayer, 0, (uint)sizeof(CompositionLayerPassthroughFB)); + + bool TryGetProcAddress(string n, out nint fptr) + { + PfnVoidFunction function; + var result = xr.GetInstanceProcAddr(instance, n, ref function); + if (result.Success()) + { + fptr = function; + return true; + } + else + { + fptr = default; + return false; + } + } + } + + public bool Enabled + { + get => passthrough_Layer.Handle != default; + set + { + if (value != Enabled) + { + if (value) + { + //start the extension + api.PassthroughStartFB(handle).CheckResult(); + + //create the layer + var passthroughLayerCreateInfo = new PassthroughLayerCreateInfoFB + { + Next = null, + Flags = PassthroughFlagsFB.IsRunningATCreationBitFB, + Passthrough = handle, + Purpose = PassthroughLayerPurposeFB.ReconstructionFB, + Type = StructureType.PassthroughLayerCreateInfoFB + }; + + api.CreatePassthroughLayerFB(session, in passthroughLayerCreateInfo, ref passthrough_Layer).CheckResult(); + } + else + { + api.DestroyPassthroughLayerFB(passthrough_Layer); + passthrough_Layer = default; + + api.PassthroughPauseFB(handle); + } + } + } + } + + internal unsafe CompositionLayerPassthroughFB* GetCompositionLayer() + { + compositionlayer->Next = null; + compositionlayer->Flags = CompositionLayerFlags.BlendTextureSourceAlphaBit; + compositionlayer->LayerHandle = passthrough_Layer; + compositionlayer->Type = StructureType.CompositionLayerPassthroughFB; + return this.compositionlayer; + } + + internal unsafe void Destroy() + { + Enabled = false; + api.DestroyPassthroughFB(handle); + Marshal.FreeHGlobal(new nint(compositionlayer)); + api.Dispose(); + } + } +} diff --git a/sources/engine/Stride.VirtualReality/OpenXR/OpenXRHmd.cs b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRHmd.cs index 35c3bc190c..078cd06148 100644 --- a/sources/engine/Stride.VirtualReality/OpenXR/OpenXRHmd.cs +++ b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRHmd.cs @@ -11,6 +11,7 @@ using System.Diagnostics; using Silk.NET.Core.Native; using System.Runtime.CompilerServices; +using Stride.Core; namespace Stride.VirtualReality { @@ -19,7 +20,11 @@ public class OpenXRHmd : VRDevice // Public static variable to add extensions to the initialization of the openXR session public static List extensions = new List(); - public static OpenXRHmd? New() + /// + /// Creates a VR device using OpenXR. + /// + /// Whether or not the XR_FB_passthrough extension should be enabled (if available). + internal static OpenXRHmd? New(bool requestPassthrough) { // Create our API object for OpenXR. var xr = XR.GetApi(); @@ -30,7 +35,7 @@ public class OpenXRHmd : VRDevice } // Takes ownership on API object - return new OpenXRHmd(xr); + return new OpenXRHmd(xr, requestPassthrough); } // API Objects for accessing OpenXR @@ -59,6 +64,10 @@ public class OpenXRHmd : VRDevice private bool sessionRunning = false; private SessionState state = SessionState.Unknown; + // Passthrough + private OpenXRExt_FB_Passthrough? passthroughExt; + private bool passthroughRequested; + private GraphicsDevice? baseDevice; private Size2 renderSize; @@ -75,6 +84,7 @@ public class OpenXRHmd : VRDevice // array of view_count containers for submitting swapchains with rendered VR frames private CompositionLayerProjectionView[]? projection_views; private View[]? views; + private readonly unsafe List compositionLayers = new(); public override Size2 ActualRenderFrameSize { @@ -124,12 +134,19 @@ public override bool CanInitialize public override TrackedItem[]? TrackedItems => null; - private OpenXRHmd(XR xr) + private OpenXRHmd(XR xr, bool requestPassthrough) { Xr = xr; VRApi = VRApi.OpenXR; - Instance = OpenXRUtils.CreateRuntime(xr, extensions, Logger); + passthroughRequested = requestPassthrough; + + var requestedExtensions = new List(extensions); + if (requestPassthrough) + requestedExtensions.Add(OpenXRUtils.XR_FB_PASSTHROUGH_EXTENSION_NAME); + + Instance = OpenXRUtils.CreateRuntime(xr, requestedExtensions, Logger); SystemId = Instance.Handle != 0 ? OpenXRUtils.GetSystem(xr, Instance, Logger) : default; + SupportsPassthrough = requestPassthrough && Xr.IsInstanceExtensionPresent(null, OpenXRUtils.XR_FB_PASSTHROUGH_EXTENSION_NAME); } public override unsafe void Enable(GraphicsDevice device, GraphicsDeviceManager graphicsDeviceManager, bool requireMirror, int mirrorWidth, int mirrorHeight) @@ -348,6 +365,25 @@ public override unsafe void Enable(GraphicsDevice device, GraphicsDeviceManager Xr.StringToPath(Instance, "/user/hand/left", ref leftHandPath); } + public override IDisposable StartPassthrough() + { + if (!passthroughRequested) + throw new InvalidOperationException("The passthrough mode needs to be enabled at device creation"); + + if (!SupportsPassthrough) + throw new NotSupportedException("The device does not support passthrough mode"); + + if (passthroughExt is null) + passthroughExt = new OpenXRExt_FB_Passthrough(Xr, globalSession, Instance); + + if (passthroughExt.Enabled) + throw new InvalidOperationException("Passthrough already started"); + + passthroughExt.Enabled = true; + + return new AnonymousDisposable(() => passthroughExt.Enabled = false); + } + private void EndNullFrame() { FrameEndInfo frame_end_info = new FrameEndInfo() @@ -492,7 +528,7 @@ public override void Commit(CommandList commandList, Texture renderFrame) #if STRIDE_GRAPHICS_API_DIRECT3D11 if (render_targets is null) return; - #endif +#endif // if we didn't wait a frame, don't commit if (begunFrame == false) @@ -503,18 +539,19 @@ public override void Commit(CommandList commandList, Texture renderFrame) if (swapImageCollected) { #if STRIDE_GRAPHICS_API_DIRECT3D11 - Debug.Assert(commandList.NativeDeviceContext == baseDevice.NativeDeviceContext); - // Logger.Warning("Blit render target"); - baseDevice.NativeDeviceContext.CopyResource(renderFrame.NativeRenderTargetView.Resource, render_targets[swapchainPointer].Resource); + Debug.Assert(commandList.NativeDeviceContext == baseDevice.NativeDeviceContext); + // Logger.Warning("Blit render target"); + baseDevice.NativeDeviceContext.CopyResource(renderFrame.NativeRenderTargetView.Resource, render_targets[swapchainPointer].Resource); #endif - // Release the swapchain image - // Logger.Warning("ReleaseSwapchainImage"); - var releaseInfo = new SwapchainImageReleaseInfo() { - Type = StructureType.SwapchainImageReleaseInfo, - Next = null, - }; - Xr.ReleaseSwapchainImage(globalSwapchain, in releaseInfo).CheckResult(); + // Release the swapchain image + // Logger.Warning("ReleaseSwapchainImage"); + var releaseInfo = new SwapchainImageReleaseInfo() + { + Type = StructureType.SwapchainImageReleaseInfo, + Next = null, + }; + Xr.ReleaseSwapchainImage(globalSwapchain, in releaseInfo).CheckResult(); for (var eye = 0; eye < 2; eye++) { @@ -524,28 +561,40 @@ public override void Commit(CommandList commandList, Texture renderFrame) unsafe { - fixed (CompositionLayerProjectionView* projection_views_ptr = &projection_views[0]) - { - var projectionLayer = new CompositionLayerProjection - ( - viewCount: (uint)projection_views.Length, - views: projection_views_ptr, - space: globalPlaySpace - ); - - var layerPointer = (CompositionLayerBaseHeader*)&projectionLayer; - var frameEndInfo = new FrameEndInfo() + // Add composition layers from extensions + compositionLayers.Clear(); + if (passthroughExt != null && passthroughExt.Enabled) { - Type = StructureType.FrameEndInfo, - DisplayTime = globalFrameState.PredictedDisplayTime, - EnvironmentBlendMode = EnvironmentBlendMode.Opaque, - LayerCount = 1, - Layers = &layerPointer, - Next = null, - }; - - //Logger.Warning("EndFrame"); - Xr.EndFrame(globalSession, in frameEndInfo).CheckResult(); + var layer = passthroughExt.GetCompositionLayer(); + compositionLayers.Add(new IntPtr(layer)); + } + + fixed (CompositionLayerProjectionView* projection_views_ptr = &projection_views[0]) + { + var projectionLayer = new CompositionLayerProjection + ( + viewCount: (uint)projection_views.Length, + views: projection_views_ptr, + space: globalPlaySpace, + layerFlags: compositionLayers.Count > 0 ? CompositionLayerFlags.BlendTextureSourceAlphaBit : 0 + ); + + compositionLayers.Add(new IntPtr(&projectionLayer)); + fixed (nint* layersPtr = CollectionsMarshal.AsSpan(compositionLayers)) + { + var frameEndInfo = new FrameEndInfo() + { + Type = StructureType.FrameEndInfo, + DisplayTime = globalFrameState.PredictedDisplayTime, + EnvironmentBlendMode = EnvironmentBlendMode.Opaque, + LayerCount = (uint)compositionLayers.Count, + Layers = (CompositionLayerBaseHeader**)layersPtr, + Next = null, + }; + + //Logger.Warning("EndFrame"); + Xr.EndFrame(globalSession, in frameEndInfo).CheckResult(); + } } } } @@ -843,6 +892,8 @@ public override void Dispose() if (globalSwapchain.Handle != 0) Xr.DestroySwapchain(globalSwapchain).CheckResult(); + passthroughExt?.Destroy(); + if (globalSession.Handle != 0) Xr.DestroySession(globalSession).CheckResult(); diff --git a/sources/engine/Stride.VirtualReality/OpenXR/OpenXRUtils.cs b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRUtils.cs index 4a1070dded..7990fdd6d2 100644 --- a/sources/engine/Stride.VirtualReality/OpenXR/OpenXRUtils.cs +++ b/sources/engine/Stride.VirtualReality/OpenXR/OpenXRUtils.cs @@ -18,6 +18,7 @@ namespace Stride.VirtualReality internal static unsafe class OpenXRUtils { public const string XR_KHR_D3D11_ENABLE_EXTENSION_NAME = "XR_KHR_D3D11_enable"; + public const string XR_FB_PASSTHROUGH_EXTENSION_NAME = "XR_FB_passthrough"; public static bool Success(this Result result) => result >= 0; diff --git a/sources/engine/Stride.VirtualReality/Stride.VirtualReality.csproj b/sources/engine/Stride.VirtualReality/Stride.VirtualReality.csproj index d7b53552a5..66feffb7ac 100644 --- a/sources/engine/Stride.VirtualReality/Stride.VirtualReality.csproj +++ b/sources/engine/Stride.VirtualReality/Stride.VirtualReality.csproj @@ -53,6 +53,7 @@ + \ No newline at end of file diff --git a/sources/engine/Stride.VirtualReality/VRDevice.cs b/sources/engine/Stride.VirtualReality/VRDevice.cs index 5fd536360f..9fd2aced22 100644 --- a/sources/engine/Stride.VirtualReality/VRDevice.cs +++ b/sources/engine/Stride.VirtualReality/VRDevice.cs @@ -54,6 +54,13 @@ protected VRDevice() public bool SupportsOverlays { get; protected set; } = false; + /// + /// Whether or not passthrough is supported by the device. + /// It needs to be requested on device creation by enabling . + /// If supported, passthrough can be started (and stopped) with the method. + /// + public bool SupportsPassthrough { get; protected set; } = false; + public virtual VROverlay CreateOverlay(int width, int height, int mipLevels, int sampleCount) { return null; @@ -63,6 +70,17 @@ public virtual void ReleaseOverlay(VROverlay overlay) { } + /// + /// Starts a passthrough. When enabled the scene is rendered on top of the camera image of the device. + /// + /// A disposable which will stop the passthrough on dispose. + /// Thrown if the passthrough mode is not supported by the device. + /// Thrown if the passthrough mode is already enabled. + public virtual IDisposable StartPassthrough() + { + throw new NotSupportedException(); + } + public abstract void Enable(GraphicsDevice device, GraphicsDeviceManager graphicsDeviceManager, bool requireMirror, int mirrorWidth, int mirrorHeight); public virtual void Recenter() diff --git a/sources/engine/Stride.VirtualReality/VRDeviceSystem.cs b/sources/engine/Stride.VirtualReality/VRDeviceSystem.cs index cfd15a3a69..c5b2147fd5 100644 --- a/sources/engine/Stride.VirtualReality/VRDeviceSystem.cs +++ b/sources/engine/Stride.VirtualReality/VRDeviceSystem.cs @@ -33,6 +33,8 @@ public VRDeviceSystem(IServiceRegistry registry) : base(registry) public int MirrorHeight; + public bool RequestPassthrough; + public bool PreviousUseCustomProjectionMatrix; public bool PreviousUseCustomViewMatrix; @@ -83,7 +85,7 @@ private void OnEnabledChanged(object sender, EventArgs eventArgs) case VRApi.OpenXR: { #if STRIDE_GRAPHICS_API_DIRECT3D11 - Device = OpenXRHmd.New(); + Device = OpenXRHmd.New(RequestPassthrough); #endif break; }