From 9c96720db1d12099f484814113eaeb3e31055de5 Mon Sep 17 00:00:00 2001 From: Gustav Sterbrant Date: Wed, 14 Aug 2024 18:00:50 +0200 Subject: [PATCH] Better submission order Fixed a bunch of compute queue swap issues. Renamed main framescript pass to Scene. Submissions are now sent in batches rather than per queue. Semaphores have debug names for easier tracking and debugging. --- .../graphicsfeature/graphicsfeatureunit.cc | 2 +- code/render/coregraphics/graphicsdevice.h | 10 +- code/render/coregraphics/semaphore.h | 5 +- code/render/coregraphics/swapchain.h | 1 + .../coregraphics/vk/vkgraphicsdevice.cc | 66 +++--- code/render/coregraphics/vk/vkloader.cc | 2 +- code/render/coregraphics/vk/vkloader.h | 5 + .../render/coregraphics/vk/vkresourcetable.cc | 2 - code/render/coregraphics/vk/vksemaphore.cc | 18 +- .../coregraphics/vk/vksubcontexthandler.cc | 222 +++++++++++------- .../coregraphics/vk/vksubcontexthandler.h | 34 ++- code/render/coregraphics/vk/vkswapchain.cc | 33 ++- code/render/frame/default.json | 3 +- code/render/graphics/graphicsserver.cc | 9 +- fips-files/generators/framescriptc.py | 6 +- toolkit/editor/editor/ui/uimanager.cc | 2 +- 16 files changed, 283 insertions(+), 137 deletions(-) diff --git a/code/addons/graphicsfeature/graphicsfeatureunit.cc b/code/addons/graphicsfeature/graphicsfeatureunit.cc index eecdbd625a..4a8fffa6ca 100644 --- a/code/addons/graphicsfeature/graphicsfeatureunit.cc +++ b/code/addons/graphicsfeature/graphicsfeatureunit.cc @@ -254,7 +254,7 @@ GraphicsFeatureUnit::OnActivate() Graphics::SetupBufferConstants(); this->gfxServer->AddPreViewCall([](IndexT frameIndex, IndexT bufferIndex) { - static auto lastFrameSubmission = FrameScript_default::Submission_ForwardShadingandPostEffects; + static auto lastFrameSubmission = FrameScript_default::Submission_Scene; FrameScript_shadows::Run(Math::rectangle(0, 0, 1024, 1024), frameIndex, bufferIndex); FrameScript_default::Bind_LastFrame(lastFrameSubmission); FrameScript_default::Bind_Shadows(FrameScript_shadows::Submission_Shadows); diff --git a/code/render/coregraphics/graphicsdevice.h b/code/render/coregraphics/graphicsdevice.h index 2ee196bc42..8b38a83b96 100644 --- a/code/render/coregraphics/graphicsdevice.h +++ b/code/render/coregraphics/graphicsdevice.h @@ -183,11 +183,15 @@ uint64 NextSubmissionIndex(const CoreGraphics::QueueType queue); bool PollSubmissionIndex(const CoreGraphics::QueueType queue, uint64 index); /// Submit a command buffer, but doesn't necessarily execute it immediately -SubmissionWaitEvent SubmitCommandBuffer(const CoreGraphics::CmdBufferId cmds, CoreGraphics::QueueType type); +SubmissionWaitEvent SubmitCommandBuffer( + const CoreGraphics::CmdBufferId cmds + , CoreGraphics::QueueType type +#if NEBULA_GRAPHICS_DEBUG + , const char* name = nullptr +#endif +); /// Wait for a submission void WaitForSubmission(SubmissionWaitEvent index, CoreGraphics::QueueType type); -/// Have a queue wait for another queue -void WaitForLastSubmission(CoreGraphics::QueueType type, CoreGraphics::QueueType waitType); /// Unlock constants void UnlockConstantUpdates(); diff --git a/code/render/coregraphics/semaphore.h b/code/render/coregraphics/semaphore.h index cbb6715e06..85169ff251 100644 --- a/code/render/coregraphics/semaphore.h +++ b/code/render/coregraphics/semaphore.h @@ -30,6 +30,9 @@ enum SemaphoreType struct SemaphoreCreateInfo { +#if NEBULA_GRAPHICS_DEBUG + const char* name = nullptr; +#endif SemaphoreType type; }; @@ -47,4 +50,4 @@ void SemaphoreReset(const SemaphoreId& semaphore); } // namespace CoreGraphics -#pragma pop_macro("CreateSemaphore") \ No newline at end of file +#pragma pop_macro("CreateSemaphore") diff --git a/code/render/coregraphics/swapchain.h b/code/render/coregraphics/swapchain.h index 021d3c81eb..425cf8f70f 100644 --- a/code/render/coregraphics/swapchain.h +++ b/code/render/coregraphics/swapchain.h @@ -21,6 +21,7 @@ struct SwapchainCreateInfo CoreGraphics::DisplayMode displayMode; bool vsync; GLFWwindow* window; + CoreGraphics::QueueType preferredQueue = CoreGraphics::ComputeQueueType; }; /// Create swapchain diff --git a/code/render/coregraphics/vk/vkgraphicsdevice.cc b/code/render/coregraphics/vk/vkgraphicsdevice.cc index dbe729ef40..7692489a76 100644 --- a/code/render/coregraphics/vk/vkgraphicsdevice.cc +++ b/code/render/coregraphics/vk/vkgraphicsdevice.cc @@ -100,7 +100,6 @@ struct GraphicsDeviceState : CoreGraphics::GraphicsDeviceState Util::FixedArray pendingDeletes; Util::Array deletesPerSubmission; Util::FixedArray> waitEvents; - CoreGraphics::SubmissionWaitEvent mostRecentEvents[CoreGraphics::QueueType::NumQueueTypes]; struct PendingMarkers { @@ -163,7 +162,6 @@ VkDebugUtilsMessengerEXT VkErrorDebugMessageHandle = nullptr; PFN_vkCreateDebugUtilsMessengerEXT VkCreateDebugMessenger = nullptr; PFN_vkDestroyDebugUtilsMessengerEXT VkDestroyDebugMessenger = nullptr; -PFN_vkSetDebugUtilsObjectNameEXT VkDebugObjectName = nullptr; PFN_vkSetDebugUtilsObjectTagEXT VkDebugObjectTag = nullptr; PFN_vkQueueBeginDebugUtilsLabelEXT VkQueueBeginLabel = nullptr; PFN_vkQueueEndDebugUtilsLabelEXT VkQueueEndLabel = nullptr; @@ -524,7 +522,6 @@ SparseTextureBind(const VkImage img, const Util::Array& opaq // Set wait events in graphics device state.waitEvents[state.currentBufferedFrameIndex].Append(sparseWait); - state.mostRecentEvents[sparseWait.queue] = sparseWait; state.sparseImageBinds.Append(GraphicsDeviceState::SparseImageBind{.img = img, .opaqueBinds = opaqueBinds, .pageBinds = pageBinds}); //state.queueHandler.AppendSparseBind(CoreGraphics::SparseQueueType, img, opaqueBinds, pageBinds); @@ -1139,7 +1136,11 @@ CreateGraphicsDevice(const GraphicsDeviceCreateInfo& info) for (i = 0; i < info.numBufferedFrames; i++) { state.presentFences[i] = CreateFence({true}); - state.renderingFinishedSemaphores[i] = CreateSemaphore({ SemaphoreType::Binary }); + state.renderingFinishedSemaphores[i] = CreateSemaphore({ +#if NEBULA_GRAPHICS_DEBUG + .name = "Present", +#endif + .type = SemaphoreType::Binary }); } #pragma pop_macro("CreateSemaphore") @@ -1579,25 +1580,29 @@ void AddSubmissionEvent(const CoreGraphics::SubmissionWaitEvent& event) { state.waitEvents[state.currentBufferedFrameIndex].Append(event); - state.mostRecentEvents[event.queue] = event; } //------------------------------------------------------------------------------ /** */ +CoreGraphics::SubmissionWaitEvent handoverWait, graphicsWait, uploadWait; CoreGraphics::SubmissionWaitEvent -SubmitCommandBuffer(const CoreGraphics::CmdBufferId cmds, CoreGraphics::QueueType type) +SubmitCommandBuffer( + const CoreGraphics::CmdBufferId cmds + , CoreGraphics::QueueType type +#if NEBULA_GRAPHICS_DEBUG + , const char* name +#endif +) { // Submit transfer and graphics commands from this frame - CoreGraphics::SubmissionWaitEvent handoverWait, graphicsWait; transferLock.Enter(); if (state.setupTransferCommandBuffer != CoreGraphics::InvalidCmdBufferId) { CmdBufferIdAcquire(state.setupTransferCommandBuffer); CmdEndMarker(state.setupTransferCommandBuffer); CmdEndRecord(state.setupTransferCommandBuffer); - CoreGraphics::SubmissionWaitEvent uploadWait; - uploadWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::TransferQueueType, CmdBufferGetVk(state.setupTransferCommandBuffer)); + uploadWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::TransferQueueType, CmdBufferGetVk(state.setupTransferCommandBuffer), "Transfer"); uploadWait.queue = CoreGraphics::TransferQueueType; // Set wait events in graphics device @@ -1616,7 +1621,7 @@ SubmitCommandBuffer(const CoreGraphics::CmdBufferId cmds, CoreGraphics::QueueTyp CmdBufferIdAcquire(state.handoverTransferCommandBuffer); CmdEndMarker(state.handoverTransferCommandBuffer); CmdEndRecord(state.handoverTransferCommandBuffer); - handoverWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::TransferQueueType, CmdBufferGetVk(state.handoverTransferCommandBuffer)); + handoverWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::TransferQueueType, CmdBufferGetVk(state.handoverTransferCommandBuffer), "Handover"); handoverWait.queue = CoreGraphics::TransferQueueType; // Set wait events in graphics device @@ -1638,12 +1643,12 @@ SubmitCommandBuffer(const CoreGraphics::CmdBufferId cmds, CoreGraphics::QueueTyp CmdEndMarker(state.setupGraphicsCommandBuffer); CmdEndRecord(state.setupGraphicsCommandBuffer); - graphicsWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::GraphicsQueueType, CmdBufferGetVk(state.setupGraphicsCommandBuffer)); + graphicsWait.timelineIndex = state.queueHandler.AppendSubmissionTimeline(CoreGraphics::GraphicsQueueType, CmdBufferGetVk(state.setupGraphicsCommandBuffer), "Setup"); graphicsWait.queue = CoreGraphics::GraphicsQueueType; // This command buffer will have handover commands, so wait for that transfer if (handoverWait != nullptr) - state.queueHandler.AppendWaitTimeline(handoverWait.timelineIndex, CoreGraphics::GraphicsQueueType, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, CoreGraphics::TransferQueueType); + state.queueHandler.AppendWaitTimeline(handoverWait.timelineIndex, CoreGraphics::GraphicsQueueType, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, handoverWait.queue); // Add wait event AddSubmissionEvent(graphicsWait); @@ -1659,8 +1664,18 @@ SubmitCommandBuffer(const CoreGraphics::CmdBufferId cmds, CoreGraphics::QueueTyp // Append submission CoreGraphics::SubmissionWaitEvent ret; - ret.timelineIndex = state.queueHandler.AppendSubmissionTimeline(type, CmdBufferGetVk(cmds)); + ret.timelineIndex = state.queueHandler.AppendSubmissionTimeline( + type + , CmdBufferGetVk(cmds) +#if NEBULA_GRAPHICS_DEBUG + , name +#endif + ); ret.queue = type; + //if (uploadWait != nullptr) + // state.queueHandler.AppendWaitTimeline(uploadWait.timelineIndex, type, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, uploadWait.queue); + //if (graphicsWait != nullptr) + // state.queueHandler.AppendWaitTimeline(graphicsWait.timelineIndex, type, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, graphicsWait.queue); // Add wait event AddSubmissionEvent(ret); @@ -1680,15 +1695,6 @@ WaitForSubmission(SubmissionWaitEvent index, CoreGraphics::QueueType type) state.queueHandler.AppendWaitTimeline(index.timelineIndex, type, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, index.queue); } -//------------------------------------------------------------------------------ -/** -*/ -void -WaitForLastSubmission(CoreGraphics::QueueType type, CoreGraphics::QueueType waitType) -{ - state.queueHandler.AppendWaitTimeline(state.mostRecentEvents[waitType].timelineIndex, type, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, waitType); -} - //------------------------------------------------------------------------------ /** */ @@ -1778,20 +1784,22 @@ FinishFrame(IndexT frameIndex) state.currentFrameIndex = frameIndex; } - // Flush all pending submissions on the queues - state.queueHandler.FlushSparseBinds(nullptr); - //state.queueHandler.FlushSubmissionsTimeline(CoreGraphics::SparseQueueType, nullptr); - state.queueHandler.FlushSubmissionsTimeline(CoreGraphics::TransferQueueType, nullptr); - state.queueHandler.FlushSubmissionsTimeline(CoreGraphics::ComputeQueueType, nullptr); + uploadWait = SubmissionWaitEvent(); + graphicsWait = SubmissionWaitEvent(); + // Flush all pending submissions on the queues + //state.queueHandler.FlushSparseBinds(nullptr); + // Signal rendering finished semaphore just before submitting graphics queue state.queueHandler.AppendPresentSignal( - GraphicsQueueType, + ComputeQueueType, SemaphoreGetVk(state.renderingFinishedSemaphores[state.currentBufferedFrameIndex]) ); + state.queueHandler.FlushSubmissions(nullptr); + // Flush graphics (main) - state.queueHandler.FlushSubmissionsTimeline(CoreGraphics::GraphicsQueueType, nullptr); + //state.queueHandler.FlushSubmissionsTimeline(CoreGraphics::GraphicsQueueType, nullptr); if (!state.sparseBufferBinds.IsEmpty() || !state.sparseImageBinds.IsEmpty()) { diff --git a/code/render/coregraphics/vk/vkloader.cc b/code/render/coregraphics/vk/vkloader.cc index 21f4586363..3fde32ceac 100644 --- a/code/render/coregraphics/vk/vkloader.cc +++ b/code/render/coregraphics/vk/vkloader.cc @@ -10,7 +10,7 @@ #endif namespace Vulkan { - +PFN_vkSetDebugUtilsObjectNameEXT VkDebugObjectName = nullptr; //------------------------------------------------------------------------------ /** */ diff --git a/code/render/coregraphics/vk/vkloader.h b/code/render/coregraphics/vk/vkloader.h index 9111ca1af8..2f36f24e31 100644 --- a/code/render/coregraphics/vk/vkloader.h +++ b/code/render/coregraphics/vk/vkloader.h @@ -26,12 +26,16 @@ extern void InitVulkan(); /// initialize Vulkan instance, loads function pointers directly from driver extern void InitInstance(VkInstance instance); +extern PFN_vkSetDebugUtilsObjectNameEXT VkDebugObjectName; + + } // namespace Vulkan #define _IMP_VK(name) name = (PFN_##name)vkGetInstanceProcAddr(instance, #name);n_assert_fmt(name != nullptr, "Unable to get function proc: %s\n",#name); #define _DEC_VK(name) extern PFN_##name name; #define _DEF_VK(name) PFN_##name name; + #ifdef __cplusplus extern "C" { #endif @@ -217,6 +221,7 @@ _DEC_VK(vkGetRayTracingShaderGroupHandlesKHR); // mesh shaders _DEC_VK(vkCmdDrawMeshTasksEXT); + #ifdef __cplusplus } #endif diff --git a/code/render/coregraphics/vk/vkresourcetable.cc b/code/render/coregraphics/vk/vkresourcetable.cc index 1d5fb20ad9..f5bcbd86ac 100644 --- a/code/render/coregraphics/vk/vkresourcetable.cc +++ b/code/render/coregraphics/vk/vkresourcetable.cc @@ -16,8 +16,6 @@ namespace Vulkan { -extern PFN_vkSetDebugUtilsObjectNameEXT VkDebugObjectName; - VkResourceTableAllocator resourceTableAllocator; VkResourceTableLayoutAllocator resourceTableLayoutAllocator; VkResourcePipelineAllocator resourcePipelineAllocator; diff --git a/code/render/coregraphics/vk/vksemaphore.cc b/code/render/coregraphics/vk/vksemaphore.cc index 2603d059a6..a9068fe774 100644 --- a/code/render/coregraphics/vk/vksemaphore.cc +++ b/code/render/coregraphics/vk/vksemaphore.cc @@ -55,9 +55,25 @@ CreateSemaphore(const SemaphoreCreateInfo& info) semaphoreAllocator.Get(id) = dev; semaphoreAllocator.Get(id) = info.type; - VkResult res = vkCreateSemaphore(dev, &cinfo, nullptr, &semaphoreAllocator.Get(id)); + VkSemaphore sem; + VkResult res = vkCreateSemaphore(dev, &cinfo, nullptr, &sem); n_assert(res == VK_SUCCESS); + semaphoreAllocator.Set(id, sem); + +#if NEBULA_GRAPHICS_DEBUG + VkDebugUtilsObjectNameInfoEXT semInfo = + { + VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT, + nullptr, + VK_OBJECT_TYPE_SEMAPHORE, + (uint64_t)sem, + info.name + }; + VkResult res2 = VkDebugObjectName(dev, &semInfo); + n_assert(res2 == VK_SUCCESS); +#endif + SemaphoreId ret = id; return ret; } diff --git a/code/render/coregraphics/vk/vksubcontexthandler.cc b/code/render/coregraphics/vk/vksubcontexthandler.cc index 85098e1ec2..d44c10a511 100644 --- a/code/render/coregraphics/vk/vksubcontexthandler.cc +++ b/code/render/coregraphics/vk/vksubcontexthandler.cc @@ -9,6 +9,7 @@ namespace Vulkan { + //------------------------------------------------------------------------------ /** */ @@ -72,7 +73,6 @@ VkSubContextHandler::Setup(VkDevice dev, const Util::FixedArray indexMap, } // setup timeline semaphores - this->timelineSubmissions.Resize(CoreGraphics::NumQueueTypes); for (IndexT i = 0; i < CoreGraphics::NumQueueTypes; i++) { VkSemaphoreTypeCreateInfo ext = @@ -91,6 +91,35 @@ VkSubContextHandler::Setup(VkDevice dev, const Util::FixedArray indexMap, VkResult res = vkCreateSemaphore(this->device, &inf, nullptr, &semaphores[i]); n_assert(res == VK_SUCCESS); +#if NEBULA_GRAPHICS_DEBUG + const char* name = nullptr; + switch (i) + { + case CoreGraphics::ComputeQueueType: + name = "Compute Semaphore"; + break; + case CoreGraphics::GraphicsQueueType: + name = "Graphics Semaphore"; + break; + case CoreGraphics::TransferQueueType: + name = "Transfer Semaphore"; + break; + case CoreGraphics::SparseQueueType: + name = "Sparse Semaphore"; + break; + } + VkDebugUtilsObjectNameInfoEXT info = + { + VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT, + nullptr, + VK_OBJECT_TYPE_SEMAPHORE, + (uint64_t)semaphores[i], + name + }; + VkResult res2 = VkDebugObjectName(this->device, &info); + n_assert(res2 == VK_SUCCESS); +#endif + semaphoreSubmissionIds[i] = 0; } @@ -106,7 +135,6 @@ VkSubContextHandler::Setup(VkDevice dev, const Util::FixedArray indexMap, void VkSubContextHandler::Discard() { - this->timelineSubmissions.Clear(); } //------------------------------------------------------------------------------ @@ -146,21 +174,46 @@ VkSubContextHandler::SetToNextContext(const CoreGraphics::QueueType type) /** */ uint64 -VkSubContextHandler::AppendSubmissionTimeline(CoreGraphics::QueueType type, VkCommandBuffer cmds) +VkSubContextHandler::AppendSubmissionTimeline( + CoreGraphics::QueueType type + , VkCommandBuffer cmds +#if NEBULA_GRAPHICS_DEBUG + , const char* name +#endif +) { n_assert(cmds != VK_NULL_HANDLE); - Util::Array& submissions = this->timelineSubmissions[type]; - submissions.Append(TimelineSubmission()); - TimelineSubmission& sub = submissions.Back(); uint64 ret = GetNextTimelineIndex(type); - - // If command buffer is present, add it - sub.buffers.Append(cmds); - // Add signal - sub.signalSemaphores.Append(this->semaphores[type]); - sub.signalIndices.Append(ret); + SubmissionList* lastList = nullptr; + if (!orderedSubmissions.IsEmpty()) + { + lastList = &this->orderedSubmissions.Back(); + } + + if (lastList != nullptr && lastList->queue == type) + { + TimelineSubmission2& sub = lastList->submissions.Emplace(); + sub.buffers.Append(cmds); + sub.signalSemaphores.Append(this->semaphores[type]); + sub.signalIndices.Append(ret); + } + else + { + SubmissionList& list = this->orderedSubmissions.Emplace(); + list.queue = type; + + // If command buffer is present, add it + TimelineSubmission2& sub2 = list.submissions.Emplace(); + sub2.buffers.Append(cmds); + sub2.signalSemaphores.Append(this->semaphores[type]); + sub2.signalIndices.Append(ret); + sub2.queue = type; +#if NEBULA_GRAPHICS_DEBUG + sub2.name = name; +#endif + } // Progress the semaphore counter this->semaphoreSubmissionIds[type] = ret; @@ -185,10 +238,19 @@ VkSubContextHandler::AppendWaitTimeline(uint64 index, CoreGraphics::QueueType ty { n_assert(type != CoreGraphics::InvalidQueueType); n_assert(index != UINT64_MAX); - TimelineSubmission& sub = this->timelineSubmissions[type].Back(); - sub.waitIndices.Append(index); - sub.waitSemaphores.Append(this->semaphores[waitType]); - sub.waitFlags.Append(waitFlags); + + auto it = this->orderedSubmissions.End() - 1; + for (; it != this->orderedSubmissions.Begin(); it--) + { + if (it->queue == type) + { + auto& sub = it->submissions.Back(); + sub.waitIndices.Append(index); + sub.waitSemaphores.Append(this->semaphores[waitType]); + sub.waitFlags.Append(waitFlags); + break; + } + } } //------------------------------------------------------------------------------ @@ -279,84 +341,82 @@ VkSubContextHandler::AppendSparseBind(CoreGraphics::QueueType type, const VkBuff void VkSubContextHandler::AppendPresentSignal(CoreGraphics::QueueType type, VkSemaphore sem) { - TimelineSubmission& sub = this->timelineSubmissions[type].Back(); - - sub.signalIndices.Append(0); - sub.signalSemaphores.Append(sem); + auto it = this->orderedSubmissions.End() - 1; + for (; it != this->orderedSubmissions.Begin(); it--) + { + if (it->queue == type) + { + auto& sub = it->submissions.Back(); + sub.signalIndices.Append(0); + sub.signalSemaphores.Append(sem); + break; + } + } } //------------------------------------------------------------------------------ /** */ -void -VkSubContextHandler::FlushSubmissionsTimeline(CoreGraphics::QueueType type, VkFence fence) +void +VkSubContextHandler::FlushSubmissions(VkFence fence) { - Util::Array& submissions = this->timelineSubmissions[type]; - - // skip flush if submission list is empty - if (submissions.IsEmpty()) - return; - - Util::FixedArray submitInfos(submissions.Size()); - Util::FixedArray extensions(submissions.Size()); - for (IndexT i = 0; i < submissions.Size(); i++) + for (auto& list : this->orderedSubmissions) { - TimelineSubmission& sub = submissions[i]; - - // if we have no work, return - if (sub.buffers.Size() == 0) + if (list.submissions.Size() == 0) continue; - VkTimelineSemaphoreSubmitInfo ext = - { - .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, - .pNext = nullptr, - .waitSemaphoreValueCount = (uint32_t)sub.waitIndices.Size(), - .pWaitSemaphoreValues = sub.waitIndices.Size() > 0 ? sub.waitIndices.Begin() : nullptr, - .signalSemaphoreValueCount = (uint32_t)sub.signalIndices.Size(), - .pSignalSemaphoreValues = sub.signalIndices.Size() > 0 ? sub.signalIndices.Begin() : nullptr - }; - extensions[i] = ext; + Util::FixedArray submitInfos(list.submissions.Size()); + Util::FixedArray timelineInfos(list.submissions.Size()); - VkSubmitInfo info = + for (int i = 0; i < list.submissions.Size(); i++) { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .pNext = &extensions[i], - .waitSemaphoreCount = (uint32_t)sub.waitSemaphores.Size(), - .pWaitSemaphores = sub.waitSemaphores.Size() > 0 ? sub.waitSemaphores.Begin() : nullptr, - .pWaitDstStageMask = sub.waitFlags.Size() > 0 ? sub.waitFlags.Begin() : nullptr, - .commandBufferCount = (uint32_t)sub.buffers.Size(), - .pCommandBuffers = sub.buffers.Size() > 0 ? sub.buffers.Begin() : nullptr, - .signalSemaphoreCount = (uint32_t)sub.signalSemaphores.Size(), // if we have a finish semaphore, add it on the submit - .pSignalSemaphores = sub.signalSemaphores.Size() > 0 ? sub.signalSemaphores.Begin() : nullptr - }; - submitInfos[i] = info; - } + auto& sub = list.submissions[i]; + timelineInfos[i] = + { + .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .waitSemaphoreValueCount = (uint32_t)sub.waitIndices.Size(), + .pWaitSemaphoreValues = sub.waitIndices.Size() > 0 ? sub.waitIndices.Begin() : nullptr, + .signalSemaphoreValueCount = (uint32_t)sub.signalIndices.Size(), + .pSignalSemaphoreValues = sub.signalIndices.Size() > 0 ? sub.signalIndices.Begin() : nullptr + }; + + submitInfos[i] = + { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + .pNext = &timelineInfos[i], + .waitSemaphoreCount = (uint32_t)sub.waitSemaphores.Size(), + .pWaitSemaphores = sub.waitSemaphores.Size() > 0 ? sub.waitSemaphores.Begin() : nullptr, + .pWaitDstStageMask = sub.waitFlags.Size() > 0 ? sub.waitFlags.Begin() : nullptr, + .commandBufferCount = (uint32_t)sub.buffers.Size(), + .pCommandBuffers = sub.buffers.Size() > 0 ? sub.buffers.Begin() : nullptr, + .signalSemaphoreCount = (uint32_t)sub.signalSemaphores.Size(), // if we have a finish semaphore, add it on the submit + .pSignalSemaphores = sub.signalSemaphores.Size() > 0 ? sub.signalSemaphores.Begin() : nullptr + }; + } + + VkQueue queue = this->GetQueue(list.queue); + switch (list.queue) + { + case CoreGraphics::ComputeQueueType: + CoreGraphics::QueueBeginMarker(list.queue, NEBULA_MARKER_COMPUTE, "Compute"); + break; + case CoreGraphics::GraphicsQueueType: + CoreGraphics::QueueBeginMarker(list.queue, NEBULA_MARKER_GRAPHICS, "Graphics"); + break; + case CoreGraphics::TransferQueueType: + CoreGraphics::QueueBeginMarker(list.queue, NEBULA_MARKER_TRANSFER, "Transfer"); + break; + case CoreGraphics::SparseQueueType: + CoreGraphics::QueueBeginMarker(list.queue, NEBULA_MARKER_TRANSFER, "Sparse"); + break; + } + VkResult res = vkQueueSubmit(queue, submitInfos.Size(), submitInfos.Begin(), fence); + n_assert(res == VK_SUCCESS); - // execute all commands - VkQueue queue = this->GetQueue(type); - switch (type) - { - case CoreGraphics::ComputeQueueType: - CoreGraphics::QueueBeginMarker(type, NEBULA_MARKER_COMPUTE, "Compute Queue"); - break; - case CoreGraphics::GraphicsQueueType: - CoreGraphics::QueueBeginMarker(type, NEBULA_MARKER_GRAPHICS, "Graphics Queue"); - break; - case CoreGraphics::TransferQueueType: - CoreGraphics::QueueBeginMarker(type, NEBULA_MARKER_TRANSFER, "Transfer Queue"); - break; - case CoreGraphics::SparseQueueType: - CoreGraphics::QueueBeginMarker(type, NEBULA_MARKER_TRANSFER, "Sparse Queue"); - break; + CoreGraphics::QueueEndMarker(list.queue); } - VkResult res = vkQueueSubmit(queue, submitInfos.Size(), submitInfos.Begin(), fence); - n_assert(res == VK_SUCCESS); - - CoreGraphics::QueueEndMarker(type); - - // clear submissions - submissions.Clear(); + this->orderedSubmissions.Clear(); } //------------------------------------------------------------------------------ diff --git a/code/render/coregraphics/vk/vksubcontexthandler.h b/code/render/coregraphics/vk/vksubcontexthandler.h index 548a4c1dd9..a3d8d2c688 100644 --- a/code/render/coregraphics/vk/vksubcontexthandler.h +++ b/code/render/coregraphics/vk/vksubcontexthandler.h @@ -50,6 +50,26 @@ class VkSubContextHandler Util::Array waitIndices; }; + struct TimelineSubmission2 + { +#if NEBULA_GRAPHICS_DEBUG + const char* name = nullptr; +#endif + CoreGraphics::QueueType queue; + Util::Array signalIndices; + Util::Array signalSemaphores; + Util::Array buffers; + Util::Array waitFlags; + Util::Array waitSemaphores; + Util::Array waitIndices; + }; + + struct SubmissionList + { + CoreGraphics::QueueType queue; + Util::Array submissions; + }; + struct SparseBindSubmission { VkBindSparseInfo bindInfo; @@ -73,7 +93,13 @@ class VkSubContextHandler void SetToNextContext(const CoreGraphics::QueueType type); /// append submission to context to execute later, supports waiting for a queue - uint64 AppendSubmissionTimeline(CoreGraphics::QueueType type, VkCommandBuffer cmds); + uint64 AppendSubmissionTimeline( + CoreGraphics::QueueType type + , VkCommandBuffer cmds +#if NEBULA_GRAPHICS_DEBUG + , const char* name = nullptr +#endif + ); /// Gets the next submission id for a specific queue uint64 GetNextTimelineIndex(CoreGraphics::QueueType type); /// Append a wait for a submission timeline index @@ -84,8 +110,8 @@ class VkSubContextHandler uint64 AppendSparseBind(CoreGraphics::QueueType type, const VkBuffer buf, const Util::Array& binds); /// Append present signal void AppendPresentSignal(CoreGraphics::QueueType type, VkSemaphore sem); - /// flush submissions - void FlushSubmissionsTimeline(CoreGraphics::QueueType type, VkFence fence); + /// Flush submissions + void FlushSubmissions(VkFence fence); /// wait for timeline index void Wait(CoreGraphics::QueueType type, uint64 index); /// check to see if timeline index has passed @@ -119,7 +145,7 @@ class VkSubContextHandler VkSemaphore semaphores[CoreGraphics::NumQueueTypes]; uint64 semaphoreSubmissionIds[CoreGraphics::NumQueueTypes]; - Util::FixedArray> timelineSubmissions; + Util::Array orderedSubmissions; Util::Array sparseBindSubmissions; }; diff --git a/code/render/coregraphics/vk/vkswapchain.cc b/code/render/coregraphics/vk/vkswapchain.cc index a9e06fa202..d1126f980c 100644 --- a/code/render/coregraphics/vk/vkswapchain.cc +++ b/code/render/coregraphics/vk/vkswapchain.cc @@ -152,19 +152,32 @@ CreateSwapchain(const SwapchainCreateInfo& info) VK_NULL_HANDLE }; - // get present queue - for (IndexT i = 0; i < NumQueueTypes; i++) + auto sets = CoreGraphics::GetQueueIndices(); + VkBool32 canPresent; + res = vkGetPhysicalDeviceSurfaceSupportKHR(physicalDev, sets.KeyAtIndex(info.preferredQueue), surface, &canPresent); + n_assert(res == VK_SUCCESS); + if (canPresent) { - VkBool32 canPresent; - res = vkGetPhysicalDeviceSurfaceSupportKHR(physicalDev, i, surface, &canPresent); - n_assert(res == VK_SUCCESS); - if (canPresent) + queue = Vulkan::GetQueue(info.preferredQueue, 0); + } + else + { + // get present queue + for (IndexT i = 0; i < NumQueueTypes; i++) { - queue = Vulkan::GetQueue((CoreGraphics::QueueType)i, 0); - break; + VkBool32 canPresent; + res = vkGetPhysicalDeviceSurfaceSupportKHR(physicalDev, i, surface, &canPresent); + n_assert(res == VK_SUCCESS); + if (canPresent) + { + queue = Vulkan::GetQueue((CoreGraphics::QueueType)i, 0); + break; + } } } + n_assert(queue != VK_NULL_HANDLE); + // create swapchain res = vkCreateSwapchainKHR(dev, &swapchainInfo, nullptr, &swapchain); n_assert(res == VK_SUCCESS); @@ -348,7 +361,7 @@ SwapchainPresent(const SwapchainId id) }; #if NEBULA_GRAPHICS_DEBUG - CoreGraphics::QueueBeginMarker(GraphicsQueueType, NEBULA_MARKER_BLACK, "Presentation"); + CoreGraphics::QueueBeginMarker(ComputeQueueType, NEBULA_MARKER_BLACK, "Presentation"); #endif const VkPresentInfoKHR info = @@ -377,7 +390,7 @@ SwapchainPresent(const SwapchainId id) #if NEBULA_GRAPHICS_DEBUG - CoreGraphics::QueueEndMarker(GraphicsQueueType); + CoreGraphics::QueueEndMarker(ComputeQueueType); #endif } diff --git a/code/render/frame/default.json b/code/render/frame/default.json index 0a85c15664..9485f531c3 100644 --- a/code/render/frame/default.json +++ b/code/render/frame/default.json @@ -2,6 +2,7 @@ "Nebula": { "version": 3, "Dependencies": [ + "LastFrame", "Shadows" ], "ImportBuffers": [ @@ -376,7 +377,7 @@ }, { - "name": "Forward Shading and Post Effects", + "name": "Scene", "wait_for_submissions": [ "Compute Prepass", "Prepass", "Shadows" ], "queue": "Graphics", "last_submit": true, diff --git a/code/render/graphics/graphicsserver.cc b/code/render/graphics/graphicsserver.cc index 9dfd4576d9..67f41a6ee6 100644 --- a/code/render/graphics/graphicsserver.cc +++ b/code/render/graphics/graphicsserver.cc @@ -605,7 +605,14 @@ GraphicsServer::EndFrame() CoreGraphics::CmdFinishQueries(cmdBuf); CoreGraphics::CmdEndRecord(cmdBuf); - auto submission = CoreGraphics::SubmitCommandBuffer(cmdBuf, CoreGraphics::ComputeQueueType); + auto submission = CoreGraphics::SubmitCommandBuffer( + cmdBuf + , CoreGraphics::ComputeQueueType +#if NEBULA_GRAPHICS_DEBUG + , "Swap" +#endif + + ); CoreGraphics::WaitForSubmission(this->swapInfo.submission, CoreGraphics::QueueType::ComputeQueueType); CoreGraphics::DestroyCmdBuffer(cmdBuf); diff --git a/fips-files/generators/framescriptc.py b/fips-files/generators/framescriptc.py index 7ae69b4900..5b18e34ecd 100644 --- a/fips-files/generators/framescriptc.py +++ b/fips-files/generators/framescriptc.py @@ -1063,7 +1063,11 @@ def FormatSource(self, file, imports): file.WriteLine("CoreGraphics::CmdEndMarker(cmdBuf);") file.WriteLine("CoreGraphics::CmdFinishQueries(cmdBuf);") file.WriteLine("CoreGraphics::CmdEndRecord(cmdBuf);") - file.WriteLine("Submission_{} = CoreGraphics::SubmitCommandBuffer(cmdBuf, CoreGraphics::QueueType::{}QueueType);".format(self.name, self.queue)) + file.WriteLine("Submission_{} = CoreGraphics::SubmitCommandBuffer(cmdBuf, CoreGraphics::QueueType::{}QueueType".format(self.name, self.queue)) + file.WriteLine("#if NEBULA_GRAPHICS_DEBUG") + file.WriteLine(', "{}"'.format(self.name)) + file.WriteLine("#endif") + file.WriteLine(");") if self.waitForQueue is not None: file.WriteLine("CoreGraphics::WaitForLastSubmission(CoreGraphics::QueueType::{}QueueType, CoreGraphics::QueueType::{}QueueType);".format(self.queue, self.waitForQueue)) for wait in self.waitForSubmissions: diff --git a/toolkit/editor/editor/ui/uimanager.cc b/toolkit/editor/editor/ui/uimanager.cc index 0dcacfdf44..3d2a11afa8 100644 --- a/toolkit/editor/editor/ui/uimanager.cc +++ b/toolkit/editor/editor/ui/uimanager.cc @@ -91,7 +91,7 @@ OnActivate() { ImGui::DockSpaceOverViewport(); windowServer->RunAll(); - FrameScript_editorframe::Bind_Scene(FrameScript_default::Submission_ForwardShadingandPostEffects); + FrameScript_editorframe::Bind_Scene(FrameScript_default::Submission_Scene); FrameScript_editorframe::Bind_SceneBuffer(Frame::TextureImport::FromExport(FrameScript_default::Export_ColorBuffer)); CoreGraphics::DisplayMode mode = CoreGraphics::WindowGetDisplayMode(CoreGraphics::CurrentWindow); Math::rectangle viewport(0, 0, mode.GetWidth(), mode.GetHeight());