Feature/detect dxvk

CMakeLists.txt

@@ -18,6 +18,7 @@ if(WIN32) # Setup some variables for Windows build
   set(PRIVATE_INGAMEOVERLAY_HEADERS
     src/VulkanHelpers.h
     src/Windows/DirectXVTables.h
+    src/Windows/DXVKDetector.h
     src/Windows/DX9Hook.h
     src/Windows/DX10Hook.h
     src/Windows/DX11Hook.h

include/InGameOverlay/RendererDetector.h

@@ -31,6 +31,7 @@ namespace InGameOverlay {
 /// Starts a detector to automatically find the renderer used by the application.
 /// </summary>
 /// <param name="timeout">The time before the future will timeout if no renderer has been found.</param>
+/// <param name="rendererToDetect">Set this to any combined RendererHookType_t value to filter the renderers you want to detect.</param>
 /// <param name="preferSystemLibraries">Prefer hooking the system libraries instead of the first one found.</param>
 /// <returns>A future nullptr or the renderer.</returns>
 std::future<RendererHook_t*> DetectRenderer(std::chrono::milliseconds timeout = std::chrono::milliseconds{ -1 }, RendererHookType_t rendererToDetect = RendererHookType_t::Any, bool preferSystemLibraries = true);

include/InGameOverlay/RendererHook.h

@@ -43,6 +43,8 @@ enum class OverlayHookState : uint8_t
 /// <summary>
 /// Only one RendererHookType_t will always be returned by RendererHook_t::GetRendererHookType
 /// but you can use them as flags to limit the detection in InGameOverlay::DetectRenderer.
+/// NOTE: Even if a DirectX hook has been found and DXVK is used, the renderer hook will only report the DirectX hook type.
+/// There is no combination like : DirectX11 | Vulkan returned by RendererHook_t::GetRendererHookType.
 /// </summary>
 enum class RendererHookType_t : uint8_t
 {

src/Linux/OpenGLXHook.cpp

@@ -27,7 +27,7 @@
 
 namespace InGameOverlay {
 
-#define TRY_HOOK_FUNCTION(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&OpenGLXHook_t::_My##NAME))) { \
+#define TRY_HOOK_FUNCTION_OR_FAIL(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&OpenGLXHook_t::_My##NAME))) { \
     INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
     UnhookAll();\
     return false;\
@@ -51,7 +51,7 @@ bool OpenGLXHook_t::StartHook(std::function<void()> keyCombinationCallback, Togg
         _X11Hooked = true;
 
         BeginHook();
-        TRY_HOOK_FUNCTION(GLXSwapBuffers);
+        TRY_HOOK_FUNCTION_OR_FAIL(GLXSwapBuffers);
         EndHook();
 
         INGAMEOVERLAY_INFO("Hooked OpenGLX");

src/Linux/VulkanHook.cpp

@@ -29,7 +29,7 @@
 
 namespace InGameOverlay {
 
-#define TRY_HOOK_FUNCTION(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&VulkanHook_t::_My##NAME))) { \
+#define TRY_HOOK_FUNCTION_OR_FAIL(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&VulkanHook_t::_My##NAME))) { \
     INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
     return false;\
 } } while(0)
@@ -74,13 +74,13 @@ bool VulkanHook_t::StartHook(std::function<void()> keyCombinationCallback, Toggl
         _X11Hooked = true;
 
         BeginHook();
-        TRY_HOOK_FUNCTION(VkAcquireNextImageKHR);
+        TRY_HOOK_FUNCTION_OR_FAIL(VkAcquireNextImageKHR);
         if (_VkAcquireNextImage2KHR != nullptr)
-            TRY_HOOK_FUNCTION(VkAcquireNextImage2KHR);
+            TRY_HOOK_FUNCTION_OR_FAIL(VkAcquireNextImage2KHR);
 
-        TRY_HOOK_FUNCTION(VkQueuePresentKHR);
-        TRY_HOOK_FUNCTION(VkCreateSwapchainKHR);
-        TRY_HOOK_FUNCTION(VkDestroyDevice);
+        TRY_HOOK_FUNCTION_OR_FAIL(VkQueuePresentKHR);
+        TRY_HOOK_FUNCTION_OR_FAIL(VkCreateSwapchainKHR);
+        TRY_HOOK_FUNCTION_OR_FAIL(VkDestroyDevice);
         EndHook();
 
         INGAMEOVERLAY_INFO("Hooked Vulkan");
@@ -269,7 +269,7 @@ bool VulkanHook_t::_CreateRenderTargets(VkSwapchainKHR swapChain)
             VkImageViewCreateInfo info = { };
             info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
             info.viewType = VK_IMAGE_VIEW_TYPE_2D;
-            info.format = VK_FORMAT_B8G8R8A8_UNORM;
+            info.format = _VulkanTargetFormat;
             info.image = frame.BackBuffer;
 
             info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
@@ -454,6 +454,7 @@ bool VulkanHook_t::_CreateVulkanInstance()
     LOAD_VULKAN_FUNCTION(vkGetPhysicalDeviceMemoryProperties);
     LOAD_VULKAN_FUNCTION(vkEnumerateDeviceExtensionProperties);
     LOAD_VULKAN_FUNCTION(vkEnumeratePhysicalDevices);
+    LOAD_VULKAN_FUNCTION(vkGetPhysicalDeviceSurfaceFormatsKHR);
     LOAD_VULKAN_FUNCTION(vkGetPhysicalDeviceProperties);
     LOAD_VULKAN_FUNCTION(vkGetPhysicalDeviceQueueFamilyProperties);
 #undef LOAD_VULKAN_FUNCTION
@@ -506,6 +507,70 @@ int32_t VulkanHook_t::_GetPhysicalDeviceFirstGraphicsQueue(VkPhysicalDevice phys
     return -1;
 }
 
+void VulkanHook_t::_SelectFormatSurface()
+{
+    _VulkanTargetFormat = VK_FORMAT_R8G8B8A8_UNORM;
+
+    // TODO: Find a way to use something like this:
+    //static constexpr VkFormat requestSurfaceImageFormat[] = { VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_B8G8R8_UNORM, VK_FORMAT_R8G8B8_UNORM };
+    //static constexpr uint32_t requestSurfaceImageFormatCount = sizeof(requestSurfaceImageFormat)/sizeof(*requestSurfaceImageFormat);
+    //static constexpr VkColorSpaceKHR requestSurfaceColorSpace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;
+
+    //VkResult err;
+    //VkWin32SurfaceCreateInfoKHR sci;
+    //PFN_vkCreateWin32SurfaceKHR vkCreateWin32SurfaceKHR;
+    //
+    //vkCreateWin32SurfaceKHR = (PFN_vkCreateWin32SurfaceKHR)
+    //    vkGetInstanceProcAddr(instance, "vkCreateWin32SurfaceKHR");
+    //if (!vkCreateWin32SurfaceKHR)
+    //{
+    //    _glfwInputError(GLFW_API_UNAVAILABLE,
+    //        "Win32: Vulkan instance missing VK_KHR_win32_surface extension");
+    //    return VK_ERROR_EXTENSION_NOT_PRESENT;
+    //}
+    //
+    //memset(&sci, 0, sizeof(sci));
+    //sci.sType = VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR;
+    //sci.hinstance = _glfw.win32.instance;
+    //sci.hwnd = window->win32.handle;
+    //
+    //err = vkCreateWin32SurfaceKHR(instance, &sci, allocator, surface);
+
+    //uint32_t availCount;
+    //_vkGetPhysicalDeviceSurfaceFormatsKHR(_VulkanPhysicalDevice, surface, &availCount, nullptr);
+    //std::vector<VkSurfaceFormatKHR> availFormat;
+    //availFormat.resize((int)availCount);
+    //_vkGetPhysicalDeviceSurfaceFormatsKHR(_VulkanPhysicalDevice, surface, &availCount, availFormat.data());
+    //
+    //// First check if only one format, VK_FORMAT_UNDEFINED, is available, which would imply that any format is available
+    //if (availCount == 1)
+    //{
+    //    if (availFormat[0].format == VK_FORMAT_UNDEFINED)
+    //    {
+    //        return VkSurfaceFormatKHR{
+    //            requestSurfaceImageFormat[0],
+    //            requestSurfaceColorSpace
+    //        };
+    //    }
+    //
+    //    // No point in searching another format
+    //    return availFormat[0];
+    //}
+    //
+    //// Request several formats, the first found will be used
+    //for (uint32_t i = 0; i < requestSurfaceImageFormatCount; ++i)
+    //{
+    //    for (uint32_t j = 0; j < availCount; ++j)
+    //    {
+    //        if (availFormat[i].format == requestSurfaceImageFormat[i] && availFormat[j].colorSpace == requestSurfaceColorSpace)
+    //            return availFormat[i];
+    //    }
+    //}
+    //
+    //// If none of the requested image formats could be found, use the first available
+    //return availFormat[0];
+}
+
 bool VulkanHook_t::_GetPhysicalDevice()
 {
     _VulkanPhysicalDevice = nullptr;
@@ -672,7 +737,7 @@ bool VulkanHook_t::_CreateRenderPass()
         return true;
 
     VkAttachmentDescription attachment = { };
-    attachment.format = VK_FORMAT_B8G8R8A8_UNORM;
+    attachment.format = _VulkanTargetFormat;
     attachment.samples = VK_SAMPLE_COUNT_1_BIT;
     attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
     attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
@@ -762,6 +827,8 @@ void VulkanHook_t::_PrepareForOverlay(VkQueue queue, const VkPresentInfoKHR* pPr
         if (!X11Hook_t::Inst()->SetInitialWindowSize((Window)_Window))
             return;
 
+        _SelectFormatSurface();
+
         if (_VulkanQueue == nullptr)
             _vkGetDeviceQueue(_VulkanDevice, _VulkanQueueFamily, 0, &_VulkanQueue);
 
@@ -939,6 +1006,7 @@ VKAPI_ATTR VkResult VKAPI_CALL VulkanHook_t::_MyVkCreateSwapchainKHR(VkDevice de
         inst->_ResetRenderState(OverlayHookState::Reset);
         inst->_DestroyRenderTargets();
     }
+    inst->_VulkanTargetFormat = pCreateInfo->imageFormat;
     auto res = inst->_VkCreateSwapchainKHR(device, pCreateInfo, pAllocator, pSwapchain);
     if (inst->_VulkanDevice == device && res == VkResult::VK_SUCCESS)
     {
@@ -974,6 +1042,7 @@ VulkanHook_t::VulkanHook_t():
     _VulkanImageSampler(VK_NULL_HANDLE),
     _VulkanImageDescriptorSetLayout(VK_NULL_HANDLE),
     _VulkanRenderPass(VK_NULL_HANDLE),
+    _VulkanTargetFormat(VK_FORMAT_R8G8B8A8_UNORM),
     _VulkanDevice(VK_NULL_HANDLE),
     _VulkanQueue(VK_NULL_HANDLE),
     _ImGuiFontAtlas(nullptr),
@@ -1039,6 +1108,7 @@ VulkanHook_t::VulkanHook_t():
     _vkGetBufferMemoryRequirements(nullptr),
     _vkGetImageMemoryRequirements(nullptr),
     _vkEnumeratePhysicalDevices(nullptr),
+    _vkGetPhysicalDeviceSurfaceFormatsKHR(nullptr),
     _vkGetPhysicalDeviceProperties(nullptr),
     _vkGetPhysicalDeviceQueueFamilyProperties(nullptr),
     _vkGetPhysicalDeviceMemoryProperties(nullptr),

src/Linux/VulkanHook.h

@@ -82,6 +82,7 @@ class VulkanHook_t :
     std::vector<VulkanFrame_t> _Frames;
     VkRenderPass _VulkanRenderPass;
     std::vector<VulkanDescriptorPool_t> _DescriptorsPools;
+    VkFormat _VulkanTargetFormat;
 
     VkDevice _VulkanDevice;
     VkQueue _VulkanQueue;
@@ -112,6 +113,7 @@ class VulkanHook_t :
     void _FreeVulkanRessources();
     bool _CreateVulkanInstance();
     int32_t _GetPhysicalDeviceFirstGraphicsQueue(VkPhysicalDevice physicalDevice);
+    void _SelectFormatSurface();
     bool _GetPhysicalDevice();
 
     bool _CreateImageSampler();
@@ -203,6 +205,7 @@ class VulkanHook_t :
     decltype(::vkGetBufferMemoryRequirements)            *_vkGetBufferMemoryRequirements;
     decltype(::vkGetImageMemoryRequirements)             *_vkGetImageMemoryRequirements;
     decltype(::vkEnumeratePhysicalDevices)               *_vkEnumeratePhysicalDevices;
+    decltype(::vkGetPhysicalDeviceSurfaceFormatsKHR)     *_vkGetPhysicalDeviceSurfaceFormatsKHR;
     decltype(::vkGetPhysicalDeviceProperties)            *_vkGetPhysicalDeviceProperties;
     decltype(::vkGetPhysicalDeviceQueueFamilyProperties) *_vkGetPhysicalDeviceQueueFamilyProperties;
     decltype(::vkGetPhysicalDeviceMemoryProperties)      *_vkGetPhysicalDeviceMemoryProperties;

src/MacOSX/OpenGLHook.mm

@@ -28,7 +28,7 @@
 
 namespace InGameOverlay {
 
-#define TRY_HOOK_FUNCTION(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&OpenGLHook_t::_My##NAME))) { \
+#define TRY_HOOK_FUNCTION_OR_FAIL(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&OpenGLHook_t::_My##NAME))) { \
     INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
     UnhookAll();\
     return false;\
@@ -63,7 +63,7 @@ static bool ImGuiOpenGL3Init()
         else if (_CGLFlushDrawable != nullptr)
         {
             BeginHook();
-            TRY_HOOK_FUNCTION(CGLFlushDrawable);
+            TRY_HOOK_FUNCTION_OR_FAIL(CGLFlushDrawable);
             EndHook();
         }
 

src/Windows/DX10Hook.cpp

@@ -27,6 +27,10 @@ namespace InGameOverlay {
 
 #define TRY_HOOK_FUNCTION(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&DX10Hook_t::_My##NAME))) { \
     INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
+} } while(0)
+
+#define TRY_HOOK_FUNCTION_OR_FAIL(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&DX10Hook_t::_My##NAME))) { \
+    INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
     UnhookAll();\
     return false;\
 } } while(0)
@@ -60,12 +64,12 @@ bool DX10Hook_t::StartHook(std::function<void()> keyCombinationCallback, ToggleK
 
         BeginHook();
         TRY_HOOK_FUNCTION(ID3D10DeviceRelease);
-        TRY_HOOK_FUNCTION(IDXGISwapChainPresent);
-        TRY_HOOK_FUNCTION(IDXGISwapChainResizeTarget);
-        TRY_HOOK_FUNCTION(IDXGISwapChainResizeBuffers);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainPresent);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainResizeTarget);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainResizeBuffers);
 
         if (_IDXGISwapChain1Present1 != nullptr)
-            TRY_HOOK_FUNCTION(IDXGISwapChain1Present1);
+            TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChain1Present1);
 
         EndHook();
 
@@ -275,6 +279,7 @@ HRESULT STDMETHODCALLTYPE DX10Hook_t::_MyIDXGISwapChain1Present1(IDXGISwapChain1
 DX10Hook_t::DX10Hook_t():
     _Hooked(false),
     _WindowsHooked(false),
+    _UsesDXVK(false),
     _DeviceReleasing(0),
     _Device(nullptr),
     _HookDeviceRefCount(0),
@@ -319,6 +324,15 @@ RendererHookType_t DX10Hook_t::GetRendererHookType() const
     return RendererHookType_t::DirectX10;
 }
 
+void DX10Hook_t::SetDXVK()
+{
+    if (!_UsesDXVK)
+    {
+        _UsesDXVK = true;
+        LibraryName += " (DXVK)";
+    }
+}
+
 void DX10Hook_t::LoadFunctions(
     decltype(_ID3D10DeviceRelease) releaseFcn,
     decltype(_IDXGISwapChainPresent) presentFcn,

src/Windows/DX10Hook.h

@@ -36,6 +36,7 @@ class DX10Hook_t :
     // Variables
     bool _Hooked;
     bool _WindowsHooked;
+    bool _UsesDXVK;
     uint32_t _DeviceReleasing;
     ID3D10Device* _Device;
     ULONG _HookDeviceRefCount;
@@ -79,6 +80,7 @@ class DX10Hook_t :
     virtual const char* GetLibraryName() const;
     virtual RendererHookType_t GetRendererHookType() const;
 
+    void SetDXVK();
     void LoadFunctions(
         decltype(_ID3D10DeviceRelease) releaseFcn,
         decltype(_IDXGISwapChainPresent) presentFcn,

src/Windows/DX11Hook.cpp

@@ -27,6 +27,10 @@ namespace InGameOverlay {
 
 #define TRY_HOOK_FUNCTION(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&DX11Hook_t::_My##NAME))) { \
     INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
+} } while(0)
+
+#define TRY_HOOK_FUNCTION_OR_FAIL(NAME) do { if (!HookFunc(std::make_pair<void**, void*>(&(void*&)_##NAME, (void*)&DX11Hook_t::_My##NAME))) { \
+    INGAMEOVERLAY_ERROR("Failed to hook {}", #NAME);\
     UnhookAll();\
     return false;\
 } } while(0)
@@ -70,12 +74,12 @@ bool DX11Hook_t::StartHook(std::function<void()> keyCombinationCallback, ToggleK
 
         BeginHook();
         TRY_HOOK_FUNCTION(ID3D11DeviceRelease);
-        TRY_HOOK_FUNCTION(IDXGISwapChainPresent);
-        TRY_HOOK_FUNCTION(IDXGISwapChainResizeTarget);
-        TRY_HOOK_FUNCTION(IDXGISwapChainResizeBuffers);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainPresent);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainResizeTarget);
+        TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChainResizeBuffers);
 
         if (_IDXGISwapChain1Present1 != nullptr)
-            TRY_HOOK_FUNCTION(IDXGISwapChain1Present1);
+            TRY_HOOK_FUNCTION_OR_FAIL(IDXGISwapChain1Present1);
 
         EndHook();
 
@@ -105,14 +109,15 @@ bool DX11Hook_t::IsStarted()
 
 void DX11Hook_t::_UpdateHookDeviceRefCount()
 {
+    const int BaseRefCount = 2;
     switch (_HookState)
     {
         // 0 ref from ImGui
-        case OverlayHookState::Removing: _HookDeviceRefCount = 2; break;
+        case OverlayHookState::Removing: _HookDeviceRefCount = BaseRefCount; break;
         // 1 refs from us, 10 refs from ImGui (device, vertex shader, input layout, vertex constant buffer, pixel shader, blend state, rasterizer state, depth stencil state, texture view, texture sample)
         //case OverlayHookState::Reset: _HookDeviceRefCount = 15 + _ImageResources.size(); break;
         // 1 refs from us, 12 refs from ImGui (device, vertex shader, input layout, vertex constant buffer, pixel shader, blend state, rasterizer state, depth stencil state, texture view, texture sample, vertex buffer, index buffer)
-        case OverlayHookState::Ready: _HookDeviceRefCount = 17 + _ImageResources.size();
+        case OverlayHookState::Ready: _HookDeviceRefCount = BaseRefCount + 15 + _ImageResources.size();
     }
 }
 
@@ -311,6 +316,7 @@ HRESULT STDMETHODCALLTYPE DX11Hook_t::_MyIDXGISwapChain1Present1(IDXGISwapChain1
 DX11Hook_t::DX11Hook_t():
     _Hooked(false),
     _WindowsHooked(false),
+    _UsesDXVK(false),
     _DeviceReleasing(0),
     _Device(nullptr),
     _HookDeviceRefCount(0),
@@ -356,6 +362,15 @@ RendererHookType_t DX11Hook_t::GetRendererHookType() const
     return RendererHookType_t::DirectX11;
 }
 
+void DX11Hook_t::SetDXVK()
+{
+    if (!_UsesDXVK)
+    {
+        _UsesDXVK = true;
+        LibraryName += " (DXVK)";
+    }
+}
+
 void DX11Hook_t::LoadFunctions(
     decltype(_ID3D11DeviceRelease) releaseFcn,
     decltype(_IDXGISwapChainPresent) presentFcn,

src/Windows/DX11Hook.h

@@ -36,6 +36,7 @@ class DX11Hook_t :
     // Variables
     bool _Hooked;
     bool _WindowsHooked;
+    bool _UsesDXVK;
     uint32_t _DeviceReleasing;
     ID3D11Device* _Device;
     ULONG _HookDeviceRefCount;
@@ -80,6 +81,7 @@ class DX11Hook_t :
     virtual const char* GetLibraryName() const;
     virtual RendererHookType_t GetRendererHookType() const;
 
+    void SetDXVK();
     void LoadFunctions(
         decltype(_ID3D11DeviceRelease) releaseFcn,
         decltype(_IDXGISwapChainPresent) presentFcn,