From eadb268940d31c1a4cd80bb915a608454c5b5fba Mon Sep 17 00:00:00 2001
From: Leonard Hecker <lhecker@microsoft.com>
Date: Tue, 30 Aug 2022 22:56:24 +0200
Subject: [PATCH] AtlasEngine: Add support for SetSoftwareRendering

---
 src/renderer/atlas/AtlasEngine.api.cpp |  5 ++++
 src/renderer/atlas/AtlasEngine.cpp     | 41 +++++++++++++++-----------
 src/renderer/atlas/AtlasEngine.h       |  1 +
 3 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/src/renderer/atlas/AtlasEngine.api.cpp b/src/renderer/atlas/AtlasEngine.api.cpp
index 647d57bb7eaf..59dbb4f5ac42 100644
--- a/src/renderer/atlas/AtlasEngine.api.cpp
+++ b/src/renderer/atlas/AtlasEngine.api.cpp
@@ -398,6 +398,11 @@ void AtlasEngine::SetSelectionBackground(const COLORREF color, const float alpha
 
 void AtlasEngine::SetSoftwareRendering(bool enable) noexcept
 {
+    if (_api.useSoftwareRendering != enable)
+    {
+        _api.useSoftwareRendering = enable;
+        WI_SetFlag(_api.invalidations, ApiInvalidations::Device);
+    }
 }
 
 void AtlasEngine::SetWarningCallback(std::function<void(HRESULT)> pfn) noexcept
diff --git a/src/renderer/atlas/AtlasEngine.cpp b/src/renderer/atlas/AtlasEngine.cpp
index d8bc042b01b8..28fe6bf657e1 100644
--- a/src/renderer/atlas/AtlasEngine.cpp
+++ b/src/renderer/atlas/AtlasEngine.cpp
@@ -578,39 +578,46 @@ void AtlasEngine::_createResources()
     {
         wil::com_ptr<ID3D11DeviceContext> deviceContext;
 
-        // Why D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS:
-        // This flag prevents the driver from creating a large thread pool for things like shader computations
-        // that would be advantageous for games. For us this has only a minimal performance benefit,
-        // but comes with a large memory usage overhead. At the time of writing the Nvidia
-        // driver launches $cpu_thread_count more worker threads without this flag.
-        static constexpr std::array driverTypes{
-            std::pair{ D3D_DRIVER_TYPE_HARDWARE, D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS },
-            std::pair{ D3D_DRIVER_TYPE_WARP, static_cast<D3D11_CREATE_DEVICE_FLAG>(0) },
-        };
         static constexpr std::array featureLevels{
             D3D_FEATURE_LEVEL_11_1,
             D3D_FEATURE_LEVEL_11_0,
             D3D_FEATURE_LEVEL_10_1,
         };
 
-        auto hr = S_OK;
-        for (const auto& [driverType, additionalFlags] : driverTypes)
+        auto hr = E_UNEXPECTED;
+
+        if (!_api.useSoftwareRendering)
         {
+            // Why D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS:
+            // This flag prevents the driver from creating a large thread pool for things like shader computations
+            // that would be advantageous for games. For us this has only a minimal performance benefit,
+            // but comes with a large memory usage overhead. At the time of writing the Nvidia
+            // driver launches $cpu_thread_count more worker threads without this flag.
             hr = D3D11CreateDevice(
                 /* pAdapter */ nullptr,
-                /* DriverType */ driverType,
+                /* DriverType */ D3D_DRIVER_TYPE_HARDWARE,
                 /* Software */ nullptr,
-                /* Flags */ deviceFlags | additionalFlags,
+                /* Flags */ deviceFlags | D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS,
+                /* pFeatureLevels */ featureLevels.data(),
+                /* FeatureLevels */ gsl::narrow_cast<UINT>(featureLevels.size()),
+                /* SDKVersion */ D3D11_SDK_VERSION,
+                /* ppDevice */ _r.device.put(),
+                /* pFeatureLevel */ nullptr,
+                /* ppImmediateContext */ deviceContext.put());
+        }
+        if (FAILED(hr))
+        {
+            hr = D3D11CreateDevice(
+                /* pAdapter */ nullptr,
+                /* DriverType */ D3D_DRIVER_TYPE_WARP,
+                /* Software */ nullptr,
+                /* Flags */ deviceFlags,
                 /* pFeatureLevels */ featureLevels.data(),
                 /* FeatureLevels */ gsl::narrow_cast<UINT>(featureLevels.size()),
                 /* SDKVersion */ D3D11_SDK_VERSION,
                 /* ppDevice */ _r.device.put(),
                 /* pFeatureLevel */ nullptr,
                 /* ppImmediateContext */ deviceContext.put());
-            if (SUCCEEDED(hr))
-            {
-                break;
-            }
         }
         THROW_IF_FAILED(hr);
 
diff --git a/src/renderer/atlas/AtlasEngine.h b/src/renderer/atlas/AtlasEngine.h
index a9632ac89cf8..251595c47d21 100644
--- a/src/renderer/atlas/AtlasEngine.h
+++ b/src/renderer/atlas/AtlasEngine.h
@@ -1105,6 +1105,7 @@ namespace Microsoft::Console::Render
 
             std::wstring customPixelShaderPath; // changes are flagged as ApiInvalidations::Device
             bool useRetroTerminalEffect = false; // changes are flagged as ApiInvalidations::Device
+            bool useSoftwareRendering = false; // changes are flagged as ApiInvalidations::Device
 
             ApiInvalidations invalidations = ApiInvalidations::Device;
         } _api;