#98441 - Ignore invalid handle/access when setting console cp (#98641)

* Ignore invalid handle/access when setting console cp

There are situations where setting the console code page is expected to
fail. For instance, if the application was started with DETACHED_PROCESS,
the attempting to set the code page will fail with ERROR_INVALID_HANDLE.
In these cases, ignore the returned error.

Fix #98441

---------

Co-authored-by: Jan Kotas <jkotas@microsoft.com>

src/libraries/Common/src/Interop/Windows/Interop.Errors.cs

@@ -13,6 +13,7 @@ internal static partial class Errors
         internal const int ERROR_ACCESS_DENIED = 0x5;
         internal const int ERROR_INVALID_HANDLE = 0x6;
         internal const int ERROR_NOT_ENOUGH_MEMORY = 0x8;
+        internal const int ERROR_INVALID_ACCESS = 0xC;
         internal const int ERROR_INVALID_DATA = 0xD;
         internal const int ERROR_OUTOFMEMORY = 0xE;
         internal const int ERROR_INVALID_DRIVE = 0xF;

src/libraries/System.Console/src/System/ConsolePal.Windows.cs

@@ -108,7 +108,9 @@ public static void SetConsoleInputEncoding(Encoding enc)
             if (enc.CodePage != UnicodeCodePage)
             {
                 if (!Interop.Kernel32.SetConsoleCP(enc.CodePage))
-                    throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError());
+                {
+                    HandleSetConsoleEncodingError(Marshal.GetLastPInvokeError());
+                }
             }
         }
 
@@ -122,10 +124,24 @@ public static void SetConsoleOutputEncoding(Encoding enc)
             if (enc.CodePage != UnicodeCodePage)
             {
                 if (!Interop.Kernel32.SetConsoleOutputCP(enc.CodePage))
-                    throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError());
+                {
+                    HandleSetConsoleEncodingError(Marshal.GetLastPInvokeError());
+                }
             }
         }
 
+        private static void HandleSetConsoleEncodingError(int lastError)
+        {
+            if (lastError == Interop.Errors.ERROR_INVALID_HANDLE
+                || lastError == Interop.Errors.ERROR_INVALID_ACCESS)
+            {
+                // no console, or not a valid handle, so fail silently
+                return;
+            }
+
+            throw Win32Marshal.GetExceptionForWin32Error(lastError);
+        }
+
         /// <summary>Gets whether Console.In is targeting a terminal display.</summary>
         public static bool IsInputRedirectedCore()
         {

src/libraries/System.Console/tests/ConsoleEncoding.Windows.cs

@@ -2,8 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
-using System.Text;
 using System.Runtime.InteropServices;
+using System.Text;
 using Microsoft.DotNet.RemoteExecutor;
 using Xunit;
 
@@ -38,6 +38,28 @@ public void InputEncoding_SetUnicodeEncoding_SilentlyIgnoredInternally()
         }).Dispose();
     }
 
+    [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+    [PlatformSpecific(TestPlatforms.Windows)]
+    public void InputEncoding_SetEncodingWhenDetached_ErrorIsSilentlyIgnored()
+    {
+        RemoteExecutor.Invoke(() =>
+        {
+            Encoding encoding = Console.InputEncoding.CodePage != Encoding.ASCII.CodePage
+                ? Encoding.ASCII
+                : Encoding.Latin1;
+
+            // use FreeConsole to detach the current console - simulating a process started with the "DETACHED_PROCESS" flag
+            FreeConsole();
+
+            // Setting the input encoding should not throw an exception
+            Console.InputEncoding = encoding;
+            // The internal state of Console should have updated, despite the failure to change the console's input encoding
+            Assert.Equal(encoding, Console.InputEncoding);
+            // Operations on the console are no longer valid - GetConsoleCP fails.
+            Assert.Equal(0u, GetConsoleCP());
+        }).Dispose();
+    }
+
     [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
     [PlatformSpecific(TestPlatforms.Windows)]
     public void OutputEncoding_SetDefaultEncoding_Success()
@@ -67,9 +89,34 @@ public void OutputEncoding_SetUnicodeEncoding_SilentlyIgnoredInternally()
         }).Dispose();
     }
 
+    [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+    [PlatformSpecific(TestPlatforms.Windows)]
+    public void OutputEncoding_SetEncodingWhenDetached_ErrorIsSilentlyIgnored()
+    {
+        RemoteExecutor.Invoke(() =>
+        {
+            Encoding encoding = Console.OutputEncoding.CodePage != Encoding.ASCII.CodePage
+                ? Encoding.ASCII
+                : Encoding.Latin1;
+
+            // use FreeConsole to detach the current console - simulating a process started with the "DETACHED_PROCESS" flag
+            FreeConsole();
+
+            // Setting the output encoding should not throw an exception
+            Console.OutputEncoding = encoding;
+            // The internal state of Console should have updated, despite the failure to change the console's output encoding
+            Assert.Equal(encoding, Console.OutputEncoding);
+            // Operations on the console are no longer valid - GetConsoleOutputCP fails.
+            Assert.Equal(0u, GetConsoleOutputCP());
+        }).Dispose();
+    }
+
     [LibraryImport("kernel32.dll")]
     public static partial uint GetConsoleCP();
 
     [LibraryImport("kernel32.dll")]
     public static partial uint GetConsoleOutputCP();
+
+    [LibraryImport("kernel32.dll")]
+    public static partial int FreeConsole();
 }