Introduce opt-in BinaryFormatter killbit

src/libraries/Common/src/System/LocalAppContextSwitches.Common.cs

@@ -51,6 +51,11 @@ private static bool GetSwitchDefaultValue(string switchName)
                 return true;
             }
 
+            if (switchName == "System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization")
+            {
+                return true;
+            }
+
             return false;
         }
     }

src/libraries/Common/tests/System/Collections/IEnumerable.Generic.Serialization.Tests.cs

@@ -9,7 +9,7 @@ namespace System.Collections.Tests
 {
     public abstract partial class IEnumerable_Generic_Tests<T> : TestBase<T>
     {
-        [Theory]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         [MemberData(nameof(ValidCollectionSizes))]
         public void IGenericSharedAPI_SerializeDeserialize(int count)
         {

src/libraries/Common/tests/System/Collections/IEnumerable.NonGeneric.Serialization.Tests.cs

@@ -11,7 +11,7 @@ namespace System.Collections.Tests
 {
     public abstract partial class IEnumerable_NonGeneric_Tests : TestBase
     {
-        [Theory]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         [MemberData(nameof(ValidCollectionSizes))]
         public void IGenericSharedAPI_SerializeDeserialize(int count)
         {

src/libraries/System.Collections/tests/Generic/Comparers/EqualityComparer.Generic.Serialization.Tests.cs

@@ -11,7 +11,7 @@ namespace System.Collections.Generic.Tests
 {
     public abstract partial class ComparersGenericTests<T>
     {
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void EqualityComparer_SerializationRoundtrip()
         {
             var bf = new BinaryFormatter();

src/libraries/System.Collections/tests/Generic/Dictionary/Dictionary.Tests.cs

@@ -399,7 +399,7 @@ private static IDictionary<T, T> CreateDictionary<T>(int size, Func<int, T> keyV
             return dict;
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void ComparerSerialization()
         {
             // Strings switch between randomized and non-randomized comparers,

src/libraries/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.cs

@@ -658,7 +658,7 @@ public void Remove_NonDefaultComparer_ComparerUsed(int capacity)
 
         #region Serialization
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void ComparerSerialization()
         {
             // Strings switch between randomized and non-randomized comparers,

src/libraries/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.Shared.xml

@@ -12,5 +12,8 @@
     <type fullname="System.LocalAppContextSwitches">
       <method signature="System.Boolean get_EnableUnsafeUTF7Encoding()" body="stub" value="false" feature="System.Text.Encoding.EnableUnsafeUTF7Encoding" featurevalue="false" />
     </type>
+    <type fullname="System.Resources.ResourceReader">
+      <method signature="System.Boolean InitializeBinaryFormatter()" body="stub" value="false" feature="System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization" featurevalue="false" />
+    </type>
   </assembly>
 </linker>

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -817,6 +817,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SafeSerializationEventArgs.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationException.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfo.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfo.SerializationGuard.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfoEnumerator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\StreamingContext.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Versioning\ComponentGuaranteesAttribute.cs" />

src/libraries/System.Private.CoreLib/src/System/LocalAppContextSwitches.cs

@@ -41,6 +41,13 @@ public static bool PreserveEventListnerObjectIdentity
             get => GetCachedSwitchValue("Switch.System.Diagnostics.EventSource.PreserveEventListnerObjectIdentity", ref s_preserveEventListnerObjectIdentity);
         }
 
+        private static int s_binaryFormatterEnabled;
+        public static bool BinaryFormatterEnabled
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => GetCachedSwitchValue("System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization", ref s_binaryFormatterEnabled);
+        }
+
         private static int s_serializationGuard;
         public static bool SerializationGuard
         {

src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.Core.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Reflection;
+using System.Runtime.Serialization;
 using System.Text;
 using System.Threading;
 
@@ -50,7 +51,10 @@ private object DeserializeObject(int typeIndex)
 
             if (_binaryFormatter == null)
             {
-                InitializeBinaryFormatter();
+                if (!InitializeBinaryFormatter())
+                {
+                    throw new NotSupportedException(SR.NotSupported_ResourceObjectSerialization);
+                }
             }
 
             Type type = FindType(typeIndex);
@@ -65,8 +69,13 @@ private object DeserializeObject(int typeIndex)
         }
 
         // Issue https://github.com/dotnet/runtime/issues/39290 tracks finding an alternative to BinaryFormatter
-        private void InitializeBinaryFormatter()
+        private bool InitializeBinaryFormatter()
         {
+            if (!LocalAppContextSwitches.BinaryFormatterEnabled)
+            {
+                return false; // initialization failed
+            }
+
             LazyInitializer.EnsureInitialized(ref s_binaryFormatterType, () =>
                 Type.GetType("System.Runtime.Serialization.Formatters.Binary.BinaryFormatter, System.Runtime.Serialization.Formatters, Version=0.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
                 throwOnError: true)!);
@@ -83,6 +92,8 @@ private void InitializeBinaryFormatter()
             });
 
             _binaryFormatter = Activator.CreateInstance(s_binaryFormatterType!)!;
+
+            return true; // initialization succeeded
         }
 
         // generic method that we specialize at runtime once we've loaded the BinaryFormatter type

src/libraries/System.Private.CoreLib/src/System/Runtime/Serialization/SerializationInfo.SerializationGuard.cs

@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Security;
+using System.Threading;
+
+namespace System.Runtime.Serialization
+{
+    /// <summary>The structure for holding all of the data needed for object serialization and deserialization.</summary>
+    public sealed partial class SerializationInfo
+    {
+        internal static AsyncLocal<bool> AsyncDeserializationInProgress { get; } = new AsyncLocal<bool>();
+
+#if !CORECLR
+        // On AoT, assume private members are reflection blocked, so there's no further protection required
+        // for the thread's DeserializationTracker
+        [ThreadStatic]
+        private static DeserializationTracker? t_deserializationTracker;
+
+        private static DeserializationTracker GetThreadDeserializationTracker() =>
+            t_deserializationTracker ??= new DeserializationTracker();
+#endif // !CORECLR
+
+        // Returns true if deserialization is currently in progress
+        public static bool DeserializationInProgress
+        {
+#if CORECLR
+            [DynamicSecurityMethod] // Methods containing StackCrawlMark local var must be marked DynamicSecurityMethod
+#endif
+            get
+            {
+                if (AsyncDeserializationInProgress.Value)
+                {
+                    return true;
+                }
+
+#if CORECLR
+                StackCrawlMark stackMark = StackCrawlMark.LookForMe;
+                DeserializationTracker tracker = Thread.GetThreadDeserializationTracker(ref stackMark);
+#else
+                DeserializationTracker tracker = GetThreadDeserializationTracker();
+#endif
+                bool result = tracker.DeserializationInProgress;
+                return result;
+            }
+        }
+
+        // Throws a SerializationException if dangerous deserialization is currently
+        // in progress
+        public static void ThrowIfDeserializationInProgress()
+        {
+            if (DeserializationInProgress)
+            {
+                throw new SerializationException(SR.Serialization_DangerousDeserialization);
+            }
+        }
+
+        // Throws a DeserializationBlockedException if dangerous deserialization is currently
+        // in progress and the AppContext switch Switch.System.Runtime.Serialization.SerializationGuard.{switchSuffix}
+        // is not true. The value of the switch is cached in cachedValue to avoid repeated lookups:
+        // 0: No value cached
+        // 1: The switch is true
+        // -1: The switch is false
+        public static void ThrowIfDeserializationInProgress(string switchSuffix, ref int cachedValue)
+        {
+            const string SwitchPrefix = "Switch.System.Runtime.Serialization.SerializationGuard.";
+            if (switchSuffix == null)
+            {
+                throw new ArgumentNullException(nameof(switchSuffix));
+            }
+            if (string.IsNullOrWhiteSpace(switchSuffix))
+            {
+                throw new ArgumentException(SR.Argument_EmptyName, nameof(switchSuffix));
+            }
+
+            if (cachedValue == 0)
+            {
+                if (AppContext.TryGetSwitch(SwitchPrefix + switchSuffix, out bool isEnabled) && isEnabled)
+                {
+                    cachedValue = 1;
+                }
+                else
+                {
+                    cachedValue = -1;
+                }
+            }
+
+            if (cachedValue == 1)
+            {
+                return;
+            }
+            else if (cachedValue == -1)
+            {
+                if (DeserializationInProgress)
+                {
+                    throw new SerializationException(SR.Format(SR.Serialization_DangerousDeserialization_Switch, SwitchPrefix + switchSuffix));
+                }
+            }
+            else
+            {
+                throw new ArgumentOutOfRangeException(nameof(cachedValue));
+            }
+        }
+
+        // Declares that the current thread and async context have begun deserialization.
+        // In this state, if the SerializationGuard or other related AppContext switches are set,
+        // actions likely to be dangerous during deserialization, such as starting a process will be blocked.
+        // Returns a DeserializationToken that must be disposed to remove the deserialization state.
+#if CORECLR
+        [DynamicSecurityMethod] // Methods containing StackCrawlMark local var must be marked DynamicSecurityMethod
+#endif
+        public static DeserializationToken StartDeserialization()
+        {
+            if (LocalAppContextSwitches.SerializationGuard)
+            {
+#if CORECLR
+                StackCrawlMark stackMark = StackCrawlMark.LookForMe;
+                DeserializationTracker tracker = Thread.GetThreadDeserializationTracker(ref stackMark);
+#else
+                DeserializationTracker tracker = GetThreadDeserializationTracker();
+#endif
+                if (!tracker.DeserializationInProgress)
+                {
+                    lock (tracker)
+                    {
+                        if (!tracker.DeserializationInProgress)
+                        {
+                            AsyncDeserializationInProgress.Value = true;
+                            tracker.DeserializationInProgress = true;
+                            return new DeserializationToken(tracker);
+                        }
+                    }
+                }
+            }
+
+            return new DeserializationToken(null);
+        }
+    }
+}