Implement RSA.GetMaxOutputSize

Co-authored-by: Jeremy Barton <jbarton@microsoft.com>

src/libraries/System.Security.Cryptography/ref/System.Security.Cryptography.cs

@@ -1873,6 +1873,7 @@ protected RSA() { }
         public virtual byte[] ExportRSAPublicKey() { throw null; }
         public string ExportRSAPublicKeyPem() { throw null; }
         public override void FromXmlString(string xmlString) { }
+        public int GetMaxOutputSize() { throw null; }
         protected virtual byte[] HashData(byte[] data, int offset, int count, System.Security.Cryptography.HashAlgorithmName hashAlgorithm) { throw null; }
         protected virtual byte[] HashData(System.IO.Stream data, System.Security.Cryptography.HashAlgorithmName hashAlgorithm) { throw null; }
         public override void ImportEncryptedPkcs8PrivateKey(System.ReadOnlySpan<byte> passwordBytes, System.ReadOnlySpan<byte> source, out int bytesRead) { throw null; }

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RSA.cs

@@ -59,6 +59,33 @@ public static RSA Create(RSAParameters parameters)
             }
         }
 
+        /// <summary>
+        ///   Gets the maximum number of bytes an RSA operation can produce.
+        /// </summary>
+        /// <returns>
+        ///  The maximum number of bytes an RSA operation can produce.
+        /// </returns>
+        /// <remarks>
+        ///   The maximum output size is defined by the RSA modulus, or key size. The key size, in bytes, is the maximum
+        ///   output size. If the key size is not an even number of bytes, then it is rounded up to the nearest number of
+        ///   whole bytes for purposes of determining the maximum output size.
+        /// </remarks>
+        /// <exception cref="CryptographicException">
+        ///   <see cref="AsymmetricAlgorithm.KeySize" /> returned a value that is not a possible RSA key size.
+        /// </exception>
+        public int GetMaxOutputSize()
+        {
+            if (KeySize <= 0)
+            {
+                throw new CryptographicException(SR.Cryptography_InvalidKeySize);
+            }
+
+            // KeySize is in bits. Add 7 before dividing by 8 to get ceil() instead of floor().
+            // There is no reality in which we will have a 2 GB RSA key. However, since KeySize is virtual,
+            // perform an unsigned shift so that we end up with the right value if the addition overflows.
+            return (KeySize + 7) >>> 3;
+        }
+
         public abstract RSAParameters ExportParameters(bool includePrivateParameters);
         public abstract void ImportParameters(RSAParameters parameters);
         public virtual byte[] Encrypt(byte[] data, RSAEncryptionPadding padding) => throw DerivedClassMustOverride();
@@ -1384,7 +1411,7 @@ private byte[] TryWithKeyBuffer<TState>(
             // In normal circumstances, the signing and encryption size is the key size.
             // In the case of decryption, it will be at most the size of the key, but the final output size is not
             // deterministic, so start with the key size.
-            int resultSize = (KeySize + 7) / 8;
+            int resultSize = GetMaxOutputSize();
             int written;
 
             // For scenarios where we are confident that we can get the output side right on the first try, we allocate

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RSABCrypt.cs

@@ -120,7 +120,7 @@ public override byte[] Encrypt(byte[] data, RSAEncryptionPadding padding)
             ArgumentNullException.ThrowIfNull(data);
             ArgumentNullException.ThrowIfNull(padding);
 
-            byte[] ret = new byte[AsymmetricAlgorithmHelpers.BitsToBytes(KeySize)];
+            byte[] ret = new byte[GetMaxOutputSize()];
             int written = Encrypt(new ReadOnlySpan<byte>(data), ret.AsSpan(), padding);
 
             VerifyWritten(ret, written);
@@ -144,7 +144,7 @@ public override byte[] SignHash(
             ArgumentException.ThrowIfNullOrEmpty(hashAlgorithm.Name, nameof(hashAlgorithm));
             ArgumentNullException.ThrowIfNull(padding);
 
-            byte[] ret = new byte[AsymmetricAlgorithmHelpers.BitsToBytes(KeySize)];
+            byte[] ret = new byte[GetMaxOutputSize()];
 
             int written = SignHash(
                 new ReadOnlySpan<byte>(hash),

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RSACryptoServiceProvider.Windows.cs

@@ -322,7 +322,7 @@ public byte[] Encrypt(byte[] rgb, bool fOAEP)
 
             if (fOAEP)
             {
-                int rsaSize = (KeySize + 7) / 8;
+                int rsaSize = GetMaxOutputSize();
                 const int OaepSha1Overhead = 20 + 20 + 2;
 
                 // Normalize the Windows 7 and Windows 8.1+ exception

src/libraries/System.Security.Cryptography/tests/RSATests.cs

@@ -1125,6 +1125,37 @@ static bool TryDecrypt(
             }
         }
 
+        [Theory]
+        [InlineData(1, 1)]
+        [InlineData(1024, 128)]
+        [InlineData(1025, 129)]
+        [InlineData(1031, 129)]
+        [InlineData(1032, 129)]
+        [InlineData(2048, 256)]
+        [InlineData(3072, 384)]
+        [InlineData(int.MaxValue, 268_435_456)]
+        public static void GetMaxOutputSize_IsModulusSizeToNearestByte(int keySize, int expectedMaxOutputSize)
+        {
+            using (DelegateRSA rsa = new DelegateRSA())
+            {
+                rsa.KeySizeGetDelegate = () => keySize;
+                Assert.Equal(expectedMaxOutputSize, rsa.GetMaxOutputSize());
+            }
+        }
+
+        [Theory]
+        [InlineData(0)]
+        [InlineData(-1)]
+        [InlineData(int.MinValue)]
+        public static void GetMaxOutputSize_InvalidKeySizes(int keySize)
+        {
+            using (DelegateRSA rsa = new DelegateRSA())
+            {
+                rsa.KeySizeGetDelegate = () => keySize;
+                Assert.Throws<CryptographicException>(() => rsa.GetMaxOutputSize());
+            }
+        }
+
         private sealed class EmptyRSA : RSA
         {
             public override RSAParameters ExportParameters(bool includePrivateParameters) => throw new NotImplementedException();
@@ -1180,6 +1211,8 @@ public delegate bool TryDecryptFunc(
             public TrySignHashFunc TrySignHashDelegate = null;
             public TryEncryptFunc TryEncryptDelegate = null;
             public TryDecryptFunc TryDecryptDelegate = null;
+            public Func<int> KeySizeGetDelegate = null;
+            public Action<int> KeySizeSetDelegate = null;
 
             public override byte[] Encrypt(byte[] data, RSAEncryptionPadding padding) =>
                 EncryptDelegate(data, padding);
@@ -1263,6 +1296,22 @@ public override bool TryDecrypt(ReadOnlySpan<byte> data, Span<byte> destination,
 
             public override RSAParameters ExportParameters(bool includePrivateParameters) => throw new NotImplementedException();
             public override void ImportParameters(RSAParameters parameters) => throw new NotImplementedException();
+
+            public override int KeySize
+            {
+                get => KeySizeGetDelegate is not null ? KeySizeGetDelegate() : base.KeySize;
+                set
+                {
+                    if (KeySizeSetDelegate is not null)
+                    {
+                        KeySizeSetDelegate(value);
+                    }
+                    else
+                    {
+                        base.KeySize = value;
+                    }
+                }
+            }
         }
     }
 }