Disable implicit rejection for RSA PKCS#1

src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs

@@ -353,19 +353,10 @@ private void RsaCryptRoundtrip(RSAEncryptionPadding paddingMode, bool expectSucc
             Assert.Equal(TestData.HelloBytes, output);
         }
 
-        [ConditionalFact]
+        [ConditionalFact(nameof(PlatformSupportsEmptyRSAEncryption))]
         [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework)]
         public void RoundtripEmptyArray()
         {
-            if (OperatingSystem.IsIOS() && !OperatingSystem.IsIOSVersionAtLeast(13, 6))
-            {
-                throw new SkipTestException("iOS prior to 13.6 does not reliably support RSA encryption of empty data.");
-            }
-            if (OperatingSystem.IsTvOS() && !OperatingSystem.IsTvOSVersionAtLeast(14, 0))
-            {
-                throw new SkipTestException("tvOS prior to 14.0 does not reliably support RSA encryption of empty data.");
-            }
-
             using (RSA rsa = RSAFactory.Create(TestData.RSA2048Params))
             {
                 void RoundtripEmpty(RSAEncryptionPadding paddingMode)
@@ -725,6 +716,26 @@ public void NotSupportedValueMethods()
             }
         }
 
+        [ConditionalTheory]
+        [InlineData(new byte[] { 1, 2, 3, 4 })]
+        [InlineData(new byte[0])]
+        public void Decrypt_Pkcs1_ErrorsForInvalidPadding(byte[] data)
+        {
+            if (data.Length == 0 && !PlatformSupportsEmptyRSAEncryption)
+            {
+                throw new SkipTestException("Platform does not support RSA encryption of empty data.");
+            }
+
+            using (RSA rsa = RSAFactory.Create(TestData.RSA2048Params))
+            {
+                byte[] encrypted = Encrypt(rsa, data, RSAEncryptionPadding.Pkcs1);
+                encrypted[1] ^= 0xFF;
+
+                // PKCS#1, the data, and the key are all deterministic so this should always throw an exception.
+                Assert.ThrowsAny<CryptographicException>(() => Decrypt(rsa, encrypted, RSAEncryptionPadding.Pkcs1));
+            }
+        }
+
         public static IEnumerable<object[]> OaepPaddingModes
         {
             get
@@ -746,5 +757,23 @@ public static IEnumerable<object[]> OaepPaddingModes
                 }
             }
         }
+
+        public static bool PlatformSupportsEmptyRSAEncryption
+        {
+            get
+            {
+                if (OperatingSystem.IsIOS() && !OperatingSystem.IsIOSVersionAtLeast(13, 6))
+                {
+                    return false;
+                }
+
+                if (OperatingSystem.IsTvOS() && !OperatingSystem.IsTvOSVersionAtLeast(14, 0))
+                {
+                    return false;
+                }
+
+                return true;
+            }
+        }
     }
 }

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -296,8 +296,10 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(ERR_peek_error) \
     REQUIRED_FUNCTION(ERR_peek_error_line) \
     REQUIRED_FUNCTION(ERR_peek_last_error) \
+    REQUIRED_FUNCTION(ERR_pop_to_mark) \
     FALLBACK_FUNCTION(ERR_put_error) \
     REQUIRED_FUNCTION(ERR_reason_error_string) \
+    REQUIRED_FUNCTION(ERR_set_mark) \
     LIGHTUP_FUNCTION(ERR_set_debug) \
     LIGHTUP_FUNCTION(ERR_set_error) \
     REQUIRED_FUNCTION(EVP_aes_128_cbc) \
@@ -353,6 +355,7 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(EVP_PKCS82PKEY) \
     REQUIRED_FUNCTION(EVP_PKEY2PKCS8) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl_str) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_free) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_get0_pkey) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_new) \
@@ -794,8 +797,10 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define ERR_peek_error_line ERR_peek_error_line_ptr
 #define ERR_peek_last_error ERR_peek_last_error_ptr
 #define ERR_put_error ERR_put_error_ptr
+#define ERR_pop_to_mark ERR_pop_to_mark_ptr
 #define ERR_reason_error_string ERR_reason_error_string_ptr
 #define ERR_set_debug ERR_set_debug_ptr
+#define ERR_set_mark ERR_set_mark_ptr
 #define ERR_set_error ERR_set_error_ptr
 #define EVP_aes_128_cbc EVP_aes_128_cbc_ptr
 #define EVP_aes_128_cfb8 EVP_aes_128_cfb8_ptr
@@ -850,6 +855,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_PKCS82PKEY EVP_PKCS82PKEY_ptr
 #define EVP_PKEY2PKCS8 EVP_PKEY2PKCS8_ptr
 #define EVP_PKEY_CTX_ctrl EVP_PKEY_CTX_ctrl_ptr
+#define EVP_PKEY_CTX_ctrl_str EVP_PKEY_CTX_ctrl_str_ptr
 #define EVP_PKEY_CTX_free EVP_PKEY_CTX_free_ptr
 #define EVP_PKEY_CTX_get0_pkey EVP_PKEY_CTX_get0_pkey_ptr
 #define EVP_PKEY_CTX_new EVP_PKEY_CTX_new_ptr

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -67,6 +67,19 @@ static bool ConfigureEncryption(EVP_PKEY_CTX* ctx, RsaPaddingMode padding, const
         {
             return false;
         }
+
+        // OpenSSL 3.2 introduced a change where PKCS#1 RSA decryption does not fail for invalid padding.
+        // If the padding is invalid, the decryption operation returns random data.
+        // See https://github.com/openssl/openssl/pull/13817 for background.
+        // Some Linux distributions backported this change to previous versions of OpenSSL.
+        // Here we do a best-effort to set a flag to revert the behavior to failing if the padding is invalid.
+        ERR_set_mark();
+
+        EVP_PKEY_CTX_ctrl_str(ctx, "rsa_pkcs1_implicit_rejection", "0");
+
+        // Undo any changes to the error queue that may have occured while configuring implicit rejection if the
+        // current version does not support implicit rejection.
+        ERR_pop_to_mark();
     }
     else
     {