Use EVP_PKEY for RSA key generation

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs

@@ -9,6 +9,22 @@ internal static partial class Interop
 {
     internal static partial class Crypto
     {
+        [DllImport(Libraries.CryptoNative)]
+        private static extern SafeEvpPKeyHandle CryptoNative_RsaGenerateKey(int keySize);
+
+        internal static SafeEvpPKeyHandle RsaGenerateKey(int keySize)
+        {
+            SafeEvpPKeyHandle pkey = CryptoNative_RsaGenerateKey(keySize);
+
+            if (pkey.IsInvalid)
+            {
+                pkey.Dispose();
+                throw CreateOpenSslCryptographicException();
+            }
+
+            return pkey;
+        }
+
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EvpPkeyGetRsa")]
         internal static extern SafeRsaHandle EvpPkeyGetRsa(SafeEvpPKeyHandle pkey);
 

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Rsa.cs

@@ -88,9 +88,6 @@ private static extern int RsaVerificationPrimitive(
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaSize")]
         internal static extern int RsaSize(SafeRsaHandle rsa);
 
-        [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_RsaGenerateKeyEx")]
-        internal static extern int RsaGenerateKeyEx(SafeRsaHandle rsa, int bits, SafeBignumHandle e);
-
         internal static bool RsaSign(int type, ReadOnlySpan<byte> m, int m_len, Span<byte> sigret, out int siglen, SafeRsaHandle rsa) =>
             RsaSign(type, ref MemoryMarshal.GetReference(m), m_len, ref MemoryMarshal.GetReference(sigret), out siglen, rsa);
 

src/libraries/Common/src/System/Security/Cryptography/RSAOpenSsl.cs

@@ -23,9 +23,6 @@ public sealed partial class RSAOpenSsl : RSA
     {
         private const int BitsPerByte = 8;
 
-        // 65537 (0x10001) in big-endian form
-        private static ReadOnlySpan<byte> DefaultExponent => new byte[] { 0x01, 0x00, 0x01 };
-
         private Lazy<SafeRsaHandle> _key;
 
         public RSAOpenSsl()
@@ -590,36 +587,18 @@ private static void CheckBoolReturn(int returnValue)
 
         private SafeRsaHandle GenerateKey()
         {
-            SafeRsaHandle key = Interop.Crypto.RsaCreate();
-            bool generated = false;
-
-            Interop.Crypto.CheckValidOpenSslHandle(key);
-
-            try
+            using (SafeEvpPKeyHandle pkey = Interop.Crypto.RsaGenerateKey(KeySize))
             {
-                using (SafeBignumHandle exponent = Interop.Crypto.CreateBignum(DefaultExponent))
-                {
-                    // The documentation for RSA_generate_key_ex does not say that it returns only
-                    // 0 or 1, so the call marshals it back as a full Int32 and checks for a value
-                    // of 1 explicitly.
-                    int response = Interop.Crypto.RsaGenerateKeyEx(
-                        key,
-                        KeySize,
-                        exponent);
-
-                    CheckBoolReturn(response);
-                    generated = true;
-                }
-            }
-            finally
-            {
-                if (!generated)
+                SafeRsaHandle rsa = Interop.Crypto.EvpPkeyGetRsa(pkey);
+
+                if (rsa.IsInvalid)
                 {
-                    key.Dispose();
+                    rsa.Dispose();
+                    throw Interop.Crypto.CreateOpenSslCryptographicException();
                 }
-            }
 
-            return key;
+                return rsa;
+            }
         }
 
         protected override byte[] HashData(byte[] data, int offset, int count, HashAlgorithmName hashAlgorithm) =>

src/libraries/Native/Unix/System.Security.Cryptography.Native/apibridge.c

@@ -788,4 +788,12 @@ int local_BIO_up_ref(BIO *bio)
 
     return CRYPTO_add_lock(&bio->references, 1, CRYPTO_LOCK_BIO, __FILE__, __LINE__) > 1;
 }
+
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2)
+{
+    // On OpenSSL 1.0.2 there aren't two different identifiers for RSA,
+    // so just pass the request on th EVP_PKEY_CTX_ctrl with the only identifier defined.
+    return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, optype, cmd, p1, p2);
+}
+
 #endif

src/libraries/Native/Unix/System.Security.Cryptography.Native/apibridge.h

@@ -26,6 +26,7 @@ int32_t local_RSA_meth_get_flags(const RSA_METHOD* meth);
 int32_t local_RSA_set0_crt_params(RSA* rsa, BIGNUM* dmp1, BIGNUM* dmq1, BIGNUM* iqmp);
 int32_t local_RSA_set0_factors(RSA* rsa, BIGNUM* p, BIGNUM* q);
 int32_t local_RSA_set0_key(RSA* rsa, BIGNUM* n, BIGNUM* e, BIGNUM* d);
+int32_t local_RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2);
 int32_t local_SSL_is_init_finished(const SSL* ssl);
 int32_t local_SSL_CTX_config(SSL_CTX* ctx, const char* name);
 unsigned long local_SSL_CTX_set_options(SSL_CTX* ctx, unsigned long options);

src/libraries/Native/Unix/System.Security.Cryptography.Native/entrypoints.c

@@ -222,7 +222,7 @@ static const Entry s_cryptoNative[] =
     DllImportEntry(CryptoNative_RecursiveFreeX509Stack)
     DllImportEntry(CryptoNative_RsaCreate)
     DllImportEntry(CryptoNative_RsaDestroy)
-    DllImportEntry(CryptoNative_RsaGenerateKeyEx)
+    DllImportEntry(CryptoNative_RsaGenerateKey)
     DllImportEntry(CryptoNative_RsaPrivateDecrypt)
     DllImportEntry(CryptoNative_RsaPublicEncrypt)
     DllImportEntry(CryptoNative_RsaSign)

src/libraries/Native/Unix/System.Security.Cryptography.Native/opensslshim.h

@@ -148,6 +148,7 @@ void RSA_get0_factors(const RSA* rsa, const BIGNUM** p, const BIGNUM** q);
 void RSA_get0_key(const RSA* rsa, const BIGNUM** n, const BIGNUM** e, const BIGNUM** d);
 int32_t RSA_meth_get_flags(const RSA_METHOD* meth);
 const RSA_METHOD* RSA_PKCS1_OpenSSL(void);
+int32_t RSA_pkey_ctx_ctrl(EVP_PKEY_CTX* ctx, int32_t optype, int32_t cmd, int32_t p1, void* p2);
 int32_t RSA_set0_crt_params(RSA* rsa, BIGNUM* dmp1, BIGNUM* dmq1, BIGNUM* iqmp);
 int32_t RSA_set0_factors(RSA* rsa, BIGNUM* p, BIGNUM* q);
 int32_t RSA_set0_key(RSA* rsa, BIGNUM* n, BIGNUM* e, BIGNUM* d);
@@ -176,6 +177,13 @@ const X509_ALGOR* X509_get0_tbs_sigalg(const X509* x509);
 X509_PUBKEY* X509_get_X509_PUBKEY(const X509* x509);
 int32_t X509_get_version(const X509* x509);
 int32_t X509_up_ref(X509* x509);
+
+// Redefine EVP_PKEY_CTX_set_rsa operations to use (local_)RSA_pkey_ctx_ctrl so the path is the same
+// for 1.0-built on 1.1 as on 1.1-built on 1.1.
+#undef EVP_PKEY_CTX_set_rsa_keygen_bits
+#define EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) \
+    RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_KEYGEN, EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
+
 #endif
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_1_0_2_RTM
@@ -362,15 +370,21 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     RENAMED_FUNCTION(EVP_MD_CTX_free, EVP_MD_CTX_destroy) \
     RENAMED_FUNCTION(EVP_MD_CTX_new, EVP_MD_CTX_create) \
     REQUIRED_FUNCTION(EVP_MD_size) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_ctrl) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_free) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_get0_pkey) \
     REQUIRED_FUNCTION(EVP_PKEY_CTX_new) \
+    REQUIRED_FUNCTION(EVP_PKEY_CTX_new_id) \
+    REQUIRED_FUNCTION(EVP_PKEY_base_id) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_set_peer) \
     REQUIRED_FUNCTION(EVP_PKEY_derive_init) \
     REQUIRED_FUNCTION(EVP_PKEY_derive) \
     REQUIRED_FUNCTION(EVP_PKEY_free) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_EC_KEY) \
     REQUIRED_FUNCTION(EVP_PKEY_get1_RSA) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen) \
+    REQUIRED_FUNCTION(EVP_PKEY_keygen_init) \
     REQUIRED_FUNCTION(EVP_PKEY_new) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_DSA) \
     REQUIRED_FUNCTION(EVP_PKEY_set1_EC_KEY) \
@@ -453,6 +467,7 @@ void SSL_get0_alpn_selected(const SSL* ssl, const unsigned char** protocol, unsi
     FALLBACK_FUNCTION(RSA_get0_key) \
     FALLBACK_FUNCTION(RSA_meth_get_flags) \
     REQUIRED_FUNCTION(RSA_new) \
+    FALLBACK_FUNCTION(RSA_pkey_ctx_ctrl) \
     RENAMED_FUNCTION(RSA_PKCS1_OpenSSL, RSA_PKCS1_SSLeay) \
     REQUIRED_FUNCTION(RSA_private_decrypt) \
     REQUIRED_FUNCTION(RSA_private_encrypt) \
@@ -773,15 +788,21 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_MD_CTX_free EVP_MD_CTX_free_ptr
 #define EVP_MD_CTX_new EVP_MD_CTX_new_ptr
 #define EVP_MD_size EVP_MD_size_ptr
+#define EVP_PKEY_CTX_ctrl EVP_PKEY_CTX_ctrl_ptr
 #define EVP_PKEY_CTX_free EVP_PKEY_CTX_free_ptr
+#define EVP_PKEY_CTX_get0_pkey EVP_PKEY_CTX_get0_pkey_ptr
 #define EVP_PKEY_CTX_new EVP_PKEY_CTX_new_ptr
+#define EVP_PKEY_CTX_new_id EVP_PKEY_CTX_new_id_ptr
+#define EVP_PKEY_base_id EVP_PKEY_base_id_ptr
 #define EVP_PKEY_derive_set_peer EVP_PKEY_derive_set_peer_ptr
 #define EVP_PKEY_derive_init EVP_PKEY_derive_init_ptr
 #define EVP_PKEY_derive EVP_PKEY_derive_ptr
 #define EVP_PKEY_free EVP_PKEY_free_ptr
 #define EVP_PKEY_get1_DSA EVP_PKEY_get1_DSA_ptr
 #define EVP_PKEY_get1_EC_KEY EVP_PKEY_get1_EC_KEY_ptr
 #define EVP_PKEY_get1_RSA EVP_PKEY_get1_RSA_ptr
+#define EVP_PKEY_keygen EVP_PKEY_keygen_ptr
+#define EVP_PKEY_keygen_init EVP_PKEY_keygen_init_ptr
 #define EVP_PKEY_new EVP_PKEY_new_ptr
 #define EVP_PKEY_set1_DSA EVP_PKEY_set1_DSA_ptr
 #define EVP_PKEY_set1_EC_KEY EVP_PKEY_set1_EC_KEY_ptr
@@ -864,6 +885,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_get_method RSA_get_method_ptr
 #define RSA_meth_get_flags RSA_meth_get_flags_ptr
 #define RSA_new RSA_new_ptr
+#define RSA_pkey_ctx_ctrl RSA_pkey_ctx_ctrl_ptr
 #define RSA_PKCS1_OpenSSL RSA_PKCS1_OpenSSL_ptr
 #define RSA_private_decrypt RSA_private_decrypt_ptr
 #define RSA_private_encrypt RSA_private_encrypt_ptr
@@ -1084,6 +1106,7 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define RSA_set0_crt_params local_RSA_set0_crt_params
 #define RSA_set0_factors local_RSA_set0_factors
 #define RSA_set0_key local_RSA_set0_key
+#define RSA_pkey_ctx_ctrl local_RSA_pkey_ctx_ctrl
 #define SSL_CTX_set_security_level local_SSL_CTX_set_security_level
 #define SSL_is_init_finished local_SSL_is_init_finished
 #define X509_CRL_get0_nextUpdate local_X509_CRL_get0_nextUpdate

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -3,6 +3,35 @@
 
 #include "pal_evp_pkey_rsa.h"
 
+EVP_PKEY* CryptoNative_RsaGenerateKey(int keySize)
+{
+    EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+
+    if (ctx == NULL)
+    {
+        return NULL;
+    }
+
+    EVP_PKEY* pkey = NULL;
+    EVP_PKEY* ret = NULL;
+
+    if (EVP_PKEY_keygen_init(ctx) == 1 &&
+        EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, keySize) == 1 &&
+        EVP_PKEY_keygen(ctx, &pkey) == 1)
+    {
+        ret = pkey;
+        pkey = NULL;
+    }
+
+    if (pkey != NULL)
+    {
+        EVP_PKEY_free(pkey);
+    }
+
+    EVP_PKEY_CTX_free(ctx);
+    return ret;
+}
+
 RSA* CryptoNative_EvpPkeyGetRsa(EVP_PKEY* pkey)
 {
     return EVP_PKEY_get1_RSA(pkey);

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h

@@ -5,6 +5,11 @@
 #include "pal_compiler.h"
 #include "opensslshim.h"
 
+/*
+Creates an RSA key of the requested size.
+*/
+PALEXPORT EVP_PKEY* CryptoNative_RsaGenerateKey(int32_t keySize);
+
 /*
 Shims the EVP_PKEY_get1_RSA method.
 

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_rsa.c

@@ -144,11 +144,6 @@ int32_t CryptoNative_RsaSize(RSA* rsa)
     return RSA_size(rsa);
 }
 
-int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e)
-{
-    return RSA_generate_key_ex(rsa, bits, e, NULL);
-}
-
 int32_t
 CryptoNative_RsaSign(int32_t type, const uint8_t* m, int32_t mlen, uint8_t* sigret, int32_t* siglen, RSA* rsa)
 {

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_rsa.h

@@ -85,13 +85,6 @@ Returns the RSA modulus size in bytes.
 */
 PALEXPORT int32_t CryptoNative_RsaSize(RSA* rsa);
 
-/*
-Shims the RSA_generate_key_ex method.
-
-Returns 1 upon success, otherwise 0.
-*/
-PALEXPORT int32_t CryptoNative_RsaGenerateKeyEx(RSA* rsa, int32_t bits, BIGNUM* e);
-
 /*
 Shims the RSA_sign method.
 

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_ssl.c

@@ -644,16 +644,22 @@ int32_t CryptoNative_SslGetCurrentCipherId(SSL* ssl, int32_t* cipherId)
 // This function generates key pair and creates simple certificate.
 static int MakeSelfSignedCertificate(X509 * cert, EVP_PKEY* evp)
 {
-    RSA* rsa = CryptoNative_RsaCreate();
+    RSA* rsa = NULL;
     ASN1_TIME* time = ASN1_TIME_new();
-    BIGNUM* bn = BN_new();
-    BN_set_word(bn, RSA_F4);
     X509_NAME * asnName;
     unsigned char * name = (unsigned char*)"localhost";
 
     int ret = 0;
 
-    if (rsa != NULL && CryptoNative_RsaGenerateKeyEx(rsa, 2048, bn) == 1)
+    EVP_PKEY* pkey = CryptoNative_RsaGenerateKey(2048);
+
+    if (pkey != NULL)
+    {
+        rsa = EVP_PKEY_get1_RSA(pkey);
+        EVP_PKEY_free(pkey);
+    }
+
+    if (rsa != NULL)
     {
         if (CryptoNative_EvpPkeySetRsa(evp, rsa) == 1)
         {
@@ -675,11 +681,6 @@ static int MakeSelfSignedCertificate(X509 * cert, EVP_PKEY* evp)
         ret = X509_sign(cert, evp, EVP_sha256());
     }
 
-    if (bn != NULL)
-    {
-        BN_free(bn);
-    }
-
     if (rsa != NULL)
     {
         RSA_free(rsa);

src/libraries/System.Security.Cryptography.Algorithms/src/System.Security.Cryptography.Algorithms.csproj

@@ -582,6 +582,8 @@
              Link="Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EcKey.cs" />
     <Compile Include="$(CommonPath)Interop\Unix\System.Security.Cryptography.Native\Interop.EcDsa.cs"
              Link="Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EcDsa.cs" />
+    <Compile Include="$(CommonPath)Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Rsa.cs"
+             Link="Common\Interop\Unix\System.Security.Cryptography.Native\Interop.EvpPkey.Rsa.cs" />
     <Compile Include="$(CommonPath)Microsoft\Win32\SafeHandles\SafeBignumHandle.Unix.cs"
              Link="Common\Microsoft\Win32\SafeHandles\SafeBignumHandle.Unix.cs" />
     <Compile Include="$(CommonPath)Microsoft\Win32\SafeHandles\SafeEcKeyHandle.Unix.cs"