Support compiling against OpenSSL 3 headers

Building against OpenSSL 3's headers fails to compile, as X509_V_ERR_INVALID_CA has changed from 24 to 79, tripping a static assert.

* Rename the managed X509VerifyStatusCode enum to X509VerifyStatusCodeUniversal, to represent the name/values that are present in all current versions of OpenSSL (1.0.2, 1.1.1, 3.0 alpha)
* Add new enums for the name/value pairs that are unique to a given version
* Add an X509VerifyStatusCode struct that just wraps the int and is a faux-union of the various enums
* Use the OpenSSL runtime version to determine which mapping table to use (after the Universal table fails)

In addition to that, there are a few const-related changes in the 3.0 headers that are addressed.

`runtime/src/libraries/Native$ ./build_native.sh -portablebuild=false` on systems where find_package(OpenSSL) maps to 3.0 succeeds with these changes.  Portable builds still fail.

Not all tests pass with OpenSSL 3.0 (alpha 13) with these changes, but it does reduce to three categories of error:

* ICryptoTransform reset/reuse tests fail (OpenSSL regression is open)
* DSA small key generation fails (OpenSSL has fixed the regression for the next alpha/beta release)
* Some OuterLoop X.509 tests are failing as positively revoked when they expect ambiguous revocation states (investigation pending)

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OCSP.cs

@@ -44,7 +44,7 @@ internal static X509VerifyStatusCode X509ChainGetCachedOcspStatus(SafeX509StoreC
         {
             X509VerifyStatusCode response = CryptoNative_X509ChainGetCachedOcspStatus(ctx, cachePath, chainDepth);
 
-            if (response < 0)
+            if (response.Code < 0)
             {
                 Debug.Fail($"Unexpected response from X509ChainGetCachedOcspSuccess: {response}");
                 throw new CryptographicException();
@@ -70,7 +70,7 @@ internal static X509VerifyStatusCode X509ChainVerifyOcsp(
         {
             X509VerifyStatusCode response = CryptoNative_X509ChainVerifyOcsp(ctx, req, resp, cachePath, chainDepth);
 
-            if (response < 0)
+            if (response.Code < 0)
             {
                 Debug.Fail($"Unexpected response from X509ChainGetCachedOcspSuccess: {response}");
                 throw new CryptographicException();

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.X509.cs

@@ -230,13 +230,13 @@ internal static bool X509StoreCtxRebuildChain(SafeX509StoreCtxHandle ctx)
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_X509StoreCtxSetVerifyCallback")]
         internal static extern void X509StoreCtxSetVerifyCallback(SafeX509StoreCtxHandle ctx, X509StoreVerifyCallback callback);
 
-        internal static string GetX509VerifyCertErrorString(X509VerifyStatusCode n)
+        internal static string GetX509VerifyCertErrorString(int n)
         {
             return Marshal.PtrToStringAnsi(X509VerifyCertErrorString(n))!;
         }
 
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_X509VerifyCertErrorString")]
-        private static extern IntPtr X509VerifyCertErrorString(X509VerifyStatusCode n);
+        private static extern IntPtr X509VerifyCertErrorString(int n);
 
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_X509CrlDestroy")]
         internal static extern void X509CrlDestroy(IntPtr a);
@@ -253,11 +253,13 @@ internal static string GetX509VerifyCertErrorString(X509VerifyStatusCode n)
         [DllImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_EncodeX509SubjectPublicKeyInfo")]
         internal static extern int EncodeX509SubjectPublicKeyInfo(SafeX509Handle x509, byte[] buf);
 
-        internal enum X509VerifyStatusCode : int
+        internal enum X509VerifyStatusCodeUniversal
         {
             X509_V_OK = 0,
+            X509_V_ERR_UNSPECIFIED = 1,
             X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT = 2,
             X509_V_ERR_UNABLE_TO_GET_CRL = 3,
+            X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE = 4,
             X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE = 5,
             X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY = 6,
             X509_V_ERR_CERT_SIGNATURE_FAILURE = 7,
@@ -277,18 +279,25 @@ internal enum X509VerifyStatusCode : int
             X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE = 21,
             X509_V_ERR_CERT_CHAIN_TOO_LONG = 22,
             X509_V_ERR_CERT_REVOKED = 23,
-            X509_V_ERR_INVALID_CA = 24,
+
+            // Code 24 varies.
+
             X509_V_ERR_PATH_LENGTH_EXCEEDED = 25,
             X509_V_ERR_INVALID_PURPOSE = 26,
             X509_V_ERR_CERT_UNTRUSTED = 27,
             X509_V_ERR_CERT_REJECTED = 28,
+            X509_V_ERR_SUBJECT_ISSUER_MISMATCH = 29,
+            X509_V_ERR_AKID_SKID_MISMATCH = 30,
+            X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH = 31,
             X509_V_ERR_KEYUSAGE_NO_CERTSIGN = 32,
             X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER = 33,
             X509_V_ERR_UNHANDLED_CRITICAL_EXTENSION = 34,
             X509_V_ERR_KEYUSAGE_NO_CRL_SIGN = 35,
             X509_V_ERR_UNHANDLED_CRITICAL_CRL_EXTENSION = 36,
             X509_V_ERR_INVALID_NON_CA = 37,
+            X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED = 38,
             X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE = 39,
+            X509_V_ERR_PROXY_CERTIFICATES_NOT_ALLOWED = 40,
             X509_V_ERR_INVALID_EXTENSION = 41,
             X509_V_ERR_INVALID_POLICY_EXTENSION = 42,
             X509_V_ERR_NO_EXPLICIT_POLICY = 43,
@@ -303,7 +312,6 @@ internal enum X509VerifyStatusCode : int
             X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX = 52,
             X509_V_ERR_UNSUPPORTED_NAME_SYNTAX = 53,
             X509_V_ERR_CRL_PATH_VALIDATION_ERROR = 54,
-            X509_V_ERR_PATH_LOOP = 55,
             X509_V_ERR_SUITE_B_INVALID_VERSION = 56,
             X509_V_ERR_SUITE_B_INVALID_ALGORITHM = 57,
             X509_V_ERR_SUITE_B_INVALID_CURVE = 58,
@@ -313,6 +321,41 @@ internal enum X509VerifyStatusCode : int
             X509_V_ERR_HOSTNAME_MISMATCH = 62,
             X509_V_ERR_EMAIL_MISMATCH = 63,
             X509_V_ERR_IP_ADDRESS_MISMATCH = 64,
+        }
+        internal enum X509VerifyStatusCode102
+        {
+            X509_V_ERR_INVALID_CA = 24,
+
+            X509_V_ERR_INVALID_CALL = 65,
+            X509_V_ERR_STORE_LOOKUP = 66,
+            X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION = 67,
+        }
+
+        internal enum X509VerifyStatusCode111
+        {
+            X509_V_ERR_INVALID_CA = 24,
+
+            X509_V_ERR_DANE_NO_MATCH = 65,
+            X509_V_ERR_EE_KEY_TOO_SMALL = 66,
+            X509_V_ERR_CA_KEY_TOO_SMALL = 67,
+            X509_V_ERR_CA_MD_TOO_WEAK = 68,
+            X509_V_ERR_INVALID_CALL = 69,
+            X509_V_ERR_STORE_LOOKUP = 70,
+            X509_V_ERR_NO_VALID_SCTS = 71,
+            X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION = 72,
+            X509_V_ERR_OCSP_VERIFY_NEEDED = 73,
+            X509_V_ERR_OCSP_VERIFY_FAILED = 74,
+            X509_V_ERR_OCSP_CERT_UNKNOWN = 75,
+            X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH = 76,
+            X509_V_ERR_NO_ISSUER_PUBLIC_KEY = 77,
+            X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM = 78,
+            X509_V_ERR_EC_KEY_EXPLICIT_PARAMS = 79,
+        }
+
+        internal enum X509VerifyStatusCode30
+        {
+            X509_V_ERR_NO_ISSUER_PUBLIC_KEY = 24,
+
             X509_V_ERR_DANE_NO_MATCH = 65,
             X509_V_ERR_EE_KEY_TOO_SMALL = 66,
             X509_V_ERR_CA_KEY_TOO_SMALL = 67,
@@ -324,6 +367,62 @@ internal enum X509VerifyStatusCode : int
             X509_V_ERR_OCSP_VERIFY_NEEDED = 73,
             X509_V_ERR_OCSP_VERIFY_FAILED = 74,
             X509_V_ERR_OCSP_CERT_UNKNOWN = 75,
+            X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM = 76,
+            X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH = 77,
+            X509_V_ERR_SIGNATURE_ALGORITHM_INCONSISTENCY = 78,
+            X509_V_ERR_INVALID_CA = 79,
+            X509_V_ERR_PATHLEN_INVALID_FOR_NON_CA = 80,
+            X509_V_ERR_PATHLEN_WITHOUT_KU_KEY_CERT_SIGN = 81,
+            X509_V_ERR_KU_KEY_CERT_SIGN_INVALID_FOR_NON_CA = 82,
+            X509_V_ERR_ISSUER_NAME_EMPTY = 83,
+            X509_V_ERR_SUBJECT_NAME_EMPTY = 84,
+            X509_V_ERR_MISSING_AUTHORITY_KEY_IDENTIFIER = 85,
+            X509_V_ERR_MISSING_SUBJECT_KEY_IDENTIFIER = 86,
+            X509_V_ERR_EMPTY_SUBJECT_ALT_NAME = 87,
+            X509_V_ERR_EMPTY_SUBJECT_SAN_NOT_CRITICAL = 88,
+            X509_V_ERR_CA_BCONS_NOT_CRITICAL = 89,
+            X509_V_ERR_AUTHORITY_KEY_IDENTIFIER_CRITICAL = 90,
+            X509_V_ERR_SUBJECT_KEY_IDENTIFIER_CRITICAL = 91,
+            X509_V_ERR_CA_CERT_MISSING_KEY_USAGE = 92,
+            X509_V_ERR_EXTENSIONS_REQUIRE_VERSION_3 = 93,
+            X509_V_ERR_EC_KEY_EXPLICIT_PARAMS = 94,
+        }
+
+        internal readonly struct X509VerifyStatusCode : IEquatable<X509VerifyStatusCode>
+        {
+            internal static readonly X509VerifyStatusCode X509_V_OK = X509VerifyStatusCodeUniversal.X509_V_OK;
+
+            public int Code { get; }
+
+            internal X509VerifyStatusCode(int code)
+            {
+                Code = code;
+            }
+
+            public X509VerifyStatusCodeUniversal UniversalCode => (X509VerifyStatusCodeUniversal)Code;
+            public X509VerifyStatusCode102 Code102 => (X509VerifyStatusCode102)Code;
+            public X509VerifyStatusCode111 Code111 => (X509VerifyStatusCode111)Code;
+            public X509VerifyStatusCode30 Code30 => (X509VerifyStatusCode30)Code;
+
+            public bool Equals(X509VerifyStatusCode other) => Code == other.Code;
+
+            public override bool Equals(object? obj) => obj is X509VerifyStatusCode other && Equals(other);
+
+            public override int GetHashCode() => Code.GetHashCode();
+
+            public static bool operator ==(X509VerifyStatusCode left, X509VerifyStatusCode right) => left.Equals(right);
+
+            public static bool operator !=(X509VerifyStatusCode left, X509VerifyStatusCode right) => !left.Equals(right);
+
+            public static explicit operator X509VerifyStatusCode(int code)
+            {
+                return new X509VerifyStatusCode(code);
+            }
+
+            public static implicit operator X509VerifyStatusCode(X509VerifyStatusCodeUniversal code)
+            {
+                return new X509VerifyStatusCode((int)code);
+            }
         }
     }
 }

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -5,7 +5,7 @@
 #include "pal_utilities.h"
 #include <assert.h>
 
-static int HasNoPrivateKey(RSA* rsa);
+static int HasNoPrivateKey(const RSA* rsa);
 
 EVP_PKEY* CryptoNative_RsaGenerateKey(int keySize)
 {
@@ -85,7 +85,7 @@ int32_t CryptoNative_RsaDecrypt(EVP_PKEY* pkey,
 
     // This check may no longer be needed on OpenSSL 3.0
     {
-        RSA* rsa = EVP_PKEY_get0_RSA(pkey);
+        const RSA* rsa = EVP_PKEY_get0_RSA(pkey);
 
         if (rsa == NULL || HasNoPrivateKey(rsa))
         {
@@ -160,7 +160,7 @@ int32_t CryptoNative_RsaSignHash(EVP_PKEY* pkey,
 
     // This check may no longer be needed on OpenSSL 3.0
     {
-        RSA* rsa = EVP_PKEY_get0_RSA(pkey);
+        const RSA* rsa = EVP_PKEY_get0_RSA(pkey);
 
         if (rsa == NULL || HasNoPrivateKey(rsa))
         {
@@ -195,7 +195,7 @@ int32_t CryptoNative_EvpPkeySetRsa(EVP_PKEY* pkey, RSA* rsa)
     return EVP_PKEY_set1_RSA(pkey, rsa);
 }
 
-static int HasNoPrivateKey(RSA* rsa)
+static int HasNoPrivateKey(const RSA* rsa)
 {
     if (rsa == NULL)
         return 1;

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.c

@@ -32,7 +32,6 @@ c_static_assert(PAL_X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY == X509_V_ERR_U
 c_static_assert(PAL_X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE == X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE);
 c_static_assert(PAL_X509_V_ERR_CERT_CHAIN_TOO_LONG == X509_V_ERR_CERT_CHAIN_TOO_LONG);
 c_static_assert(PAL_X509_V_ERR_CERT_REVOKED == X509_V_ERR_CERT_REVOKED);
-c_static_assert(PAL_X509_V_ERR_INVALID_CA == X509_V_ERR_INVALID_CA);
 c_static_assert(PAL_X509_V_ERR_PATH_LENGTH_EXCEEDED == X509_V_ERR_PATH_LENGTH_EXCEEDED);
 c_static_assert(PAL_X509_V_ERR_INVALID_PURPOSE == X509_V_ERR_INVALID_PURPOSE);
 c_static_assert(PAL_X509_V_ERR_CERT_UNTRUSTED == X509_V_ERR_CERT_UNTRUSTED);
@@ -47,6 +46,26 @@ c_static_assert(PAL_X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE == X509_V_ERR_KEYUS
 c_static_assert(PAL_X509_V_ERR_INVALID_EXTENSION == X509_V_ERR_INVALID_EXTENSION);
 c_static_assert(PAL_X509_V_ERR_INVALID_POLICY_EXTENSION == X509_V_ERR_INVALID_POLICY_EXTENSION);
 c_static_assert(PAL_X509_V_ERR_NO_EXPLICIT_POLICY == X509_V_ERR_NO_EXPLICIT_POLICY);
+c_static_assert(PAL_X509_V_ERR_DIFFERENT_CRL_SCOPE == X509_V_ERR_DIFFERENT_CRL_SCOPE);
+c_static_assert(PAL_X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE == X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE);
+c_static_assert(PAL_X509_V_ERR_UNNESTED_RESOURCE == X509_V_ERR_UNNESTED_RESOURCE);
+c_static_assert(PAL_X509_V_ERR_PERMITTED_VIOLATION == X509_V_ERR_PERMITTED_VIOLATION);
+c_static_assert(PAL_X509_V_ERR_EXCLUDED_VIOLATION == X509_V_ERR_EXCLUDED_VIOLATION);
+c_static_assert(PAL_X509_V_ERR_SUBTREE_MINMAX == X509_V_ERR_SUBTREE_MINMAX);
+c_static_assert(PAL_X509_V_ERR_APPLICATION_VERIFICATION == X509_V_ERR_APPLICATION_VERIFICATION);
+c_static_assert(PAL_X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE == X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE);
+c_static_assert(PAL_X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX == X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX);
+c_static_assert(PAL_X509_V_ERR_UNSUPPORTED_NAME_SYNTAX == X509_V_ERR_UNSUPPORTED_NAME_SYNTAX);
+c_static_assert(PAL_X509_V_ERR_CRL_PATH_VALIDATION_ERROR == X509_V_ERR_CRL_PATH_VALIDATION_ERROR);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_INVALID_VERSION == X509_V_ERR_SUITE_B_INVALID_VERSION);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_INVALID_ALGORITHM == X509_V_ERR_SUITE_B_INVALID_ALGORITHM);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_INVALID_CURVE == X509_V_ERR_SUITE_B_INVALID_CURVE);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM == X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED == X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED);
+c_static_assert(PAL_X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 == X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256);
+c_static_assert(PAL_X509_V_ERR_HOSTNAME_MISMATCH == X509_V_ERR_HOSTNAME_MISMATCH);
+c_static_assert(PAL_X509_V_ERR_EMAIL_MISMATCH == X509_V_ERR_EMAIL_MISMATCH);
+c_static_assert(PAL_X509_V_ERR_IP_ADDRESS_MISMATCH == X509_V_ERR_IP_ADDRESS_MISMATCH);
 
 EVP_PKEY* CryptoNative_GetX509EvpPublicKey(X509* x509)
 {
@@ -1115,7 +1134,10 @@ CryptoNative_X509ChainVerifyOcsp(X509_STORE_CTX* storeCtx, OCSP_REQUEST* req, OC
 
                     if (bio != NULL)
                     {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcast-qual"
                         if (i2d_OCSP_RESPONSE_bio(bio, resp))
+#pragma clang diagnostic pop
                         {
                             clearErr = 0;
                         }

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_x509.h

@@ -17,7 +17,10 @@ typedef enum {
 /*
 The error codes used when verifying X509 certificate chains.
 
-These values should be kept in sync with Interop.Crypto.X509VerifyStatusCode.
+These values should be kept in sync with Interop.Crypto.X509VerifyStatusCodeUniversal.
+
+Codes specific to specific versions of OpenSSL can also be returned,
+but are not represented in this enum due to their non-constant nature.
 */
 typedef enum {
     PAL_X509_V_OK = 0,
@@ -42,7 +45,9 @@ typedef enum {
     PAL_X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE = 21,
     PAL_X509_V_ERR_CERT_CHAIN_TOO_LONG = 22,
     PAL_X509_V_ERR_CERT_REVOKED = 23,
-    PAL_X509_V_ERR_INVALID_CA = 24,
+
+    // Code 24 varies
+
     PAL_X509_V_ERR_PATH_LENGTH_EXCEEDED = 25,
     PAL_X509_V_ERR_INVALID_PURPOSE = 26,
     PAL_X509_V_ERR_CERT_UNTRUSTED = 27,
@@ -57,6 +62,26 @@ typedef enum {
     PAL_X509_V_ERR_INVALID_EXTENSION = 41,
     PAL_X509_V_ERR_INVALID_POLICY_EXTENSION = 42,
     PAL_X509_V_ERR_NO_EXPLICIT_POLICY = 43,
+    PAL_X509_V_ERR_DIFFERENT_CRL_SCOPE = 44,
+    PAL_X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE = 45,
+    PAL_X509_V_ERR_UNNESTED_RESOURCE = 46,
+    PAL_X509_V_ERR_PERMITTED_VIOLATION = 47,
+    PAL_X509_V_ERR_EXCLUDED_VIOLATION = 48,
+    PAL_X509_V_ERR_SUBTREE_MINMAX = 49,
+    PAL_X509_V_ERR_APPLICATION_VERIFICATION = 50,
+    PAL_X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE = 51,
+    PAL_X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX = 52,
+    PAL_X509_V_ERR_UNSUPPORTED_NAME_SYNTAX = 53,
+    PAL_X509_V_ERR_CRL_PATH_VALIDATION_ERROR = 54,
+    PAL_X509_V_ERR_SUITE_B_INVALID_VERSION = 56,
+    PAL_X509_V_ERR_SUITE_B_INVALID_ALGORITHM = 57,
+    PAL_X509_V_ERR_SUITE_B_INVALID_CURVE = 58,
+    PAL_X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM = 59,
+    PAL_X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED = 60,
+    PAL_X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 = 61,
+    PAL_X509_V_ERR_HOSTNAME_MISMATCH = 62,
+    PAL_X509_V_ERR_EMAIL_MISMATCH = 63,
+    PAL_X509_V_ERR_IP_ADDRESS_MISMATCH = 64,
 } X509VerifyStatusCode;
 
 typedef int32_t (*X509StoreVerifyCallback)(int32_t, X509_STORE_CTX*);