RSA SHA2 256/512 key upgrade support RFC 8332 libssh2#536 (libssh2#626)

Notes:
* Host Key RSA 256/512 support libssh2#536
* Client side key hash upgrading for RFC 8332
* Support for server-sig-algs, ext-info-c server messages
* Customizing preferred server-sig-algs via the preference LIBSSH2_METHOD_SIGN_ALGO

Credit: Anders Borum, Will Cosgrove

docs/HACKING-CRYPTO

@@ -637,6 +637,32 @@ Note: this procedure is not used if macro _libssh2_rsa_sha1_signv() is defined.
 void _libssh2_rsa_free(libssh2_rsa_ctx *rsactx);
 Releases the RSA computation context at rsactx.
 
+LIBSSH2_RSA_SHA2
+#define as 1 if the crypto library supports RSA SHA2 256/512, else 0.
+If defined as 0, the rest of this section can be omitted.
+
+int _libssh2_rsa_sha2_sign(LIBSSH2_SESSION * session,
+                           libssh2_rsa_ctx * rsactx,
+                           const unsigned char *hash,
+                           size_t hash_len,
+                           unsigned char **signature,
+                           size_t *signature_len);
+RSA signs the (hash, hashlen) SHA-2 hash bytes based on hash length and stores
+the allocated signature at (signature, signature_len).
+Signature buffer must be allocated from the given session.
+Returns 0 if OK, else -1.
+This procedure is already prototyped in crypto.h.
+Note: this procedure is not used if macro _libssh2_rsa_sha1_signv() is defined.
+
+int _libssh2_rsa_sha2_verify(libssh2_rsa_ctx * rsa,
+                             size_t hash_len,
+                             const unsigned char *sig,
+                             unsigned long sig_len,
+                             const unsigned char *m, unsigned long m_len);
+Verify (sig, sig_len) signature of (m, m_len) using an SHA-2 hash based on
+hash length and the RSA context.
+Return 0 if OK, else -1.
+This procedure is already prototyped in crypto.h.
 
 7.2) DSA
 LIBSSH2_DSA
@@ -900,3 +926,14 @@ If this is not needed, it should be defined as an empty macro.
 int _libssh2_random(unsigned char *buf, int len);
 Store len random bytes at buf.
 Returns 0 if OK, else -1.
+
+const char * _libssh2_supported_key_sign_algorithms(LIBSSH2_SESSION *session,
+                                                    unsigned char *key_method,
+                                                    size_t key_method_len);
+
+This function is for implementing key hash upgrading as defined in RFC 8332.
+
+Based on the incoming key_method value, this function will return a
+list of supported algorithms that can upgrade the original key method algorithm
+as a comma seperated list, if there is no upgrade option this function should
+return NULL.

docs/libssh2_session_methods.3

@@ -1,4 +1,4 @@
-.TH libssh2_session_methods 3 "1 Jun 2007" "libssh2 0.15" "libssh2 manual"
+.TH libssh2_session_methods 3 "8 Nov 2021" "libssh2 1.11" "libssh2 manual"
 .SH NAME
 libssh2_session_methods - return the currently active algorithms
 .SH SYNOPSIS
@@ -8,13 +8,14 @@ const char *
 libssh2_session_methods(LIBSSH2_SESSION *session, int method_type);
 
 .SH DESCRIPTION
-\fIsession\fP - Session instance as returned by 
+\fIsession\fP - Session instance as returned by
 .BR libssh2_session_init_ex(3)
 
 \fImethod_type\fP - one of the method type constants: LIBSSH2_METHOD_KEX,
 LIBSSH2_METHOD_HOSTKEY, LIBSSH2_METHOD_CRYPT_CS, LIBSSH2_METHOD_CRYPT_SC,
 LIBSSH2_METHOD_MAC_CS, LIBSSH2_METHOD_MAC_SC, LIBSSH2_METHOD_COMP_CS,
-LIBSSH2_METHOD_COMP_SC, LIBSSH2_METHOD_LANG_CS, LIBSSH2_METHOD_LANG_SC.
+LIBSSH2_METHOD_COMP_SC, LIBSSH2_METHOD_LANG_CS, LIBSSH2_METHOD_LANG_SC,
+LIBSSH2_METHOD_SIGN_ALGO.
 
 Returns the actual method negotiated for a particular transport parameter.
 .SH RETURN VALUE

include/libssh2.h

@@ -356,6 +356,7 @@ typedef struct _LIBSSH2_USERAUTH_KBDINT_RESPONSE
 #define LIBSSH2_METHOD_COMP_SC      7
 #define LIBSSH2_METHOD_LANG_CS      8
 #define LIBSSH2_METHOD_LANG_SC      9
+#define LIBSSH2_METHOD_SIGN_ALGO    10
 
 /* flags */
 #define LIBSSH2_FLAG_SIGPIPE        1

src/crypto.h

@@ -93,6 +93,19 @@ int _libssh2_rsa_sha1_sign(LIBSSH2_SESSION * session,
                            size_t hash_len,
                            unsigned char **signature,
                            size_t *signature_len);
+#if LIBSSH2_RSA_SHA2
+int _libssh2_rsa_sha2_sign(LIBSSH2_SESSION * session,
+                           libssh2_rsa_ctx * rsactx,
+                           const unsigned char *hash,
+                           size_t hash_len,
+                           unsigned char **signature,
+                           size_t *signature_len);
+int _libssh2_rsa_sha2_verify(libssh2_rsa_ctx * rsa,
+                             size_t hash_len,
+                             const unsigned char *sig,
+                             unsigned long sig_len,
+                             const unsigned char *m, unsigned long m_len);
+#endif
 int _libssh2_rsa_new_private_frommemory(libssh2_rsa_ctx ** rsa,
                                         LIBSSH2_SESSION * session,
                                         const char *filedata,
@@ -245,4 +258,23 @@ int _libssh2_pub_priv_keyfilememory(LIBSSH2_SESSION *session,
                                     size_t privatekeydata_len,
                                     const char *passphrase);
 
+
+/**
+ * @function _libssh2_supported_key_sign_algorithms
+ * @abstract Returns supported algorithms used for upgrading public
+ * key signing RFC 8332
+ * @discussion Based on the incoming key_method value, this function
+ * will return supported algorithms that can upgrade the key method
+ * @related _libssh2_key_sign_algorithm()
+ * @param key_method current key method, usually the default key sig method
+ * @param key_method_len length of the key method buffer
+ * @result comma seperated list of supported upgrade options per RFC 8332, if
+ * there is no upgrade option return NULL
+ */
+
+const char *
+_libssh2_supported_key_sign_algorithms(LIBSSH2_SESSION *session,
+                                       unsigned char *key_method,
+                                       size_t key_method_len);
+
 #endif /* __LIBSSH2_CRYPTO_H */

src/hostkey.c

@@ -64,8 +64,8 @@ hostkey_method_ssh_rsa_init(LIBSSH2_SESSION * session,
                             void **abstract)
 {
     libssh2_rsa_ctx *rsactx;
-    unsigned char *e, *n;
-    size_t e_len, n_len;
+    unsigned char *e, *n, *type;
+    size_t e_len, n_len, type_len;
     struct string_buf buf;
 
     if(*abstract) {
@@ -83,8 +83,27 @@ hostkey_method_ssh_rsa_init(LIBSSH2_SESSION * session,
     buf.dataptr = buf.data;
     buf.len = hostkey_data_len;
 
-    if(_libssh2_match_string(&buf, "ssh-rsa"))
+    if(_libssh2_get_string(&buf, &type, &type_len)) {
         return -1;
+    }
+
+    /* we accept one of 3 header types */
+    if(type_len == 7 && strncmp("ssh-rsa", (char *)type, 7) == 0) {
+        /* ssh-rsa */
+    }
+#if LIBSSH2_RSA_SHA2
+    else if(type_len == 12 && strncmp("rsa-sha2-256", (char *)type, 12) == 0) {
+        /* rsa-sha2-256 */
+    }
+    else if(type_len == 12 && strncmp("rsa-sha2-512", (char *)type, 12) == 0) {
+        /* rsa-sha2-512 */
+    }
+#endif
+    else {
+        _libssh2_debug(session, LIBSSH2_TRACE_ERROR,
+                       "unexpected rsa type: %.*s", type_len, type);
+        return -1;
+    }
 
     if(_libssh2_get_string(&buf, &e, &e_len))
         return -1;
@@ -227,6 +246,146 @@ hostkey_method_ssh_rsa_signv(LIBSSH2_SESSION * session,
 #endif
 }
 
+/*
+ * hostkey_method_ssh_rsa_sha2_256_sig_verify
+ *
+ * Verify signature created by remote
+ */
+#if LIBSSH2_RSA_SHA2
+
+static int
+hostkey_method_ssh_rsa_sha2_256_sig_verify(LIBSSH2_SESSION * session,
+                                   const unsigned char *sig,
+                                   size_t sig_len,
+                                   const unsigned char *m,
+                                   size_t m_len, void **abstract)
+{
+    libssh2_rsa_ctx *rsactx = (libssh2_rsa_ctx *) (*abstract);
+    (void) session;
+
+    /* Skip past keyname_len(4) + keyname(12){"rsa-sha2-256"} +
+    signature_len(4) */
+    if(sig_len < 20)
+        return -1;
+
+    sig += 20;
+    sig_len -= 20;
+    return _libssh2_rsa_sha2_verify(rsactx, SHA256_DIGEST_LENGTH, sig, sig_len,
+                                    m, m_len);
+}
+
+/*
+ * hostkey_method_ssh_rsa_sha2_256_signv
+ *
+ * Construct a signature from an array of vectors
+ */
+
+static int
+hostkey_method_ssh_rsa_sha2_256_signv(LIBSSH2_SESSION * session,
+                             unsigned char **signature,
+                             size_t *signature_len,
+                             int veccount,
+                             const struct iovec datavec[],
+                             void **abstract)
+{
+    libssh2_rsa_ctx *rsactx = (libssh2_rsa_ctx *) (*abstract);
+
+#ifdef _libssh2_rsa_sha2_256_signv
+    return _libssh2_rsa_sha2_256_signv(session, signature, signature_len,
+                                       veccount, datavec, rsactx);
+#else
+    int ret;
+    int i;
+    unsigned char hash[SHA256_DIGEST_LENGTH];
+    libssh2_sha256_ctx ctx;
+
+    libssh2_sha256_init(&ctx);
+    for(i = 0; i < veccount; i++) {
+        libssh2_sha256_update(ctx, datavec[i].iov_base, datavec[i].iov_len);
+    }
+    libssh2_sha256_final(ctx, hash);
+
+    ret = _libssh2_rsa_sha2_sign(session, rsactx, hash, SHA256_DIGEST_LENGTH,
+                                 signature, signature_len);
+    if(ret) {
+        return -1;
+    }
+
+    return 0;
+#endif
+}
+
+/*
+ * hostkey_method_ssh_rsa_sha2_512_sig_verify
+ *
+ * Verify signature created by remote
+ */
+
+static int
+hostkey_method_ssh_rsa_sha2_512_sig_verify(LIBSSH2_SESSION * session,
+                                   const unsigned char *sig,
+                                   size_t sig_len,
+                                   const unsigned char *m,
+                                   size_t m_len, void **abstract)
+{
+    libssh2_rsa_ctx *rsactx = (libssh2_rsa_ctx *) (*abstract);
+    (void) session;
+
+    /* Skip past keyname_len(4) + keyname(12){"rsa-sha2-512"} +
+    signature_len(4) */
+    if(sig_len < 20)
+        return -1;
+
+    sig += 20;
+    sig_len -= 20;
+    return _libssh2_rsa_sha2_verify(rsactx, SHA512_DIGEST_LENGTH, sig,
+                                    sig_len, m, m_len);
+}
+
+
+/*
+ * hostkey_method_ssh_rsa_sha2_512_signv
+ *
+ * Construct a signature from an array of vectors
+ */
+static int
+hostkey_method_ssh_rsa_sha2_512_signv(LIBSSH2_SESSION * session,
+                             unsigned char **signature,
+                             size_t *signature_len,
+                             int veccount,
+                             const struct iovec datavec[],
+                             void **abstract)
+{
+    libssh2_rsa_ctx *rsactx = (libssh2_rsa_ctx *) (*abstract);
+
+#ifdef _libssh2_rsa_sha2_512_signv
+    return _libssh2_rsa_sha2_512_signv(session, signature, signature_len,
+                                       veccount, datavec, rsactx);
+#else
+    int ret;
+    int i;
+    unsigned char hash[SHA512_DIGEST_LENGTH];
+    libssh2_sha512_ctx ctx;
+
+    libssh2_sha512_init(&ctx);
+    for(i = 0; i < veccount; i++) {
+        libssh2_sha512_update(ctx, datavec[i].iov_base, datavec[i].iov_len);
+    }
+    libssh2_sha512_final(ctx, hash);
+
+    ret = _libssh2_rsa_sha2_sign(session, rsactx, hash, SHA512_DIGEST_LENGTH,
+                                 signature, signature_len);
+    if(ret) {
+        return -1;
+    }
+
+    return 0;
+#endif
+}
+
+#endif /* LIBSSH2_RSA_SHA2 */
+
+
 /*
  * hostkey_method_ssh_rsa_dtor
  *
@@ -260,6 +419,35 @@ static const LIBSSH2_HOSTKEY_METHOD hostkey_method_ssh_rsa = {
     NULL,                       /* encrypt */
     hostkey_method_ssh_rsa_dtor,
 };
+
+#if LIBSSH2_RSA_SHA2
+
+static const LIBSSH2_HOSTKEY_METHOD hostkey_method_ssh_rsa_sha2_256 = {
+    "rsa-sha2-256",
+    SHA256_DIGEST_LENGTH,
+    hostkey_method_ssh_rsa_init,
+    hostkey_method_ssh_rsa_initPEM,
+    hostkey_method_ssh_rsa_initPEMFromMemory,
+    hostkey_method_ssh_rsa_sha2_256_sig_verify,
+    hostkey_method_ssh_rsa_sha2_256_signv,
+    NULL,                       /* encrypt */
+    hostkey_method_ssh_rsa_dtor,
+};
+
+static const LIBSSH2_HOSTKEY_METHOD hostkey_method_ssh_rsa_sha2_512 = {
+    "rsa-sha2-512",
+    SHA512_DIGEST_LENGTH,
+    hostkey_method_ssh_rsa_init,
+    hostkey_method_ssh_rsa_initPEM,
+    hostkey_method_ssh_rsa_initPEMFromMemory,
+    hostkey_method_ssh_rsa_sha2_512_sig_verify,
+    hostkey_method_ssh_rsa_sha2_512_signv,
+    NULL,                       /* encrypt */
+    hostkey_method_ssh_rsa_dtor,
+};
+
+#endif /* LIBSSH2_RSA_SHA2 */
+
 #endif /* LIBSSH2_RSA */
 
 #if LIBSSH2_DSA
@@ -1041,6 +1229,10 @@ static const LIBSSH2_HOSTKEY_METHOD *hostkey_methods[] = {
     &hostkey_method_ssh_ed25519,
 #endif
 #if LIBSSH2_RSA
+#if LIBSSH2_RSA_SHA2
+    &hostkey_method_ssh_rsa_sha2_512,
+    &hostkey_method_ssh_rsa_sha2_256,
+#endif /* LIBSSH2_RSA_SHA2 */
     &hostkey_method_ssh_rsa,
 #endif /* LIBSSH2_RSA */
 #if LIBSSH2_DSA