kex: do not ignore failure of libssh2_md5_init()

The MD5 algorithm is disabled when running in FIPS mode.
mallexxx · Sep 17, 2012 · 43b730c · 43b730c
1 parent 6af85b6
commit 43b730c
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 6 deletions.
diff --git a/src/hostkey.c b/src/hostkey.c
@@ -429,7 +429,9 @@ libssh2_hostkey_hash(LIBSSH2_SESSION * session, int hash_type)
     switch (hash_type) {
 #if LIBSSH2_MD5
     case LIBSSH2_HOSTKEY_HASH_MD5:
-        return (char *) session->server_hostkey_md5;
+        return (session->server_hostkey_md5_valid)
+          ? (char *) session->server_hostkey_md5
+          : NULL;
         break;
 #endif /* LIBSSH2_MD5 */
     case LIBSSH2_HOSTKEY_HASH_SHA1:

diff --git a/src/kex.c b/src/kex.c
@@ -218,10 +218,15 @@ static int diffie_hellman_sha1(LIBSSH2_SESSION *session,
         {
             libssh2_md5_ctx fingerprint_ctx;
 
-            libssh2_md5_init(&fingerprint_ctx);
-            libssh2_md5_update(fingerprint_ctx, session->server_hostkey,
-                               session->server_hostkey_len);
-            libssh2_md5_final(fingerprint_ctx, session->server_hostkey_md5);
+            if (libssh2_md5_init(&fingerprint_ctx)) {
+                libssh2_md5_update(fingerprint_ctx, session->server_hostkey,
+                                   session->server_hostkey_len);
+                libssh2_md5_final(fingerprint_ctx, session->server_hostkey_md5);
+                session->server_hostkey_md5_valid = TRUE;
+            }
+            else {
+                session->server_hostkey_md5_valid = FALSE;
+            }
         }
 #ifdef LIBSSH2DEBUG
         {

diff --git a/src/libgcrypt.h b/src/libgcrypt.h
@@ -68,7 +68,11 @@
   gcry_md_hash_buffer (GCRY_MD_SHA1, out, message, len)
 
 #define libssh2_md5_ctx gcry_md_hd_t
-#define libssh2_md5_init(ctx) gcry_md_open (ctx,  GCRY_MD_MD5, 0);
+
+/* returns 0 in case of failure */
+#define libssh2_md5_init(ctx) \
+  (GPG_ERR_NO_ERROR == gcry_md_open (ctx,  GCRY_MD_MD5, 0))
+
 #define libssh2_md5_update(ctx, data, len) gcry_md_write (ctx, data, len)
 #define libssh2_md5_final(ctx, out) \
   memcpy (out, gcry_md_read (ctx, 0), MD5_DIGEST_LENGTH), gcry_md_close (ctx)

diff --git a/src/libssh2_priv.h b/src/libssh2_priv.h
@@ -597,6 +597,7 @@ struct _LIBSSH2_SESSION
     uint32_t server_hostkey_len;
 #if LIBSSH2_MD5
     unsigned char server_hostkey_md5[MD5_DIGEST_LENGTH];
+    int server_hostkey_md5_valid;
 #endif                          /* ! LIBSSH2_MD5 */
     unsigned char server_hostkey_sha1[SHA_DIGEST_LENGTH];
 

diff --git a/src/openssl.h b/src/openssl.h
@@ -113,7 +113,10 @@
 void libssh2_sha1(const unsigned char *message, unsigned long len, unsigned char *out);
 
 #define libssh2_md5_ctx EVP_MD_CTX
+
+/* returns 0 in case of failure */
 #define libssh2_md5_init(ctx) EVP_DigestInit(ctx, EVP_get_digestbyname("md5"))
+
 #define libssh2_md5_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_md5_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 void libssh2_md5(const unsigned char *message, unsigned long len, unsigned char *out);