[PATCH 1/4] kex: do not ignore failure of libssh2_md5_init()

The MD5 algorithm is disabled when running in FIPS mode.

---
 src/hostkey.c      |    4 +++-
 src/kex.c          |   13 +++++++++----
 src/libgcrypt.h    |    6 +++++-
 src/libssh2_priv.h |    1 +
 src/openssl.h      |    3 +++
 5 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/src/hostkey.c b/src/hostkey.c
index 53f7479..753563d 100644
--- a/src/hostkey.c
+++ b/src/hostkey.c
@@ -429,7 +429,9 @@ libssh2_hostkey_hash(LIBSSH2_SESSION * session, int hash_type)
     switch (hash_type) {
 #if LIBSSH2_MD5
     case LIBSSH2_HOSTKEY_HASH_MD5:
-        return (char *) session->server_hostkey_md5;
+        return (session->server_hostkey_md5_valid)
+          ? (char *) session->server_hostkey_md5
+          : NULL;
         break;
 #endif /* LIBSSH2_MD5 */
     case LIBSSH2_HOSTKEY_HASH_SHA1:
diff --git a/src/kex.c b/src/kex.c
index 0a72cb7..a3691ce 100644
--- a/src/kex.c
+++ b/src/kex.c
@@ -218,10 +218,15 @@ static int diffie_hellman_sha1(LIBSSH2_SESSION *session,
         {
             libssh2_md5_ctx fingerprint_ctx;
 
-            libssh2_md5_init(&fingerprint_ctx);
-            libssh2_md5_update(fingerprint_ctx, session->server_hostkey,
-                               session->server_hostkey_len);
-            libssh2_md5_final(fingerprint_ctx, session->server_hostkey_md5);
+            if (libssh2_md5_init(&fingerprint_ctx)) {
+              libssh2_md5_update(fingerprint_ctx, session->server_hostkey,
+                                 session->server_hostkey_len);
+              libssh2_md5_final(fingerprint_ctx, session->server_hostkey_md5);
+              session->server_hostkey_md5_valid = TRUE;
+            }
+            else {
+              session->server_hostkey_md5_valid = FALSE;
+            }
         }
 #ifdef LIBSSH2DEBUG
         {
diff --git a/src/libgcrypt.h b/src/libgcrypt.h
index 04516e5..1f0276e 100644
--- a/src/libgcrypt.h
+++ b/src/libgcrypt.h
@@ -68,7 +68,11 @@
   gcry_md_hash_buffer (GCRY_MD_SHA1, out, message, len)
 
 #define libssh2_md5_ctx gcry_md_hd_t
-#define libssh2_md5_init(ctx) gcry_md_open (ctx,  GCRY_MD_MD5, 0);
+
+/* returns 0 in case of failure */
+#define libssh2_md5_init(ctx) \
+  (GPG_ERR_NO_ERROR == gcry_md_open (ctx,  GCRY_MD_MD5, 0))
+
 #define libssh2_md5_update(ctx, data, len) gcry_md_write (ctx, data, len)
 #define libssh2_md5_final(ctx, out) \
   memcpy (out, gcry_md_read (ctx, 0), MD5_DIGEST_LENGTH), gcry_md_close (ctx)
diff --git a/src/libssh2_priv.h b/src/libssh2_priv.h
index 23fbc65..196864d 100644
--- a/src/libssh2_priv.h
+++ b/src/libssh2_priv.h
@@ -597,6 +597,7 @@ struct _LIBSSH2_SESSION
     uint32_t server_hostkey_len;
 #if LIBSSH2_MD5
     unsigned char server_hostkey_md5[MD5_DIGEST_LENGTH];
+    int server_hostkey_md5_valid;
 #endif                          /* ! LIBSSH2_MD5 */
     unsigned char server_hostkey_sha1[SHA_DIGEST_LENGTH];
 
diff --git a/src/openssl.h b/src/openssl.h
index 6d2aeed..4835ab6 100644
--- a/src/openssl.h
+++ b/src/openssl.h
@@ -113,7 +113,10 @@
 void libssh2_sha1(const unsigned char *message, unsigned long len, unsigned char *out);
 
 #define libssh2_md5_ctx EVP_MD_CTX
+
+/* returns 0 in case of failure */
 #define libssh2_md5_init(ctx) EVP_DigestInit(ctx, EVP_get_digestbyname("md5"))
+
 #define libssh2_md5_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_md5_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 void libssh2_md5(const unsigned char *message, unsigned long len, unsigned char *out);
-- 
1.7.1
_______________________________________________
libssh2-devel http://cool.haxx.se/cgi-bin/mailman/listinfo/libssh2-devel