Curl_ssl_md5sum: return CURLcode

... since the funciton can fail on OOM. Check this return code.

Coverity CID 1291705.

3 files changed, 16 insertions, 13 deletions

diff --git a/lib/curl_ntlm_msgs.c b/lib/curl_ntlm_msgs.c
index b5f3935d8..7f07decb8 100644
--- a/lib/curl_ntlm_msgs.c
+++ b/lib/curl_ntlm_msgs.c
@@ -591,11 +591,11 @@ CURLcode Curl_sasl_create_ntlm_type3_message(struct SessionHandle *data,
     memcpy(tmp, &ntlm->nonce[0], 8);
     memcpy(tmp + 8, entropy, 8);
 
-    Curl_ssl_md5sum(tmp, 16, md5sum, MD5_DIGEST_LENGTH);
-
-    /* We shall only use the first 8 bytes of md5sum, but the des
-       code in Curl_ntlm_core_lm_resp only encrypt the first 8 bytes */
-    result = Curl_ntlm_core_mk_nt_hash(data, passwdp, ntbuffer);
+    result = Curl_ssl_md5sum(tmp, 16, md5sum, MD5_DIGEST_LENGTH);
+    if(!result)
+      /* We shall only use the first 8 bytes of md5sum, but the des code in
+         Curl_ntlm_core_lm_resp only encrypt the first 8 bytes */
+      result = Curl_ntlm_core_mk_nt_hash(data, passwdp, ntbuffer);
     if(result)
       return result;
 
diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c
index 6973de42a..12427c17c 100644
--- a/lib/vtls/vtls.c
+++ b/lib/vtls/vtls.c
@@ -828,10 +828,10 @@ CURLcode Curl_pin_peer_pubkey(const char *pinnedpubkey,
 }
 
 #ifndef CURL_DISABLE_CRYPTO_AUTH
-void Curl_ssl_md5sum(unsigned char *tmp, /* input */
-                     size_t tmplen,
-                     unsigned char *md5sum, /* output */
-                     size_t md5len)
+CURLcode Curl_ssl_md5sum(unsigned char *tmp, /* input */
+                         size_t tmplen,
+                         unsigned char *md5sum, /* output */
+                         size_t md5len)
 {
 #ifdef curlssl_md5sum
   curlssl_md5sum(tmp, tmplen, md5sum, md5len);
@@ -841,9 +841,12 @@ void Curl_ssl_md5sum(unsigned char *tmp, /* input */
   (void) md5len;
 
   MD5pw = Curl_MD5_init(Curl_DIGEST_MD5);
+  if(!MD5pw)
+    return CURLE_OUT_OF_MEMORY;
   Curl_MD5_update(MD5pw, tmp, curlx_uztoui(tmplen));
   Curl_MD5_final(MD5pw, md5sum);
 #endif
+  return CURLE_OK;
 }
 #endif
 
diff --git a/lib/vtls/vtls.h b/lib/vtls/vtls.h
index 1a5f54fe4..b9741d7af 100644
--- a/lib/vtls/vtls.h
+++ b/lib/vtls/vtls.h
@@ -108,10 +108,10 @@ void Curl_ssl_delsessionid(struct connectdata *conn, void *ssl_sessionid);
    in */
 int Curl_ssl_random(struct SessionHandle *data, unsigned char *buffer,
                     size_t length);
-void Curl_ssl_md5sum(unsigned char *tmp, /* input */
-                     size_t tmplen,
-                     unsigned char *md5sum, /* output */
-                     size_t md5len);
+CURLcode Curl_ssl_md5sum(unsigned char *tmp, /* input */
+                         size_t tmplen,
+                         unsigned char *md5sum, /* output */
+                         size_t md5len);
 /* Check pinned public key. */
 CURLcode Curl_pin_peer_pubkey(const char *pinnedpubkey,
                               const unsigned char *pubkey, size_t pubkeylen);