sha256/md5: return errors when init fails

Closes #8133

9 files changed, 73 insertions, 40 deletions

diff --git a/lib/curl_hmac.h b/lib/curl_hmac.h
index 84c73121b..5755655d0 100644
--- a/lib/curl_hmac.h
+++ b/lib/curl_hmac.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -26,7 +26,7 @@
 
 #define HMAC_MD5_LENGTH 16
 
-typedef void    (* HMAC_hinit_func)(void *context);
+typedef CURLcode (* HMAC_hinit_func)(void *context);
 typedef void    (* HMAC_hupdate_func)(void *context,
                                       const unsigned char *data,
                                       unsigned int len);
diff --git a/lib/curl_md5.h b/lib/curl_md5.h
index 5739c89ca..b7d7c1f5d 100644
--- a/lib/curl_md5.h
+++ b/lib/curl_md5.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -27,7 +27,7 @@
 
 #define MD5_DIGEST_LEN  16
 
-typedef void (* Curl_MD5_init_func)(void *context);
+typedef CURLcode (* Curl_MD5_init_func)(void *context);
 typedef void (* Curl_MD5_update_func)(void *context,
                                       const unsigned char *data,
                                       unsigned int len);
@@ -49,8 +49,8 @@ struct MD5_context {
 extern const struct MD5_params Curl_DIGEST_MD5[1];
 extern const struct HMAC_params Curl_HMAC_MD5[1];
 
-void Curl_md5it(unsigned char *output, const unsigned char *input,
-                const size_t len);
+CURLcode Curl_md5it(unsigned char *output, const unsigned char *input,
+                    const size_t len);
 
 struct MD5_context *Curl_MD5_init(const struct MD5_params *md5params);
 CURLcode Curl_MD5_update(struct MD5_context *context,
diff --git a/lib/curl_sha256.h b/lib/curl_sha256.h
index b14c475ef..55dc30ad7 100644
--- a/lib/curl_sha256.h
+++ b/lib/curl_sha256.h
@@ -37,8 +37,8 @@ extern const struct HMAC_params Curl_HMAC_SHA256[1];
 #define SHA256_DIGEST_LENGTH 32
 #endif
 
-void Curl_sha256it(unsigned char *outbuffer, const unsigned char *input,
-                   const size_t len);
+CURLcode Curl_sha256it(unsigned char *outbuffer, const unsigned char *input,
+                       const size_t len);
 
 #endif
 
diff --git a/lib/http_aws_sigv4.c b/lib/http_aws_sigv4.c
index cbbecb712..751e5af5f 100644
--- a/lib/http_aws_sigv4.c
+++ b/lib/http_aws_sigv4.c
@@ -286,8 +286,11 @@ CURLcode Curl_output_aws_sigv4(struct Curl_easy *data, bool proxy)
     post_data_len = strlen(post_data);
   else
     post_data_len = (size_t)data->set.postfieldsize;
-  Curl_sha256it(sha_hash,
-                (const unsigned char *) post_data, post_data_len);
+  if(Curl_sha256it(sha_hash, (const unsigned char *) post_data,
+                   post_data_len)) {
+    goto fail;
+  }
+
   sha256_to_hex(sha_hex, sha_hash, sizeof(sha_hex));
 
   Curl_http_method(data, conn, &method, &httpreq);
@@ -320,8 +323,11 @@ CURLcode Curl_output_aws_sigv4(struct Curl_easy *data, bool proxy)
     goto fail;
   }
 
-  Curl_sha256it(sha_hash, (unsigned char *) canonical_request,
-                strlen(canonical_request));
+  if(Curl_sha256it(sha_hash, (unsigned char *) canonical_request,
+                   strlen(canonical_request))) {
+    goto fail;
+  }
+
   sha256_to_hex(sha_hex, sha_hash, sizeof(sha_hex));
 
   /*
diff --git a/lib/md5.c b/lib/md5.c
index 810c5fba8..c6923e036 100644
--- a/lib/md5.c
+++ b/lib/md5.c
@@ -62,9 +62,10 @@
 
 typedef struct md5_ctx MD5_CTX;
 
-static void MD5_Init(MD5_CTX *ctx)
+static CURLcode MD5_Init(MD5_CTX *ctx)
 {
   md5_init(ctx);
+  return CURLE_OK;
 }
 
 static void MD5_Update(MD5_CTX *ctx,
@@ -98,13 +99,14 @@ static void MD5_Final(unsigned char *digest, MD5_CTX *ctx)
 
 typedef mbedtls_md5_context MD5_CTX;
 
-static void MD5_Init(MD5_CTX *ctx)
+static CURLcode MD5_Init(MD5_CTX *ctx)
 {
 #if !defined(HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS)
   (void) mbedtls_md5_starts(ctx);
 #else
   (void) mbedtls_md5_starts_ret(ctx);
 #endif
+  return CURLE_OK;
 }
 
 static void MD5_Update(MD5_CTX *ctx,
@@ -146,9 +148,10 @@ static void MD5_Final(unsigned char *digest, MD5_CTX *ctx)
 /* The last #include file should be: */
 #include "memdebug.h"
 
-static void MD5_Init(MD5_CTX *ctx)
+static CURLcode MD5_Init(MD5_CTX *ctx)
 {
   CC_MD5_Init(ctx);
+  return CURLE_OK;
 }
 
 static void MD5_Update(MD5_CTX *ctx,
@@ -176,12 +179,13 @@ struct md5_ctx {
 };
 typedef struct md5_ctx MD5_CTX;
 
-static void MD5_Init(MD5_CTX *ctx)
+static CURLcode MD5_Init(MD5_CTX *ctx)
 {
   if(CryptAcquireContext(&ctx->hCryptProv, NULL, NULL, PROV_RSA_FULL,
                          CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
     CryptCreateHash(ctx->hCryptProv, CALG_MD5, 0, 0, &ctx->hHash);
   }
+  return CURLE_OK;
 }
 
 static void MD5_Update(MD5_CTX *ctx,
@@ -261,7 +265,7 @@ struct md5_ctx {
 };
 typedef struct md5_ctx MD5_CTX;
 
-static void MD5_Init(MD5_CTX *ctx);
+static CURLcode MD5_Init(MD5_CTX *ctx);
 static void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size);
 static void MD5_Final(unsigned char *result, MD5_CTX *ctx);
 
@@ -422,7 +426,7 @@ static const void *body(MD5_CTX *ctx, const void *data, unsigned long size)
   return ptr;
 }
 
-static void MD5_Init(MD5_CTX *ctx)
+static CURLcode MD5_Init(MD5_CTX *ctx)
 {
   ctx->a = 0x67452301;
   ctx->b = 0xefcdab89;
@@ -431,6 +435,8 @@ static void MD5_Init(MD5_CTX *ctx)
 
   ctx->lo = 0;
   ctx->hi = 0;
+
+  return CURLE_OK;
 }
 
 static void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size)
@@ -555,8 +561,9 @@ const struct MD5_params Curl_DIGEST_MD5[] = {
 
 /*
  * @unittest: 1601
+ * Returns CURLE_OK on success.
  */
-void Curl_md5it(unsigned char *outbuffer, const unsigned char *input,
+CURLcode Curl_md5it(unsigned char *outbuffer, const unsigned char *input,
                 const size_t len)
 {
   MD5_CTX ctx;
@@ -564,6 +571,8 @@ void Curl_md5it(unsigned char *outbuffer, const unsigned char *input,
   MD5_Init(&ctx);
   MD5_Update(&ctx, input, curlx_uztoui(len));
   MD5_Final(outbuffer, &ctx);
+
+  return CURLE_OK;
 }
 
 struct MD5_context *Curl_MD5_init(const struct MD5_params *md5params)
diff --git a/lib/sha256.c b/lib/sha256.c
index 1391412ce..cf7ea4f54 100644
--- a/lib/sha256.c
+++ b/lib/sha256.c
@@ -82,10 +82,14 @@ struct sha256_ctx {
 };
 typedef struct sha256_ctx my_sha256_ctx;
 
-static void my_sha256_init(my_sha256_ctx *ctx)
+static CURLcode my_sha256_init(my_sha256_ctx *ctx)
 {
   ctx->openssl_ctx = EVP_MD_CTX_create();
+  if(!ctx->openssl_ctx)
+    return CURLE_OUT_OF_MEMORY;
+
   EVP_DigestInit_ex(ctx->openssl_ctx, EVP_sha256(), NULL);
+  return CURLE_OK;
 }
 
 static void my_sha256_update(my_sha256_ctx *ctx,
@@ -112,9 +116,10 @@ static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx)
 
 typedef struct sha256_ctx my_sha256_ctx;
 
-static void my_sha256_init(my_sha256_ctx *ctx)
+static CURLcode my_sha256_init(my_sha256_ctx *ctx)
 {
   sha256_init(ctx);
+  return CURLE_OK;
 }
 
 static void my_sha256_update(my_sha256_ctx *ctx,
@@ -140,13 +145,14 @@ static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx)
 
 typedef mbedtls_sha256_context my_sha256_ctx;
 
-static void my_sha256_init(my_sha256_ctx *ctx)
+static CURLcode my_sha256_init(my_sha256_ctx *ctx)
 {
 #if !defined(HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS)
   (void) mbedtls_sha256_starts(ctx, 0);
 #else
   (void) mbedtls_sha256_starts_ret(ctx, 0);
 #endif
+  return CURLE_OK;
 }
 
 static void my_sha256_update(my_sha256_ctx *ctx,
@@ -183,9 +189,10 @@ static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx)
 
 typedef CC_SHA256_CTX my_sha256_ctx;
 
-static void my_sha256_init(my_sha256_ctx *ctx)
+static CURLcode my_sha256_init(my_sha256_ctx *ctx)
 {
   (void) CC_SHA256_Init(ctx);
+  return CURLE_OK;
 }
 
 static void my_sha256_update(my_sha256_ctx *ctx,
@@ -214,12 +221,14 @@ typedef struct sha256_ctx my_sha256_ctx;
 #define CALG_SHA_256 0x0000800c
 #endif
 
-static void my_sha256_init(my_sha256_ctx *ctx)
+static CURLcode my_sha256_init(my_sha256_ctx *ctx)
 {
   if(CryptAcquireContext(&ctx->hCryptProv, NULL, NULL, PROV_RSA_AES,
                          CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
     CryptCreateHash(ctx->hCryptProv, CALG_SHA_256, 0, 0, &ctx->hHash);
   }
+
+  return CURLE_OK;
 }
 
 static void my_sha256_update(my_sha256_ctx *ctx,
@@ -375,7 +384,7 @@ static int sha256_compress(struct sha256_state *md,
 }
 
 /* Initialize the hash state */
-static void my_sha256_init(struct sha256_state *md)
+static CURLcode my_sha256_init(struct sha256_state *md)
 {
   md->curlen = 0;
   md->length = 0;
@@ -387,6 +396,8 @@ static void my_sha256_init(struct sha256_state *md)
   md->state[5] = 0x9B05688CUL;
   md->state[6] = 0x1F83D9ABUL;
   md->state[7] = 0x5BE0CD19UL;
+
+  return CURLE_OK;
 }
 
 /*
@@ -394,7 +405,7 @@ static void my_sha256_init(struct sha256_state *md)
    @param md     The hash state
    @param in     The data to hash
    @param inlen  The length of the data (octets)
-   @return CRYPT_OK if successful
+   @return 0 if successful
 */
 static int my_sha256_update(struct sha256_state *md,
                             const unsigned char *in,
@@ -435,7 +446,7 @@ static int my_sha256_update(struct sha256_state *md,
    Terminate the hash to get the digest
    @param md  The hash state
    @param out [out] The destination of the hash (32 bytes)
-   @return CRYPT_OK if successful
+   @return 0 if successful
 */
 static int my_sha256_final(unsigned char *out,
                            struct sha256_state *md)
@@ -491,15 +502,21 @@ static int my_sha256_final(unsigned char *out,
  * output [in/out] - The output buffer.
  * input  [in]     - The input data.
  * length [in]     - The input length.
+ *
+ * Returns CURLE_OK on success.
  */
-void Curl_sha256it(unsigned char *output, const unsigned char *input,
+CURLcode Curl_sha256it(unsigned char *output, const unsigned char *input,
                    const size_t length)
 {
+  CURLcode result;
   my_sha256_ctx ctx;
 
-  my_sha256_init(&ctx);
-  my_sha256_update(&ctx, input, curlx_uztoui(length));
-  my_sha256_final(output, &ctx);
+  result = my_sha256_init(&ctx);
+  if(!result) {
+    my_sha256_update(&ctx, input, curlx_uztoui(length));
+    my_sha256_final(output, &ctx);
+  }
+  return result;
 }
 
 
diff --git a/lib/vauth/digest.c b/lib/vauth/digest.c
index 52179c265..62484be9d 100644
--- a/lib/vauth/digest.c
+++ b/lib/vauth/digest.c
@@ -666,8 +666,8 @@ static CURLcode auth_create_digest_http_message(
                   struct digestdata *digest,
                   char **outptr, size_t *outlen,
                   void (*convert_to_ascii)(unsigned char *, unsigned char *),
-                  void (*hash)(unsigned char *, const unsigned char *,
-                               const size_t))
+                  CURLcode (*hash)(unsigned char *, const unsigned char *,
+                                   const size_t))
 {
   CURLcode result;
   unsigned char hashbuf[32]; /* 32 bytes/256 bits */
diff --git a/lib/vauth/ntlm.c b/lib/vauth/ntlm.c
index 0aa3f1c8c..04f6590ac 100644
--- a/lib/vauth/ntlm.c
+++ b/lib/vauth/ntlm.c
@@ -603,7 +603,9 @@ CURLcode Curl_auth_create_ntlm_type3_message(struct Curl_easy *data,
     memcpy(tmp, &ntlm->nonce[0], 8);
     memcpy(tmp + 8, entropy, 8);
 
-    Curl_md5it(md5sum, tmp, 16);
+    result = Curl_md5it(md5sum, tmp, 16);
+    if(result)
+      return result;
 
     /* We shall only use the first 8 bytes of md5sum, but the des code in
        Curl_ntlm_core_lm_resp only encrypt the first 8 bytes */
diff --git a/lib/vssh/libssh2.c b/lib/vssh/libssh2.c
index eb2f53a0d..581bc1be8 100644
--- a/lib/vssh/libssh2.c
+++ b/lib/vssh/libssh2.c
@@ -645,8 +645,8 @@ static CURLcode ssh_check_fingerprint(struct Curl_easy *data)
 
     hostkey = libssh2_session_hostkey(sshc->ssh_session, &len, NULL);
     if(hostkey) {
-      Curl_sha256it(hash, (const unsigned char *) hostkey, len);
-      fingerprint = (char *) hash;
+      if(!Curl_sha256it(hash, (const unsigned char *) hostkey, len))
+        fingerprint = (char *) hash;
     }
 #endif
 
@@ -661,16 +661,15 @@ static CURLcode ssh_check_fingerprint(struct Curl_easy *data)
 
     /* The length of fingerprint is 32 bytes for SHA256.
      * See libssh2_hostkey_hash documentation. */
-    if(Curl_base64_encode (data, fingerprint, 32, &fingerprint_b64,
-        &fingerprint_b64_len) != CURLE_OK) {
+    if(Curl_base64_encode(data, fingerprint, 32, &fingerprint_b64,
+                          &fingerprint_b64_len) != CURLE_OK) {
       state(data, SSH_SESSION_FREE);
       sshc->actualcode = CURLE_PEER_FAILED_VERIFICATION;
       return sshc->actualcode;
     }
 
     if(!fingerprint_b64) {
-      failf(data,
-          "sha256 fingerprint could not be encoded");
+      failf(data, "sha256 fingerprint could not be encoded");
       state(data, SSH_SESSION_FREE);
       sshc->actualcode = CURLE_PEER_FAILED_VERIFICATION;
       return sshc->actualcode;