crypto: raise error on malloc failure

clean a few style nits.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14806)

7 files changed, 72 insertions, 31 deletions

diff --git a/crypto/asn1/a_object.c b/crypto/asn1/a_object.c
index f8bc315026..3740f608c5 100644
--- a/crypto/asn1/a_object.c
+++ b/crypto/asn1/a_object.c
@@ -135,8 +135,10 @@ int a2d_ASN1_OBJECT(unsigned char *out, int olen, const char *buf, int num)
                     OPENSSL_free(tmp);
                 tmpsize = blsize + 32;
                 tmp = OPENSSL_malloc(tmpsize);
-                if (tmp == NULL)
+                if (tmp == NULL) {
+                    ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
                     goto err;
+                }
             }
             while (blsize--) {
                 BN_ULONG t = BN_div_word(bl, 0x80L);
diff --git a/crypto/asn1/a_time.c b/crypto/asn1/a_time.c
index 1a0c91ef9f..fb3bd2aca6 100644
--- a/crypto/asn1/a_time.c
+++ b/crypto/asn1/a_time.c
@@ -420,8 +420,10 @@ int ASN1_TIME_set_string_X509(ASN1_TIME *s, const char *str)
              * new t.data would be freed after ASN1_STRING_copy is done.
              */
             t.data = OPENSSL_zalloc(t.length + 1);
-            if (t.data == NULL)
+            if (t.data == NULL) {
+                ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
                 goto out;
+            }
             memcpy(t.data, str + 2, t.length);
             t.type = V_ASN1_UTCTIME;
         }
diff --git a/crypto/asn1/ameth_lib.c b/crypto/asn1/ameth_lib.c
index 8614610333..dea5fc9482 100644
--- a/crypto/asn1/ameth_lib.c
+++ b/crypto/asn1/ameth_lib.c
@@ -223,8 +223,10 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags,
 {
     EVP_PKEY_ASN1_METHOD *ameth = OPENSSL_zalloc(sizeof(*ameth));
 
-    if (ameth == NULL)
+    if (ameth == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
         return NULL;
+    }
 
     ameth->pkey_id = id;
     ameth->pkey_base_id = id;
@@ -232,13 +234,13 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags,
 
     if (info) {
         ameth->info = OPENSSL_strdup(info);
-        if (!ameth->info)
+        if (ameth->info == NULL)
             goto err;
     }
 
     if (pem_str) {
         ameth->pem_str = OPENSSL_strdup(pem_str);
-        if (!ameth->pem_str)
+        if (ameth->pem_str == NULL)
             goto err;
     }
 
@@ -246,8 +248,8 @@ EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_new(int id, int flags,
 
  err:
     EVP_PKEY_asn1_free(ameth);
+    ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
     return NULL;
-
 }
 
 void EVP_PKEY_asn1_copy(EVP_PKEY_ASN1_METHOD *dst,
diff --git a/crypto/asn1/bio_asn1.c b/crypto/asn1/bio_asn1.c
index f84b7624b8..dc99e2d7c2 100644
--- a/crypto/asn1/bio_asn1.c
+++ b/crypto/asn1/bio_asn1.c
@@ -102,8 +102,10 @@ static int asn1_bio_new(BIO *b)
 {
     BIO_ASN1_BUF_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));
 
-    if (ctx == NULL)
+    if (ctx == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
         return 0;
+    }
     if (!asn1_bio_init(ctx, DEFAULT_ASN1_BUF_SIZE)) {
         OPENSSL_free(ctx);
         return 0;
diff --git a/crypto/asn1/tasn_enc.c b/crypto/asn1/tasn_enc.c
index 9d72b16779..2d24320af9 100644
--- a/crypto/asn1/tasn_enc.c
+++ b/crypto/asn1/tasn_enc.c
@@ -216,9 +216,9 @@ int ASN1_item_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
 static int asn1_template_ex_i2d(const ASN1_VALUE **pval, unsigned char **out,
                                 const ASN1_TEMPLATE *tt, int tag, int iclass)
 {
-    int i, ret, flags, ttag, tclass, ndef;
+    const int flags = tt->flags;
+    int i, ret, ttag, tclass, ndef;
     const ASN1_VALUE *tval;
-    flags = tt->flags;
 
     /*
      * If field is embedded then val needs fixing so it is a pointer to
@@ -391,10 +391,11 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk,
                             int skcontlen, const ASN1_ITEM *item,
                             int do_sort, int iclass)
 {
-    int i;
+    int i, ret = 0;
     const ASN1_VALUE *skitem;
     unsigned char *tmpdat = NULL, *p = NULL;
     DER_ENC *derlst = NULL, *tder;
+
     if (do_sort) {
         /* Don't need to sort less than 2 items */
         if (sk_const_ASN1_VALUE_num(sk) < 2)
@@ -402,12 +403,14 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk,
         else {
             derlst = OPENSSL_malloc(sk_const_ASN1_VALUE_num(sk)
                                     * sizeof(*derlst));
-            if (derlst == NULL)
+            if (derlst == NULL) {
+                ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
                 return 0;
+            }
             tmpdat = OPENSSL_malloc(skcontlen);
             if (tmpdat == NULL) {
-                OPENSSL_free(derlst);
-                return 0;
+                ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
+                goto err;
             }
         }
     }
@@ -443,9 +446,11 @@ static int asn1_set_seq_out(STACK_OF(const_ASN1_VALUE) *sk,
         for (i = 0, tder = derlst; i < sk_const_ASN1_VALUE_num(sk); i++, tder++)
             (void)sk_const_ASN1_VALUE_set(sk, i, tder->field);
     }
+    ret = 1;
+err:
     OPENSSL_free(derlst);
     OPENSSL_free(tmpdat);
-    return 1;
+    return ret;
 }
 
 static int asn1_i2d_ex_primitive(const ASN1_VALUE **pval, unsigned char **out,
diff --git a/crypto/bn/bn_gf2m.c b/crypto/bn/bn_gf2m.c
index 622d41302f..8f91fa05e6 100644
--- a/crypto/bn/bn_gf2m.c
+++ b/crypto/bn/bn_gf2m.c
@@ -467,12 +467,17 @@ int BN_GF2m_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
 {
     int ret = 0;
     const int max = BN_num_bits(p) + 1;
-    int *arr = NULL;
+    int *arr;
+
     bn_check_top(a);
     bn_check_top(b);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
-        goto err;
+
+    arr = OPENSSL_malloc(sizeof(*arr) * max);
+    if (arr == NULL) {
+        ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
         ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH);
@@ -525,12 +530,16 @@ int BN_GF2m_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 {
     int ret = 0;
     const int max = BN_num_bits(p) + 1;
-    int *arr = NULL;
+    int *arr;
 
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
-        goto err;
+
+    arr = OPENSSL_malloc(sizeof(*arr) * max);
+    if (arr == NULL) {
+        ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
         ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH);
@@ -899,12 +908,17 @@ int BN_GF2m_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
 {
     int ret = 0;
     const int max = BN_num_bits(p) + 1;
-    int *arr = NULL;
+    int *arr;
+
     bn_check_top(a);
     bn_check_top(b);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
-        goto err;
+
+    arr = OPENSSL_malloc(sizeof(*arr) * max);
+    if (arr == NULL) {
+        ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
         ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH);
@@ -959,11 +973,16 @@ int BN_GF2m_mod_sqrt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 {
     int ret = 0;
     const int max = BN_num_bits(p) + 1;
-    int *arr = NULL;
+    int *arr;
+
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
-        goto err;
+
+    arr = OPENSSL_malloc(sizeof(*arr) * max);
+    if (arr == NULL) {
+        ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
         ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH);
@@ -1090,11 +1109,16 @@ int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
 {
     int ret = 0;
     const int max = BN_num_bits(p) + 1;
-    int *arr = NULL;
+    int *arr;
+
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
+
+    arr = OPENSSL_malloc(sizeof(*arr) * max);
+    if (arr == NULL) {
+        ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
         goto err;
+    }
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
         ERR_raise(ERR_LIB_BN, BN_R_INVALID_LENGTH);
diff --git a/crypto/bn/bn_mod.c b/crypto/bn/bn_mod.c
index fbc42168bd..d083ed27a3 100644
--- a/crypto/bn/bn_mod.c
+++ b/crypto/bn/bn_mod.c
@@ -56,9 +56,13 @@ int bn_mod_add_fixed_top(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
     if (bn_wexpand(r, mtop) == NULL)
         return 0;
 
-    if (mtop > sizeof(storage) / sizeof(storage[0])
-        && (tp = OPENSSL_malloc(mtop * sizeof(BN_ULONG))) == NULL)
-        return 0;
+    if (mtop > sizeof(storage) / sizeof(storage[0])) {
+        tp = OPENSSL_malloc(mtop * sizeof(BN_ULONG));
+        if (tp == NULL) {
+            ERR_raise(ERR_LIB_BN, ERR_R_MALLOC_FAILURE);
+            return 0;
+        }
+    }
 
     ap = a->d != NULL ? a->d : tp;
     bp = b->d != NULL ? b->d : tp;