Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

3 files changed, 15 insertions, 28 deletions

diff --git a/crypto/conf/conf_def.c b/crypto/conf/conf_def.c
index 5370e06bf9..b6b56cd967 100644
--- a/crypto/conf/conf_def.c
+++ b/crypto/conf/conf_def.c
@@ -233,13 +233,11 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
     }
 
     section = OPENSSL_strdup("default");
-    if (section == NULL) {
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+    if (section == NULL)
         goto err;
-    }
 
     if (_CONF_new_data(conf) == 0) {
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB);
         goto err;
     }
 
@@ -425,10 +423,8 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
                         goto err;
                 } else if (strcmp(p, "includedir") == 0) {
                     OPENSSL_free(conf->includedir);
-                    if ((conf->includedir = OPENSSL_strdup(pval)) == NULL) {
-                        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+                    if ((conf->includedir = OPENSSL_strdup(pval)) == NULL)
                         goto err;
-                    }
                 }
 
                 /*
@@ -458,7 +454,6 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
 
                     include_path = OPENSSL_malloc(newlen);
                     if (include_path == NULL) {
-                        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
                         OPENSSL_free(include);
                         goto err;
                     }
@@ -495,13 +490,13 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
                     /* push the currently processing BIO onto stack */
                     if (biosk == NULL) {
                         if ((biosk = sk_BIO_new_null()) == NULL) {
-                            ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+                            ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB);
                             BIO_free(next);
                             goto err;
                         }
                     }
                     if (!sk_BIO_push(biosk, in)) {
-                        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+                        ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB);
                         BIO_free(next);
                         goto err;
                     }
@@ -519,16 +514,12 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
             start = eat_ws(conf, p);
             trim_ws(conf, start);
 
-            if ((v = OPENSSL_malloc(sizeof(*v))) == NULL) {
-                ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+            if ((v = OPENSSL_malloc(sizeof(*v))) == NULL)
                 goto err;
-            }
             v->name = OPENSSL_strdup(pname);
             v->value = NULL;
-            if (v->name == NULL) {
-                ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+            if (v->name == NULL)
                 goto err;
-            }
             if (!str_copy(conf, psection, &(v->value), start))
                 goto err;
 
@@ -544,7 +535,7 @@ static int def_load_bio(CONF *conf, BIO *in, long *line)
             } else
                 tv = sv;
             if (_CONF_add_string(conf, tv, v) == 0) {
-                ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB);
                 goto err;
             }
             v = NULL;
@@ -757,7 +748,7 @@ static int str_copy(CONF *conf, char *section, char **pto, char *from)
                 goto err;
             }
             if (!BUF_MEM_grow_clean(buf, newsize)) {
-                ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_CONF, ERR_R_BUF_LIB);
                 goto err;
             }
             while (*p)
@@ -849,10 +840,8 @@ static BIO *get_next_file(const char *path, OPENSSL_DIR_CTX **dirctx)
 
             newlen = pathlen + namelen + 2;
             newpath = OPENSSL_zalloc(newlen);
-            if (newpath == NULL) {
-                ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+            if (newpath == NULL)
                 break;
-            }
 #ifdef OPENSSL_SYS_VMS
             /*
              * If the given path isn't clear VMS syntax,
diff --git a/crypto/conf/conf_lib.c b/crypto/conf/conf_lib.c
index 1766facd65..05e7bf19cc 100644
--- a/crypto/conf/conf_lib.c
+++ b/crypto/conf/conf_lib.c
@@ -188,7 +188,7 @@ CONF *NCONF_new_ex(OSSL_LIB_CTX *libctx, CONF_METHOD *meth)
 
     ret = meth->create(meth);
     if (ret == NULL) {
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CONF, ERR_R_CONF_LIB);
         return NULL;
     }
     ret->libctx = libctx;
diff --git a/crypto/conf/conf_mod.c b/crypto/conf/conf_mod.c
index 2de47e8e42..148ce6b524 100644
--- a/crypto/conf/conf_mod.c
+++ b/crypto/conf/conf_mod.c
@@ -100,7 +100,7 @@ DEFINE_RUN_ONCE_STATIC(do_init_module_list_lock)
 {
     module_list_lock = CRYPTO_THREAD_lock_new();
     if (module_list_lock == NULL) {
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB);
         return 0;
     }
 
@@ -332,10 +332,8 @@ static CONF_MODULE *module_add(DSO *dso, const char *name,
         supported_modules = sk_CONF_MODULE_new_null();
     if (supported_modules == NULL)
         goto err;
-    if ((tmod = OPENSSL_zalloc(sizeof(*tmod))) == NULL) {
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+    if ((tmod = OPENSSL_zalloc(sizeof(*tmod))) == NULL)
         goto err;
-    }
 
     tmod->dso = dso;
     tmod->name = OPENSSL_strdup(name);
@@ -435,14 +433,14 @@ static int module_init(CONF_MODULE *pmod, const char *name, const char *value,
         initialized_modules = sk_CONF_IMODULE_new_null();
         if (initialized_modules == NULL) {
             CRYPTO_THREAD_unlock(module_list_lock);
-            ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB);
             goto err;
         }
     }
 
     if (!sk_CONF_IMODULE_push(initialized_modules, imod)) {
         CRYPTO_THREAD_unlock(module_list_lock);
-        ERR_raise(ERR_LIB_CONF, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CONF, ERR_R_CRYPTO_LIB);
         goto err;
     }