Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 21 insertions, 34 deletions

diff --git a/crypto/ui/ui_lib.c b/crypto/ui/ui_lib.c
index 1ff8c6fa35..9c779df48d 100644
--- a/crypto/ui/ui_lib.c
+++ b/crypto/ui/ui_lib.c
@@ -24,14 +24,12 @@ UI *UI_new_method(const UI_METHOD *method)
 {
     UI *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
-        ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_UI, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ret);
         return NULL;
     }
@@ -210,10 +208,8 @@ int UI_dup_input_string(UI *ui, const char *prompt, int flags,
 
     if (prompt != NULL) {
         prompt_copy = OPENSSL_strdup(prompt);
-        if (prompt_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (prompt_copy == NULL)
             return 0;
-        }
     }
 
     return general_allocate_string(ui, prompt_copy, 1,
@@ -238,10 +234,8 @@ int UI_dup_verify_string(UI *ui, const char *prompt, int flags,
 
     if (prompt != NULL) {
         prompt_copy = OPENSSL_strdup(prompt);
-        if (prompt_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (prompt_copy == NULL)
             return -1;
-        }
     }
 
     return general_allocate_string(ui, prompt_copy, 1,
@@ -269,34 +263,26 @@ int UI_dup_input_boolean(UI *ui, const char *prompt, const char *action_desc,
 
     if (prompt != NULL) {
         prompt_copy = OPENSSL_strdup(prompt);
-        if (prompt_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (prompt_copy == NULL)
             goto err;
-        }
     }
 
     if (action_desc != NULL) {
         action_desc_copy = OPENSSL_strdup(action_desc);
-        if (action_desc_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (action_desc_copy == NULL)
             goto err;
-        }
     }
 
     if (ok_chars != NULL) {
         ok_chars_copy = OPENSSL_strdup(ok_chars);
-        if (ok_chars_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (ok_chars_copy == NULL)
             goto err;
-        }
     }
 
     if (cancel_chars != NULL) {
         cancel_chars_copy = OPENSSL_strdup(cancel_chars);
-        if (cancel_chars_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (cancel_chars_copy == NULL)
             goto err;
-        }
     }
 
     return general_allocate_boolean(ui, prompt_copy, action_desc_copy,
@@ -322,10 +308,8 @@ int UI_dup_info_string(UI *ui, const char *text)
 
     if (text != NULL) {
         text_copy = OPENSSL_strdup(text);
-        if (text_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (text_copy == NULL)
             return -1;
-        }
     }
 
     return general_allocate_string(ui, text_copy, 1, UIT_INFO, 0, NULL,
@@ -344,10 +328,8 @@ int UI_dup_error_string(UI *ui, const char *text)
 
     if (text != NULL) {
         text_copy = OPENSSL_strdup(text);
-        if (text_copy == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if (text_copy == NULL)
             return -1;
-        }
     }
     return general_allocate_string(ui, text_copy, 1, UIT_ERROR, 0, NULL,
                                    0, 0, NULL);
@@ -373,10 +355,8 @@ char *UI_construct_prompt(UI *ui, const char *phrase_desc,
             len += sizeof(prompt2) - 1 + strlen(object_name);
         len += sizeof(prompt3) - 1;
 
-        if ((prompt = OPENSSL_malloc(len + 1)) == NULL) {
-            ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        if ((prompt = OPENSSL_malloc(len + 1)) == NULL)
             return NULL;
-        }
         OPENSSL_strlcpy(prompt, prompt1, len + 1);
         OPENSSL_strlcat(prompt, phrase_desc, len + 1);
         if (object_name != NULL) {
@@ -413,7 +393,7 @@ int UI_dup_user_data(UI *ui, void *user_data)
 
     duplicate = ui->meth->ui_duplicate_data(ui, user_data);
     if (duplicate == NULL) {
-        ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_UI, ERR_R_UI_LIB);
         return -1;
     }
 
@@ -599,10 +579,17 @@ UI_METHOD *UI_create_method(const char *name)
         || (ui_method->name = OPENSSL_strdup(name)) == NULL
         || !CRYPTO_new_ex_data(CRYPTO_EX_INDEX_UI_METHOD, ui_method,
                                &ui_method->ex_data)) {
-        if (ui_method)
+
+        if (ui_method != NULL) {
+            if (ui_method->name != NULL)
+                /*
+                 * These conditions indicate that the CRYPTO_new_ex_data()
+                 * call failed.
+                 */
+                ERR_raise(ERR_LIB_UI, ERR_R_CRYPTO_LIB);
             OPENSSL_free(ui_method->name);
+        }
         OPENSSL_free(ui_method);
-        ERR_raise(ERR_LIB_UI, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
     return ui_method;