Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 12 insertions, 8 deletions

diff --git a/crypto/x509/x509_v3.c b/crypto/x509/x509_v3.c
index 262061a20f..e9f256cee2 100644
--- a/crypto/x509/x509_v3.c
+++ b/crypto/x509/x509_v3.c
@@ -102,12 +102,14 @@ STACK_OF(X509_EXTENSION) *X509v3_add_ext(STACK_OF(X509_EXTENSION) **x,
 
     if (x == NULL) {
         ERR_raise(ERR_LIB_X509, ERR_R_PASSED_NULL_PARAMETER);
-        goto err2;
+        goto err;
     }
 
     if (*x == NULL) {
-        if ((sk = sk_X509_EXTENSION_new_null()) == NULL)
+        if ((sk = sk_X509_EXTENSION_new_null()) == NULL) {
+            ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB);
             goto err;
+        }
     } else
         sk = *x;
 
@@ -117,16 +119,18 @@ STACK_OF(X509_EXTENSION) *X509v3_add_ext(STACK_OF(X509_EXTENSION) **x,
     else if (loc < 0)
         loc = n;
 
-    if ((new_ex = X509_EXTENSION_dup(ex)) == NULL)
-        goto err2;
-    if (!sk_X509_EXTENSION_insert(sk, new_ex, loc))
+    if ((new_ex = X509_EXTENSION_dup(ex)) == NULL) {
+        ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB);
         goto err;
+    }
+    if (!sk_X509_EXTENSION_insert(sk, new_ex, loc)) {
+        ERR_raise(ERR_LIB_X509, ERR_R_CRYPTO_LIB);
+        goto err;
+    }
     if (*x == NULL)
         *x = sk;
     return sk;
  err:
-    ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
- err2:
     X509_EXTENSION_free(new_ex);
     if (x != NULL && *x == NULL)
         sk_X509_EXTENSION_free(sk);
@@ -159,7 +163,7 @@ X509_EXTENSION *X509_EXTENSION_create_by_OBJ(X509_EXTENSION **ex,
 
     if ((ex == NULL) || (*ex == NULL)) {
         if ((ret = X509_EXTENSION_new()) == NULL) {
-            ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_X509, ERR_R_ASN1_LIB);
             return NULL;
         }
     } else