Add testcase for OSSL_trace_set_callback()

Also test the OSSL_TRACE_CATEGORY_TRACE tracing - this fails
on address sanitizer runs without the fix for #19915

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/19917)

1 files changed, 39 insertions, 10 deletions

diff --git a/test/trace_api_test.c b/test/trace_api_test.c
index cfef47eda1..ba9ba226c3 100644
--- a/test/trace_api_test.c
+++ b/test/trace_api_test.c
@@ -33,10 +33,8 @@ static int test_trace_categories(void)
         CASE(TLS);
         CASE(TLS_CIPHER);
         CASE(CONF);
-#ifndef OPENSSL_NO_ENGINE
         CASE(ENGINE_TABLE);
         CASE(ENGINE_REF_COUNT);
-#endif
         CASE(PKCS5V2);
         CASE(PKCS12_KEYGEN);
         CASE(PKCS12_DECRYPT);
@@ -69,10 +67,10 @@ static int test_trace_categories(void)
 #ifndef OPENSSL_NO_TRACE
 static void put_trace_output(void)
 {
-    OSSL_TRACE_BEGIN(TLS) {
+    OSSL_TRACE_BEGIN(HTTP) {
         BIO_printf(trc_out, "Hello World\n");
         BIO_printf(trc_out, "Good Bye Universe\n");
-    } OSSL_TRACE_END(TLS);
+    } OSSL_TRACE_END(HTTP);
 }
 
 static int test_trace_channel(void)
@@ -88,30 +86,60 @@ static int test_trace_channel(void)
     if (!TEST_ptr(bio))
         goto end;
 
-    if (!TEST_int_eq(OSSL_trace_set_channel(OSSL_TRACE_CATEGORY_TLS, bio), 1))
+    if (!TEST_int_eq(OSSL_trace_set_channel(OSSL_TRACE_CATEGORY_HTTP, bio), 1))
         goto end;
 
-    if (!TEST_true(OSSL_trace_enabled(OSSL_TRACE_CATEGORY_TLS)))
+    if (!TEST_true(OSSL_trace_enabled(OSSL_TRACE_CATEGORY_HTTP)))
         goto end;
 
-    if (!TEST_int_eq(OSSL_trace_set_prefix(OSSL_TRACE_CATEGORY_TLS, "xyz-"), 1))
+    if (!TEST_int_eq(OSSL_trace_set_prefix(OSSL_TRACE_CATEGORY_HTTP, "xyz-"), 1))
         goto end;
-    if (!TEST_int_eq(OSSL_trace_set_suffix(OSSL_TRACE_CATEGORY_TLS, "-abc"), 1))
+    if (!TEST_int_eq(OSSL_trace_set_suffix(OSSL_TRACE_CATEGORY_HTTP, "-abc"), 1))
         goto end;
 
     put_trace_output();
     len = BIO_get_mem_data(bio, &p_buf);
     if (!TEST_strn2_eq(p_buf, len, expected, expected_len))
         goto end;
-    if (!TEST_int_eq(OSSL_trace_set_channel(OSSL_TRACE_CATEGORY_TLS, NULL), 1))
+    if (!TEST_int_eq(OSSL_trace_set_channel(OSSL_TRACE_CATEGORY_HTTP, NULL), 1))
         goto end;
     bio = NULL;
 
     ret = 1;
-end:
+ end:
     BIO_free(bio);
     return ret;
 }
+
+static int trace_cb_failure;
+static int trace_cb_called;
+
+static size_t trace_cb(const char *buffer, size_t count,
+                       int category, int cmd, void *data)
+{
+    trace_cb_called = 1;
+    if (!TEST_true(category == OSSL_TRACE_CATEGORY_TRACE))
+        trace_cb_failure = 1;
+    return count;
+}
+
+static int test_trace_callback(void)
+{
+    int ret = 0;
+
+    if (!TEST_true(OSSL_trace_set_callback(OSSL_TRACE_CATEGORY_TRACE, trace_cb,
+                                           NULL)))
+        goto end;
+
+    put_trace_output();
+
+    if (!TEST_false(trace_cb_failure) || !TEST_true(trace_cb_called))
+        goto end;
+
+    ret = 1;
+ end:
+    return ret;
+}
 #endif
 
 OPT_TEST_DECLARE_USAGE("\n")
@@ -126,6 +154,7 @@ int setup_tests(void)
     ADD_TEST(test_trace_categories);
 #ifndef OPENSSL_NO_TRACE
     ADD_TEST(test_trace_channel);
+    ADD_TEST(test_trace_callback);
 #endif
     return 1;
 }