Initial implementation for MDEV-10332:MDEV-10332

  support for OpenSSL 1.1 and LibreSSL
  tested against OpenSSL 1.0.1, 1.0.2, 1.1.0, Yassl and LibreSSL
  not working on Windows with native SChannel support, due to wrong cipher mapping: Latter one requires push of CONC-241 fixes.
  Please note that OpenSSL 0.9.8 and OpenSSL 1.1.0 will not work: Even if the build succeeds, test cases will fail with various errors, especially when using different tls libraries or versions for client and server.

1 files changed, 27 insertions, 12 deletions

diff --git a/mysys_ssl/my_md5.cc b/mysys_ssl/my_md5.cc
index 7139ea9b6ff..02c01dd7148 100644
--- a/mysys_ssl/my_md5.cc
+++ b/mysys_ssl/my_md5.cc
@@ -27,6 +27,8 @@
 #include <my_md5.h>
 #include <stdarg.h>
 
+#define MA_HASH_CTX_SIZE 512
+
 #if defined(HAVE_YASSL)
 #include "md5.hpp"
 
@@ -57,11 +59,18 @@ static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
 }
 
 #elif defined(HAVE_OPENSSL)
+
+
 #include <openssl/evp.h>
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+#define EVP_MD_CTX_reset(X) EVP_MD_CTX_cleanup(X)
+#endif
 typedef EVP_MD_CTX MD5_CONTEXT;
 
 static void md5_init(MD5_CONTEXT *context)
 {
+  memset(context, 0, my_md5_context_size());
   EVP_MD_CTX_init(context);
 #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
   /* Ok to ignore FIPS: MD5 is not used for crypto here */
@@ -83,7 +92,7 @@ static void md5_input(MD5_CONTEXT *context, const uchar *buf, unsigned len)
 static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
 {
   EVP_DigestFinal_ex(context, digest, NULL);
-  EVP_MD_CTX_cleanup(context);
+  EVP_MD_CTX_reset(context);
 }
 
 #endif /* HAVE_YASSL */
@@ -99,11 +108,14 @@ static void md5_result(MD5_CONTEXT *context, uchar digest[MD5_HASH_SIZE])
 */
 void my_md5(uchar *digest, const char *buf, size_t len)
 {
+#ifdef HAVE_YASSL
   MD5_CONTEXT md5_context;
-
-  md5_init_fast(&md5_context);
-  md5_input(&md5_context, (const uchar *)buf, len);
-  md5_result(&md5_context, digest);
+#else
+  unsigned char md5_context[MA_HASH_CTX_SIZE];
+#endif
+  md5_init_fast((MD5_CONTEXT *)&md5_context);
+  md5_input((MD5_CONTEXT *)&md5_context, (const uchar *)buf, len);
+  md5_result((MD5_CONTEXT *)&md5_context, digest);
 }
 
 
@@ -122,22 +134,25 @@ void my_md5(uchar *digest, const char *buf, size_t len)
 void my_md5_multi(uchar *digest, ...)
 {
   va_list args;
-  va_start(args, digest);
-
-  MD5_CONTEXT md5_context;
   const uchar *str;
+#ifdef HAVE_YASSL
+  MD5_CONTEXT md5_context;
+#else
+  unsigned char md5_context[MA_HASH_CTX_SIZE];
+#endif
+  va_start(args, digest);
 
-  md5_init_fast(&md5_context);
+  md5_init_fast((MD5_CONTEXT *)&md5_context);
   for (str= va_arg(args, const uchar*); str; str= va_arg(args, const uchar*))
-    md5_input(&md5_context, str, va_arg(args, size_t));
+    md5_input((MD5_CONTEXT *)&md5_context, str, va_arg(args, size_t));
 
-  md5_result(&md5_context, digest);
+  md5_result((MD5_CONTEXT *)&md5_context, digest);
   va_end(args);
 }
 
 size_t my_md5_context_size()
 {
-  return sizeof(MD5_CONTEXT);
+  return MA_HASH_CTX_SIZE;
 }
 
 void my_md5_init(void *context)