MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

2 files changed, 25 insertions, 43 deletions

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index c6be6bfba4c..ab84f3eabda 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -111,7 +111,6 @@
 #endif
 
 #include <my_systemd.h>
-#include <my_crypt.h>
 
 #define mysqld_charset &my_charset_latin1
 
@@ -121,7 +120,6 @@
 #define HAVE_CLOSE_SERVER_SOCK 1
 #endif
 
-
 extern "C" {					// Because of SCO 3.2V4.2
 #include <sys/stat.h>
 #ifndef __GNU_LIBRARY__
@@ -339,9 +337,13 @@ static PSI_thread_key key_thread_handle_con_sockets;
 static PSI_thread_key key_thread_handle_shutdown;
 #endif /* __WIN__ */
 
-#if defined (HAVE_OPENSSL) && !defined(HAVE_YASSL)
+#ifdef HAVE_OPENSSL
+#include <ssl_compat.h>
+
+#ifdef HAVE_OPENSSL10
 static PSI_rwlock_key key_rwlock_openssl;
 #endif
+#endif
 #endif /* HAVE_PSI_INTERFACE */
 
 #ifdef HAVE_NPTL
@@ -1026,7 +1028,7 @@ PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger,
 
 static PSI_rwlock_info all_server_rwlocks[]=
 {
-#if defined (HAVE_OPENSSL) && !defined(HAVE_YASSL)
+#ifdef HAVE_OPENSSL10
   { &key_rwlock_openssl, "CRYPTO_dynlock_value::lock", 0},
 #endif
   { &key_rwlock_LOCK_grant, "LOCK_grant", PSI_FLAG_GLOBAL},
@@ -1502,9 +1504,7 @@ scheduler_functions *thread_scheduler= &thread_scheduler_struct,
 
 #ifdef HAVE_OPENSSL
 #include <openssl/crypto.h>
-#ifndef HAVE_YASSL
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+#ifdef HAVE_OPENSSL10
 typedef struct CRYPTO_dynlock_value
 {
   mysql_rwlock_t lock;
@@ -1515,8 +1515,7 @@ static openssl_lock_t *openssl_dynlock_create(const char *, int);
 static void openssl_dynlock_destroy(openssl_lock_t *, const char *, int);
 static void openssl_lock_function(int, int, const char *, int);
 static void openssl_lock(int, openssl_lock_t *, const char *, int);
-#endif
-#endif
+#endif /* HAVE_OPENSSL10 */
 char *des_key_file;
 #ifndef EMBEDDED_LIBRARY
 struct st_VioSSLFd *ssl_acceptor_fd;
@@ -2335,13 +2334,11 @@ static void clean_up_mutexes()
   mysql_mutex_destroy(&LOCK_global_index_stats);
 #ifdef HAVE_OPENSSL
   mysql_mutex_destroy(&LOCK_des_key_file);
-#ifndef HAVE_YASSL
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+#ifdef HAVE_OPENSSL10
   for (int i= 0; i < CRYPTO_num_locks(); ++i)
     mysql_rwlock_destroy(&openssl_stdlocks[i].lock);
   OPENSSL_free(openssl_stdlocks);
-#endif
-#endif /* HAVE_YASSL */
+#endif /* HAVE_OPENSSL10 */
 #endif /* HAVE_OPENSSL */
 #ifdef HAVE_REPLICATION
   mysql_mutex_destroy(&LOCK_rpl_status);
@@ -4142,6 +4139,14 @@ static int init_common_variables()
     return 1;
   }
 
+#ifdef HAVE_OPENSSL
+  if (check_openssl_compatibility())
+  {
+    sql_print_error("Incompatible OpenSSL version. Cannot continue...");
+    return 1;
+  }
+#endif
+
   if (init_thread_environment() ||
       mysql_init_variables())
     return 1;
@@ -4701,8 +4706,7 @@ static int init_thread_environment()
 #ifdef HAVE_OPENSSL
   mysql_mutex_init(key_LOCK_des_key_file,
                    &LOCK_des_key_file, MY_MUTEX_INIT_FAST);
-#ifndef HAVE_YASSL
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+#ifdef HAVE_OPENSSL10
   openssl_stdlocks= (openssl_lock_t*) OPENSSL_malloc(CRYPTO_num_locks() *
                                                      sizeof(openssl_lock_t));
   for (int i= 0; i < CRYPTO_num_locks(); ++i)
@@ -4711,9 +4715,8 @@ static int init_thread_environment()
   CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy);
   CRYPTO_set_dynlock_lock_callback(openssl_lock);
   CRYPTO_set_locking_callback(openssl_lock_function);
-#endif
-#endif
-#endif
+#endif /* HAVE_OPENSSL10 */
+#endif /* HAVE_OPENSSL */
   mysql_rwlock_init(key_rwlock_LOCK_sys_init_connect, &LOCK_sys_init_connect);
   mysql_rwlock_init(key_rwlock_LOCK_sys_init_slave, &LOCK_sys_init_slave);
   mysql_rwlock_init(key_rwlock_LOCK_grant, &LOCK_grant);
@@ -4747,8 +4750,7 @@ static int init_thread_environment()
 }
 
 
-#if defined(HAVE_OPENSSL) && !defined(HAVE_YASSL)
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+#ifdef HAVE_OPENSSL10
 static openssl_lock_t *openssl_dynlock_create(const char *file, int line)
 {
   openssl_lock_t *lock= new openssl_lock_t;
@@ -4808,9 +4810,7 @@ static void openssl_lock(int mode, openssl_lock_t *lock, const char *file,
     abort();
   }
 }
-#endif
-#endif /* HAVE_OPENSSL */
-
+#endif /* HAVE_OPENSSL10 */
 
 static void init_ssl()
 {
@@ -4838,9 +4838,8 @@ static void init_ssl()
       while ((err= ERR_get_error()))
         sql_print_warning("SSL error: %s", ERR_error_string(err, NULL));
     }
-    else {
+    else
       ERR_remove_state(0);
-    }
   }
   else
   {
diff --git a/sql/slave.cc b/sql/slave.cc
index 611baf6c9bc..ec2d441e197 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -40,6 +40,7 @@
 #include <my_dir.h>
 #include <sql_common.h>
 #include <errmsg.h>
+#include <ssl_compat.h>
 #include <mysqld_error.h>
 #include <mysys_err.h>
 #include "rpl_handler.h"
@@ -60,12 +61,6 @@
 #include "debug_sync.h"
 #include "rpl_parallel.h"
 
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-#define ERR_remove_state(X)
-#elif defined(HAVE_ERR_remove_thread_state)
-#define ERR_remove_state(X) ERR_remove_thread_state(NULL)
-#endif
-
 #define FLAGSTR(V,F) ((V)&(F)?#F" ":"")
 
 #define MAX_SLAVE_RETRY_PAUSE 5
@@ -4677,13 +4672,7 @@ err_during_init:
 
   DBUG_LEAVE;                                   // Must match DBUG_ENTER()
   my_thread_end();
-#ifdef HAVE_OPENSSL
-#if OPENSSL_VERSION_NUMBER < 0x10000000L
   ERR_remove_state(0);
-#elif OPENSSL_VERSION_NUMBER < 0x10100000L
-  ERR_remove_thread_state(0);
-#endif
-#endif
   pthread_exit(0);
   return 0;                                     // Avoid compiler warnings
 }
@@ -5346,13 +5335,7 @@ err_during_init:
 
   DBUG_LEAVE;                                   // Must match DBUG_ENTER()
   my_thread_end();
-#ifdef HAVE_OPENSSL
-#if OPENSSL_VERSION_NUMBER < 0x10000000L
   ERR_remove_state(0);
-#elif OPENSSL_VERSION_NUMBER < 0x10100000L
-  ERR_remove_thread_state(0);
-#endif
-#endif
   pthread_exit(0);
   return 0;                                     // Avoid compiler warnings
 }