MDEV-10332 support for OpenSSL 1.1 and LibreSSL

post-review fixes:
* move all ssl implementation related ifdefs/defines to one file
  (ssl_compat.h)
* work around OpenSSL-1.1 desire to malloc every EVP context by
  run-time checking that context allocated on the stack is big enough
  (openssl.c)
* use newer version of the AWS SDK for OpenSSL 1.1
* use get_dh2048() function as generated by openssl 1.1
  (viosslfactories.c)

1 files changed, 39 insertions, 63 deletions

diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc
index 9cacee4b450..6ad9171bfbc 100644
--- a/mysys_ssl/my_crypt.cc
+++ b/mysys_ssl/my_crypt.cc
@@ -1,6 +1,6 @@
 /*
  Copyright (c) 2014 Google Inc.
- Copyright (c) 2014, 2015 MariaDB Corporation
+ Copyright (c) 2014, 2017 MariaDB Corporation
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -21,30 +21,31 @@
 #ifdef HAVE_YASSL
 #include "yassl.cc"
 #else
-
 #include <openssl/evp.h>
 #include <openssl/aes.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
-
 #endif
-#include <my_crypt.h>
 
-#define MY_CIPHER_CTX_SIZE 384
+#include <my_crypt.h>
+#include <ssl_compat.h>
 
 class MyCTX
 {
 public:
+  char ctx_buf[EVP_CIPHER_CTX_SIZE];
   EVP_CIPHER_CTX *ctx;
-  const uchar *key;
-  unsigned int klen;
-  MyCTX() {
-            ctx= EVP_CIPHER_CTX_new();
-          }
-  virtual ~MyCTX() {
-                     EVP_CIPHER_CTX_free(ctx);
-                     ERR_remove_state(0);
-                    }
+
+  MyCTX()
+  {
+    ctx= (EVP_CIPHER_CTX *)ctx_buf;
+    EVP_CIPHER_CTX_init(ctx);
+  }
+  virtual ~MyCTX()
+  {
+    EVP_CIPHER_CTX_cleanup(ctx);
+    ERR_remove_state(0);
+  }
 
   virtual int init(const EVP_CIPHER *cipher, int encrypt, const uchar *key,
                    uint klen, const uchar *iv, uint ivlen)
@@ -78,9 +79,12 @@ public:
 class MyCTX_nopad : public MyCTX
 {
 public:
+  const uchar *key;
+  uint klen, buf_len;
+  uchar oiv[MY_AES_BLOCK_SIZE];
+
   MyCTX_nopad() : MyCTX() { }
   ~MyCTX_nopad() { }
-  unsigned int buf_len;
 
   int init(const EVP_CIPHER *cipher, int encrypt, const uchar *key, uint klen,
            const uchar *iv, uint ivlen)
@@ -89,19 +93,8 @@ public:
     this->key= key;
     this->klen= klen;
     this->buf_len= 0;
-    /* FIX-ME:
-       For the sake of backward compatibility we do some strange hack here:
-       Since ECB doesn't need an IV (and therefore is considered kind of
-       insecure) we need to store the specified iv.
-       The last nonpadding block will be encrypted with an additional
-       expensive crypt_call in ctr mode instead
-       of encrypting the entire plain text in ctr-mode */
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-    const unsigned char *oiv= EVP_CIPHER_CTX_original_iv(ctx);
-#else
-    const unsigned char *oiv= ctx->oiv;
-#endif
-    memcpy((char *)oiv, iv, ivlen);
+    memcpy(oiv, iv, ivlen);
+    DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv));
 
     int res= MyCTX::init(cipher, encrypt, key, klen, iv, ivlen);
 
@@ -111,34 +104,30 @@ public:
 
   int update(const uchar *src, uint slen, uchar *dst, uint *dlen)
   {
-    buf_len= slen % MY_AES_BLOCK_SIZE;
+    buf_len+= slen;
     return MyCTX::update(src, slen, dst, dlen);
   }
 
   int finish(uchar *dst, uint *dlen)
   {
+    buf_len %= MY_AES_BLOCK_SIZE;
     if (buf_len)
     {
-      const uchar *org_iv;
-      unsigned char *buf;
+      uchar *buf= EVP_CIPHER_CTX_buf_noconst(ctx);
       /*
         Not much we can do, block ciphers cannot encrypt data that aren't
         a multiple of the block length. At least not without padding.
         Let's do something CTR-like for the last partial block.
+
+        NOTE this assumes that there are only buf_len bytes in the buf.
+        If OpenSSL will change that, we'll need to change the implementation
+        of this class too.
       */
       uchar mask[MY_AES_BLOCK_SIZE];
       uint mlen;
 
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
-      org_iv= EVP_CIPHER_CTX_original_iv(ctx);
-      buf= EVP_CIPHER_CTX_buf_noconst(ctx);
-#else
-      org_iv= ctx->oiv;
-      buf= ctx->buf;
-#endif
-
       my_aes_crypt(MY_AES_ECB, ENCRYPTION_FLAG_ENCRYPT | ENCRYPTION_FLAG_NOPAD,
-                   org_iv, sizeof(mask), mask, &mlen, key, klen, 0, 0);
+                   oiv, sizeof(mask), mask, &mlen, key, klen, 0, 0);
       DBUG_ASSERT(mlen == sizeof(mask));
 
       for (uint i=0; i < buf_len; i++)
@@ -178,9 +167,8 @@ make_aes_dispatcher(gcm)
 class MyCTX_gcm : public MyCTX
 {
 public:
-  const uchar *aad= NULL;
+  const uchar *aad;
   int aadlen;
-  my_bool encrypt;
   MyCTX_gcm() : MyCTX() { }
   ~MyCTX_gcm() { }
 
@@ -192,7 +180,6 @@ public:
     int real_ivlen= EVP_CIPHER_CTX_iv_length(ctx);
     aad= iv + real_ivlen;
     aadlen= ivlen - real_ivlen;
-    this->encrypt= encrypt;
     return res;
   }
 
@@ -204,7 +191,7 @@ public:
       before decrypting the data. it can encrypt data piecewise, like, first
       half, then the second half, but it must decrypt all at once
     */
-    if (!this->encrypt)
+    if (!EVP_CIPHER_CTX_encrypting(ctx))
     {
       /* encrypted string must contain authenticaton tag (see MDEV-11174) */
       if (slen < MY_AES_BLOCK_SIZE)
@@ -214,7 +201,7 @@ public:
                               (void*)(src + slen)))
         return MY_AES_OPENSSL_ERROR;
     }
-    int unused= 0;
+    int unused;
     if (aadlen && !EVP_CipherUpdate(ctx, NULL, &unused, aad, aadlen))
       return MY_AES_OPENSSL_ERROR;
     aadlen= 0;
@@ -223,12 +210,12 @@ public:
 
   int finish(uchar *dst, uint *dlen)
   {
-    int fin= 0;
+    int fin;
     if (!EVP_CipherFinal_ex(ctx, dst, &fin))
       return MY_AES_BAD_DATA;
     DBUG_ASSERT(fin == 0);
 
-    if (this->encrypt)
+    if (EVP_CIPHER_CTX_encrypting(ctx))
     {
       if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, MY_AES_BLOCK_SIZE, dst))
         return MY_AES_OPENSSL_ERROR;
@@ -298,20 +285,15 @@ int my_aes_crypt(enum my_aes_mode mode, int flags,
 {
   void *ctx= alloca(MY_AES_CTX_SIZE);
   int res1, res2;
-  uint d1= 0, d2= 0;
+  uint d1= 0, d2;
   if ((res1= my_aes_crypt_init(ctx, mode, flags, key, klen, iv, ivlen)))
     return res1;
   res1= my_aes_crypt_update(ctx, src, slen, dst, &d1);
   res2= my_aes_crypt_finish(ctx, dst + d1, &d2);
-  *dlen= d1 + d2;
-  /* in case of failure clear error queue */
-#ifndef HAVE_YASSL
-  /* since we don't check the crypto error messages we need to
-     clear the error queue - otherwise subsequent crypto or tls/ssl
-     calls will fail */
-  if (!*dlen)
-    ERR_clear_error();
-#endif
+  if (res1 || res2)
+    ERR_remove_state(0); /* in case of failure clear error queue */
+  else
+    *dlen= d1 + d2;
   return res1 ? res1 : res2;
 }
 
@@ -353,12 +335,6 @@ int my_random_bytes(uchar* buf, int num)
 
 int my_random_bytes(uchar *buf, int num)
 {
-  /*
-    Unfortunately RAND_bytes manual page does not provide any guarantees
-    in relation to blocking behavior. Here we explicitly use SSLeay random
-    instead of whatever random engine is currently set in OpenSSL. That way
-    we are guaranteed to have a non-blocking random.
-  */
   RAND_METHOD *rand = RAND_OpenSSL();
   if (rand == NULL || rand->bytes(buf, num) != 1)
     return MY_AES_OPENSSL_ERROR;