MDEV-25785 Add support for OpenSSL 3.0

Summary of changes

- MD_CTX_SIZE is increased

- EVP_CIPHER_CTX_buf_noconst(ctx) does not work anymore, points
  to nobody knows where. The assumption made previously was that
  (since the function does not seem to be documented)
  was that it points to the last partial source block.
  Add own partial block buffer for NOPAD encryption instead

- SECLEVEL in CipherString in openssl.cnf
  had been downgraded to 0, from 1, to make TLSv1.0 and TLSv1.1 possible
   (according to https://github.com/openssl/openssl/blob/openssl-3.0.0/NEWS.md
   even though the manual for SSL_CTX_get_security_level claims that it
   should not be necessary)

- Workaround Ssl_cipher_list issue, it now returns TLSv1.3 ciphers,
  in addition to what was set in --ssl-cipher

- ctx_buf buffer now must be aligned to 16 bytes with openssl(
  previously with WolfSSL only), ot crashes will happen

- updated aes-t , to be better debuggable
  using function, rather than a huge multiline macro
  added test that does "nopad" encryption piece-wise, to test
  replacement of EVP_CIPHER_CTX_buf_noconst

part of MDEV-29000

1 files changed, 29 insertions, 17 deletions

diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc
index e512eee9066..4d7ebc7bd27 100644
--- a/mysys_ssl/my_crypt.cc
+++ b/mysys_ssl/my_crypt.cc
@@ -29,11 +29,7 @@
 #include <ssl_compat.h>
 #include <cstdint>
 
-#ifdef HAVE_WOLFSSL
 #define CTX_ALIGN 16
-#else
-#define CTX_ALIGN 0
-#endif
 
 class MyCTX
 {
@@ -100,8 +96,9 @@ class MyCTX_nopad : public MyCTX
 {
 public:
   const uchar *key;
-  uint klen, buf_len;
+  uint klen, source_tail_len;
   uchar oiv[MY_AES_BLOCK_SIZE];
+  uchar source_tail[MY_AES_BLOCK_SIZE];
 
   MyCTX_nopad() : MyCTX() { }
   ~MyCTX_nopad() { }
@@ -112,7 +109,7 @@ public:
     compile_time_assert(MY_AES_CTX_SIZE >= sizeof(MyCTX_nopad));
     this->key= key;
     this->klen= klen;
-    this->buf_len= 0;
+    this->source_tail_len= 0;
     if (ivlen)
       memcpy(oiv, iv, ivlen);
     DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv));
@@ -123,26 +120,41 @@ public:
     return res;
   }
 
+  /** Update last partial source block, stored in source_tail array. */
+  void update_source_tail(const uchar* src, uint slen)
+  {
+    if (!slen)
+      return;
+    uint new_tail_len= (source_tail_len + slen) % MY_AES_BLOCK_SIZE;
+    if (new_tail_len)
+    {
+      if (slen + source_tail_len < MY_AES_BLOCK_SIZE)
+      {
+        memcpy(source_tail + source_tail_len, src, slen);
+      }
+      else
+      {
+        DBUG_ASSERT(slen > new_tail_len);
+        memcpy(source_tail, src + slen - new_tail_len, new_tail_len);
+      }
+    }
+    source_tail_len= new_tail_len;
+  }
+
   int update(const uchar *src, uint slen, uchar *dst, uint *dlen)
   {
-    buf_len+= slen;
+    update_source_tail(src, slen);
     return MyCTX::update(src, slen, dst, dlen);
   }
 
   int finish(uchar *dst, uint *dlen)
   {
-    buf_len %= MY_AES_BLOCK_SIZE;
-    if (buf_len)
+    if (source_tail_len)
     {
-      uchar *buf= EVP_CIPHER_CTX_buf_noconst(ctx);
       /*
         Not much we can do, block ciphers cannot encrypt data that aren't
         a multiple of the block length. At least not without padding.
         Let's do something CTR-like for the last partial block.
-
-        NOTE this assumes that there are only buf_len bytes in the buf.
-        If OpenSSL will change that, we'll need to change the implementation
-        of this class too.
       */
       uchar mask[MY_AES_BLOCK_SIZE];
       uint mlen;
@@ -154,10 +166,10 @@ public:
         return rc;
       DBUG_ASSERT(mlen == sizeof(mask));
 
-      for (uint i=0; i < buf_len; i++)
-        dst[i]= buf[i] ^ mask[i];
+      for (uint i=0; i < source_tail_len; i++)
+        dst[i]= source_tail[i] ^ mask[i];
     }
-    *dlen= buf_len;
+    *dlen= source_tail_len;
     return MY_AES_OK;
   }
 };