Add a test case for the engine crash with AES-256-CTR

Implement the AES-256-CTR cipher in the dasync engine.

Use that to reproduce the reported problems with the
devcrypto engine in our normal test environment.

See #17995 and #17532 for details.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19386)

1 files changed, 67 insertions, 0 deletions

diff --git a/engines/e_dasync.c b/engines/e_dasync.c
index b0a08654cc..f6771abad1 100644
--- a/engines/e_dasync.c
+++ b/engines/e_dasync.c
@@ -139,6 +139,14 @@ static int dasync_aes128_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                                     const unsigned char *in, size_t inl);
 static int dasync_aes128_cbc_cleanup(EVP_CIPHER_CTX *ctx);
 
+static int dasync_aes256_ctr_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg,
+                                  void *ptr);
+static int dasync_aes256_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
+                                  const unsigned char *iv, int enc);
+static int dasync_aes256_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
+                                    const unsigned char *in, size_t inl);
+static int dasync_aes256_ctr_cleanup(EVP_CIPHER_CTX *ctx);
+
 static int dasync_aes128_cbc_hmac_sha1_ctrl(EVP_CIPHER_CTX *ctx, int type,
                                              int arg, void *ptr);
 static int dasync_aes128_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx,
@@ -171,6 +179,12 @@ static const EVP_CIPHER *dasync_aes_128_cbc(void)
     return _hidden_aes_128_cbc;
 }
 
+static EVP_CIPHER *_hidden_aes_256_ctr = NULL;
+static const EVP_CIPHER *dasync_aes_256_ctr(void)
+{
+    return _hidden_aes_256_ctr;
+}
+
 /*
  * Holds the EVP_CIPHER object for aes_128_cbc_hmac_sha1 in this engine. Set up
  * once only during engine bind and can then be reused many times.
@@ -192,8 +206,10 @@ static const EVP_CIPHER *dasync_aes_128_cbc_hmac_sha1(void)
 static void destroy_ciphers(void)
 {
     EVP_CIPHER_meth_free(_hidden_aes_128_cbc);
+    EVP_CIPHER_meth_free(_hidden_aes_256_ctr);
     EVP_CIPHER_meth_free(_hidden_aes_128_cbc_hmac_sha1);
     _hidden_aes_128_cbc = NULL;
+    _hidden_aes_256_ctr = NULL;
     _hidden_aes_128_cbc_hmac_sha1 = NULL;
 }
 
@@ -202,6 +218,7 @@ static int dasync_ciphers(ENGINE *e, const EVP_CIPHER **cipher,
 
 static int dasync_cipher_nids[] = {
     NID_aes_128_cbc,
+    NID_aes_256_ctr,
     NID_aes_128_cbc_hmac_sha1,
     0
 };
@@ -283,6 +300,30 @@ static int bind_dasync(ENGINE *e)
         _hidden_aes_128_cbc = NULL;
     }
 
+    _hidden_aes_256_ctr = EVP_CIPHER_meth_new(NID_aes_256_ctr,
+                                              1  /* block size */,
+                                              32 /* key len */);
+    if (_hidden_aes_256_ctr == NULL
+            || !EVP_CIPHER_meth_set_iv_length(_hidden_aes_256_ctr,16)
+            || !EVP_CIPHER_meth_set_flags(_hidden_aes_256_ctr,
+                                          EVP_CIPH_FLAG_DEFAULT_ASN1
+                                          | EVP_CIPH_CTR_MODE
+                                          | EVP_CIPH_FLAG_PIPELINE
+                                          | EVP_CIPH_CUSTOM_COPY)
+            || !EVP_CIPHER_meth_set_init(_hidden_aes_256_ctr,
+                                         dasync_aes256_init_key)
+            || !EVP_CIPHER_meth_set_do_cipher(_hidden_aes_256_ctr,
+                                              dasync_aes256_ctr_cipher)
+            || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_256_ctr,
+                                            dasync_aes256_ctr_cleanup)
+            || !EVP_CIPHER_meth_set_ctrl(_hidden_aes_256_ctr,
+                                         dasync_aes256_ctr_ctrl)
+            || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_256_ctr,
+                                sizeof(struct dasync_pipeline_ctx))) {
+        EVP_CIPHER_meth_free(_hidden_aes_256_ctr);
+        _hidden_aes_256_ctr = NULL;
+    }
+
     _hidden_aes_128_cbc_hmac_sha1 = EVP_CIPHER_meth_new(
                                                 NID_aes_128_cbc_hmac_sha1,
                                                 16 /* block size */,
@@ -444,6 +485,9 @@ static int dasync_ciphers(ENGINE *e, const EVP_CIPHER **cipher,
     case NID_aes_128_cbc:
         *cipher = dasync_aes_128_cbc();
         break;
+    case NID_aes_256_ctr:
+        *cipher = dasync_aes_256_ctr();
+        break;
     case NID_aes_128_cbc_hmac_sha1:
         *cipher = dasync_aes_128_cbc_hmac_sha1();
         break;
@@ -775,6 +819,29 @@ static int dasync_aes128_cbc_cleanup(EVP_CIPHER_CTX *ctx)
     return dasync_cipher_cleanup_helper(ctx, EVP_aes_128_cbc());
 }
 
+static int dasync_aes256_ctr_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg,
+                                  void *ptr)
+{
+    return dasync_cipher_ctrl_helper(ctx, type, arg, ptr, 0, EVP_aes_256_ctr());
+}
+
+static int dasync_aes256_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
+                             const unsigned char *iv, int enc)
+{
+    return dasync_cipher_init_key_helper(ctx, key, iv, enc, EVP_aes_256_ctr());
+}
+
+static int dasync_aes256_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
+                               const unsigned char *in, size_t inl)
+{
+    return dasync_cipher_helper(ctx, out, in, inl, EVP_aes_256_ctr());
+}
+
+static int dasync_aes256_ctr_cleanup(EVP_CIPHER_CTX *ctx)
+{
+    return dasync_cipher_cleanup_helper(ctx, EVP_aes_256_ctr());
+}
+
 
 /*
  * AES128 CBC HMAC SHA1 Implementation