SM4 AESE optimization for ARMv8

Signed-off-by: Xu Yizhou <xuyizhou1@huawei.com>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19914)

4 files changed, 47 insertions, 18 deletions

diff --git a/providers/implementations/ciphers/cipher_sm4_hw.c b/providers/implementations/ciphers/cipher_sm4_hw.c
index 1fd916a565..d8bc5a1e85 100644
--- a/providers/implementations/ciphers/cipher_sm4_hw.c
+++ b/providers/implementations/ciphers/cipher_sm4_hw.c
@@ -42,6 +42,19 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx,
             (void)0;            /* terminate potentially open 'else' */
         } else
 #endif
+#ifdef VPSM4_EX_CAPABLE
+        if (VPSM4_EX_CAPABLE) {
+            vpsm4_ex_set_encrypt_key(key, ks);
+            ctx->block = (block128_f)vpsm4_ex_encrypt;
+            ctx->stream.cbc = NULL;
+            if (ctx->mode == EVP_CIPH_CBC_MODE)
+                ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt;
+            else if (ctx->mode == EVP_CIPH_ECB_MODE)
+                ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt;
+            else if (ctx->mode == EVP_CIPH_CTR_MODE)
+                ctx->stream.ctr = (ctr128_f)vpsm4_ex_ctr32_encrypt_blocks;
+        } else
+#endif
 #ifdef VPSM4_CAPABLE
         if (VPSM4_CAPABLE) {
             vpsm4_set_encrypt_key(key, ks);
@@ -75,6 +88,17 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx,
 #endif
         } else
 #endif
+#ifdef VPSM4_EX_CAPABLE
+        if (VPSM4_EX_CAPABLE) {
+            vpsm4_ex_set_decrypt_key(key, ks);
+            ctx->block = (block128_f)vpsm4_ex_decrypt;
+            ctx->stream.cbc = NULL;
+            if (ctx->mode == EVP_CIPH_CBC_MODE)
+                ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt;
+            else if (ctx->mode == EVP_CIPH_ECB_MODE)
+                ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt;
+        } else
+#endif
 #ifdef VPSM4_CAPABLE
         if (VPSM4_CAPABLE) {
             vpsm4_set_decrypt_key(key, ks);
@@ -82,7 +106,7 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx,
             ctx->stream.cbc = NULL;
             if (ctx->mode == EVP_CIPH_CBC_MODE)
                 ctx->stream.cbc = (cbc128_f)vpsm4_cbc_encrypt;
-        else if (ctx->mode == EVP_CIPH_ECB_MODE)
+            else if (ctx->mode == EVP_CIPH_ECB_MODE)
                 ctx->stream.ecb = (ecb128_f)vpsm4_ecb_encrypt;
         } else
 #endif
diff --git a/providers/implementations/ciphers/cipher_sm4_xts.c b/providers/implementations/ciphers/cipher_sm4_xts.c
index 3c568d4d18..037055fce8 100644
--- a/providers/implementations/ciphers/cipher_sm4_xts.c
+++ b/providers/implementations/ciphers/cipher_sm4_xts.c
@@ -145,14 +145,14 @@ static int sm4_xts_cipher(void *vctx, unsigned char *out, size_t *outl,
     if (ctx->xts_standard) {
         if (ctx->stream != NULL)
             (*ctx->stream)(in, out, inl, ctx->xts.key1, ctx->xts.key2,
-                           ctx->base.iv);
+                           ctx->base.iv, ctx->base.enc);
         else if (CRYPTO_xts128_encrypt(&ctx->xts, ctx->base.iv, in, out, inl,
                                        ctx->base.enc))
             return 0;
     } else {
         if (ctx->stream_gb != NULL)
             (*ctx->stream_gb)(in, out, inl, ctx->xts.key1, ctx->xts.key2,
-                              ctx->base.iv);
+                              ctx->base.iv, ctx->base.enc);
         else if (ossl_crypto_xts128gb_encrypt(&ctx->xts, ctx->base.iv, in, out,
                                               inl, ctx->base.enc))
             return 0;
diff --git a/providers/implementations/ciphers/cipher_sm4_xts.h b/providers/implementations/ciphers/cipher_sm4_xts.h
index 4c369183e2..cfca596979 100644
--- a/providers/implementations/ciphers/cipher_sm4_xts.h
+++ b/providers/implementations/ciphers/cipher_sm4_xts.h
@@ -14,7 +14,7 @@
 PROV_CIPHER_FUNC(void, xts_stream,
                  (const unsigned char *in, unsigned char *out, size_t len,
                   const SM4_KEY *key1, const SM4_KEY *key2,
-                  const unsigned char iv[16]));
+                  const unsigned char iv[16], const int enc));
 
 typedef struct prov_sm4_xts_ctx_st {
     /* Must be first */
diff --git a/providers/implementations/ciphers/cipher_sm4_xts_hw.c b/providers/implementations/ciphers/cipher_sm4_xts_hw.c
index 403eb879b1..67a9923d94 100644
--- a/providers/implementations/ciphers/cipher_sm4_xts_hw.c
+++ b/providers/implementations/ciphers/cipher_sm4_xts_hw.c
@@ -11,8 +11,7 @@
 
 #define XTS_SET_KEY_FN(fn_set_enc_key, fn_set_dec_key,                         \
                        fn_block_enc, fn_block_dec,                             \
-                       fn_stream_enc, fn_stream_dec,                           \
-                       fn_stream_gb_enc, fn_stream_gb_dec) {                   \
+                       fn_stream, fn_stream_gb) {                              \
     size_t bytes = keylen / 2;                                                 \
                                                                                \
     if (ctx->enc) {                                                            \
@@ -26,8 +25,8 @@
     xctx->xts.block2 = (block128_f)fn_block_enc;                               \
     xctx->xts.key1 = &xctx->ks1;                                               \
     xctx->xts.key2 = &xctx->ks2;                                               \
-    xctx->stream = ctx->enc ? fn_stream_enc : fn_stream_dec;                   \
-    xctx->stream_gb = ctx->enc ? fn_stream_gb_enc : fn_stream_gb_dec;          \
+    xctx->stream = fn_stream;                                                  \
+    xctx->stream_gb = fn_stream_gb;                                            \
 }
 
 static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
@@ -35,23 +34,30 @@ static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
                                              size_t keylen)
 {
     PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)ctx;
-    OSSL_xts_stream_fn stream_enc = NULL;
-    OSSL_xts_stream_fn stream_dec = NULL;
-    OSSL_xts_stream_fn stream_gb_enc = NULL;
-    OSSL_xts_stream_fn stream_gb_dec = NULL;
+    OSSL_xts_stream_fn stream = NULL;
+    OSSL_xts_stream_fn stream_gb = NULL;
 #ifdef HWSM4_CAPABLE
     if (HWSM4_CAPABLE) {
         XTS_SET_KEY_FN(HWSM4_set_encrypt_key, HWSM4_set_decrypt_key,
-                       HWSM4_encrypt, HWSM4_decrypt, stream_enc, stream_dec,
-                       stream_gb_enc, stream_gb_dec);
+                       HWSM4_encrypt, HWSM4_decrypt, stream, stream_gb);
         return 1;
     } else
 #endif /* HWSM4_CAPABLE */
+#ifdef VPSM4_EX_CAPABLE
+    if (VPSM4_EX_CAPABLE) {
+        stream = vpsm4_ex_xts_encrypt;
+        stream_gb = vpsm4_ex_xts_encrypt_gb;
+        XTS_SET_KEY_FN(vpsm4_ex_set_encrypt_key, vpsm4_ex_set_decrypt_key,
+                       vpsm4_ex_encrypt, vpsm4_ex_decrypt, stream, stream_gb);
+        return 1;
+    } else
+#endif /* VPSM4_EX_CAPABLE */
 #ifdef VPSM4_CAPABLE
     if (VPSM4_CAPABLE) {
+        stream = vpsm4_xts_encrypt;
+        stream_gb = vpsm4_xts_encrypt_gb;
         XTS_SET_KEY_FN(vpsm4_set_encrypt_key, vpsm4_set_decrypt_key,
-                       vpsm4_encrypt, vpsm4_decrypt, stream_enc, stream_dec,
-                       stream_gb_enc, stream_gb_dec);
+                       vpsm4_encrypt, vpsm4_decrypt, stream, stream_gb);
         return 1;
     } else
 #endif /* VPSM4_CAPABLE */
@@ -60,8 +66,7 @@ static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx,
     }
     {
         XTS_SET_KEY_FN(ossl_sm4_set_key, ossl_sm4_set_key, ossl_sm4_encrypt,
-                       ossl_sm4_decrypt, stream_enc, stream_dec, stream_gb_enc,
-                       stream_gb_dec);
+                       ossl_sm4_decrypt, stream, stream_gb);
     }
     return 1;
 }