From c007203b94b6921ebc8103cb7ae51af554c86afe Mon Sep 17 00:00:00 2001 From: Xu Yizhou Date: Wed, 18 Jan 2023 09:55:02 +0800 Subject: SM4 AESE optimization for ARMv8 Signed-off-by: Xu Yizhou Reviewed-by: Tomas Mraz Reviewed-by: Paul Dale (Merged from https://github.com/openssl/openssl/pull/19914) --- providers/implementations/ciphers/cipher_sm4_hw.c | 26 ++++++++++++++++- providers/implementations/ciphers/cipher_sm4_xts.c | 4 +-- providers/implementations/ciphers/cipher_sm4_xts.h | 2 +- .../implementations/ciphers/cipher_sm4_xts_hw.c | 33 +++++++++++++--------- 4 files changed, 47 insertions(+), 18 deletions(-) (limited to 'providers') diff --git a/providers/implementations/ciphers/cipher_sm4_hw.c b/providers/implementations/ciphers/cipher_sm4_hw.c index 1fd916a565..d8bc5a1e85 100644 --- a/providers/implementations/ciphers/cipher_sm4_hw.c +++ b/providers/implementations/ciphers/cipher_sm4_hw.c @@ -42,6 +42,19 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx, (void)0; /* terminate potentially open 'else' */ } else #endif +#ifdef VPSM4_EX_CAPABLE + if (VPSM4_EX_CAPABLE) { + vpsm4_ex_set_encrypt_key(key, ks); + ctx->block = (block128_f)vpsm4_ex_encrypt; + ctx->stream.cbc = NULL; + if (ctx->mode == EVP_CIPH_CBC_MODE) + ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt; + else if (ctx->mode == EVP_CIPH_ECB_MODE) + ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt; + else if (ctx->mode == EVP_CIPH_CTR_MODE) + ctx->stream.ctr = (ctr128_f)vpsm4_ex_ctr32_encrypt_blocks; + } else +#endif #ifdef VPSM4_CAPABLE if (VPSM4_CAPABLE) { vpsm4_set_encrypt_key(key, ks); @@ -75,6 +88,17 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx, #endif } else #endif +#ifdef VPSM4_EX_CAPABLE + if (VPSM4_EX_CAPABLE) { + vpsm4_ex_set_decrypt_key(key, ks); + ctx->block = (block128_f)vpsm4_ex_decrypt; + ctx->stream.cbc = NULL; + if (ctx->mode == EVP_CIPH_CBC_MODE) + ctx->stream.cbc = (cbc128_f)vpsm4_ex_cbc_encrypt; + else if (ctx->mode == EVP_CIPH_ECB_MODE) + ctx->stream.ecb = (ecb128_f)vpsm4_ex_ecb_encrypt; + } else +#endif #ifdef VPSM4_CAPABLE if (VPSM4_CAPABLE) { vpsm4_set_decrypt_key(key, ks); @@ -82,7 +106,7 @@ static int cipher_hw_sm4_initkey(PROV_CIPHER_CTX *ctx, ctx->stream.cbc = NULL; if (ctx->mode == EVP_CIPH_CBC_MODE) ctx->stream.cbc = (cbc128_f)vpsm4_cbc_encrypt; - else if (ctx->mode == EVP_CIPH_ECB_MODE) + else if (ctx->mode == EVP_CIPH_ECB_MODE) ctx->stream.ecb = (ecb128_f)vpsm4_ecb_encrypt; } else #endif diff --git a/providers/implementations/ciphers/cipher_sm4_xts.c b/providers/implementations/ciphers/cipher_sm4_xts.c index 3c568d4d18..037055fce8 100644 --- a/providers/implementations/ciphers/cipher_sm4_xts.c +++ b/providers/implementations/ciphers/cipher_sm4_xts.c @@ -145,14 +145,14 @@ static int sm4_xts_cipher(void *vctx, unsigned char *out, size_t *outl, if (ctx->xts_standard) { if (ctx->stream != NULL) (*ctx->stream)(in, out, inl, ctx->xts.key1, ctx->xts.key2, - ctx->base.iv); + ctx->base.iv, ctx->base.enc); else if (CRYPTO_xts128_encrypt(&ctx->xts, ctx->base.iv, in, out, inl, ctx->base.enc)) return 0; } else { if (ctx->stream_gb != NULL) (*ctx->stream_gb)(in, out, inl, ctx->xts.key1, ctx->xts.key2, - ctx->base.iv); + ctx->base.iv, ctx->base.enc); else if (ossl_crypto_xts128gb_encrypt(&ctx->xts, ctx->base.iv, in, out, inl, ctx->base.enc)) return 0; diff --git a/providers/implementations/ciphers/cipher_sm4_xts.h b/providers/implementations/ciphers/cipher_sm4_xts.h index 4c369183e2..cfca596979 100644 --- a/providers/implementations/ciphers/cipher_sm4_xts.h +++ b/providers/implementations/ciphers/cipher_sm4_xts.h @@ -14,7 +14,7 @@ PROV_CIPHER_FUNC(void, xts_stream, (const unsigned char *in, unsigned char *out, size_t len, const SM4_KEY *key1, const SM4_KEY *key2, - const unsigned char iv[16])); + const unsigned char iv[16], const int enc)); typedef struct prov_sm4_xts_ctx_st { /* Must be first */ diff --git a/providers/implementations/ciphers/cipher_sm4_xts_hw.c b/providers/implementations/ciphers/cipher_sm4_xts_hw.c index 403eb879b1..67a9923d94 100644 --- a/providers/implementations/ciphers/cipher_sm4_xts_hw.c +++ b/providers/implementations/ciphers/cipher_sm4_xts_hw.c @@ -11,8 +11,7 @@ #define XTS_SET_KEY_FN(fn_set_enc_key, fn_set_dec_key, \ fn_block_enc, fn_block_dec, \ - fn_stream_enc, fn_stream_dec, \ - fn_stream_gb_enc, fn_stream_gb_dec) { \ + fn_stream, fn_stream_gb) { \ size_t bytes = keylen / 2; \ \ if (ctx->enc) { \ @@ -26,8 +25,8 @@ xctx->xts.block2 = (block128_f)fn_block_enc; \ xctx->xts.key1 = &xctx->ks1; \ xctx->xts.key2 = &xctx->ks2; \ - xctx->stream = ctx->enc ? fn_stream_enc : fn_stream_dec; \ - xctx->stream_gb = ctx->enc ? fn_stream_gb_enc : fn_stream_gb_dec; \ + xctx->stream = fn_stream; \ + xctx->stream_gb = fn_stream_gb; \ } static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx, @@ -35,23 +34,30 @@ static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx, size_t keylen) { PROV_SM4_XTS_CTX *xctx = (PROV_SM4_XTS_CTX *)ctx; - OSSL_xts_stream_fn stream_enc = NULL; - OSSL_xts_stream_fn stream_dec = NULL; - OSSL_xts_stream_fn stream_gb_enc = NULL; - OSSL_xts_stream_fn stream_gb_dec = NULL; + OSSL_xts_stream_fn stream = NULL; + OSSL_xts_stream_fn stream_gb = NULL; #ifdef HWSM4_CAPABLE if (HWSM4_CAPABLE) { XTS_SET_KEY_FN(HWSM4_set_encrypt_key, HWSM4_set_decrypt_key, - HWSM4_encrypt, HWSM4_decrypt, stream_enc, stream_dec, - stream_gb_enc, stream_gb_dec); + HWSM4_encrypt, HWSM4_decrypt, stream, stream_gb); return 1; } else #endif /* HWSM4_CAPABLE */ +#ifdef VPSM4_EX_CAPABLE + if (VPSM4_EX_CAPABLE) { + stream = vpsm4_ex_xts_encrypt; + stream_gb = vpsm4_ex_xts_encrypt_gb; + XTS_SET_KEY_FN(vpsm4_ex_set_encrypt_key, vpsm4_ex_set_decrypt_key, + vpsm4_ex_encrypt, vpsm4_ex_decrypt, stream, stream_gb); + return 1; + } else +#endif /* VPSM4_EX_CAPABLE */ #ifdef VPSM4_CAPABLE if (VPSM4_CAPABLE) { + stream = vpsm4_xts_encrypt; + stream_gb = vpsm4_xts_encrypt_gb; XTS_SET_KEY_FN(vpsm4_set_encrypt_key, vpsm4_set_decrypt_key, - vpsm4_encrypt, vpsm4_decrypt, stream_enc, stream_dec, - stream_gb_enc, stream_gb_dec); + vpsm4_encrypt, vpsm4_decrypt, stream, stream_gb); return 1; } else #endif /* VPSM4_CAPABLE */ @@ -60,8 +66,7 @@ static int cipher_hw_sm4_xts_generic_initkey(PROV_CIPHER_CTX *ctx, } { XTS_SET_KEY_FN(ossl_sm4_set_key, ossl_sm4_set_key, ossl_sm4_encrypt, - ossl_sm4_decrypt, stream_enc, stream_dec, stream_gb_enc, - stream_gb_dec); + ossl_sm4_decrypt, stream, stream_gb); } return 1; } -- cgit v1.2.1