// rijndael-simd.cpp - written and placed in the public domain by // Jeffrey Walton, Uri Blumenthal and Marcel Raad. // AES-NI code originally written by Wei Dai. // // This source file uses intrinsics and built-ins to gain access to // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source // file is needed because additional CXXFLAGS are required to enable // the appropriate instructions sets in some build configurations. // // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders, // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping // mbedTLS under a debugger was helped for us to determine problems // with our subkey generation and scheduling. // // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/ // For Power8 do not remove the casts, even when const-ness is cast away. It causes // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering // questions and filling the gaps in the IBM documentation. // #include "pch.h" #include "config.h" #include "misc.h" #include "adv-simd.h" // We set CRYPTOPP_POWER8_CRYPTO_AVAILABLE based on compiler version. // If the crypto is not available, then we have to disable it here. #if !(defined(__CRYPTO) || defined(_ARCH_PWR8) || defined(_ARCH_PWR9)) # undef CRYPTOPP_POWER8_CRYPTO_AVAILABLE # undef CRYPTOPP_POWER8_AES_AVAILABLE #endif #if (CRYPTOPP_AESNI_AVAILABLE) # include # include #endif // Use ARMv8 rather than NEON due to compiler inconsistencies #if (CRYPTOPP_ARM_AES_AVAILABLE) # include #endif // Can't use CRYPTOPP_ARM_XXX_AVAILABLE because too many // compilers don't follow ACLE conventions for the include. #if defined(CRYPTOPP_ARM_ACLE_AVAILABLE) # include # include #endif #if defined(CRYPTOPP_POWER8_AES_AVAILABLE) # include "ppc-simd.h" #endif #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY # include # include #endif #ifndef EXCEPTION_EXECUTE_HANDLER # define EXCEPTION_EXECUTE_HANDLER 1 #endif // Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670 #define M128_CAST(x) ((__m128i *)(void *)(x)) #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x)) // Squash MS LNK4221 and libtool warnings extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__; NAMESPACE_BEGIN(CryptoPP) #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY extern "C" { typedef void (*SigHandler)(int); static jmp_buf s_jmpSIGILL; static void SigIllHandler(int) { longjmp(s_jmpSIGILL, 1); } } #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64) bool CPU_ProbeAES() { #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES) return false; #elif (CRYPTOPP_ARM_AES_AVAILABLE) # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY) volatile bool result = true; __try { // AES encrypt and decrypt uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0); uint8x16_t r1 = vaeseq_u8(data, key); uint8x16_t r2 = vaesdq_u8(data, key); r1 = vaesmcq_u8(r1); r2 = vaesimcq_u8(r2); result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7)); } __except (EXCEPTION_EXECUTE_HANDLER) { return false; } return result; # else // longjmp and clobber warnings. Volatile is required. // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854 volatile bool result = true; volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler); if (oldHandler == SIG_ERR) return false; volatile sigset_t oldMask; if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask)) return false; if (setjmp(s_jmpSIGILL)) result = false; else { uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0); uint8x16_t r1 = vaeseq_u8(data, key); uint8x16_t r2 = vaesdq_u8(data, key); r1 = vaesmcq_u8(r1); r2 = vaesimcq_u8(r2); // Hack... GCC optimizes away the code and returns true result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7)); } sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR); signal(SIGILL, oldHandler); return result; # endif #else return false; #endif // CRYPTOPP_ARM_AES_AVAILABLE } #endif // ARM32 or ARM64 // ***************************** ARMv8 ***************************** // #if (CRYPTOPP_ARM_AES_AVAILABLE) ANONYMOUS_NAMESPACE_BEGIN static inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds) { CRYPTOPP_ASSERT(subkeys); const byte *keys = reinterpret_cast(subkeys); uint8x16_t block = vreinterpretq_u8_u64(data); // AES single round encryption block = vaeseq_u8(block, vld1q_u8(keys+0*16)); // AES mix columns block = vaesmcq_u8(block); for (unsigned int i=1; i(subkeys); uint8x16_t block0 = vreinterpretq_u8_u64(data0); uint8x16_t block1 = vreinterpretq_u8_u64(data1); uint8x16_t block2 = vreinterpretq_u8_u64(data2); uint8x16_t block3 = vreinterpretq_u8_u64(data3); uint8x16_t block4 = vreinterpretq_u8_u64(data4); uint8x16_t block5 = vreinterpretq_u8_u64(data5); uint8x16_t key; for (unsigned int i=0; i(subkeys); uint8x16_t block = vreinterpretq_u8_u64(data); // AES single round decryption block = vaesdq_u8(block, vld1q_u8(keys+0*16)); // AES inverse mix columns block = vaesimcq_u8(block); for (unsigned int i=1; i(subkeys); uint8x16_t block0 = vreinterpretq_u8_u64(data0); uint8x16_t block1 = vreinterpretq_u8_u64(data1); uint8x16_t block2 = vreinterpretq_u8_u64(data2); uint8x16_t block3 = vreinterpretq_u8_u64(data3); uint8x16_t block4 = vreinterpretq_u8_u64(data4); uint8x16_t block5 = vreinterpretq_u8_u64(data5); uint8x16_t key; for (unsigned int i=0; i(subkeys); block = _mm_xor_si128(block, skeys[0]); for (unsigned int i=1; i(subkeys); __m128i rk = skeys[0]; block0 = _mm_xor_si128(block0, rk); block1 = _mm_xor_si128(block1, rk); block2 = _mm_xor_si128(block2, rk); block3 = _mm_xor_si128(block3, rk); for (unsigned int i=1; i(subkeys); block = _mm_xor_si128(block, skeys[0]); for (unsigned int i=1; i(subkeys); __m128i rk = skeys[0]; block0 = _mm_xor_si128(block0, rk); block1 = _mm_xor_si128(block1, rk); block2 = _mm_xor_si128(block2, rk); block3 = _mm_xor_si128(block3, rk); for (unsigned int i=1; i(subkeys); uint32x4_p k = VectorLoadKey(keys); block = VectorXor(block, k); for (size_t i=1; i(subkeys); uint32x4_p k = VectorLoadKey(keys); block0 = VectorXor(block0, k); block1 = VectorXor(block1, k); block2 = VectorXor(block2, k); block3 = VectorXor(block3, k); block4 = VectorXor(block4, k); block5 = VectorXor(block5, k); for (size_t i=1; i(subkeys); uint32x4_p k = VectorLoadKey(rounds*16, keys); block = VectorXor(block, k); for (size_t i=rounds-1; i>1; i-=2) { block = VectorDecrypt(block, VectorLoadKey( i*16, keys)); block = VectorDecrypt(block, VectorLoadKey((i-1)*16, keys)); } block = VectorDecrypt(block, VectorLoadKey(16, keys)); block = VectorDecryptLast(block, VectorLoadKey(0, keys)); } static inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1, uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4, uint32x4_p &block5, const word32 *subkeys, unsigned int rounds) { CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16)); const byte *keys = reinterpret_cast(subkeys); uint32x4_p k = VectorLoadKey(rounds*16, keys); block0 = VectorXor(block0, k); block1 = VectorXor(block1, k); block2 = VectorXor(block2, k); block3 = VectorXor(block3, k); block4 = VectorXor(block4, k); block5 = VectorXor(block5, k); for (size_t i=rounds-1; i>0; --i) { k = VectorLoadKey(i*16, keys); block0 = VectorDecrypt(block0, k); block1 = VectorDecrypt(block1, k); block2 = VectorDecrypt(block2, k); block3 = VectorDecrypt(block3, k); block4 = VectorDecrypt(block4, k); block5 = VectorDecrypt(block5, k); } k = VectorLoadKey(0, keys); block0 = VectorDecryptLast(block0, k); block1 = VectorDecryptLast(block1, k); block2 = VectorDecryptLast(block2, k); block3 = VectorDecryptLast(block3, k); block4 = VectorDecryptLast(block4, k); block5 = VectorDecryptLast(block5, k); } ANONYMOUS_NAMESPACE_END void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se) { const size_t rounds = keyLen / 4 + 6; const word32 *rc = s_rconBE; GetUserKey(BIG_ENDIAN_ORDER, rk, keyLen/4, userKey, keyLen); word32 *rk_saved = rk, temp; // unused in big-endian CRYPTOPP_UNUSED(rk_saved); // keySize: m_key allocates 4*(rounds+1) word32's. const size_t keySize = 4*(rounds+1); const word32* end = rk + keySize; while (true) { temp = rk[keyLen/4-1]; word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^ (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)]; rk[keyLen/4] = rk[0] ^ x ^ *(rc++); rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4]; rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1]; rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2]; if (rk + keyLen/4 + 4 == end) break; if (keyLen == 24) { rk[10] = rk[ 4] ^ rk[ 9]; rk[11] = rk[ 5] ^ rk[10]; } else if (keyLen == 32) { temp = rk[11]; rk[12] = rk[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)]; rk[13] = rk[ 5] ^ rk[12]; rk[14] = rk[ 6] ^ rk[13]; rk[15] = rk[ 7] ^ rk[14]; } rk += keyLen/4; } #if defined(CRYPTOPP_LITTLE_ENDIAN) rk = rk_saved; const uint8x16_p mask = ((uint8x16_p){12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3}); const uint8x16_p zero = {0}; unsigned int i=0; for (i=0; i