// ppc_simd.h - written and placed in public domain by Jeffrey Walton /// \file ppc_simd.h /// \brief Support functions for PowerPC and vector operations /// \details This header provides an agnostic interface into Clang, GCC /// and IBM XL C/C++ compilers modulo their different built-in functions /// for accessing vector intructions. /// \details The abstractions are necesssary to support back to GCC 4.8 and /// XLC 11 and 12. GCC 4.8 and 4.9 are still popular, and they are the /// default compiler for GCC112, GCC118 and others on the compile farm. /// Older IBM XL C/C++ compilers also experience it due to lack of /// vec_xl and vec_xst support on some platforms. Modern /// compilers provide best support and don't need many of the hacks /// below. /// \details The library is tested with the following PowerPC machines and /// compilers. GCC110, GCC111, GCC112, GCC119 and GCC135 are provided by /// the GCC Compile Farm /// - PowerMac G5, OSX 10.5, POWER4, Apple GCC 4.0 /// - PowerMac G5, OSX 10.5, POWER4, Macports GCC 5.0 /// - GCC110, Linux, POWER7, GCC 4.8.5 /// - GCC110, Linux, POWER7, XLC 12.01 /// - GCC111, AIX, POWER7, GCC 4.8.1 /// - GCC111, AIX, POWER7, XLC 12.01 /// - GCC112, Linux, POWER8, GCC 4.8.5 /// - GCC112, Linux, POWER8, XLC 13.01 /// - GCC112, Linux, POWER8, Clang 7.0 /// - GCC119, AIX, POWER8, GCC 7.2.0 /// - GCC119, AIX, POWER8, XLC 13.01 /// - GCC135, Linux, POWER9, GCC 7.0 /// \details 12 machines are used for testing because the three compilers form /// five profiles. The profiles are listed below. /// - GCC (Linux GCC, Macports GCC, etc. Consistent across machines) /// - XLC 13.0 and earlier (all IBM components) /// - XLC 13.1 and later on Linux (LLVM front-end, no compatibility macros) /// - XLC 13.1 and later on Linux (LLVM front-end, -qxlcompatmacros option) /// - LLVM Clang (traditional Clang compiler) /// \details The LLVM front-end makes it tricky to write portable code because /// LLVM pretends to be other compilers but cannot consume other compiler's /// builtins. When using XLC with -qxlcompatmacros the compiler pretends to /// be GCC, Clang and XLC all at once but it can only consume it's variety /// of builtins. /// \details At Crypto++ 8.0 the various Vector{FuncName} were /// renamed to Vec{FuncName}. For example, VectorAnd was /// changed to VecAnd. The name change helped consolidate two /// slightly different implementations. /// \since Crypto++ 6.0, LLVM Clang compiler support since Crypto++ 8.0 // Use __ALTIVEC__, _ARCH_PWR7 and _ARCH_PWR8 when detecting actual // availaibility of the feature for the source file being compiled. The // preprocessor macros depend on compiler options like -maltivec; and // not compiler versions. // DO NOT USE this pattern in VecLoad and VecStore. We have to use the // spaghetti code tangled in preprocessor macros because XLC 12 generates // bad code in some places. To verify the bad code generation test on // GCC111 with XLC 12.01 installed. XLC 13.01 on GCC112 and GCC119 are OK. // // inline uint32x4_p VecLoad(const byte src[16]) // { // #if defined(_ARCH_PWR7) // return (uint32x4_p) *(uint8x16_p*)((byte*)src); // #else // return VecLoad_ALTIVEC(src); // #endif // } #ifndef CRYPTOPP_PPC_CRYPTO_H #define CRYPTOPP_PPC_CRYPTO_H #include "config.h" #include "misc.h" #if defined(__ALTIVEC__) # include # undef vector # undef pixel # undef bool #endif // IBM XLC on AIX does not define __CRYPTO__ like it should with -qarch=pwr8. // Crypto is available in XLC 13.1 and above. More LLVM front-end goodness. #if defined(_AIX) && defined(_ARCH_PWR8) && (__xlC__ >= 0xd01) # undef __CRYPTO__ # define __CRYPTO__ 1 #endif // Hack to detect early XLC compilers. XLC compilers for POWER7 use // vec_xlw4 and vec_xstw4 (and ld2 variants); not vec_xl and vec_st. // Some XLC compilers for POWER7 and above use vec_xl and vec_xst. // The way to tell the difference is, XLC compilers version 13.0 and // earlier use vec_xlw4 and vec_xstw4. XLC compilers 13.1 and later // are use vec_xl and vec_xst. The open question is, how to handle // early Clang compilers for POWER7. We know the latest Clang // compilers support vec_xl and vec_xst. Also see // https://www-01.ibm.com/support/docview.wss?uid=swg21683541. #if defined(__xlc__) && (__xlc__ < 0x0d01) # define __early_xlc__ 1 #endif #if defined(__xlC__) && (__xlC__ < 0x0d01) # define __early_xlC__ 1 #endif // VecLoad_ALTIVEC and VecStore_ALTIVEC are // too noisy on modern compilers #if CRYPTOPP_GCC_DIAGNOSTIC_AVAILABLE # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wdeprecated" #endif NAMESPACE_BEGIN(CryptoPP) #if defined(__ALTIVEC__) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \brief Vector of 8-bit elements /// \par Wraps /// __vector unsigned char /// \since Crypto++ 6.0 typedef __vector unsigned char uint8x16_p; /// \brief Vector of 16-bit elements /// \par Wraps /// __vector unsigned short /// \since Crypto++ 6.0 typedef __vector unsigned short uint16x8_p; /// \brief Vector of 32-bit elements /// \par Wraps /// __vector unsigned int /// \since Crypto++ 6.0 typedef __vector unsigned int uint32x4_p; #if defined(_ARCH_PWR7) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \brief Vector of 64-bit elements /// \details uint64x2_p is available on POWER7 and above. Some supporting /// functions, like 64-bit vec_add (vaddudm), did not /// arrive until POWER8. /// \par Wraps /// __vector unsigned long long /// \since Crypto++ 6.0 typedef __vector unsigned long long uint64x2_p; #endif // _ARCH_PWR7 /// \brief The 0 vector /// \returns a 32-bit vector of 0's /// \since Crypto++ 8.0 inline uint32x4_p VecZero() { const uint32x4_p v = {0,0,0,0}; return v; } /// \brief The 1 vector /// \returns a 32-bit vector of 1's /// \since Crypto++ 8.0 inline uint32x4_p VecOne() { const uint32x4_p v = {1,1,1,1}; return v; } /// \brief Reverse bytes in a vector /// \tparam T vector type /// \param data the vector /// \returns vector /// \details VecReverse() reverses the bytes in a vector /// \par Wraps /// vec_perm /// \since Crypto++ 6.0 template inline T VecReverse(const T data) { #if (_ARCH_PWR9) return (T)vec_revb((uint8x16_p)data); #else const uint8x16_p mask = {15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0}; return (T)vec_perm(data, data, mask); #endif } /// \name LOAD OPERATIONS //@{ /// \brief Loads a vector from a byte array /// \param src the byte array /// \details Loads a vector in native endian format from a byte array. /// \details VecLoad_ALTIVEC() uses vec_ld if the effective address /// of src is aligned. If unaligned it uses vec_lvsl, /// vec_ld, vec_perm and src. The fixups using /// vec_lvsl and vec_perm are relatively expensive so /// you should provide aligned memory adresses. /// \par Wraps /// vec_ld, vec_lvsl, vec_perm /// \since Crypto++ 6.0 inline uint32x4_p VecLoad_ALTIVEC(const byte src[16]) { // Avoid IsAlignedOn for convenience. uintptr_t eff = reinterpret_cast(src)+0; if (eff % 16 == 0) { return (uint32x4_p)vec_ld(0, src); } else { // http://www.nxp.com/docs/en/reference-manual/ALTIVECPEM.pdf const uint8x16_p perm = vec_lvsl(0, src); const uint8x16_p low = vec_ld(0, src); const uint8x16_p high = vec_ld(15, src); return (uint32x4_p)vec_perm(low, high, perm); } } /// \brief Loads a vector from a byte array /// \param src the byte array /// \param off offset into the src byte array /// \details Loads a vector in native endian format from a byte array. /// \details VecLoad_ALTIVEC() uses vec_ld if the effective address /// of src is aligned. If unaligned it uses vec_lvsl, /// vec_ld, vec_perm and src. /// \details The fixups using vec_lvsl and vec_perm are /// relatively expensive so you should provide aligned memory adresses. /// \par Wraps /// vec_ld, vec_lvsl, vec_perm /// \since Crypto++ 6.0 inline uint32x4_p VecLoad_ALTIVEC(int off, const byte src[16]) { // Avoid IsAlignedOn for convenience. uintptr_t eff = reinterpret_cast(src)+off; if (eff % 16 == 0) { return (uint32x4_p)vec_ld(off, src); } else { // http://www.nxp.com/docs/en/reference-manual/ALTIVECPEM.pdf const uint8x16_p perm = vec_lvsl(off, src); const uint8x16_p low = vec_ld(off, src); const uint8x16_p high = vec_ld(15, src); return (uint32x4_p)vec_perm(low, high, perm); } } /// \brief Loads a vector from a byte array /// \param src the byte array /// \details VecLoad() loads a vector in from a byte array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 6.0 inline uint32x4_p VecLoad(const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) return (uint32x4_p)vec_xlw4(0, (byte*)src); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl(0, (byte*)src); # else return (uint32x4_p)vec_vsx_ld(0, (byte*)src); # endif #else return VecLoad_ALTIVEC(src); #endif } /// \brief Loads a vector from a byte array /// \param src the byte array /// \param off offset into the byte array /// \details VecLoad() loads a vector in from a byte array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 6.0 inline uint32x4_p VecLoad(int off, const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) return (uint32x4_p)vec_xlw4(off, (byte*)src); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl(off, (byte*)src); # else return (uint32x4_p)vec_vsx_ld(off, (byte*)src); # endif #else return VecLoad_ALTIVEC(off, src); #endif } /// \brief Loads a vector from a word array /// \param src the word array /// \details VecLoad() loads a vector in from a word array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 8.0 inline uint32x4_p VecLoad(const word32 src[4]) { return VecLoad((const byte*)src); } /// \brief Loads a vector from a word array /// \param src the word array /// \param off offset into the word array /// \details VecLoad() loads a vector in from a word array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 8.0 inline uint32x4_p VecLoad(int off, const word32 src[4]) { return VecLoad(off, (const byte*)src); } #if defined(_ARCH_PWR7) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \brief Loads a vector from a word array /// \param src the word array /// \details VecLoad() loads a vector in from a word array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \details VecLoad() with 64-bit elements is available on POWER7 and above. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 8.0 inline uint64x2_p VecLoad(const word64 src[2]) { return (uint64x2_p)VecLoad((const byte*)src); } /// \brief Loads a vector from a word array /// \param src the word array /// \param off offset into the word array /// \details VecLoad() loads a vector in from a word array. /// \details VecLoad() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \details VecLoad() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 8.0 inline uint64x2_p VecLoad(int off, const word64 src[2]) { return (uint64x2_p)VecLoad(off, (const byte*)src); } #endif // _ARCH_PWR7 /// \brief Loads a vector from an aligned byte array /// \param src the byte array /// \details VecLoadAligned() loads a vector in from an aligned byte array. /// \details VecLoadAligned() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. Altivec's vec_ld is used /// if POWER7 is not available. The effective address of src must /// be aligned. /// \par Wraps /// vec_ld, vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld /// \since Crypto++ 8.0 inline uint32x4_p VecLoadAligned(const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) return (uint32x4_p)vec_xlw4(0, (byte*)src); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl(0, (byte*)src); # else return (uint32x4_p)vec_vsx_ld(0, (byte*)src); # endif #else // _ARCH_PWR7 CRYPTOPP_ASSERT(((uintptr_t)src) % 16 == 0); return (uint32x4_p)vec_ld(0, (byte*)src); #endif // _ARCH_PWR7 } /// \brief Loads a vector from an aligned byte array /// \param src the byte array /// \param off offset into the byte array /// \details VecLoadAligned() loads a vector in from an aligned byte array. /// \details VecLoadAligned() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. Altivec's vec_ld is used /// if POWER7 is not available. The effective address of src must /// be aligned. /// \par Wraps /// vec_ld, vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld /// \since Crypto++ 8.0 inline uint32x4_p VecLoadAligned(int off, const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) return (uint32x4_p)vec_xlw4(off, (byte*)src); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl(off, (byte*)src); # else return (uint32x4_p)vec_vsx_ld(off, (byte*)src); # endif #else // _ARCH_PWR7 CRYPTOPP_ASSERT((((uintptr_t)src)+off) % 16 == 0); return (uint32x4_p)vec_ld(off, (byte*)src); #endif // _ARCH_PWR7 } /// \brief Loads a vector from a byte array /// \param src the byte array /// \details VecLoadBE() loads a vector in from a byte array. VecLoadBE /// will reverse all bytes in the array on a little endian system. /// \details VecLoadBE() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 6.0 inline uint32x4_p VecLoadBE(const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)vec_xlw4(0, (byte*)src); # else return (uint32x4_p)VecReverse(vec_xlw4(0, (byte*)src)); # endif # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl_be(0, (byte*)src); # else # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)vec_vsx_ld(0, (byte*)src); # else return (uint32x4_p)VecReverse(vec_vsx_ld(0, (byte*)src)); # endif # endif #else // _ARCH_PWR7 # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)VecLoad((const byte*)src); # else return (uint32x4_p)VecReverse(VecLoad((const byte*)src)); # endif #endif // _ARCH_PWR7 } /// \brief Loads a vector from a byte array /// \param src the byte array /// \param off offset into the src byte array /// \details VecLoadBE() loads a vector in from a byte array. VecLoadBE /// will reverse all bytes in the array on a little endian system. /// \details VecLoadBE() uses POWER7's vec_xl or /// vec_vsx_ld if available. The instructions do not require /// aligned effective memory addresses. VecLoad_ALTIVEC() is used if POWER7 /// is not available. VecLoad_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xlw4, vec_xld2, vec_xl, vec_vsx_ld (and Altivec load) /// \since Crypto++ 6.0 inline uint32x4_p VecLoadBE(int off, const byte src[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)vec_xlw4(off, (byte*)src); # else return (uint32x4_p)VecReverse(vec_xlw4(off, (byte*)src)); # endif # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) return (uint32x4_p)vec_xl_be(off, (byte*)src); # else # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)vec_vsx_ld(off, (byte*)src); # else return (uint32x4_p)VecReverse(vec_vsx_ld(off, (byte*)src)); # endif # endif #else // _ARCH_PWR7 # if (CRYPTOPP_BIG_ENDIAN) return (uint32x4_p)VecLoad(off, (const byte*)src); # else return (uint32x4_p)VecReverse(VecLoad(off, (const byte*)src)); # endif #endif // _ARCH_PWR7 } //@} /// \name STORE OPERATIONS //@{ /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param dest the byte array /// \details VecStore_ALTIVEC() stores a vector to a byte array. /// \details VecStore_ALTIVEC() uses vec_st if the effective address /// of dest is aligned, and uses vec_ste otherwise. /// vec_ste is relatively expensive so you should provide aligned /// memory adresses. /// \details VecStore_ALTIVEC() is used automatically when POWER7 or above /// and unaligned loads is not available. /// \par Wraps /// vec_st, vec_ste, vec_lvsr, vec_perm /// \since Crypto++ 8.0 template inline void VecStore_ALTIVEC(const T data, byte dest[16]) { // Avoid IsAlignedOn for convenience. uintptr_t eff = reinterpret_cast(dest)+0; if (eff % 16 == 0) { vec_st((uint8x16_p)data, 0, dest); } else { // http://www.nxp.com/docs/en/reference-manual/ALTIVECPEM.pdf uint8x16_p perm = (uint8x16_p)vec_perm(data, data, vec_lvsr(0, dest)); vec_ste((uint8x16_p) perm, 0, (unsigned char*) dest); vec_ste((uint16x8_p) perm, 1, (unsigned short*)dest); vec_ste((uint32x4_p) perm, 3, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 4, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 8, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 12, (unsigned int*) dest); vec_ste((uint16x8_p) perm, 14, (unsigned short*)dest); vec_ste((uint8x16_p) perm, 15, (unsigned char*) dest); } } /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param off the byte offset into the array /// \param dest the byte array /// \details VecStore_ALTIVEC() stores a vector to a byte array. /// \details VecStore_ALTIVEC() uses vec_st if the effective address /// of dest is aligned, and uses vec_ste otherwise. /// vec_ste is relatively expensive so you should provide aligned /// memory adresses. /// \details VecStore_ALTIVEC() is used automatically when POWER7 or above /// and unaligned loads is not available. /// \par Wraps /// vec_st, vec_ste, vec_lvsr, vec_perm /// \since Crypto++ 8.0 template inline void VecStore_ALTIVEC(const T data, int off, byte dest[16]) { // Avoid IsAlignedOn for convenience. uintptr_t eff = reinterpret_cast(dest)+off; if (eff % 16 == 0) { vec_st((uint8x16_p)data, off, dest); } else { // http://www.nxp.com/docs/en/reference-manual/ALTIVECPEM.pdf uint8x16_p perm = (uint8x16_p)vec_perm(data, data, vec_lvsr(off, dest)); vec_ste((uint8x16_p) perm, 0, (unsigned char*) dest); vec_ste((uint16x8_p) perm, 1, (unsigned short*)dest); vec_ste((uint32x4_p) perm, 3, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 4, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 8, (unsigned int*) dest); vec_ste((uint32x4_p) perm, 12, (unsigned int*) dest); vec_ste((uint16x8_p) perm, 14, (unsigned short*)dest); vec_ste((uint8x16_p) perm, 15, (unsigned char*) dest); } } /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param dest the byte array /// \details VecStore() stores a vector to a byte array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 6.0 template inline void VecStore(const T data, byte dest[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) vec_xstw4((uint8x16_p)data, 0, (byte*)dest); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) vec_xst((uint8x16_p)data, 0, (byte*)dest); # else vec_vsx_st((uint8x16_p)data, 0, (byte*)dest); # endif #else VecStore_ALTIVEC((uint8x16_p)data, 0, (byte*)dest); #endif } /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param off the byte offset into the array /// \param dest the byte array /// \details VecStore() stores a vector to a byte array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 6.0 template inline void VecStore(const T data, int off, byte dest[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) vec_xstw4((uint8x16_p)data, off, (byte*)dest); # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) vec_xst((uint8x16_p)data, off, (byte*)dest); # else vec_vsx_st((uint8x16_p)data, off, (byte*)dest); # endif #else VecStore_ALTIVEC((uint8x16_p)data, off, (byte*)dest); #endif } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param dest the word array /// \details VecStore() stores a vector to a word array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStore(const T data, word32 dest[4]) { VecStore((uint8x16_p)data, 0, (byte*)dest); } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param off the byte offset into the array /// \param dest the word array /// \details VecStore() stores a vector to a word array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStore(const T data, int off, word32 dest[4]) { VecStore((uint8x16_p)data, off, (byte*)dest); } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param dest the word array /// \details VecStore() stores a vector to a word array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \details VecStore() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStore(const T data, word64 dest[2]) { VecStore((uint8x16_p)data, 0, (byte*)dest); } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param off the byte offset into the array /// \param dest the word array /// \details VecStore() stores a vector to a word array. /// \details VecStore() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \details VecStore() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStore(const T data, int off, word64 dest[2]) { VecStore((uint8x16_p)data, off, (byte*)dest); } /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param dest the byte array /// \details VecStoreBE() stores a vector to a byte array. VecStoreBE /// will reverse all bytes in the array on a little endian system. /// \details VecStoreBE() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 6.0 template inline void VecStoreBE(const T data, byte dest[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) # if (CRYPTOPP_BIG_ENDIAN) vec_xstw4((uint8x16_p)data, 0, (byte*)dest); # else vec_xstw4((uint8x16_p)VecReverse(data), 0, (byte*)dest); # endif # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) vec_xst_be((uint8x16_p)data, 0, (byte*)dest); # else # if (CRYPTOPP_BIG_ENDIAN) vec_vsx_st((uint8x16_p)data, 0, (byte*)dest); # else vec_vsx_st((uint8x16_p)VecReverse(data), 0, (byte*)dest); # endif # endif #else // _ARCH_PWR7 # if (CRYPTOPP_BIG_ENDIAN) VecStore_ALTIVEC((uint8x16_p)data, 0, (byte*)dest); # else VecStore_ALTIVEC((uint8x16_p)VecReverse(data), 0, (byte*)dest); # endif #endif // _ARCH_PWR7 } /// \brief Stores a vector to a byte array /// \tparam T vector type /// \param data the vector /// \param off offset into the dest byte array /// \param dest the byte array /// \details VecStoreBE() stores a vector to a byte array. VecStoreBE /// will reverse all bytes in the array on a little endian system. /// \details VecStoreBE() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 6.0 template inline void VecStoreBE(const T data, int off, byte dest[16]) { #if defined(_ARCH_PWR7) # if defined(__early_xlc__) || defined(__early_xlC__) # if (CRYPTOPP_BIG_ENDIAN) vec_xstw4((uint8x16_p)data, off, (byte*)dest); # else vec_xstw4((uint8x16_p)VecReverse(data), off, (byte*)dest); # endif # elif defined(__xlc__) || defined(__xlC__) || defined(__clang__) vec_xst_be((uint8x16_p)data, off, (byte*)dest); # else # if (CRYPTOPP_BIG_ENDIAN) vec_vsx_st((uint8x16_p)data, off, (byte*)dest); # else vec_vsx_st((uint8x16_p)VecReverse(data), off, (byte*)dest); # endif # endif #else // _ARCH_PWR7 # if (CRYPTOPP_BIG_ENDIAN) VecStore_ALTIVEC((uint8x16_p)data, off, (byte*)dest); # else VecStore_ALTIVEC((uint8x16_p)VecReverse(data), off, (byte*)dest); # endif #endif // _ARCH_PWR7 } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param dest the word array /// \details VecStoreBE() stores a vector to a word array. VecStoreBE /// will reverse all bytes in the array on a little endian system. /// \details VecStoreBE() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStoreBE(const T data, word32 dest[4]) { return VecStoreBE((uint8x16_p)data, (byte*)dest); } /// \brief Stores a vector to a word array /// \tparam T vector type /// \param data the vector /// \param off offset into the dest word array /// \param dest the word array /// \details VecStoreBE() stores a vector to a word array. VecStoreBE /// will reverse all words in the array on a little endian system. /// \details VecStoreBE() uses POWER7's vec_xst or /// vec_vsx_st if available. The instructions do not require /// aligned effective memory addresses. VecStore_ALTIVEC() is used if POWER7 /// is not available. VecStore_ALTIVEC() can be relatively expensive if /// extra instructions are required to fix up unaligned memory /// addresses. /// \par Wraps /// vec_xstw4, vec_xstld2, vec_xst, vec_vsx_st (and Altivec store) /// \since Crypto++ 8.0 template inline void VecStoreBE(const T data, int off, word32 dest[4]) { return VecStoreBE((uint8x16_p)data, off, (byte*)dest); } //@} /// \name LOGICAL OPERATIONS //@{ /// \brief AND two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \details VecAnd() returns a new vector from vec1 and vec2. The return /// vector is the same type as vec1. /// \par Wraps /// vec_and /// \since Crypto++ 6.0 template inline T1 VecAnd(const T1 vec1, const T2 vec2) { return (T1)vec_and(vec1, (T1)vec2); } /// \brief OR two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \details VecOr() returns a new vector from vec1 and vec2. The return /// vector is the same type as vec1. /// \par Wraps /// vec_or /// \since Crypto++ 6.0 template inline T1 VecOr(const T1 vec1, const T2 vec2) { return (T1)vec_or(vec1, (T1)vec2); } /// \brief XOR two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \details VecXor() returns a new vector from vec1 and vec2. The return /// vector is the same type as vec1. /// \par Wraps /// vec_xor /// \since Crypto++ 6.0 template inline T1 VecXor(const T1 vec1, const T2 vec2) { return (T1)vec_xor(vec1, (T1)vec2); } //@} /// \name ARITHMETIC OPERATIONS //@{ /// \brief Add two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \details VecAdd() returns a new vector from vec1 and vec2. /// vec2 is cast to the same type as vec1. The return vector /// is the same type as vec1. /// \par Wraps /// vec_add /// \since Crypto++ 6.0 template inline T1 VecAdd(const T1 vec1, const T2 vec2) { return (T1)vec_add(vec1, (T1)vec2); } /// \brief Subtract two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \details VecSub() returns a new vector from vec1 and vec2. /// vec2 is cast to the same type as vec1. The return vector /// is the same type as vec1. /// \par Wraps /// vec_sub /// \since Crypto++ 6.0 template inline T1 VecSub(const T1 vec1, const T2 vec2) { return (T1)vec_sub(vec1, (T1)vec2); } /// \brief Add two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \details VecAdd64() returns a new vector from vec1 and vec2. /// vec1 and vec2 are added as if uint64x2_p vectors. On POWER7 /// and below VecAdd64() manages the carries from two elements in /// a uint32x4_p vector. /// \par Wraps /// vec_add for POWER8, vec_addc, vec_perm, vec_add for Altivec /// \since Crypto++ 8.0 inline uint32x4_p VecAdd64(const uint32x4_p& vec1, const uint32x4_p& vec2) { // 64-bit elements available at POWER7, but addudm requires POWER8 #if defined(_ARCH_PWR8) return (uint32x4_p)vec_add((uint64x2_p)vec1, (uint64x2_p)vec2); #else // The carry mask selects carries from elements 1 and 3 and sets remaining // elements to 0. The mask also shifts the carried values left by 4 bytes // so the carries are added to elements 0 and 2. const uint8x16_p cmask = {4,5,6,7, 16,16,16,16, 12,13,14,15, 16,16,16,16}; const uint32x4_p zero = {0, 0, 0, 0}; uint32x4_p cy = vec_addc(vec1, vec2); cy = vec_perm(cy, zero, cmask); return vec_add(vec_add(vec1, vec2), cy); #endif } //@} /// \name OTHER OPERATIONS //@{ /// \brief Permutes a vector /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec the vector /// \param mask vector mask /// \returns vector /// \details VecPermute() returns a new vector from vec based on /// mask. mask is an uint8x16_p type vector. The return /// vector is the same type as vec. /// \par Wraps /// vec_perm /// \since Crypto++ 6.0 template inline T1 VecPermute(const T1 vec, const T2 mask) { return (T1)vec_perm(vec, vec, (uint8x16_p)mask); } /// \brief Permutes two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \param mask vector mask /// \returns vector /// \details VecPermute() returns a new vector from vec1 and vec2 /// based on mask. mask is an uint8x16_p type vector. The return /// vector is the same type as vec1. /// \par Wraps /// vec_perm /// \since Crypto++ 6.0 template inline T1 VecPermute(const T1 vec1, const T1 vec2, const T2 mask) { return (T1)vec_perm(vec1, (T1)vec2, (uint8x16_p)mask); } /// \brief Shift a vector left /// \tparam C shift byte count /// \tparam T vector type /// \param vec the vector /// \returns vector /// \details VecShiftLeftOctet() returns a new vector after shifting the /// concatenation of the zero vector and the source vector by the specified /// number of bytes. The return vector is the same type as vec. /// \details On big endian machines VecShiftLeftOctet() is vec_sld(a, z, /// c). On little endian machines VecShiftLeftOctet() is translated to /// vec_sld(z, a, 16-c). You should always call the function as /// if on a big endian machine as shown below. ///
///    uint8x16_p x = VecLoad(ptr);
///    uint8x16_p y = VecShiftLeftOctet<12>(x);
/// 
/// \par Wraps /// vec_sld /// \sa Is vec_sld /// endian sensitive? on Stack Overflow /// \since Crypto++ 6.0 template inline T VecShiftLeftOctet(const T vec) { const T zero = {0}; if (C >= 16) { // Out of range return zero; } else if (C == 0) { // Noop return vec; } else { #if (CRYPTOPP_BIG_ENDIAN) enum { R=C&0xf }; return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)zero, R); #else enum { R=(16-C)&0xf }; // Linux xlC 13.1 workaround in Debug builds return (T)vec_sld((uint8x16_p)zero, (uint8x16_p)vec, R); #endif } } /// \brief Shift a vector right /// \tparam C shift byte count /// \tparam T vector type /// \param vec the vector /// \returns vector /// \details VecShiftRightOctet() returns a new vector after shifting the /// concatenation of the zero vector and the source vector by the specified /// number of bytes. The return vector is the same type as vec. /// \details On big endian machines VecShiftRightOctet() is vec_sld(a, z, /// c). On little endian machines VecShiftRightOctet() is translated to /// vec_sld(z, a, 16-c). You should always call the function as /// if on a big endian machine as shown below. ///
///    uint8x16_p x = VecLoad(ptr);
///    uint8x16_p y = VecShiftRightOctet<12>(y);
/// 
/// \par Wraps /// vec_sld /// \sa Is vec_sld /// endian sensitive? on Stack Overflow /// \since Crypto++ 6.0 template inline T VecShiftRightOctet(const T vec) { const T zero = {0}; if (C >= 16) { // Out of range return zero; } else if (C == 0) { // Noop return vec; } else { #if (CRYPTOPP_BIG_ENDIAN) enum { R=(16-C)&0xf }; // Linux xlC 13.1 workaround in Debug builds return (T)vec_sld((uint8x16_p)zero, (uint8x16_p)vec, R); #else enum { R=C&0xf }; return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)zero, R); #endif } } /// \brief Rotate a vector left /// \tparam C shift byte count /// \tparam T vector type /// \param vec the vector /// \returns vector /// \details VecRotateLeftOctet() returns a new vector after rotating the /// concatenation of the source vector with itself by the specified /// number of bytes. The return vector is the same type as vec. /// \par Wraps /// vec_sld /// \sa Is vec_sld /// endian sensitive? on Stack Overflow /// \since Crypto++ 6.0 template inline T VecRotateLeftOctet(const T vec) { #if (CRYPTOPP_BIG_ENDIAN) enum { R = C&0xf }; return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)vec, R); #else enum { R=(16-C)&0xf }; // Linux xlC 13.1 workaround in Debug builds return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)vec, R); #endif } /// \brief Rotate a vector right /// \tparam C shift byte count /// \tparam T vector type /// \param vec the vector /// \returns vector /// \details VecRotateRightOctet() returns a new vector after rotating the /// concatenation of the source vector with itself by the specified /// number of bytes. The return vector is the same type as vec. /// \par Wraps /// vec_sld /// \sa Is vec_sld /// endian sensitive? on Stack Overflow /// \since Crypto++ 6.0 template inline T VecRotateRightOctet(const T vec) { #if (CRYPTOPP_BIG_ENDIAN) enum { R=(16-C)&0xf }; // Linux xlC 13.1 workaround in Debug builds return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)vec, R); #else enum { R = C&0xf }; return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)vec, R); #endif } /// \brief Rotate a packed vector left /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecRotateLeft() rotates each element in a packed vector by bit count. /// \par Wraps /// vec_rl /// \since Crypto++ 7.0 template inline uint32x4_p VecRotateLeft(const uint32x4_p vec) { const uint32x4_p m = {C, C, C, C}; return vec_rl(vec, m); } /// \brief Shift a packed vector left /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecShiftLeft() rotates each element in a packed vector by bit count. /// \par Wraps /// vec_sl /// \since Crypto++ 8.1 template inline uint32x4_p VecShiftLeft(const uint32x4_p vec) { const uint32x4_p m = {C, C, C, C}; return vec_sl(vec, m); } /// \brief Merge two vectors /// \tparam T vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \par Wraps /// vec_mergeh /// \since Crypto++ 8.1 template inline T VecMergeHigh(const T vec1, const T vec2) { return vec_mergeh(vec1, vec2); } /// \brief Merge two vectors /// \tparam T vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns vector /// \par Wraps /// vec_mergel /// \since Crypto++ 8.1 template inline T VecMergeLow(const T vec1, const T vec2) { return vec_mergel(vec1, vec2); } #if defined(_ARCH_PWR8) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \brief Rotate a packed vector left /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecRotateLeft() rotates each element in a packed vector by bit count. /// \details VecRotateLeft() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_rl /// \since Crypto++ 8.0 template inline uint64x2_p VecRotateLeft(const uint64x2_p vec) { const uint64x2_p m = {C, C}; return vec_rl(vec, m); } /// \brief Shift a packed vector left /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecShiftLeft() rotates each element in a packed vector by bit count. /// \details VecShiftLeft() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_sl /// \since Crypto++ 8.1 template inline uint64x2_p VecShiftLeft(const uint64x2_p vec) { const uint64x2_p m = {C, C}; return vec_sl(vec, m); } #endif /// \brief Rotate a packed vector right /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecRotateRight() rotates each element in a packed vector by bit count. /// \par Wraps /// vec_rl /// \since Crypto++ 7.0 template inline uint32x4_p VecRotateRight(const uint32x4_p vec) { const uint32x4_p m = {32-C, 32-C, 32-C, 32-C}; return vec_rl(vec, m); } /// \brief Shift a packed vector right /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecShiftRight() rotates each element in a packed vector by bit count. /// \par Wraps /// vec_rl /// \since Crypto++ 8.1 template inline uint32x4_p VecShiftRight(const uint32x4_p vec) { const uint32x4_p m = {C, C, C, C}; return vec_sr(vec, m); } #if defined(_ARCH_PWR8) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \brief Rotate a packed vector right /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecRotateRight() rotates each element in a packed vector by bit count. /// \details VecRotateRight() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_rl /// \since Crypto++ 8.0 template inline uint64x2_p VecRotateRight(const uint64x2_p vec) { const uint64x2_p m = {64-C, 64-C}; return vec_rl(vec, m); } /// \brief Shift a packed vector right /// \tparam C shift bit count /// \param vec the vector /// \returns vector /// \details VecShiftRight() rotates each element in a packed vector by bit count. /// \details VecShiftRight() with 64-bit elements is available on POWER8 and above. /// \par Wraps /// vec_sr /// \since Crypto++ 8.1 template inline uint64x2_p VecShiftRight(const uint64x2_p vec) { const uint64x2_p m = {C, C}; return vec_sr(vec, m); } #endif /// \brief Exchange high and low double words /// \tparam T vector type /// \param vec the vector /// \returns vector /// \par Wraps /// vec_sld /// \since Crypto++ 7.0 template inline T VecSwapWords(const T vec) { return (T)vec_sld((uint8x16_p)vec, (uint8x16_p)vec, 8); } /// \brief Extract a dword from a vector /// \tparam T vector type /// \param val the vector /// \returns vector created from low dword /// \details VecGetLow() extracts the low dword from a vector. The low dword /// is composed of the least significant bits and occupies bytes 8 through 15 /// when viewed as a big endian array. The return vector is the same type as /// the original vector and padded with 0's in the most significant bit positions. /// \par Wraps /// vec_sld /// \since Crypto++ 7.0 template inline T VecGetLow(const T val) { #if (CRYPTOPP_BIG_ENDIAN) && (_ARCH_PWR8) const T zero = {0}; return (T)VecMergeLow((uint64x2_p)zero, (uint64x2_p)val); #else return VecShiftRightOctet<8>(VecShiftLeftOctet<8>(val)); #endif } /// \brief Extract a dword from a vector /// \tparam T vector type /// \param val the vector /// \returns vector created from high dword /// \details VecGetHigh() extracts the high dword from a vector. The high dword /// is composed of the most significant bits and occupies bytes 0 through 7 /// when viewed as a big endian array. The return vector is the same type as /// the original vector and padded with 0's in the most significant bit positions. /// \par Wraps /// vec_sld /// \since Crypto++ 7.0 template inline T VecGetHigh(const T val) { #if (CRYPTOPP_BIG_ENDIAN) && (_ARCH_PWR8) const T zero = {0}; return (T)VecMergeHigh((uint64x2_p)zero, (uint64x2_p)val); #else return VecShiftRightOctet<8>(val); #endif } /// \brief Compare two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns true if vec1 equals vec2, false otherwise /// \details VecEqual() performs a bitwise compare. The vector element types do /// not matter. /// \par Wraps /// vec_all_eq /// \since Crypto++ 8.0 template inline bool VecEqual(const T1 vec1, const T2 vec2) { return 1 == vec_all_eq((uint32x4_p)vec1, (uint32x4_p)vec2); } /// \brief Compare two vectors /// \tparam T1 vector type /// \tparam T2 vector type /// \param vec1 the first vector /// \param vec2 the second vector /// \returns true if vec1 does not equal vec2, false otherwise /// \details VecNotEqual() performs a bitwise compare. The vector element types do /// not matter. /// \par Wraps /// vec_all_eq /// \since Crypto++ 8.0 template inline bool VecNotEqual(const T1 vec1, const T2 vec2) { return 0 == vec_all_eq((uint32x4_p)vec1, (uint32x4_p)vec2); } //@} //////////////////////// Power8 Crypto //////////////////////// #if defined(__CRYPTO__) || defined(CRYPTOPP_DOXYGEN_PROCESSING) /// \name POLYNOMIAL MULTIPLICATION //@{ /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply() performs polynomial multiplication. POWER8 /// polynomial multiplication multiplies the high and low terms, and then /// XOR's the high and low products. That is, the result is ah*bh XOR /// al*bl. It is different behavior than Intel polynomial /// multiplication. To obtain a single product without the XOR, then set /// one of the high or low terms to 0. For example, setting ah=0 /// results in 0*bh XOR al*bl = al*bl. /// \par Wraps /// __vpmsumw, __builtin_altivec_crypto_vpmsumw and __builtin_crypto_vpmsumw. /// \since Crypto++ 8.1 inline uint32x4_p VecPolyMultiply(const uint32x4_p& a, const uint32x4_p& b) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return __vpmsumw (a, b); #elif defined(__clang__) return __builtin_altivec_crypto_vpmsumw (a, b); #else return __builtin_crypto_vpmsumw (a, b); #endif } /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply() performs polynomial multiplication. POWER8 /// polynomial multiplication multiplies the high and low terms, and then /// XOR's the high and low products. That is, the result is ah*bh XOR /// al*bl. It is different behavior than Intel polynomial /// multiplication. To obtain a single product without the XOR, then set /// one of the high or low terms to 0. For example, setting ah=0 /// results in 0*bh XOR al*bl = al*bl. /// \par Wraps /// __vpmsumd, __builtin_altivec_crypto_vpmsumd and __builtin_crypto_vpmsumd. /// \since Crypto++ 8.1 inline uint64x2_p VecPolyMultiply(const uint64x2_p& a, const uint64x2_p& b) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return __vpmsumd (a, b); #elif defined(__clang__) return __builtin_altivec_crypto_vpmsumd (a, b); #else return __builtin_crypto_vpmsumd (a, b); #endif } /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply00LE() performs polynomial multiplication and presents /// the result like Intel's c = _mm_clmulepi64_si128(a, b, 0x00). /// The 0x00 indicates the low 64-bits of a and b /// are multiplied. /// \note An Intel XMM register is composed of 128-bits. The leftmost bit /// is MSB and numbered 127, while the the rightmost bit is LSB and numbered 0. /// \par Wraps /// __vpmsumd, __builtin_altivec_crypto_vpmsumd and __builtin_crypto_vpmsumd. /// \since Crypto++ 8.0 inline uint64x2_p VecPolyMultiply00LE(const uint64x2_p& a, const uint64x2_p& b) { #if (CRYPTOPP_BIG_ENDIAN) return VecSwapWords(VecPolyMultiply(VecGetHigh(a), VecGetHigh(b))); #else return VecPolyMultiply(VecGetHigh(a), VecGetHigh(b)); #endif } /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply01LE performs() polynomial multiplication and presents /// the result like Intel's c = _mm_clmulepi64_si128(a, b, 0x01). /// The 0x01 indicates the low 64-bits of a and high /// 64-bits of b are multiplied. /// \note An Intel XMM register is composed of 128-bits. The leftmost bit /// is MSB and numbered 127, while the the rightmost bit is LSB and numbered 0. /// \par Wraps /// __vpmsumd, __builtin_altivec_crypto_vpmsumd and __builtin_crypto_vpmsumd. /// \since Crypto++ 8.0 inline uint64x2_p VecPolyMultiply01LE(const uint64x2_p& a, const uint64x2_p& b) { #if (CRYPTOPP_BIG_ENDIAN) return VecSwapWords(VecPolyMultiply(a, VecGetHigh(b))); #else return VecPolyMultiply(a, VecGetHigh(b)); #endif } /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply10LE() performs polynomial multiplication and presents /// the result like Intel's c = _mm_clmulepi64_si128(a, b, 0x10). /// The 0x10 indicates the high 64-bits of a and low /// 64-bits of b are multiplied. /// \note An Intel XMM register is composed of 128-bits. The leftmost bit /// is MSB and numbered 127, while the the rightmost bit is LSB and numbered 0. /// \par Wraps /// __vpmsumd, __builtin_altivec_crypto_vpmsumd and __builtin_crypto_vpmsumd. /// \since Crypto++ 8.0 inline uint64x2_p VecPolyMultiply10LE(const uint64x2_p& a, const uint64x2_p& b) { #if (CRYPTOPP_BIG_ENDIAN) return VecSwapWords(VecPolyMultiply(VecGetHigh(a), b)); #else return VecPolyMultiply(VecGetHigh(a), b); #endif } /// \brief Polynomial multiplication /// \param a the first term /// \param b the second term /// \returns vector product /// \details VecPolyMultiply11LE() performs polynomial multiplication and presents /// the result like Intel's c = _mm_clmulepi64_si128(a, b, 0x11). /// The 0x11 indicates the high 64-bits of a and b /// are multiplied. /// \note An Intel XMM register is composed of 128-bits. The leftmost bit /// is MSB and numbered 127, while the the rightmost bit is LSB and numbered 0. /// \par Wraps /// __vpmsumd, __builtin_altivec_crypto_vpmsumd and __builtin_crypto_vpmsumd. /// \since Crypto++ 8.0 inline uint64x2_p VecPolyMultiply11LE(const uint64x2_p& a, const uint64x2_p& b) { #if (CRYPTOPP_BIG_ENDIAN) return VecSwapWords(VecPolyMultiply(VecGetLow(a), b)); #else return VecPolyMultiply(VecGetLow(a), b); #endif } //@} /// \name AES ENCRYPTION //@{ /// \brief One round of AES encryption /// \tparam T1 vector type /// \tparam T2 vector type /// \param state the state vector /// \param key the subkey vector /// \details VecEncrypt() performs one round of AES encryption of state /// using subkey key. The return vector is the same type as vec1. /// \details VecEncrypt() is available on POWER8 and above. /// \par Wraps /// __vcipher, __builtin_altivec_crypto_vcipher, __builtin_crypto_vcipher /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T1 VecEncrypt(const T1 state, const T2 key) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T1)__vcipher((uint8x16_p)state, (uint8x16_p)key); #elif defined(__clang__) return (T1)__builtin_altivec_crypto_vcipher((uint64x2_p)state, (uint64x2_p)key); #elif defined(__GNUC__) return (T1)__builtin_crypto_vcipher((uint64x2_p)state, (uint64x2_p)key); #else CRYPTOPP_ASSERT(0); #endif } /// \brief Final round of AES encryption /// \tparam T1 vector type /// \tparam T2 vector type /// \param state the state vector /// \param key the subkey vector /// \details VecEncryptLast() performs the final round of AES encryption /// of state using subkey key. The return vector is the same type as vec1. /// \details VecEncryptLast() is available on POWER8 and above. /// \par Wraps /// __vcipherlast, __builtin_altivec_crypto_vcipherlast, __builtin_crypto_vcipherlast /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T1 VecEncryptLast(const T1 state, const T2 key) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T1)__vcipherlast((uint8x16_p)state, (uint8x16_p)key); #elif defined(__clang__) return (T1)__builtin_altivec_crypto_vcipherlast((uint64x2_p)state, (uint64x2_p)key); #elif defined(__GNUC__) return (T1)__builtin_crypto_vcipherlast((uint64x2_p)state, (uint64x2_p)key); #else CRYPTOPP_ASSERT(0); #endif } /// \brief One round of AES decryption /// \tparam T1 vector type /// \tparam T2 vector type /// \param state the state vector /// \param key the subkey vector /// \details VecDecrypt() performs one round of AES decryption of state /// using subkey key. The return vector is the same type as vec1. /// \details VecDecrypt() is available on POWER8 and above. /// \par Wraps /// __vncipher, __builtin_altivec_crypto_vncipher, __builtin_crypto_vncipher /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T1 VecDecrypt(const T1 state, const T2 key) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T1)__vncipher((uint8x16_p)state, (uint8x16_p)key); #elif defined(__clang__) return (T1)__builtin_altivec_crypto_vncipher((uint64x2_p)state, (uint64x2_p)key); #elif defined(__GNUC__) return (T1)__builtin_crypto_vncipher((uint64x2_p)state, (uint64x2_p)key); #else CRYPTOPP_ASSERT(0); #endif } /// \brief Final round of AES decryption /// \tparam T1 vector type /// \tparam T2 vector type /// \param state the state vector /// \param key the subkey vector /// \details VecDecryptLast() performs the final round of AES decryption /// of state using subkey key. The return vector is the same type as vec1. /// \details VecDecryptLast() is available on POWER8 and above. /// \par Wraps /// __vncipherlast, __builtin_altivec_crypto_vncipherlast, __builtin_crypto_vncipherlast /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T1 VecDecryptLast(const T1 state, const T2 key) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T1)__vncipherlast((uint8x16_p)state, (uint8x16_p)key); #elif defined(__clang__) return (T1)__builtin_altivec_crypto_vncipherlast((uint64x2_p)state, (uint64x2_p)key); #elif defined(__GNUC__) return (T1)__builtin_crypto_vncipherlast((uint64x2_p)state, (uint64x2_p)key); #else CRYPTOPP_ASSERT(0); #endif } //@} /// \name SHA DIGESTS //@{ /// \brief SHA256 Sigma functions /// \tparam func function /// \tparam fmask function mask /// \tparam T vector type /// \param vec the block to transform /// \details VecSHA256() selects sigma0, sigma1, Sigma0, Sigma1 based on /// func and fmask. The return vector is the same type as vec. /// \details VecSHA256() is available on POWER8 and above. /// \par Wraps /// __vshasigmaw, __builtin_altivec_crypto_vshasigmaw, __builtin_crypto_vshasigmaw /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T VecSHA256(const T vec) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T)__vshasigmaw((uint32x4_p)vec, func, fmask); #elif defined(__clang__) return (T)__builtin_altivec_crypto_vshasigmaw((uint32x4_p)vec, func, fmask); #elif defined(__GNUC__) return (T)__builtin_crypto_vshasigmaw((uint32x4_p)vec, func, fmask); #else CRYPTOPP_ASSERT(0); #endif } /// \brief SHA512 Sigma functions /// \tparam func function /// \tparam fmask function mask /// \tparam T vector type /// \param vec the block to transform /// \details VecSHA512() selects sigma0, sigma1, Sigma0, Sigma1 based on /// func and fmask. The return vector is the same type as vec. /// \details VecSHA512() is available on POWER8 and above. /// \par Wraps /// __vshasigmad, __builtin_altivec_crypto_vshasigmad, __builtin_crypto_vshasigmad /// \since GCC and XLC since Crypto++ 6.0, LLVM Clang since Crypto++ 8.0 template inline T VecSHA512(const T vec) { #if defined(__ibmxl__) || (defined(_AIX) && defined(__xlC__)) return (T)__vshasigmad((uint64x2_p)vec, func, fmask); #elif defined(__clang__) return (T)__builtin_altivec_crypto_vshasigmad((uint64x2_p)vec, func, fmask); #elif defined(__GNUC__) return (T)__builtin_crypto_vshasigmad((uint64x2_p)vec, func, fmask); #else CRYPTOPP_ASSERT(0); #endif } //@} #endif // __CRYPTO__ #endif // _ALTIVEC_ NAMESPACE_END #if CRYPTOPP_GCC_DIAGNOSTIC_AVAILABLE # pragma GCC diagnostic pop #endif #endif // CRYPTOPP_PPC_CRYPTO_H