reduce memory usage. move sbox macros to serpentp.h

1 files changed, 20 insertions, 441 deletions

diff --git a/serpent.cpp b/serpent.cpp
index 69956464..4d8cd1d0 100644
--- a/serpent.cpp
+++ b/serpent.cpp
@@ -4,454 +4,27 @@
 #include "serpent.h"
 #include "misc.h"
 
-NAMESPACE_BEGIN(CryptoPP)
-
-// linear transformation
-#define LT(i,a,b,c,d,e)	{\
-	a = rotlFixed(a, 13);	\
-	c = rotlFixed(c, 3); 	\
-	d = rotlFixed(d ^ c ^ (a << 3), 7); 	\
-	b = rotlFixed(b ^ a ^ c, 1); 	\
-	a = rotlFixed(a ^ b ^ d, 5); 		\
-	c = rotlFixed(c ^ d ^ (b << 7), 22);}
-
-// inverse linear transformation
-#define ILT(i,a,b,c,d,e)	{\
-	c = rotrFixed(c, 22);	\
-	a = rotrFixed(a, 5); 	\
-	c ^= d ^ (b << 7);	\
-	a ^= b ^ d; 		\
-	b = rotrFixed(b, 1); 	\
-	d = rotrFixed(d, 7) ^ c ^ (a << 3);	\
-	b ^= a ^ c; 		\
-	c = rotrFixed(c, 3); 	\
-	a = rotrFixed(a, 13);}
-
-// order of output from S-box functions
-#define beforeS0(f) f(0,a,b,c,d,e)
-#define afterS0(f) f(1,b,e,c,a,d)
-#define afterS1(f) f(2,c,b,a,e,d)
-#define afterS2(f) f(3,a,e,b,d,c)
-#define afterS3(f) f(4,e,b,d,c,a)
-#define afterS4(f) f(5,b,a,e,c,d)
-#define afterS5(f) f(6,a,c,b,e,d)
-#define afterS6(f) f(7,a,c,d,b,e)
-#define afterS7(f) f(8,d,e,b,a,c)
-
-// order of output from inverse S-box functions
-#define beforeI7(f) f(8,a,b,c,d,e)
-#define afterI7(f) f(7,d,a,b,e,c)
-#define afterI6(f) f(6,a,b,c,e,d)
-#define afterI5(f) f(5,b,d,e,c,a)
-#define afterI4(f) f(4,b,c,e,a,d)
-#define afterI3(f) f(3,a,b,e,c,d)
-#define afterI2(f) f(2,b,d,e,c,a)
-#define afterI1(f) f(1,a,b,c,e,d)
-#define afterI0(f) f(0,a,d,b,e,c)
-
-// The instruction sequences for the S-box functions 
-// come from Dag Arne Osvik's paper "Speeding up Serpent".
-
-#define S0(i, r0, r1, r2, r3, r4) \
-       {           \
-    r3 ^= r0;   \
-    r4 = r1;   \
-    r1 &= r3;   \
-    r4 ^= r2;   \
-    r1 ^= r0;   \
-    r0 |= r3;   \
-    r0 ^= r4;   \
-    r4 ^= r3;   \
-    r3 ^= r2;   \
-    r2 |= r1;   \
-    r2 ^= r4;   \
-    r4 = ~r4;      \
-    r4 |= r1;   \
-    r1 ^= r3;   \
-    r1 ^= r4;   \
-    r3 |= r0;   \
-    r1 ^= r3;   \
-    r4 ^= r3;   \
-            }
-
-#define I0(i, r0, r1, r2, r3, r4) \
-       {           \
-    r2 = ~r2;      \
-    r4 = r1;   \
-    r1 |= r0;   \
-    r4 = ~r4;      \
-    r1 ^= r2;   \
-    r2 |= r4;   \
-    r1 ^= r3;   \
-    r0 ^= r4;   \
-    r2 ^= r0;   \
-    r0 &= r3;   \
-    r4 ^= r0;   \
-    r0 |= r1;   \
-    r0 ^= r2;   \
-    r3 ^= r4;   \
-    r2 ^= r1;   \
-    r3 ^= r0;   \
-    r3 ^= r1;   \
-    r2 &= r3;   \
-    r4 ^= r2;   \
-            }
-
-#define S1(i, r0, r1, r2, r3, r4) \
-       {           \
-    r0 = ~r0;      \
-    r2 = ~r2;      \
-    r4 = r0;   \
-    r0 &= r1;   \
-    r2 ^= r0;   \
-    r0 |= r3;   \
-    r3 ^= r2;   \
-    r1 ^= r0;   \
-    r0 ^= r4;   \
-    r4 |= r1;   \
-    r1 ^= r3;   \
-    r2 |= r0;   \
-    r2 &= r4;   \
-    r0 ^= r1;   \
-    r1 &= r2;   \
-    r1 ^= r0;   \
-    r0 &= r2;   \
-    r0 ^= r4;   \
-            }
-
-#define I1(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r1;   \
-    r1 ^= r3;   \
-    r3 &= r1;   \
-    r4 ^= r2;   \
-    r3 ^= r0;   \
-    r0 |= r1;   \
-    r2 ^= r3;   \
-    r0 ^= r4;   \
-    r0 |= r2;   \
-    r1 ^= r3;   \
-    r0 ^= r1;   \
-    r1 |= r3;   \
-    r1 ^= r0;   \
-    r4 = ~r4;      \
-    r4 ^= r1;   \
-    r1 |= r0;   \
-    r1 ^= r0;   \
-    r1 |= r4;   \
-    r3 ^= r1;   \
-            }
-
-#define S2(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r0;   \
-    r0 &= r2;   \
-    r0 ^= r3;   \
-    r2 ^= r1;   \
-    r2 ^= r0;   \
-    r3 |= r4;   \
-    r3 ^= r1;   \
-    r4 ^= r2;   \
-    r1 = r3;   \
-    r3 |= r4;   \
-    r3 ^= r0;   \
-    r0 &= r1;   \
-    r4 ^= r0;   \
-    r1 ^= r3;   \
-    r1 ^= r4;   \
-    r4 = ~r4;      \
-            }
-
-#define I2(i, r0, r1, r2, r3, r4) \
-       {           \
-    r2 ^= r3;   \
-    r3 ^= r0;   \
-    r4 = r3;   \
-    r3 &= r2;   \
-    r3 ^= r1;   \
-    r1 |= r2;   \
-    r1 ^= r4;   \
-    r4 &= r3;   \
-    r2 ^= r3;   \
-    r4 &= r0;   \
-    r4 ^= r2;   \
-    r2 &= r1;   \
-    r2 |= r0;   \
-    r3 = ~r3;      \
-    r2 ^= r3;   \
-    r0 ^= r3;   \
-    r0 &= r1;   \
-    r3 ^= r4;   \
-    r3 ^= r0;   \
-            }
-
-#define S3(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r0;   \
-    r0 |= r3;   \
-    r3 ^= r1;   \
-    r1 &= r4;   \
-    r4 ^= r2;   \
-    r2 ^= r3;   \
-    r3 &= r0;   \
-    r4 |= r1;   \
-    r3 ^= r4;   \
-    r0 ^= r1;   \
-    r4 &= r0;   \
-    r1 ^= r3;   \
-    r4 ^= r2;   \
-    r1 |= r0;   \
-    r1 ^= r2;   \
-    r0 ^= r3;   \
-    r2 = r1;   \
-    r1 |= r3;   \
-    r1 ^= r0;   \
-            }
-
-#define I3(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r2;   \
-    r2 ^= r1;   \
-    r1 &= r2;   \
-    r1 ^= r0;   \
-    r0 &= r4;   \
-    r4 ^= r3;   \
-    r3 |= r1;   \
-    r3 ^= r2;   \
-    r0 ^= r4;   \
-    r2 ^= r0;   \
-    r0 |= r3;   \
-    r0 ^= r1;   \
-    r4 ^= r2;   \
-    r2 &= r3;   \
-    r1 |= r3;   \
-    r1 ^= r2;   \
-    r4 ^= r0;   \
-    r2 ^= r4;   \
-            }
-
-#define S4(i, r0, r1, r2, r3, r4) \
-       {           \
-    r1 ^= r3;   \
-    r3 = ~r3;      \
-    r2 ^= r3;   \
-    r3 ^= r0;   \
-    r4 = r1;   \
-    r1 &= r3;   \
-    r1 ^= r2;   \
-    r4 ^= r3;   \
-    r0 ^= r4;   \
-    r2 &= r4;   \
-    r2 ^= r0;   \
-    r0 &= r1;   \
-    r3 ^= r0;   \
-    r4 |= r1;   \
-    r4 ^= r0;   \
-    r0 |= r3;   \
-    r0 ^= r2;   \
-    r2 &= r3;   \
-    r0 = ~r0;      \
-    r4 ^= r2;   \
-            }
-
-#define I4(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r2;   \
-    r2 &= r3;   \
-    r2 ^= r1;   \
-    r1 |= r3;   \
-    r1 &= r0;   \
-    r4 ^= r2;   \
-    r4 ^= r1;   \
-    r1 &= r2;   \
-    r0 = ~r0;      \
-    r3 ^= r4;   \
-    r1 ^= r3;   \
-    r3 &= r0;   \
-    r3 ^= r2;   \
-    r0 ^= r1;   \
-    r2 &= r0;   \
-    r3 ^= r0;   \
-    r2 ^= r4;   \
-    r2 |= r3;   \
-    r3 ^= r0;   \
-    r2 ^= r1;   \
-            }
-
-#define S5(i, r0, r1, r2, r3, r4) \
-       {           \
-    r0 ^= r1;   \
-    r1 ^= r3;   \
-    r3 = ~r3;      \
-    r4 = r1;   \
-    r1 &= r0;   \
-    r2 ^= r3;   \
-    r1 ^= r2;   \
-    r2 |= r4;   \
-    r4 ^= r3;   \
-    r3 &= r1;   \
-    r3 ^= r0;   \
-    r4 ^= r1;   \
-    r4 ^= r2;   \
-    r2 ^= r0;   \
-    r0 &= r3;   \
-    r2 = ~r2;      \
-    r0 ^= r4;   \
-    r4 |= r3;   \
-    r2 ^= r4;   \
-            }
+#include "serpentp.h"
 
-#define I5(i, r0, r1, r2, r3, r4) \
-       {           \
-    r1 = ~r1;      \
-    r4 = r3;   \
-    r2 ^= r1;   \
-    r3 |= r0;   \
-    r3 ^= r2;   \
-    r2 |= r1;   \
-    r2 &= r0;   \
-    r4 ^= r3;   \
-    r2 ^= r4;   \
-    r4 |= r0;   \
-    r4 ^= r1;   \
-    r1 &= r2;   \
-    r1 ^= r3;   \
-    r4 ^= r2;   \
-    r3 &= r4;   \
-    r4 ^= r1;   \
-    r3 ^= r0;   \
-    r3 ^= r4;   \
-    r4 = ~r4;      \
-            }
-
-#define S6(i, r0, r1, r2, r3, r4) \
-       {           \
-    r2 = ~r2;      \
-    r4 = r3;   \
-    r3 &= r0;   \
-    r0 ^= r4;   \
-    r3 ^= r2;   \
-    r2 |= r4;   \
-    r1 ^= r3;   \
-    r2 ^= r0;   \
-    r0 |= r1;   \
-    r2 ^= r1;   \
-    r4 ^= r0;   \
-    r0 |= r3;   \
-    r0 ^= r2;   \
-    r4 ^= r3;   \
-    r4 ^= r0;   \
-    r3 = ~r3;      \
-    r2 &= r4;   \
-    r2 ^= r3;   \
-            }
-
-#define I6(i, r0, r1, r2, r3, r4) \
-       {           \
-    r0 ^= r2;   \
-    r4 = r2;   \
-    r2 &= r0;   \
-    r4 ^= r3;   \
-    r2 = ~r2;      \
-    r3 ^= r1;   \
-    r2 ^= r3;   \
-    r4 |= r0;   \
-    r0 ^= r2;   \
-    r3 ^= r4;   \
-    r4 ^= r1;   \
-    r1 &= r3;   \
-    r1 ^= r0;   \
-    r0 ^= r3;   \
-    r0 |= r2;   \
-    r3 ^= r1;   \
-    r4 ^= r0;   \
-            }
-
-#define S7(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r2;   \
-    r2 &= r1;   \
-    r2 ^= r3;   \
-    r3 &= r1;   \
-    r4 ^= r2;   \
-    r2 ^= r1;   \
-    r1 ^= r0;   \
-    r0 |= r4;   \
-    r0 ^= r2;   \
-    r3 ^= r1;   \
-    r2 ^= r3;   \
-    r3 &= r0;   \
-    r3 ^= r4;   \
-    r4 ^= r2;   \
-    r2 &= r0;   \
-    r4 = ~r4;      \
-    r2 ^= r4;   \
-    r4 &= r0;   \
-    r1 ^= r3;   \
-    r4 ^= r1;   \
-            }
-
-#define I7(i, r0, r1, r2, r3, r4) \
-       {           \
-    r4 = r2;   \
-    r2 ^= r0;   \
-    r0 &= r3;   \
-    r2 = ~r2;      \
-    r4 |= r3;   \
-    r3 ^= r1;   \
-    r1 |= r0;   \
-    r0 ^= r2;   \
-    r2 &= r4;   \
-    r1 ^= r2;   \
-    r2 ^= r0;   \
-    r0 |= r2;   \
-    r3 &= r4;   \
-    r0 ^= r3;   \
-    r4 ^= r1;   \
-    r3 ^= r4;   \
-    r4 |= r0;   \
-    r3 ^= r2;   \
-    r4 ^= r2;   \
-            }
-
-// key xor
-#define KX(r, a, b, c, d, e)	{\
-	a ^= k[4 * r + 0]; \
-	b ^= k[4 * r + 1]; \
-	c ^= k[4 * r + 2]; \
-	d ^= k[4 * r + 3];}
+NAMESPACE_BEGIN(CryptoPP)
 
-void Serpent::Base::UncheckedSetKey(const byte *userKey, unsigned int keylen, const NameValuePairs &)
+void Serpent_KeySchedule(word32 *k, unsigned int rounds, const byte *userKey, unsigned int keylen)
 {
-	AssertValidKeyLength(keylen);
-
-	word32 *k = m_key;
-	GetUserKey(LITTLE_ENDIAN_ORDER, k, 8, userKey, keylen);
-
+	FixedSizeSecBlock<word32, 8> k0;
+	GetUserKey(LITTLE_ENDIAN_ORDER, k0.begin(), 8, userKey, keylen);
 	if (keylen < 32)
-		k[keylen/4] |= word32(1) << ((keylen%4)*8);
+		k0[keylen/4] |= word32(1) << ((keylen%4)*8);
 
-	k += 8;
-	word32 t = k[-1];
-	signed int i;
-	for (i = 0; i < 132; ++i)
+	word32 t = k0[7];
+	unsigned int i;
+	for (i = 0; i < 8; ++i)
+		k[i] = k0[i] = t = rotlFixed(k0[i] ^ k0[(i+3)%8] ^ k0[(i+5)%8] ^ t ^ 0x9e3779b9 ^ i, 11);
+	for (i = 8; i < 4*(rounds+1); ++i)
 		k[i] = t = rotlFixed(k[i-8] ^ k[i-5] ^ k[i-3] ^ t ^ 0x9e3779b9 ^ i, 11);
 	k -= 20;
 
-#define LK(r, a, b, c, d, e)	{\
-	a = k[(8-r)*4 + 0];		\
-	b = k[(8-r)*4 + 1];		\
-	c = k[(8-r)*4 + 2];		\
-	d = k[(8-r)*4 + 3];}
-
-#define SK(r, a, b, c, d, e)	{\
-	k[(8-r)*4 + 4] = a;		\
-	k[(8-r)*4 + 5] = b;		\
-	k[(8-r)*4 + 6] = c;		\
-	k[(8-r)*4 + 7] = d;}	\
-
 	word32 a,b,c,d,e;
-	for (i=0; i<4; i++)
+	for (i=0; i<rounds/8; i++)
 	{
 		afterS2(LK); afterS2(S3); afterS3(SK);
 		afterS1(LK); afterS1(S2); afterS2(SK);
@@ -466,6 +39,12 @@ void Serpent::Base::UncheckedSetKey(const byte *userKey, unsigned int keylen, co
 	afterS2(LK); afterS2(S3); afterS3(SK);
 }
 
+void Serpent::Base::UncheckedSetKey(const byte *userKey, unsigned int keylen, const NameValuePairs &)
+{
+	AssertValidKeyLength(keylen);
+	Serpent_KeySchedule(m_key, 32, userKey, keylen);
+}
+
 typedef BlockGetAndPut<word32, LittleEndian> Block;
 
 void Serpent::Enc::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock, byte *outBlock) const
@@ -474,7 +53,7 @@ void Serpent::Enc::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock,
 	
 	Block::Get(inBlock)(a)(b)(c)(d);
 
-	const word32 *k = m_key + 8;
+	const word32 *k = m_key;
 	unsigned int i=1;
 
 	do
@@ -513,7 +92,7 @@ void Serpent::Dec::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock,
 	
 	Block::Get(inBlock)(a)(b)(c)(d);
 
-	const word32 *k = m_key + 104;
+	const word32 *k = m_key + 96;
 	unsigned int i=4;
 
 	beforeI7(KX);