When PCLMUL isn't available use the SSSE3 implementation of AES to optimize GCM.

8 files changed, 203 insertions, 28 deletions

diff --git a/lib/accelerated/x86/Makefile.am b/lib/accelerated/x86/Makefile.am
index e231103dad..d5525903b2 100644
--- a/lib/accelerated/x86/Makefile.am
+++ b/lib/accelerated/x86/Makefile.am
@@ -36,13 +36,14 @@ EXTRA_DIST = README license.txt files.mk
 noinst_LTLIBRARIES = libx86.la
 
 libx86_la_SOURCES = sha-padlock.c hmac-padlock.c aes-x86.c aes-padlock.c aes-gcm-padlock.c \
-	aes-padlock.h aes-x86.h x86.h sha-padlock.h sha-x86.c sha-x86.h hmac-x86.c
+	aes-padlock.h aes-x86.h x86.h sha-padlock.h sha-x86.c sha-x86.h hmac-x86.c \
+	aes-gcm-x86-ssse3.c
 
 include files.mk
 
 if ASM_X86_64
 AM_CFLAGS += -DASM_X86_64 -DASM_X86
-libx86_la_SOURCES += aes-gcm-x86.c
+libx86_la_SOURCES += aes-gcm-x86-pclmul.c
 
 if WINDOWS
 libx86_la_SOURCES += $(X86_64_FILES_COFF)
diff --git a/lib/accelerated/x86/aes-gcm-x86.c b/lib/accelerated/x86/aes-gcm-x86-pclmul.c
index d506553465..b33cf395c8 100644
--- a/lib/accelerated/x86/aes-gcm-x86.c
+++ b/lib/accelerated/x86/aes-gcm-x86-pclmul.c
@@ -255,7 +255,7 @@ static void aes_gcm_tag(void *_ctx, void *tag, size_t tagsize)
 	memcpy(tag, ctx->gcm.Xi.c, MIN(GCM_BLOCK_SIZE, tagsize));
 }
 
-const gnutls_crypto_cipher_st aes_gcm_struct = {
+const gnutls_crypto_cipher_st aes_gcm_pclmul = {
 	.init = aes_gcm_cipher_init,
 	.setkey = aes_gcm_cipher_setkey,
 	.setiv = aes_gcm_setiv,
diff --git a/lib/accelerated/x86/aes-gcm-x86-ssse3.c b/lib/accelerated/x86/aes-gcm-x86-ssse3.c
new file mode 100644
index 0000000000..14942e2028
--- /dev/null
+++ b/lib/accelerated/x86/aes-gcm-x86-ssse3.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2011-2012 Free Software Foundation, Inc.
+ *
+ * Author: Nikos Mavrogiannopoulos
+ *
+ * This file is part of GnuTLS.
+ *
+ * The GnuTLS is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+/*
+ * The following code is an implementation of the AES-128-GCM cipher
+ */
+
+#include <gnutls_errors.h>
+#include <gnutls_int.h>
+
+#ifdef HAVE_LIBNETTLE
+
+#include <gnutls/crypto.h>
+#include <gnutls_errors.h>
+#include <aes-x86.h>
+#include <x86.h>
+#include <byteswap.h>
+#include <nettle/gcm.h>
+#include <aes-x86.h>
+
+/* GCM mode 
+ * It is used when the CPU doesn't include the PCLMUL instructions.
+ */
+struct gcm_x86_aes_ctx GCM_CTX(AES_KEY);
+
+static void x86_aes_encrypt(void *_ctx,
+				unsigned length, uint8_t * dst,
+				const uint8_t * src)
+{
+	AES_KEY *ctx = _ctx;
+
+	vpaes_encrypt(src, dst, ctx);
+}
+
+static void x86_aes_set_encrypt_key(void *_ctx,
+					unsigned length,
+					const uint8_t * key)
+{
+	AES_KEY *ctx = _ctx;
+
+	vpaes_set_encrypt_key(key, length*8, ctx);
+}
+
+static int
+aes_gcm_cipher_init(gnutls_cipher_algorithm_t algorithm, void **_ctx,
+		    int enc)
+{
+	/* we use key size to distinguish */
+	if (algorithm != GNUTLS_CIPHER_AES_128_GCM &&
+	    algorithm != GNUTLS_CIPHER_AES_256_GCM)
+		return GNUTLS_E_INVALID_REQUEST;
+
+	*_ctx = gnutls_calloc(1, sizeof(struct gcm_x86_aes_ctx));
+	if (*_ctx == NULL) {
+		gnutls_assert();
+		return GNUTLS_E_MEMORY_ERROR;
+	}
+
+	return 0;
+}
+
+static int
+aes_gcm_cipher_setkey(void *_ctx, const void *userkey, size_t keysize)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	GCM_SET_KEY(ctx, x86_aes_set_encrypt_key, x86_aes_encrypt,
+		    keysize, userkey);
+
+	return 0;
+}
+
+static int aes_gcm_setiv(void *_ctx, const void *iv, size_t iv_size)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	if (iv_size != GCM_BLOCK_SIZE - 4)
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
+	GCM_SET_IV(ctx, iv_size, iv);
+
+	return 0;
+}
+
+static int
+aes_gcm_encrypt(void *_ctx, const void *src, size_t src_size,
+		void *dst, size_t length)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	GCM_ENCRYPT(ctx, x86_aes_encrypt, src_size, dst, src);
+
+	return 0;
+}
+
+static int
+aes_gcm_decrypt(void *_ctx, const void *src, size_t src_size,
+		void *dst, size_t dst_size)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	GCM_DECRYPT(ctx, x86_aes_encrypt, src_size, dst, src);
+	return 0;
+}
+
+static int aes_gcm_auth(void *_ctx, const void *src, size_t src_size)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	GCM_UPDATE(ctx, src_size, src);
+
+	return 0;
+}
+
+static void aes_gcm_tag(void *_ctx, void *tag, size_t tagsize)
+{
+	struct gcm_x86_aes_ctx *ctx = _ctx;
+
+	GCM_DIGEST(ctx, x86_aes_encrypt, tagsize, tag);
+}
+
+static void aes_gcm_deinit(void *ctx)
+{
+	gnutls_free(ctx);
+}
+
+const gnutls_crypto_cipher_st aes_gcm_x86_ssse3 = {
+	.init = aes_gcm_cipher_init,
+	.setkey = aes_gcm_cipher_setkey,
+	.setiv = aes_gcm_setiv,
+	.encrypt = aes_gcm_encrypt,
+	.decrypt = aes_gcm_decrypt,
+	.deinit = aes_gcm_deinit,
+	.tag = aes_gcm_tag,
+	.auth = aes_gcm_auth,
+};
+
+#endif
diff --git a/lib/accelerated/x86/aes-x86.c b/lib/accelerated/x86/aes-x86.c
index 9e00df5ee7..298ab5a511 100644
--- a/lib/accelerated/x86/aes-x86.c
+++ b/lib/accelerated/x86/aes-x86.c
@@ -118,7 +118,7 @@ static void aes_deinit(void *_ctx)
 	gnutls_free(_ctx);
 }
 
-static const gnutls_crypto_cipher_st aesni_struct = {
+static const gnutls_crypto_cipher_st aesni_x86 = {
 	.init = aes_cipher_init,
 	.setkey = aes_cipher_setkey,
 	.setiv = aes_setiv,
@@ -171,7 +171,7 @@ aes_ssse3_decrypt(void *_ctx, const void *src, size_t src_size,
 	return 0;
 }
 
-static const gnutls_crypto_cipher_st aes_ssse3_struct = {
+static const gnutls_crypto_cipher_st aes_ssse3 = {
 	.init = aes_cipher_init,
 	.setkey = aes_ssse3_cipher_setkey,
 	.setiv = aes_setiv,
@@ -228,21 +228,37 @@ void register_x86_crypto(void)
 
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_128_CBC, 79, &aes_ssse3_struct);
+		    (GNUTLS_CIPHER_AES_128_GCM, 79,
+		     &aes_gcm_x86_ssse3);
+			if (ret < 0) {
+				gnutls_assert();
+			}
+
+		ret =
+		    gnutls_crypto_single_cipher_register
+		    (GNUTLS_CIPHER_AES_256_GCM, 79,
+		     &aes_gcm_x86_ssse3);
+		if (ret < 0) {
+			gnutls_assert();
+		}
+
+		ret =
+		    gnutls_crypto_single_cipher_register
+		    (GNUTLS_CIPHER_AES_128_CBC, 79, &aes_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
 
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_192_CBC, 79, &aes_ssse3_struct);
+		    (GNUTLS_CIPHER_AES_192_CBC, 79, &aes_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
 
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_256_CBC, 79, &aes_ssse3_struct);
+		    (GNUTLS_CIPHER_AES_256_CBC, 79, &aes_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
@@ -250,7 +266,7 @@ void register_x86_crypto(void)
 		ret =
 		    gnutls_crypto_single_digest_register(GNUTLS_DIG_SHA1,
 							 80,
-							 &sha_x86_struct);
+							 &sha_x86_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
@@ -258,7 +274,7 @@ void register_x86_crypto(void)
 		ret =
 		    gnutls_crypto_single_digest_register(GNUTLS_DIG_SHA224,
 							 80,
-							 &sha_x86_struct);
+							 &sha_x86_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
@@ -266,7 +282,7 @@ void register_x86_crypto(void)
 		ret =
 		    gnutls_crypto_single_digest_register(GNUTLS_DIG_SHA256,
 							 80,
-							 &sha_x86_struct);
+							 &sha_x86_ssse3);
 		if (ret < 0) {
 			gnutls_assert();
 		}
@@ -275,21 +291,21 @@ void register_x86_crypto(void)
 		ret =
 		    gnutls_crypto_single_mac_register(GNUTLS_DIG_SHA1,
 							 80,
-							 &hmac_sha_x86_struct);
+							 &hmac_sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 
 		ret =
 		    gnutls_crypto_single_mac_register(GNUTLS_DIG_SHA224,
 							 80,
-							 &hmac_sha_x86_struct);
+							 &hmac_sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 
 		ret =
 		    gnutls_crypto_single_mac_register(GNUTLS_DIG_SHA256,
 							 80,
-							 &hmac_sha_x86_struct);
+							 &hmac_sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 
@@ -297,27 +313,27 @@ void register_x86_crypto(void)
 		ret =
 		    gnutls_crypto_single_digest_register(GNUTLS_DIG_SHA384,
 							 80,
-							 &sha_x86_struct);
+							 &sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 
 		ret =
 		    gnutls_crypto_single_digest_register(GNUTLS_DIG_SHA512,
 							 80,
-							 &sha_x86_struct);
+							 &sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 		ret =
 		    gnutls_crypto_single_mac_register(GNUTLS_DIG_SHA384,
 							 80,
-							 &hmac_sha_x86_struct);
+							 &hmac_sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 
 		ret =
 		    gnutls_crypto_single_mac_register(GNUTLS_DIG_SHA512,
 							 80,
-							 &hmac_sha_x86_struct);
+							 &hmac_sha_x86_ssse3);
 		if (ret < 0)
 			gnutls_assert();
 #endif
@@ -327,21 +343,21 @@ void register_x86_crypto(void)
 		_gnutls_debug_log("Intel AES accelerator was detected\n");
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_128_CBC, 80, &aesni_struct);
+		    (GNUTLS_CIPHER_AES_128_CBC, 80, &aesni_x86);
 		if (ret < 0) {
 			gnutls_assert();
 		}
 
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_192_CBC, 80, &aesni_struct);
+		    (GNUTLS_CIPHER_AES_192_CBC, 80, &aesni_x86);
 		if (ret < 0) {
 			gnutls_assert();
 		}
 
 		ret =
 		    gnutls_crypto_single_cipher_register
-		    (GNUTLS_CIPHER_AES_256_CBC, 80, &aesni_struct);
+		    (GNUTLS_CIPHER_AES_256_CBC, 80, &aesni_x86);
 		if (ret < 0) {
 			gnutls_assert();
 		}
@@ -353,7 +369,7 @@ void register_x86_crypto(void)
 			ret =
 			    gnutls_crypto_single_cipher_register
 			    (GNUTLS_CIPHER_AES_128_GCM, 80,
-			     &aes_gcm_struct);
+			     &aes_gcm_pclmul);
 			if (ret < 0) {
 				gnutls_assert();
 			}
@@ -361,7 +377,7 @@ void register_x86_crypto(void)
 			ret =
 			    gnutls_crypto_single_cipher_register
 			    (GNUTLS_CIPHER_AES_256_GCM, 80,
-			     &aes_gcm_struct);
+			     &aes_gcm_pclmul);
 			if (ret < 0) {
 				gnutls_assert();
 			}
diff --git a/lib/accelerated/x86/aes-x86.h b/lib/accelerated/x86/aes-x86.h
index 2c81bb36c1..cb3fed4bc5 100644
--- a/lib/accelerated/x86/aes-x86.h
+++ b/lib/accelerated/x86/aes-x86.h
@@ -43,6 +43,7 @@ void vpaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
 void vpaes_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key);
 void vpaes_decrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key);
 
-extern const gnutls_crypto_cipher_st aes_gcm_struct;
+extern const gnutls_crypto_cipher_st aes_gcm_pclmul;
+extern const gnutls_crypto_cipher_st aes_gcm_x86_ssse3;
 
 #endif
diff --git a/lib/accelerated/x86/hmac-x86.c b/lib/accelerated/x86/hmac-x86.c
index 73b21cc25d..3058ad67f2 100644
--- a/lib/accelerated/x86/hmac-x86.c
+++ b/lib/accelerated/x86/hmac-x86.c
@@ -287,7 +287,7 @@ static int wrap_x86_hmac_fast(gnutls_mac_algorithm_t algo,
 	return 0;
 }
 
-const gnutls_crypto_mac_st hmac_sha_x86_struct = {
+const gnutls_crypto_mac_st hmac_sha_x86_ssse3 = {
 	.init = wrap_x86_hmac_init,
 	.setkey = wrap_x86_hmac_setkey,
 	.setnonce = NULL,
diff --git a/lib/accelerated/x86/sha-x86.c b/lib/accelerated/x86/sha-x86.c
index 054dc6ce73..29dcd2dbd2 100644
--- a/lib/accelerated/x86/sha-x86.c
+++ b/lib/accelerated/x86/sha-x86.c
@@ -356,7 +356,7 @@ const struct nettle_hash x86_sha512 =
 NN_HASH(sha512, x86_sha512_update, sha512_digest, SHA512);
 #endif
 
-const gnutls_crypto_digest_st sha_x86_struct = {
+const gnutls_crypto_digest_st sha_x86_ssse3 = {
 	.init = wrap_x86_hash_init,
 	.hash = wrap_x86_hash_update,
 	.output = wrap_x86_hash_output,
diff --git a/lib/accelerated/x86/sha-x86.h b/lib/accelerated/x86/sha-x86.h
index a288962da5..55994dce9e 100644
--- a/lib/accelerated/x86/sha-x86.h
+++ b/lib/accelerated/x86/sha-x86.h
@@ -16,7 +16,7 @@ void x86_sha1_update(struct sha1_ctx *ctx, size_t length, const uint8_t * data);
 void x86_sha256_update(struct sha256_ctx *ctx, size_t length, const uint8_t * data);
 void x86_sha512_update(struct sha512_ctx *ctx, size_t length, const uint8_t * data);
 
-extern const gnutls_crypto_digest_st sha_x86_struct;
-extern const gnutls_crypto_mac_st hmac_sha_x86_struct;
+extern const gnutls_crypto_digest_st sha_x86_ssse3;
+extern const gnutls_crypto_mac_st hmac_sha_x86_ssse3;
 
 #endif