Merge main -> VMS_64BITBRANCH_VMS_64BIT

1 files changed, 63 insertions, 34 deletions

diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index e3f7c608ec..553d212ebe 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -28,7 +28,7 @@
 #include <openssl/rand.h>
 #include <openssl/sha.h>
 
-int MGF1(unsigned char *mask, long len,
+static int MGF1(unsigned char *mask, long len,
 	const unsigned char *seed, long seedlen);
 
 int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
@@ -52,18 +52,12 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
 		return 0;
 		}
 
-	dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH);
-	if (dbmask == NULL)
-		{
-		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
-		return 0;
-		}
-
 	to[0] = 0;
 	seed = to + 1;
 	db = to + SHA_DIGEST_LENGTH + 1;
 
-	EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL);
+	if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
+		return 0;
 	memset(db + SHA_DIGEST_LENGTH, 0,
 		emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
 	db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
@@ -76,11 +70,20 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
 	   20);
 #endif
 
-	MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
+	dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH);
+	if (dbmask == NULL)
+		{
+		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
+		return 0;
+		}
+
+	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH) < 0)
+		return 0;
 	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
 		db[i] ^= dbmask[i];
 
-	MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);
+	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH) < 0)
+		return 0;
 	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
 		seed[i] ^= seedmask[i];
 
@@ -96,6 +99,7 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 	const unsigned char *maskeddb;
 	int lzero;
 	unsigned char *db = NULL, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
+	unsigned char *padded_from;
 	int bad = 0;
 
 	if (--num < 2 * SHA_DIGEST_LENGTH + 1)
@@ -106,8 +110,6 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 	lzero = num - flen;
 	if (lzero < 0)
 		{
-		/* lzero == -1 */
-
 		/* signalling this error immediately after detection might allow
 		 * for side-channel attacks (e.g. timing if 'plen' is huge
 		 * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA Optimal
@@ -115,26 +117,37 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 		 * so we use a 'bad' flag */
 		bad = 1;
 		lzero = 0;
+		flen = num; /* don't overflow the memcpy to padded_from */
 		}
-	maskeddb = from - lzero + SHA_DIGEST_LENGTH;
 
 	dblen = num - SHA_DIGEST_LENGTH;
-	db = OPENSSL_malloc(dblen);
+	db = OPENSSL_malloc(dblen + num);
 	if (db == NULL)
 		{
-		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
+		RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
 		return -1;
 		}
 
-	MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
-	for (i = lzero; i < SHA_DIGEST_LENGTH; i++)
-		seed[i] ^= from[i - lzero];
+	/* Always do this zero-padding copy (even when lzero == 0)
+	 * to avoid leaking timing info about the value of lzero. */
+	padded_from = db + dblen;
+	memset(padded_from, 0, lzero);
+	memcpy(padded_from + lzero, from, flen);
+
+	maskeddb = padded_from + SHA_DIGEST_LENGTH;
+
+	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
+		return -1;
+	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
+		seed[i] ^= padded_from[i];
   
-	MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
+	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
+		return -1;
 	for (i = 0; i < dblen; i++)
 		db[i] ^= maskeddb[i];
 
-	EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL);
+	if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
+		return -1;
 
 	if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
 		goto decoding_err;
@@ -143,13 +156,13 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 		for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
 			if (db[i] != 0x00)
 				break;
-		if (db[i] != 0x01 || i++ >= dblen)
+		if (i == dblen || db[i] != 0x01)
 			goto decoding_err;
 		else
 			{
 			/* everything looks OK */
 
-			mlen = dblen - i;
+			mlen = dblen - ++i;
 			if (tlen < mlen)
 				{
 				RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);
@@ -170,37 +183,53 @@ decoding_err:
 	return -1;
 	}
 
-int MGF1(unsigned char *mask, long len,
-	const unsigned char *seed, long seedlen)
+int PKCS1_MGF1(unsigned char *mask, long len,
+	const unsigned char *seed, long seedlen, const EVP_MD *dgst)
 	{
 	long i, outlen = 0;
 	unsigned char cnt[4];
 	EVP_MD_CTX c;
-	unsigned char md[SHA_DIGEST_LENGTH];
+	unsigned char md[EVP_MAX_MD_SIZE];
+	int mdlen;
+	int rv = -1;
 
 	EVP_MD_CTX_init(&c);
+	mdlen = EVP_MD_size(dgst);
+	if (mdlen < 0)
+		goto err;
 	for (i = 0; outlen < len; i++)
 		{
 		cnt[0] = (unsigned char)((i >> 24) & 255);
 		cnt[1] = (unsigned char)((i >> 16) & 255);
 		cnt[2] = (unsigned char)((i >> 8)) & 255;
 		cnt[3] = (unsigned char)(i & 255);
-		EVP_DigestInit_ex(&c,EVP_sha1(), NULL);
-		EVP_DigestUpdate(&c, seed, seedlen);
-		EVP_DigestUpdate(&c, cnt, 4);
-		if (outlen + SHA_DIGEST_LENGTH <= len)
+		if (!EVP_DigestInit_ex(&c,dgst, NULL)
+			|| !EVP_DigestUpdate(&c, seed, seedlen)
+			|| !EVP_DigestUpdate(&c, cnt, 4))
+			goto err;
+		if (outlen + mdlen <= len)
 			{
-			EVP_DigestFinal_ex(&c, mask + outlen, NULL);
-			outlen += SHA_DIGEST_LENGTH;
+			if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL))
+				goto err;
+			outlen += mdlen;
 			}
 		else
 			{
-			EVP_DigestFinal_ex(&c, md, NULL);
+			if (!EVP_DigestFinal_ex(&c, md, NULL))
+				goto err;
 			memcpy(mask + outlen, md, len - outlen);
 			outlen = len;
 			}
 		}
+	rv = 0;
+	err:
 	EVP_MD_CTX_cleanup(&c);
-	return 0;
+	return rv;
+	}
+
+static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
+		 long seedlen)
+	{
+	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
 	}
 #endif