nettle: use rsa_*_key_prepare on key import

Previously we calculated the size of the key directly, but
by using the rsa_*_key_prepare we benefit from any checks that
may be introduced in the future. Specifically any checks for invalid
public keys (e.g., keys that may crash the underlying gmp functions).

This patch avoids calling rsa_private_key_prepare every time we construct
a nettle private key struct, because this function requires a bigint
multiplication. We call that function once on private key import.

1 files changed, 52 insertions, 10 deletions

diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index f6dc7399e0..8a5785803b 100644
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -126,18 +126,24 @@ _rsa_params_to_privkey(const gnutls_pk_params_st * pk_params,
 	memcpy(priv->c, pk_params->params[5], SIZEOF_MPZT);
 	memcpy(priv->a, pk_params->params[6], SIZEOF_MPZT);
 	memcpy(priv->b, pk_params->params[7], SIZEOF_MPZT);
+	/* we do not rsa_private_key_prepare() because it involves a multiplication.
+	 * we call it once when we import the parameters */
 	priv->size =
 	    nettle_mpz_sizeinbase_256_u(TOMPZ
 					(pk_params->params[RSA_MODULUS]));
 }
 
-static void
+/* returns a negative value on invalid pubkey */
+static int
 _rsa_params_to_pubkey(const gnutls_pk_params_st * pk_params,
 		      struct rsa_public_key *pub)
 {
 	memcpy(pub->n, pk_params->params[RSA_MODULUS], SIZEOF_MPZT);
 	memcpy(pub->e, pk_params->params[RSA_PUB], SIZEOF_MPZT);
-	pub->size = nettle_mpz_sizeinbase_256_u(pub->n);
+	if (rsa_public_key_prepare(pub) == 0)
+		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+
+	return 0;
 }
 
 static int
@@ -340,7 +346,13 @@ _wrap_nettle_pk_encrypt(gnutls_pk_algorithm_t algo,
 		{
 			struct rsa_public_key pub;
 
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret < 0) {
+				ret =
+				    gnutls_assert_val
+				    (GNUTLS_E_ENCRYPTION_FAILED);
+				goto cleanup;
+			}
 
 			ret =
 			    rsa_encrypt(&pub, NULL, rnd_func,
@@ -402,7 +414,12 @@ _wrap_nettle_pk_decrypt(gnutls_pk_algorithm_t algo,
 			bigint_t c;
 
 			_rsa_params_to_privkey(pk_params, &priv);
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret < 0)
+				return
+				    gnutls_assert_val
+				    (GNUTLS_E_DECRYPTION_FAILED);
+
 
 			if (ciphertext->size != pub.size)
 				return
@@ -623,7 +640,11 @@ _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
 			mpz_t s;
 
 			_rsa_params_to_privkey(pk_params, &priv);
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret < 0)
+				return
+				    gnutls_assert_val
+				    (GNUTLS_E_PK_SIGN_FAILED);
 
 			mpz_init(s);
 
@@ -795,7 +816,11 @@ _wrap_nettle_pk_verify(gnutls_pk_algorithm_t algo,
 		{
 			struct rsa_public_key pub;
 
-			_rsa_params_to_pubkey(pk_params, &pub);
+			ret = _rsa_params_to_pubkey(pk_params, &pub);
+			if (ret < 0)
+				return
+				    gnutls_assert_val
+				    (GNUTLS_E_PK_SIG_VERIFY_FAILED);
 
 			if (signature->size != pub.size)
 				return
@@ -1851,7 +1876,12 @@ wrap_nettle_pk_fixup(gnutls_pk_algorithm_t algo,
 {
 	int ret;
 
-	if (direction == GNUTLS_IMPORT && algo == GNUTLS_PK_RSA) {
+	if (direction != GNUTLS_IMPORT)
+		return 0;
+
+	if (algo == GNUTLS_PK_RSA) {
+		struct rsa_private_key priv;
+
 		/* do not trust the generated values. Some old private keys
 		 * generated by us have mess on the values. Those were very
 		 * old but it seemed some of the shipped example private
@@ -1865,9 +1895,14 @@ wrap_nettle_pk_fixup(gnutls_pk_algorithm_t algo,
 			if (ret < 0)
 				return gnutls_assert_val(ret);
 		}
-		mpz_invert(TOMPZ(params->params[RSA_COEF]),
-			   TOMPZ(params->params[RSA_PRIME2]),
-			   TOMPZ(params->params[RSA_PRIME1]));
+
+		if (mpz_cmp_ui(TOMPZ(params->params[RSA_PRIME1]), 0) == 0)
+			return gnutls_assert_val(GNUTLS_E_ILLEGAL_PARAMETER);
+
+		if (mpz_invert(TOMPZ(params->params[RSA_COEF]),
+			       TOMPZ(params->params[RSA_PRIME2]),
+			       TOMPZ(params->params[RSA_PRIME1])) == 0)
+			return gnutls_assert_val(GNUTLS_E_ILLEGAL_PARAMETER);
 
 		/* calculate exp1 [6] and exp2 [7] */
 		zrelease_mpi_key(&params->params[RSA_E1]);
@@ -1878,6 +1913,13 @@ wrap_nettle_pk_fixup(gnutls_pk_algorithm_t algo,
 			return gnutls_assert_val(ret);
 
 		params->params_nr = RSA_PRIVATE_PARAMS;
+
+		/* perform nettle's internal checks */
+		_rsa_params_to_privkey(params, &priv);
+		ret = rsa_private_key_prepare(&priv);
+		if (ret == 0) {
+			return gnutls_assert_val(GNUTLS_E_ILLEGAL_PARAMETER);
+		}
 	}
 
 	return 0;