_gnutls_find_rsa_pss_salt_size: add a validity check for salt size

That is, in order to reject invalid parameters.

Resolves #402

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

7 files changed, 48 insertions, 15 deletions

diff --git a/lib/pk.c b/lib/pk.c
index be1d8b6a85..3014396bc0 100644
--- a/lib/pk.c
+++ b/lib/pk.c
@@ -382,19 +382,29 @@ void gnutls_pk_params_clear(gnutls_pk_params_st * p)
 	}
 }
 
-unsigned
+int
 _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
 			       unsigned salt_size)
 {
-	unsigned max_salt_size, digest_size;
+	unsigned digest_size;
+	int max_salt_size;
+	unsigned key_size;
 
 	digest_size = _gnutls_hash_get_algo_len(me);
-	max_salt_size = (bits + 7) / 8 - digest_size - 2;
+	key_size = (bits + 7) / 8;
+
+	if (key_size == 0) {
+		return gnutls_assert_val(GNUTLS_E_PK_INVALID_PUBKEY);
+	} else {
+		max_salt_size = key_size - digest_size - 2;
+		if (max_salt_size < 0)
+			return gnutls_assert_val(GNUTLS_E_CONSTRAINT_ERROR);
+	}
 
 	if (salt_size < digest_size)
 		salt_size = digest_size;
 
-	if (salt_size > max_salt_size)
+	if (salt_size > (unsigned)max_salt_size)
 		salt_size = max_salt_size;
 
 	return salt_size;
diff --git a/lib/pk.h b/lib/pk.h
index 7ff76d12df..a6eb043333 100644
--- a/lib/pk.h
+++ b/lib/pk.h
@@ -104,7 +104,7 @@ int pk_hash_data(gnutls_pk_algorithm_t pk, const mac_entry_st * hash,
 		 gnutls_pk_params_st * params, const gnutls_datum_t * data,
 		 gnutls_datum_t * digest);
 
-unsigned _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
-					unsigned salt_size);
+int _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
+				   unsigned salt_size);
 
 #endif				/* GNUTLS_PK_H */
diff --git a/lib/privkey.c b/lib/privkey.c
index 6c1a52ee30..63cc7fcbd3 100644
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -333,6 +333,7 @@ _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 
 	if (pk == GNUTLS_PK_RSA_PSS) {
 		const mac_entry_st *me;
+		int ret;
 
 		me = hash_to_entry(dig);
 		if (unlikely(me == NULL))
@@ -350,8 +351,12 @@ _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 
 		if (flags & GNUTLS_PRIVKEY_FLAG_REPRODUCIBLE)
 			params->salt_size = 0;
-		else
-			params->salt_size = _gnutls_find_rsa_pss_salt_size(bits, me, salt_size);
+		else {
+			ret = _gnutls_find_rsa_pss_salt_size(bits, me, salt_size);
+			if (ret < 0)
+				return gnutls_assert_val(ret);
+			params->salt_size = ret;
+		}
 		params->rsa_pss_dig = dig;
 	}
 
diff --git a/lib/pubkey.c b/lib/pubkey.c
index a53122f68a..466e2dee63 100644
--- a/lib/pubkey.c
+++ b/lib/pubkey.c
@@ -1527,7 +1527,7 @@ int fixup_spki_params(const gnutls_pk_params_st *key_params, const gnutls_sign_e
 	}
 
 	if (params->pk == GNUTLS_PK_RSA_PSS) {
-
+		int ret;
 		if (!GNUTLS_PK_IS_RSA(key_params->algo))
 			return gnutls_assert_val(GNUTLS_E_CONSTRAINT_ERROR);
 
@@ -1537,7 +1537,11 @@ int fixup_spki_params(const gnutls_pk_params_st *key_params, const gnutls_sign_e
 		if (key_params->algo == GNUTLS_PK_RSA || params->rsa_pss_dig == 0) {
 			bits = pubkey_to_bits(key_params);
 			params->rsa_pss_dig = se->hash;
-			params->salt_size = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+			ret = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+			if (ret < 0)
+				return gnutls_assert_val(ret);
+
+			params->salt_size = ret;
 		}
 
 		if (params->rsa_pss_dig != se->hash)
diff --git a/lib/x509/crq.c b/lib/x509/crq.c
index ac58529f6f..417d630405 100644
--- a/lib/x509/crq.c
+++ b/lib/x509/crq.c
@@ -3052,9 +3052,14 @@ gnutls_x509_crq_set_spki(gnutls_x509_crq_t crq,
 
 		/* If salt size is zero, find the optimal salt size. */
 		if (spki->salt_size == 0) {
-			tpki.salt_size =
+			ret =
 			    _gnutls_find_rsa_pss_salt_size(bits, me,
 							   spki->salt_size);
+			if (ret < 0) {
+				gnutls_assert();
+				goto cleanup;
+			}
+			tpki.salt_size = ret;
 		} else
 			tpki.salt_size = spki->salt_size;
 	} else if (crq_pk == GNUTLS_PK_RSA_PSS) {
diff --git a/lib/x509/privkey.c b/lib/x509/privkey.c
index badeb945df..d4be99ef05 100644
--- a/lib/x509/privkey.c
+++ b/lib/x509/privkey.c
@@ -1682,8 +1682,13 @@ gnutls_x509_privkey_generate2(gnutls_x509_privkey_t key,
 			goto cleanup;
 		}
 
-		key->params.spki.salt_size =
-		    _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+		ret = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+		if (ret < 0) {
+			gnutls_assert();
+			goto cleanup;
+		}
+
+		key->params.spki.salt_size = ret;
 	}
 
 	ret = _gnutls_pk_generate_keys(algo, bits, &key->params, 0);
diff --git a/lib/x509/x509_write.c b/lib/x509/x509_write.c
index 6d5ed6a52e..db90dab9c4 100644
--- a/lib/x509/x509_write.c
+++ b/lib/x509/x509_write.c
@@ -2080,9 +2080,13 @@ gnutls_x509_crt_set_spki(gnutls_x509_crt_t crt,
 
 		/* If salt size is zero, find the optimal salt size. */
 		if (spki->salt_size == 0) {
-			tpki.salt_size =
-			    _gnutls_find_rsa_pss_salt_size(bits, me,
+			ret = _gnutls_find_rsa_pss_salt_size(bits, me,
 							   spki->salt_size);
+			if (ret < 0) {
+				gnutls_assert();
+				goto cleanup;
+			}
+			tpki.salt_size = ret;
 		} else
 			tpki.salt_size = spki->salt_size;
 	} else if (crt_pk == GNUTLS_PK_RSA_PSS) {