prior to negotiating a signature check compatibility with private key

That is, check if the private key can support the public key operation
needed for the signature. That in particular includes, excluding the
Ed25519 and RSA-PSS from being used with the 'EXT' keys as the
current API cannot handle them, and RSA-PSS from being used by PKCS#11
RSA keys which do not provide the CKM_RSA_PKCS_PSS mechanism.

Relates #234
Resolves #209

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

8 files changed, 85 insertions, 23 deletions

diff --git a/lib/abstract_int.h b/lib/abstract_int.h
index 878856782a..bfe0baa7bd 100644
--- a/lib/abstract_int.h
+++ b/lib/abstract_int.h
@@ -77,6 +77,8 @@ int _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 				     unsigned flags,
 				     gnutls_x509_spki_st *params);
 
+unsigned _gnutls_privkey_compatible_with_sig(gnutls_privkey_t key, gnutls_sign_algorithm_t sig);
+
 void _gnutls_privkey_cleanup(gnutls_privkey_t key);
 
 int privkey_sign_and_hash_data(gnutls_privkey_t signer,
diff --git a/lib/auth/cert.c b/lib/auth/cert.c
index bcf7ffff4b..c66f4f2950 100644
--- a/lib/auth/cert.c
+++ b/lib/auth/cert.c
@@ -1448,6 +1448,7 @@ unsigned pubkey_is_compat_with_cs(gnutls_session_t session,
 static
 int select_sign_algorithm(gnutls_session_t session,
 			  gnutls_pcert_st * cert,
+			  gnutls_privkey_t pkey,
 			  const gnutls_cipher_suite_entry_st *cs)
 {
 	gnutls_sign_algorithm_t algo;
@@ -1464,7 +1465,7 @@ int select_sign_algorithm(gnutls_session_t session,
 		return 0;
 	}
 
-	algo = _gnutls_session_get_sign_algo(session, cert, 0);
+	algo = _gnutls_session_get_sign_algo(session, cert, pkey, 0);
 	if (algo == GNUTLS_SIGN_UNKNOWN)
 		return gnutls_assert_val(GNUTLS_E_INCOMPATIBLE_SIG_WITH_KEY);
 
@@ -1526,7 +1527,10 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 			return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
 		}
 
-		ret = select_sign_algorithm(session, &session->internals.selected_cert_list[0], cs);
+		ret = select_sign_algorithm(session,
+					    &session->internals.selected_cert_list[0],
+					    session->internals.selected_key,
+					    cs);
 		if (ret < 0) {
 			return gnutls_assert_val(ret);
 		}
@@ -1561,7 +1565,10 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 					continue;
 				}
 
-				ret = select_sign_algorithm(session, &cred->certs[i].cert_list[0], cs);
+				ret = select_sign_algorithm(session,
+							    &cred->certs[i].cert_list[0],
+							    cred->pkey[i],
+							    cs);
 				if (ret >= 0) {
 					idx = i;
 					_gnutls_debug_log("Selected (%s) cert based on ciphersuite %x.%x: %s\n",
@@ -1595,7 +1602,10 @@ _gnutls_server_select_cert(gnutls_session_t session, const gnutls_cipher_suite_e
 			continue;
 		}
 
-		ret = select_sign_algorithm(session, &cred->certs[i].cert_list[0], cs);
+		ret = select_sign_algorithm(session,
+					    &cred->certs[i].cert_list[0],
+					    cred->pkey[i],
+					    cs);
 		if (ret >= 0) {
 			idx = i;
 			_gnutls_debug_log("Selected (%s) cert based on ciphersuite %x.%x: %s\n",
diff --git a/lib/ext/signature.c b/lib/ext/signature.c
index 765a475b1a..61a67b0d31 100644
--- a/lib/ext/signature.c
+++ b/lib/ext/signature.c
@@ -255,7 +255,9 @@ _gnutls_signature_algorithm_send_params(gnutls_session_t session,
  */
 gnutls_sign_algorithm_t
 _gnutls_session_get_sign_algo(gnutls_session_t session,
-			      gnutls_pcert_st * cert, unsigned client_cert)
+			      gnutls_pcert_st * cert,
+			      gnutls_privkey_t privkey,
+			      unsigned client_cert)
 {
 	unsigned i;
 	int ret;
@@ -285,6 +287,9 @@ _gnutls_session_get_sign_algo(gnutls_session_t session,
 	}
 
 	for (i = 0; i < priv->sign_algorithms_size; i++) {
+		if (_gnutls_privkey_compatible_with_sig(privkey, priv->sign_algorithms[i]) == 0)
+			continue;
+
 		if (gnutls_sign_supports_pk_algorithm(priv->sign_algorithms[i], cert_algo) != 0) {
 			if (_gnutls_pubkey_compatible_with_sig
 			    (session, cert->pubkey, ver,
diff --git a/lib/ext/signature.h b/lib/ext/signature.h
index fe1c1ee1c9..5e8f710c13 100644
--- a/lib/ext/signature.h
+++ b/lib/ext/signature.h
@@ -31,7 +31,9 @@ extern const extension_entry_st ext_mod_sig;
 
 gnutls_sign_algorithm_t
 _gnutls_session_get_sign_algo(gnutls_session_t session,
-			      gnutls_pcert_st * cert, unsigned client_cert);
+			      gnutls_pcert_st * cert,
+			      gnutls_privkey_t privkey,
+			      unsigned client_cert);
 int _gnutls_sign_algorithm_parse_data(gnutls_session_t session,
 				      const uint8_t * data,
 				      size_t data_size);
diff --git a/lib/pkcs11_int.h b/lib/pkcs11_int.h
index baa28b772f..2c2de3463f 100644
--- a/lib/pkcs11_int.h
+++ b/lib/pkcs11_int.h
@@ -61,6 +61,23 @@ struct gnutls_pkcs11_obj_st {
 	struct pin_info_st pin;
 };
 
+struct gnutls_pkcs11_privkey_st {
+	gnutls_pk_algorithm_t pk_algorithm;
+	unsigned int rsa_pss_ok; /* if it is an RSA key, it can do RSA-PSS */
+
+	unsigned int flags;
+	struct p11_kit_uri *uinfo;
+	char *url;
+
+	struct pkcs11_session_info sinfo;
+	ck_object_handle_t ref;	/* the key in the session */
+	unsigned reauth; /* whether we need to login on each operation */
+
+	void *mutex; /* lock for operations requiring co-ordination */
+
+	struct pin_info_st pin;
+};
+
 /* This must be called on every function that uses a PKCS #11 function
  * directly. It can be provided a callback function to run when a reinitialization
  * occurs. */
diff --git a/lib/pkcs11_privkey.c b/lib/pkcs11_privkey.c
index b6765fcec8..9e1d1de1fa 100644
--- a/lib/pkcs11_privkey.c
+++ b/lib/pkcs11_privkey.c
@@ -59,22 +59,6 @@
 		break; \
 	} while (1);
 
-struct gnutls_pkcs11_privkey_st {
-	gnutls_pk_algorithm_t pk_algorithm;
-	unsigned int rsa_pss_ok; /* if it is an RSA key, it can do RSA-PSS */
-
-	unsigned int flags;
-	struct p11_kit_uri *uinfo;
-	char *url;
-
-	struct pkcs11_session_info sinfo;
-	ck_object_handle_t ref;	/* the key in the session */
-	unsigned reauth; /* whether we need to login on each operation */
-
-	void *mutex; /* lock for operations requiring co-ordination */
-
-	struct pin_info_st pin;
-};
 
 /**
  * gnutls_pkcs11_privkey_init:
diff --git a/lib/privkey.c b/lib/privkey.c
index 2011ca8a45..d1bccc30b8 100644
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -35,6 +35,7 @@
 #include <fips.h>
 #include <system-keys.h>
 #include "urls.h"
+#include "pkcs11_int.h"
 #include <abstract_int.h>
 
 static int
@@ -1667,3 +1668,44 @@ gnutls_privkey_set_spki(gnutls_privkey_t privkey, const gnutls_x509_spki_t spki,
 
 	return gnutls_x509_privkey_set_spki(privkey->key.x509, spki, flags);
 }
+
+/* Checks whether the public key given is compatible with the
+ * signature algorithm used. The session is only used for audit logging, and
+ * it may be null.
+ */
+unsigned _gnutls_privkey_compatible_with_sig(gnutls_privkey_t privkey,
+				        gnutls_sign_algorithm_t sign)
+{
+	const gnutls_sign_entry_st *se;
+
+	se = _gnutls_sign_to_entry(sign);
+	if (unlikely(se == NULL))
+		return gnutls_assert_val(0);
+
+	/* Prevent RSA-PSS private keys from negotiating an RSA signature,
+	 * and RSA keys which cannot do RSA-PSS (e.g., smart card) from
+	 * negotiating RSA-PSS sig.
+	 */
+	if (privkey->pk_algorithm == GNUTLS_PK_RSA_PSS && se->pk != GNUTLS_PK_RSA_PSS) {
+			return 0;
+	}
+
+	if (privkey->type == GNUTLS_PRIVKEY_EXT) {
+		/* This key type is very limited on what it can handle */
+		if (se->pk == GNUTLS_PK_EDDSA_ED25519)
+			return 0;
+
+		if (se->pk == GNUTLS_PK_RSA_PSS)
+			return 0;
+	}
+#ifdef ENABLE_PKCS11
+	else if (privkey->type == GNUTLS_PRIVKEY_PKCS11) {
+		if (privkey->pk_algorithm == GNUTLS_PK_RSA && se->pk == GNUTLS_PK_RSA_PSS) {
+			if (!privkey->key.pkcs11->rsa_pss_ok)
+				return 0;
+		}
+	}
+#endif
+
+	return 1;
+}
diff --git a/lib/tls-sig.c b/lib/tls-sig.c
index 95a7b3ea64..a452cdfb77 100644
--- a/lib/tls-sig.c
+++ b/lib/tls-sig.c
@@ -554,7 +554,7 @@ _gnutls_handshake_sign_crt_vrfy12(gnutls_session_t session,
 	if (sign_algo == GNUTLS_SIGN_UNKNOWN || 
 	    _gnutls_session_sign_algo_enabled(session, sign_algo) < 0) {
 
-		sign_algo = _gnutls_session_get_sign_algo(session, cert, 1);
+		sign_algo = _gnutls_session_get_sign_algo(session, cert, pkey, 1);
 		if (sign_algo == GNUTLS_SIGN_UNKNOWN) {
 			gnutls_assert();
 			return GNUTLS_E_UNWANTED_ALGORITHM;