Merge branch 'tmp-key-usage' into 'master'

When negotiating TLS1.3 enforce certificate key usage

Closes #690

See merge request gnutls/gnutls!902

15 files changed, 320 insertions, 59 deletions

diff --git a/NEWS b/NEWS
index bfc895f0f3..af6aee6872 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,18 @@ Copyright (C) 2000-2016 Free Software Foundation, Inc.
 Copyright (C) 2013-2017 Nikos Mavrogiannopoulos
 See the end for copying conditions.
 
+* Version 3.6.7 (unreleased)
+
+** libgnutls: enforce key usage limitations on certificates more actively.
+   Previously we would enforce it for TLS1.2 protocol, now we enforce it
+   even when TLS1.3 is negotiated, or on client certificates as well. When
+   an inappropriate for TLS1.3 certificate is seen on the credentials structure
+   GnuTLS will disable TLS1.3 support for that session (#690).
+
+** API and ABI modifications:
+No changes since last version.
+
+
 * Version 3.6.6 (released 2019-01-25)
 
 ** libgnutls: gnutls_pubkey_import_ecc_raw() was fixed to set the number bits
diff --git a/lib/algorithms/ciphersuites.c b/lib/algorithms/ciphersuites.c
index b97bbc82db..7269861ffe 100644
--- a/lib/algorithms/ciphersuites.c
+++ b/lib/algorithms/ciphersuites.c
@@ -1578,7 +1578,7 @@ _gnutls_figure_common_ciphersuite(gnutls_session_t session,
 	/* RFC7919 requires that we reply with insufficient security if we have
 	 * negotiated an FFDHE group, but cannot find a common ciphersuite. However,
 	 * we must also distinguish between not matching a ciphersuite due to an
-	 * incompatible certificate which we traditionally return GNUTLS_E_INSUFFICIENT_SECURITY.
+	 * incompatible certificate which we traditionally return GNUTLS_E_NO_CIPHER_SUITES.
 	 */
 	if (!no_cert_found && (session->internals.hsk_flags & HSK_HAVE_FFDHE) &&
 	    session->internals.priorities->groups.have_ffdhe && !version->tls13_sem)
diff --git a/lib/auth.c b/lib/auth.c
index 91a67c9afa..dd3fc861fb 100644
--- a/lib/auth.c
+++ b/lib/auth.c
@@ -138,6 +138,29 @@ gnutls_credentials_set(gnutls_session_t session,
 		}
 	}
 
+	/* sanity tests */
+	if (type == GNUTLS_CRD_CERTIFICATE) {
+		gnutls_certificate_credentials_t c = cred;
+		unsigned i;
+		bool allow_tls13 = 0;
+		unsigned key_usage;
+
+		if (c != NULL && c->ncerts != 0) {
+			for (i = 0; i < c->ncerts; i++) {
+				key_usage = get_key_usage(session, c->certs[i].cert_list[0].pubkey);
+				if (key_usage == 0 || (key_usage & GNUTLS_KEY_DIGITAL_SIGNATURE)) {
+					allow_tls13 = 1;
+					break;
+				}
+			}
+
+			if (!allow_tls13) {
+				/* to prevent the server random indicate TLS1.3 support */
+				session->internals.flags |= INT_FLAG_NO_TLS13;
+			}
+		}
+	}
+
 	return 0;
 }
 
diff --git a/lib/auth/cert.c b/lib/auth/cert.c
index b6bd3bf91e..5868ca244d 100644
--- a/lib/auth/cert.c
+++ b/lib/auth/cert.c
@@ -181,6 +181,7 @@ find_x509_client_cert(gnutls_session_t session,
 	ssize_t data_size = _data_size;
 	unsigned i, j;
 	int result, cert_pk;
+	unsigned key_usage;
 
 	*indx = -1;
 
@@ -191,8 +192,16 @@ find_x509_client_cert(gnutls_session_t session,
 	    (data_size == 0
 	     || (session->internals.flags & GNUTLS_FORCE_CLIENT_CERT))) {
 		if (cred->certs[0].cert_list[0].type == GNUTLS_CRT_X509) {
-			/* This check is necessary to prevent sending other certificate
-			 * credentials that are set (e.g. raw public-key). */
+
+			key_usage = get_key_usage(session, cred->certs[0].cert_list[0].pubkey);
+
+			/* For client certificates we require signatures */
+			result = _gnutls_check_key_usage_for_sig(session, key_usage, 1);
+			if (result < 0) {
+				_gnutls_debug_log("Client certificate is not suitable for signing\n");
+				return gnutls_assert_val(result);
+			}
+
 			*indx = 0;
 			return 0;
 		}
@@ -221,6 +230,14 @@ find_x509_client_cert(gnutls_session_t session,
 				if (odn.size == 0 || odn.size != asked_dn.size)
 					continue;
 
+				key_usage = get_key_usage(session, cred->certs[i].cert_list[0].pubkey);
+
+				/* For client certificates we require signatures */
+				if (_gnutls_check_key_usage_for_sig(session, key_usage, 1) < 0) {
+					_gnutls_debug_log("Client certificate is not suitable for signing\n");
+					continue;
+				}
+
 				/* If the DN matches and
 				 * the *_SIGN algorithm matches
 				 * the cert is our cert!
@@ -268,6 +285,7 @@ find_rawpk_client_cert(gnutls_session_t session,
 			int pk_algos_length, int* indx)
 {
 	unsigned i;
+	int ret;
 	gnutls_pk_algorithm_t pk;
 
 	*indx = -1;
@@ -276,15 +294,22 @@ find_rawpk_client_cert(gnutls_session_t session,
 		/* We know that our list length will be 1, therefore we can
 		 * ignore the rest.
 		 */
-		if (cred->certs[i].cert_list_length == 1) {
-			pk = gnutls_pubkey_get_pk_algorithm(cred->certs[i].
-																					cert_list[0].pubkey, NULL);
+		if (cred->certs[i].cert_list_length == 1 && cred->certs[i].cert_list[0].type == GNUTLS_CRT_RAWPK) {
+			pk = gnutls_pubkey_get_pk_algorithm(cred->certs[i].cert_list[0].pubkey, NULL);
+
+			/* For client certificates we require signatures */
+			ret = _gnutls_check_key_usage_for_sig(session, get_key_usage(session, cred->certs[i].cert_list[0].pubkey), 1);
+			if (ret < 0) {
+				/* we return an error instead of skipping so that the user is notified about
+				 * the key incompatibility */
+				_gnutls_debug_log("Client certificate is not suitable for signing\n");
+				return gnutls_assert_val(ret);
+			}
 
 			/* Check whether the public-key algorithm of our credential is in
 			 * the list with supported public-key algorithms and whether the
 			 * cert type matches. */
-			if ((check_pk_algo_in_list(pk_algos, pk_algos_length, pk) == 0)
-			   && (cred->certs[i].cert_list[0].type == GNUTLS_CRT_RAWPK))	{
+			if ((check_pk_algo_in_list(pk_algos, pk_algos_length, pk) == 0)) {
 				// We found a compatible credential
 				*indx = i;
 				break;
@@ -643,21 +668,24 @@ _gnutls_gen_rawpk_crt(gnutls_session_t session, gnutls_buffer_st* data)
 	/* Since we are transmitting a raw public key with no additional
 	 * certificate credentials attached to it, it doesn't make sense to
 	 * have more than one certificate set (i.e. to have a certificate chain).
-	 * This is enforced by the API so having a value other than 1 should
-	 * be an impossible situation.
 	 */
-	assert(apr_cert_list_length == 1);
-	
+	assert(apr_cert_list_length <= 1);
+
 	/* Write our certificate containing only the SubjectPublicKeyInfo to
 	 * the output buffer. We always have exactly one certificate that
 	 * contains our raw public key. Our message looks like:
 	 * <length++certificate> where
-	 * length = 3 bytes and
+	 * length = 3 bytes (or 24 bits) and
 	 * certificate = length bytes.
 	 */
-	ret = _gnutls_buffer_append_data_prefix(data, 24,
-					apr_cert_list[0].cert.data,
-					apr_cert_list[0].cert.size);
+	if (apr_cert_list_length == 0) {
+		ret = _gnutls_buffer_append_prefix(data, 24, 0);
+	} else {
+		ret = _gnutls_buffer_append_data_prefix(data, 24,
+							apr_cert_list[0].cert.data,
+							apr_cert_list[0].cert.size);
+	}
+
 
 	if (ret < 0) return gnutls_assert_val(ret);
 
@@ -1462,11 +1490,11 @@ int cert_select_sign_algorithm(gnutls_session_t session,
 		return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
 	}
 
-	if (unlikely(session->internals.priorities->allow_server_key_usage_violation)) {
-		key_usage = 0;
-	} else {
-		key_usage = pubkey->key_usage;
-	}
+	key_usage = get_key_usage(session, pubkey);
+
+	/* In TLS1.3 we support only signatures; ensure the selected key supports them */
+	if (ver->tls13_sem && _gnutls_check_key_usage_for_sig(session, key_usage, 1) < 0)
+		return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
 
 	if (!ver->tls13_sem && !_gnutls_kx_supports_pk_usage(cs->kx_algorithm, pk, key_usage)) {
 		return gnutls_assert_val(GNUTLS_E_INSUFFICIENT_CREDENTIALS);
diff --git a/lib/auth/cert.h b/lib/auth/cert.h
index 3f57ec1c74..fd66628820 100644
--- a/lib/auth/cert.h
+++ b/lib/auth/cert.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2002-2012 Free Software Foundation, Inc.
- * Copyright (C) 2016-2017 Red Hat, Inc.
+ * Copyright (C) 2016-2019 Red Hat, Inc.
  *
  * Author: Nikos Mavrogiannopoulos
  *
@@ -29,6 +29,7 @@
 #include <gnutls/abstract.h>
 #include <gnutls/compat.h>
 #include <str_array.h>
+#include "abstract_int.h"
 
 #define MAX_OCSP_RESPONSES 8
 
@@ -171,5 +172,13 @@ int _gnutls_gen_rawpk_crt(gnutls_session_t session, gnutls_buffer_st* data);
 int _gnutls_proc_rawpk_crt(gnutls_session_t session,
 				uint8_t * data, size_t data_size);
 
+inline static unsigned get_key_usage(gnutls_session_t session, gnutls_pubkey_t pubkey)
+{
+	if (unlikely(session->internals.priorities &&
+	    session->internals.priorities->allow_server_key_usage_violation))
+		return 0;
+	else
+		return pubkey->key_usage;
+}
 
 #endif
diff --git a/lib/gnutls_int.h b/lib/gnutls_int.h
index a0c47efa0f..93ffd7cee9 100644
--- a/lib/gnutls_int.h
+++ b/lib/gnutls_int.h
@@ -1566,9 +1566,9 @@ inline static size_t max_user_send_size(gnutls_session_t session,
  *
  * This function is made static inline for optimization reasons.
  */
-static inline gnutls_certificate_type_t
+inline static gnutls_certificate_type_t
 get_certificate_type(gnutls_session_t session,
-								gnutls_ctype_target_t target)
+		     gnutls_ctype_target_t target)
 {
 	switch (target) {
 		case GNUTLS_CTYPE_CLIENT:
diff --git a/lib/handshake.c b/lib/handshake.c
index 70b4486266..481210ebc0 100644
--- a/lib/handshake.c
+++ b/lib/handshake.c
@@ -444,6 +444,9 @@ _gnutls_negotiate_version(gnutls_session_t session,
 
 		if (aversion && aversion->id == GNUTLS_TLS1_2) {
 			vers = _gnutls_version_max(session);
+			if (unlikely(vers == NULL))
+				return gnutls_assert_val(GNUTLS_E_NO_CIPHER_SUITES);
+
 			if (vers->id >= GNUTLS_TLS1_2) {
 				session->security_parameters.pversion = aversion;
 				return 0;
@@ -2138,7 +2141,10 @@ static int send_client_hello(gnutls_session_t session, int again)
 
 		if (hver == NULL) {
 			gnutls_assert();
-			ret = GNUTLS_E_NO_PRIORITIES_WERE_SET;
+			if (session->internals.flags & INT_FLAG_NO_TLS13)
+				ret = GNUTLS_E_INSUFFICIENT_CREDENTIALS;
+			else
+				ret = GNUTLS_E_NO_PRIORITIES_WERE_SET;
 			goto cleanup;
 		}
 
diff --git a/lib/tls-sig.c b/lib/tls-sig.c
index 19357c06a1..73ca4314a3 100644
--- a/lib/tls-sig.c
+++ b/lib/tls-sig.c
@@ -40,8 +40,7 @@
 #include <x509/common.h>
 #include <abstract_int.h>
 
-static
-int check_key_usage_for_sig(gnutls_session_t session, unsigned key_usage, unsigned our_cert)
+int _gnutls_check_key_usage_for_sig(gnutls_session_t session, unsigned key_usage, unsigned our_cert)
 {
 	const char *lstr;
 	unsigned allow_key_usage_violation;
@@ -192,7 +191,7 @@ _gnutls_handshake_sign_data(gnutls_session_t session,
 
 	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
 
-	ret = check_key_usage_for_sig(session, key_usage, 1);
+	ret = _gnutls_check_key_usage_for_sig(session, key_usage, 1);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
@@ -336,7 +335,7 @@ _gnutls_handshake_verify_data(gnutls_session_t session,
 
 	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
 
-	ret = check_key_usage_for_sig(session, key_usage, 0);
+	ret = _gnutls_check_key_usage_for_sig(session, key_usage, 0);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
@@ -497,7 +496,7 @@ _gnutls_handshake_verify_crt_vrfy(gnutls_session_t session,
 
 	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
 
-	ret = check_key_usage_for_sig(session, key_usage, 0);
+	ret = _gnutls_check_key_usage_for_sig(session, key_usage, 0);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
@@ -708,7 +707,7 @@ _gnutls_handshake_sign_crt_vrfy(gnutls_session_t session,
 
 	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
 
-	ret = check_key_usage_for_sig(session, key_usage, 1);
+	ret = _gnutls_check_key_usage_for_sig(session, key_usage, 1);
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
diff --git a/lib/tls-sig.h b/lib/tls-sig.h
index afea4ebaf9..6e0564de1b 100644
--- a/lib/tls-sig.h
+++ b/lib/tls-sig.h
@@ -32,6 +32,9 @@
  */
 #define MAX_SIG_SIZE (19 + MAX_HASH_SIZE)
 
+int _gnutls_check_key_usage_for_sig(gnutls_session_t session, unsigned key_usage,
+				    unsigned our_cert);
+
 int _gnutls_handshake_sign_crt_vrfy(gnutls_session_t session,
 				    gnutls_pcert_st * cert,
 				    gnutls_privkey_t pkey,
diff --git a/lib/tls13-sig.c b/lib/tls13-sig.c
index 8eea6166b3..1f3a74bb5e 100644
--- a/lib/tls13-sig.c
+++ b/lib/tls13-sig.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017-2018 Red Hat, Inc.
+ * Copyright (C) 2017-2019 Red Hat, Inc.
  *
  * Author: Nikos Mavrogiannopoulos
  *
@@ -27,6 +27,7 @@
 #include <ext/signature.h>
 #include <abstract_int.h>
 #include "tls13-sig.h"
+#include "tls-sig.h"
 #include "hash_int.h"
 
 #undef PREFIX_SIZE
@@ -48,6 +49,7 @@ _gnutls13_handshake_verify_data(gnutls_session_t session,
 	const version_entry_st *ver = get_version(session);
 	gnutls_buffer_st buf;
 	uint8_t prefix[PREFIX_SIZE];
+	unsigned key_usage = 0;
 	gnutls_datum_t p;
 
 	_gnutls_handshake_log
@@ -75,6 +77,12 @@ _gnutls13_handshake_verify_data(gnutls_session_t session,
 	if (se->tls13_ok == 0) /* explicitly prohibited */
 		return gnutls_assert_val(GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER);
 
+	gnutls_pubkey_get_key_usage(cert->pubkey, &key_usage);
+
+	ret = _gnutls_check_key_usage_for_sig(session, key_usage, 0);
+	if (ret < 0)
+		return gnutls_assert_val(ret);
+
 	_gnutls_buffer_init(&buf);
 
 	memset(prefix, 0x20, sizeof(prefix));
@@ -150,6 +158,7 @@ _gnutls13_handshake_sign_data(gnutls_session_t session,
 	if (unlikely(sign_supports_priv_pk_algorithm(se, pkey->pk_algorithm) == 0))
 		return gnutls_assert_val(GNUTLS_E_RECEIVED_ILLEGAL_PARAMETER);
 
+	/* when we reach here we know we have a signing certificate */
 	_gnutls_handshake_log
 	    ("HSK[%p]: signing TLS 1.3 handshake data: using %s and PRF: %s\n", session, se->name,
 	     session->security_parameters.prf->name);
diff --git a/tests/cipher-neg-common.c b/tests/cipher-neg-common.c
index bfbda8b05b..1fcd6048b3 100644
--- a/tests/cipher-neg-common.c
+++ b/tests/cipher-neg-common.c
@@ -54,8 +54,8 @@ static void try(test_case_st *test)
 	gnutls_certificate_set_known_dh_params(s_cert_cred, GNUTLS_SEC_PARAM_MEDIUM);
 
 	assert(gnutls_certificate_set_x509_key_mem(s_cert_cred, &server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, GNUTLS_X509_FMT_PEM) >= 0);
-	assert(gnutls_certificate_set_x509_key_mem(s_cert_cred, &server_ca3_localhost_ecc_cert, &server_ca3_ecc_key, GNUTLS_X509_FMT_PEM) >= 0);
 	assert(gnutls_certificate_set_x509_key_mem(s_cert_cred, &server_ca3_localhost_rsa_sign_cert, &server_ca3_key, GNUTLS_X509_FMT_PEM) >= 0);
+	assert(gnutls_certificate_set_x509_key_mem(s_cert_cred, &server_ca3_localhost_ecc_cert, &server_ca3_ecc_key, GNUTLS_X509_FMT_PEM) >= 0);
 
 	gnutls_credentials_set(client, GNUTLS_CRD_CERTIFICATE, c_cert_cred);
 
diff --git a/tests/common-cert-key-exchange.c b/tests/common-cert-key-exchange.c
index 468475f846..de4b57ac4c 100644
--- a/tests/common-cert-key-exchange.c
+++ b/tests/common-cert-key-exchange.c
@@ -74,7 +74,7 @@ void try_with_key_fail(const char *name, const char *client_prio,
 
 	reset_buffers();
 	/* Init server */
-	gnutls_certificate_allocate_credentials(&serverx509cred);
+	assert(gnutls_certificate_allocate_credentials(&serverx509cred)>=0);
 
 	ret = gnutls_certificate_set_x509_key_mem(serverx509cred,
 						  serv_cert, serv_key,
@@ -82,16 +82,15 @@ void try_with_key_fail(const char *name, const char *client_prio,
 	if (ret < 0)
 		fail("Could not set key/cert: %s\n", gnutls_strerror(ret));
 
-	gnutls_init(&server, GNUTLS_SERVER);
-	gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
-			       serverx509cred);
-
-
+	assert(gnutls_init(&server, GNUTLS_SERVER)>=0);
 	if (server_priority)
 		assert(gnutls_priority_set_direct(server, server_priority, NULL) >= 0);
 	else
 		assert(gnutls_priority_set_direct(server, client_prio, NULL) >= 0);
 
+	assert(gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
+				      serverx509cred)>=0);
+
 	gnutls_transport_set_push_function(server, server_push);
 	gnutls_transport_set_pull_function(server, server_pull);
 	gnutls_transport_set_ptr(server, server);
@@ -112,11 +111,98 @@ void try_with_key_fail(const char *name, const char *client_prio,
 	if (ret < 0)
 		exit(1);
 
+	gnutls_transport_set_push_function(client, client_push);
+	gnutls_transport_set_pull_function(client, client_pull);
+	gnutls_transport_set_ptr(client, client);
+
+	ret = gnutls_priority_set_direct(client, client_prio, &err);
+	if (ret < 0) {
+		if (ret == GNUTLS_E_INVALID_REQUEST)
+			fprintf(stderr, "Error in %s\n", err);
+		exit(1);
+	}
+
 	ret = gnutls_credentials_set(client, GNUTLS_CRD_CERTIFICATE,
 				clientx509cred);
 	if (ret < 0)
 		exit(1);
 
+	success("negotiating %s\n", name);
+	HANDSHAKE_EXPECT(client, server, client_err, server_err);
+
+	gnutls_deinit(client);
+	gnutls_deinit(server);
+
+	gnutls_certificate_free_credentials(serverx509cred);
+	gnutls_certificate_free_credentials(clientx509cred);
+}
+
+void try_with_rawpk_key_fail(const char *name, const char *client_prio,
+			     int server_err, int client_err,
+			     const gnutls_datum_t *serv_cert,
+			     const gnutls_datum_t *serv_key,
+			     unsigned server_ku,
+			     const gnutls_datum_t *cli_cert,
+			     const gnutls_datum_t *cli_key,
+			     unsigned client_ku)
+{
+	int ret;
+	/* Server stuff. */
+	gnutls_certificate_credentials_t server_cred;
+	gnutls_session_t server;
+	int sret = GNUTLS_E_AGAIN;
+	/* Client stuff. */
+	gnutls_certificate_credentials_t client_cred;
+	gnutls_session_t client;
+	int cret = GNUTLS_E_AGAIN;
+	const char *err;
+
+	/* General init. */
+	gnutls_global_set_log_function(tls_log_func);
+	if (debug)
+		gnutls_global_set_log_level(6);
+
+	reset_buffers();
+	/* Init server */
+	gnutls_certificate_allocate_credentials(&server_cred);
+
+	ret = gnutls_certificate_set_rawpk_key_mem(server_cred,
+						   serv_cert, serv_key, GNUTLS_X509_FMT_PEM, NULL, server_ku,
+						   NULL, 0, 0);
+	if (ret < 0)
+		fail("Could not set key/cert: %s\n", gnutls_strerror(ret));
+
+	assert(gnutls_init(&server, GNUTLS_SERVER | GNUTLS_ENABLE_RAWPK) >= 0);
+	if (server_priority)
+		assert(gnutls_priority_set_direct(server, server_priority, NULL) >= 0);
+	else
+		assert(gnutls_priority_set_direct(server, client_prio, NULL) >= 0);
+
+	gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
+			       server_cred);
+
+	gnutls_transport_set_push_function(server, server_push);
+	gnutls_transport_set_pull_function(server, server_pull);
+	gnutls_transport_set_ptr(server, server);
+
+	/* Init client */
+	ret = gnutls_certificate_allocate_credentials(&client_cred);
+	if (ret < 0)
+		exit(1);
+
+	if (cli_cert) {
+		ret = gnutls_certificate_set_rawpk_key_mem(client_cred,
+							   cli_cert, cli_key, GNUTLS_X509_FMT_PEM, NULL, client_ku,
+							   NULL, 0, 0);
+		if (ret < 0)
+			fail("Could not set key/cert: %s\n", gnutls_strerror(ret));
+		gnutls_certificate_server_set_request(server, GNUTLS_CERT_REQUIRE);
+	}
+
+	ret = gnutls_init(&client, GNUTLS_CLIENT|GNUTLS_ENABLE_RAWPK);
+	if (ret < 0)
+		exit(1);
+
 	gnutls_transport_set_push_function(client, client_push);
 	gnutls_transport_set_pull_function(client, client_pull);
 	gnutls_transport_set_ptr(client, client);
@@ -127,14 +213,20 @@ void try_with_key_fail(const char *name, const char *client_prio,
 			fprintf(stderr, "Error in %s\n", err);
 		exit(1);
 	}
+
+	ret = gnutls_credentials_set(client, GNUTLS_CRD_CERTIFICATE,
+				client_cred);
+	if (ret < 0)
+		exit(1);
+
 	success("negotiating %s\n", name);
 	HANDSHAKE_EXPECT(client, server, client_err, server_err);
 
 	gnutls_deinit(client);
 	gnutls_deinit(server);
 
-	gnutls_certificate_free_credentials(serverx509cred);
-	gnutls_certificate_free_credentials(clientx509cred);
+	gnutls_certificate_free_credentials(server_cred);
+	gnutls_certificate_free_credentials(client_cred);
 }
 
 void try_with_key_ks(const char *name, const char *client_prio, gnutls_kx_algorithm_t client_kx,
@@ -173,8 +265,8 @@ void try_with_key_ks(const char *name, const char *client_prio, gnutls_kx_algori
 
 	reset_buffers();
 	/* Init server */
-	gnutls_anon_allocate_server_credentials(&s_anoncred);
-	gnutls_certificate_allocate_credentials(&server_cred);
+	assert(gnutls_anon_allocate_server_credentials(&s_anoncred)>=0);
+	assert(gnutls_certificate_allocate_credentials(&server_cred)>=0);
 
 	// Set server crt creds based on ctype
 	switch (server_ctype) {
@@ -201,11 +293,10 @@ void try_with_key_ks(const char *name, const char *client_prio, gnutls_kx_algori
 	gnutls_certificate_set_dh_params(server_cred, dh_params);
 	gnutls_anon_set_server_dh_params(s_anoncred, dh_params);
 
-	gnutls_init(&server, GNUTLS_SERVER | GNUTLS_ENABLE_RAWPK);
-	gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
-				server_cred);
-	gnutls_credentials_set(server, GNUTLS_CRD_ANON, s_anoncred);
-
+	assert(gnutls_init(&server, GNUTLS_SERVER | GNUTLS_ENABLE_RAWPK)>=0);
+	assert(gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
+				      server_cred)>=0);
+	assert(gnutls_credentials_set(server, GNUTLS_CRD_ANON, s_anoncred)>=0);
 
 	if (server_priority)
 		assert(gnutls_priority_set_direct(server, server_priority, NULL) >= 0);
@@ -254,8 +345,8 @@ void try_with_key_ks(const char *name, const char *client_prio, gnutls_kx_algori
 		exit(1);
 
 
-	gnutls_anon_allocate_client_credentials(&c_anoncred);
-	gnutls_credentials_set(client, GNUTLS_CRD_ANON, c_anoncred);
+	assert(gnutls_anon_allocate_client_credentials(&c_anoncred)>=0);
+	assert(gnutls_credentials_set(client, GNUTLS_CRD_ANON, c_anoncred)>=0);
 	ret = gnutls_credentials_set(client, GNUTLS_CRD_CERTIFICATE,
 				client_cred);
 	if (ret < 0)
@@ -397,14 +488,14 @@ void dtls_try_with_key_mtu(const char *name, const char *client_prio, gnutls_kx_
 	gnutls_certificate_set_dh_params(serverx509cred, dh_params);
 	gnutls_anon_set_server_dh_params(s_anoncred, dh_params);
 
-	gnutls_init(&server, GNUTLS_SERVER|GNUTLS_DATAGRAM|GNUTLS_NONBLOCK);
-	gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
-				serverx509cred);
-	gnutls_credentials_set(server, GNUTLS_CRD_ANON, s_anoncred);
+	assert(gnutls_init(&server, GNUTLS_SERVER|GNUTLS_DATAGRAM|GNUTLS_NONBLOCK)>=0);
+	assert(gnutls_credentials_set(server, GNUTLS_CRD_CERTIFICATE,
+				      serverx509cred)>=0);
+	assert(gnutls_credentials_set(server, GNUTLS_CRD_ANON, s_anoncred)>=0);
 
-	gnutls_priority_set_direct(server,
-				   "NORMAL:+ANON-ECDH:+ANON-DH:+ECDHE-RSA:+DHE-RSA:+RSA:+ECDHE-ECDSA:+CURVE-X25519",
-				   NULL);
+	assert(gnutls_priority_set_direct(server,
+					  "NORMAL:+ANON-ECDH:+ANON-DH:+ECDHE-RSA:+DHE-RSA:+RSA:+ECDHE-ECDSA:+CURVE-X25519",
+					   NULL)>=0);
 	gnutls_transport_set_push_function(server, server_push);
 	gnutls_transport_set_pull_function(server, server_pull);
 	gnutls_transport_set_pull_timeout_function(server, server_pull_timeout_func);
@@ -440,8 +531,8 @@ void dtls_try_with_key_mtu(const char *name, const char *client_prio, gnutls_kx_
 	if (ret < 0)
 		exit(1);
 
-	gnutls_anon_allocate_client_credentials(&c_anoncred);
-	gnutls_credentials_set(client, GNUTLS_CRD_ANON, c_anoncred);
+	assert(gnutls_anon_allocate_client_credentials(&c_anoncred)>=0);
+	assert(gnutls_credentials_set(client, GNUTLS_CRD_ANON, c_anoncred)>=0);
 	ret = gnutls_credentials_set(client, GNUTLS_CRD_CERTIFICATE,
 				clientx509cred);
 	if (ret < 0)
diff --git a/tests/common-cert-key-exchange.h b/tests/common-cert-key-exchange.h
index 8fb5ab754e..5d68d72ff8 100644
--- a/tests/common-cert-key-exchange.h
+++ b/tests/common-cert-key-exchange.h
@@ -51,6 +51,14 @@ extern const char *server_priority;
 	try_with_key(name, client_prio, client_kx, server_sign_algo, client_sign_algo, \
 		&rawpk_public_key1, &rawpk_private_key1, &rawpk_public_key2, &rawpk_private_key2, client_cert, GNUTLS_CRT_RAWPK, GNUTLS_CRT_RAWPK)
 
+void try_with_rawpk_key_fail(const char *name, const char *client_prio,
+			     int server_err, int client_err,
+			     const gnutls_datum_t *serv_cert,
+			     const gnutls_datum_t *serv_key,
+			     unsigned server_ku,
+			     const gnutls_datum_t *cli_cert,
+			     const gnutls_datum_t *cli_key,
+			     unsigned client_ku);
 
 void try_with_key_ks(const char *name, const char *client_prio, gnutls_kx_algorithm_t client_kx,
 		gnutls_sign_algorithm_t server_sign_algo,
diff --git a/tests/tls12-cert-key-exchange.c b/tests/tls12-cert-key-exchange.c
index 7811ae85bb..d8563c0943 100644
--- a/tests/tls12-cert-key-exchange.c
+++ b/tests/tls12-cert-key-exchange.c
@@ -120,5 +120,35 @@ void doit(void)
 			GNUTLS_E_AGAIN, GNUTLS_E_UNWANTED_ALGORITHM,
 			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &cli_ca3_cert, &cli_ca3_key);
 
+	try_with_key_fail("TLS 1.2 with rsa encryption cert without RSA",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2:-RSA",
+			GNUTLS_E_NO_CIPHER_SUITES, GNUTLS_E_AGAIN,
+			&server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, NULL, NULL);
+
+	try_with_key_fail("TLS 1.2 with (forced) rsa encryption cert and no RSA - client should detect",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2:-RSA:%DEBUG_ALLOW_KEY_USAGE_VIOLATIONS",
+			GNUTLS_E_AGAIN, GNUTLS_E_KEY_USAGE_VIOLATION,
+			&server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, NULL, NULL);
+
+	try_with_key_fail("TLS 1.2 with client rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2",
+			GNUTLS_E_AGAIN, GNUTLS_E_KEY_USAGE_VIOLATION,
+			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key);
+
+	try_with_key_fail("TLS 1.2 with (forced) client rsa encryption cert - server should detect",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2:%DEBUG_ALLOW_KEY_USAGE_VIOLATIONS",
+			GNUTLS_E_KEY_USAGE_VIOLATION, GNUTLS_E_AGAIN,
+			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key);
+
+	try_with_rawpk_key_fail("rawpk TLS 1.2 with rsa encryption cert without KX-RSA",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2:+CTYPE-RAWPK:-RSA",
+			GNUTLS_E_NO_CIPHER_SUITES, GNUTLS_E_AGAIN,
+			&rawpk_public_key1, &rawpk_private_key1, GNUTLS_KEY_KEY_ENCIPHERMENT, NULL, NULL, 0);
+
+	try_with_rawpk_key_fail("rawpk TLS 1.2 with client rsa encryption cert without KX-RSA",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.2:+CTYPE-RAWPK:-RSA",
+			GNUTLS_E_AGAIN, GNUTLS_E_KEY_USAGE_VIOLATION,
+			&rawpk_public_key2, &rawpk_private_key2, 0, &rawpk_public_key1, &rawpk_private_key1, GNUTLS_KEY_KEY_ENCIPHERMENT);
+
 	gnutls_global_deinit();
 }
diff --git a/tests/tls13-cert-key-exchange.c b/tests/tls13-cert-key-exchange.c
index 8b72b8a8d6..2b3d581ffd 100644
--- a/tests/tls13-cert-key-exchange.c
+++ b/tests/tls13-cert-key-exchange.c
@@ -138,5 +138,48 @@ void doit(void)
 			GNUTLS_E_AGAIN, GNUTLS_E_INCOMPATIBLE_SIG_WITH_KEY,
 			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &cli_ca3_cert, &cli_ca3_key);
 
+	try_with_key_fail("TLS 1.3 with rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3",
+			GNUTLS_E_NO_CIPHER_SUITES, GNUTLS_E_AGAIN,
+			&server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, NULL, NULL);
+
+	try_with_key_fail("TLS 1.3 and TLS 1.2 with rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:+VERS-TLS1.2",
+			GNUTLS_E_SUCCESS, GNUTLS_E_SUCCESS,
+			&server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, NULL, NULL);
+
+	try_with_key_fail("TLS 1.3 with (forced) rsa encryption cert - client should detect",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:%DEBUG_ALLOW_KEY_USAGE_VIOLATIONS",
+			GNUTLS_E_AGAIN, GNUTLS_E_KEY_USAGE_VIOLATION,
+			&server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key, NULL, NULL);
+
+	try_with_key_fail("TLS 1.3 with client rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3",
+			GNUTLS_E_AGAIN, GNUTLS_E_INSUFFICIENT_CREDENTIALS,
+			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key);
+
+	try_with_key_fail("TLS 1.3 with (forced) client rsa encryption cert - server should detect",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:%DEBUG_ALLOW_KEY_USAGE_VIOLATIONS",
+			GNUTLS_E_KEY_USAGE_VIOLATION, GNUTLS_E_SUCCESS,
+			&server_ca3_rsa_pss_cert, &server_ca3_rsa_pss_key, &server_ca3_localhost_rsa_decrypt_cert, &server_ca3_key);
+
+	try_with_rawpk_key_fail("rawpk TLS 1.3 with rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:+CTYPE-RAWPK",
+			GNUTLS_E_NO_CIPHER_SUITES, GNUTLS_E_AGAIN,
+			&rawpk_public_key1, &rawpk_private_key1, GNUTLS_KEY_KEY_ENCIPHERMENT, NULL, NULL, 0);
+
+	try_with_rawpk_key_fail("rawpk TLS 1.3 and TLS 1.2 with rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:+VERS-TLS1.2:+CTYPE-RAWPK",
+			GNUTLS_E_SUCCESS, GNUTLS_E_SUCCESS,
+			&rawpk_public_key1, &rawpk_private_key1, GNUTLS_KEY_KEY_ENCIPHERMENT, NULL, NULL, 0);
+
+	try_with_rawpk_key_fail("rawpk TLS 1.3 with client rsa encryption cert",
+			"NORMAL:-VERS-ALL:+VERS-TLS1.3:+CTYPE-RAWPK",
+			GNUTLS_E_AGAIN, GNUTLS_E_INSUFFICIENT_CREDENTIALS,
+			&rawpk_public_key2, &rawpk_private_key2, 0, &rawpk_public_key1, &rawpk_private_key1, GNUTLS_KEY_KEY_ENCIPHERMENT);
+
+	/* we do not test TLS 1.3 with (forced) rsa encryption cert - client should detect, because
+	 * there is no way under raw public keys for the client or server  to know the intended type. */
+
 	gnutls_global_deinit();
 }