pkcs11: added support for signatures with RSA-PSS

Relates #209

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

4 files changed, 139 insertions, 64 deletions

diff --git a/lib/pkcs11_int.h b/lib/pkcs11_int.h
index 885a69ff00..ffeb23e161 100644
--- a/lib/pkcs11_int.h
+++ b/lib/pkcs11_int.h
@@ -156,9 +156,11 @@ int pkcs11_token_matches_info(struct p11_kit_uri *info,
 unsigned int pkcs11_obj_flags_to_int(unsigned int flags);
 
 int
-_gnutls_pkcs11_privkey_sign_hash(gnutls_pkcs11_privkey_t key,
-				 const gnutls_datum_t * hash,
-				 gnutls_datum_t * signature);
+_gnutls_pkcs11_privkey_sign(gnutls_pkcs11_privkey_t key,
+			    const gnutls_sign_entry_st *se,
+			    const gnutls_datum_t * hash,
+			    gnutls_datum_t * signature,
+			    gnutls_x509_spki_st *spki_params);
 
 int
 _gnutls_pkcs11_privkey_decrypt_data(gnutls_pkcs11_privkey_t key,
@@ -175,8 +177,12 @@ static inline int pk_to_mech(gnutls_pk_algorithm_t pk)
 		return CKM_DSA;
 	else if (pk == GNUTLS_PK_EC)
 		return CKM_ECDSA;
-	else
+	else if (pk == GNUTLS_PK_RSA)
 		return CKM_RSA_PKCS;
+	else if (pk == GNUTLS_PK_RSA_PSS)
+		return CKM_RSA_PKCS_PSS;
+	else
+		return -1;
 }
 
 static inline int pk_to_key_type(gnutls_pk_algorithm_t pk)
@@ -185,8 +191,10 @@ static inline int pk_to_key_type(gnutls_pk_algorithm_t pk)
 		return CKK_DSA;
 	else if (pk == GNUTLS_PK_EC)
 		return CKK_ECDSA;
-	else
+	else if (pk == GNUTLS_PK_RSA_PSS || pk == GNUTLS_PK_RSA)
 		return CKK_RSA;
+	else
+		return -1;
 }
 
 static inline gnutls_pk_algorithm_t key_type_to_pk(ck_key_type_t m)
@@ -209,9 +217,12 @@ static inline int pk_to_genmech(gnutls_pk_algorithm_t pk, ck_key_type_t *type)
 	} else if (pk == GNUTLS_PK_EC) {
 		*type = CKK_ECDSA;
 		return CKM_ECDSA_KEY_PAIR_GEN;
-	} else {
+	} else if (pk == GNUTLS_PK_RSA_PSS || pk == GNUTLS_PK_RSA) {
 		*type = CKK_RSA;
 		return CKM_RSA_PKCS_KEY_PAIR_GEN;
+	} else {
+		*type = -1;
+		return -1;
 	}
 }
 
diff --git a/lib/pkcs11_privkey.c b/lib/pkcs11_privkey.c
index 86bdff4ef8..60786855a6 100644
--- a/lib/pkcs11_privkey.c
+++ b/lib/pkcs11_privkey.c
@@ -266,6 +266,57 @@ static int reopen_privkey_session(void * _privkey)
 		expr; \
 	}
 
+struct hash_mappings_st {
+	gnutls_digest_algorithm_t id;
+	unsigned long phash; /* pkcs11 hash ID */
+	unsigned long mgf_id;
+};
+
+
+#ifndef CKG_MGF1_SHA224
+# define CKG_MGF1_SHA224 0x00000005UL
+# define CKG_MGF1_SHA256 0x00000002UL
+# define CKG_MGF1_SHA384 0x00000003UL
+# define CKG_MGF1_SHA512 0x00000004UL
+
+struct ck_rsa_pkcs_pss_params {
+	ck_mechanism_type_t hash_alg;
+	/* ck_rsa_pkcs_mgf_type_t is not defined in old versions of p11-kit */
+	unsigned long mgf;
+	unsigned long s_len;
+};
+#endif
+
+static const struct hash_mappings_st hash_mappings[] =
+{
+	{.id = GNUTLS_DIG_SHA224,
+	 .phash = CKM_SHA224,
+	 .mgf_id = CKG_MGF1_SHA224
+	},
+	{.id = GNUTLS_DIG_SHA256,
+	 .phash = CKM_SHA256,
+	 .mgf_id = CKG_MGF1_SHA256
+	},
+	{.id = GNUTLS_DIG_SHA384,
+	 .phash = CKM_SHA384,
+	 .mgf_id = CKG_MGF1_SHA384
+	},
+	{.id = GNUTLS_DIG_SHA512,
+	 .phash = CKM_SHA512,
+	 .mgf_id = CKG_MGF1_SHA512
+	}
+};
+
+static const struct hash_mappings_st *hash_to_map(gnutls_digest_algorithm_t hash)
+{
+	unsigned i;
+	for (i=0;i<sizeof(hash_mappings)/sizeof(hash_mappings[0]);i++) {
+		if (hash == hash_mappings[i].id)
+			return &hash_mappings[i];
+	}
+	return NULL;
+}
+
 /*-
  * _gnutls_pkcs11_privkey_sign_hash:
  * @key: Holds the key
@@ -280,9 +331,11 @@ static int reopen_privkey_session(void * _privkey)
  *   negative error value.
  -*/
 int
-_gnutls_pkcs11_privkey_sign_hash(gnutls_pkcs11_privkey_t key,
-				 const gnutls_datum_t * hash,
-				 gnutls_datum_t * signature)
+_gnutls_pkcs11_privkey_sign(gnutls_pkcs11_privkey_t key,
+			    const gnutls_sign_entry_st *se,
+			    const gnutls_datum_t * hash,
+			    gnutls_datum_t * signature,
+			    gnutls_x509_spki_st *spki_params)
 {
 	ck_rv_t rv;
 	int ret;
@@ -292,14 +345,35 @@ _gnutls_pkcs11_privkey_sign_hash(gnutls_pkcs11_privkey_t key,
 	struct pkcs11_session_info *sinfo;
 	unsigned req_login = 0;
 	unsigned login_flags = SESSION_LOGIN|SESSION_CONTEXT_SPECIFIC;
+	struct ck_rsa_pkcs_pss_params rsa_pss_params;
 
 	PKCS11_CHECK_INIT_PRIVKEY(key);
 
 	sinfo = &key->sinfo;
 
-	mech.mechanism = pk_to_mech(key->pk_algorithm);
-	mech.parameter = NULL;
-	mech.parameter_len = 0;
+	if (se->pk == GNUTLS_PK_RSA_PSS) {
+		const struct hash_mappings_st *map = hash_to_map(se->hash);
+
+		if (map == NULL)
+			return gnutls_assert_val(GNUTLS_E_UNKNOWN_PK_ALGORITHM);
+
+		rsa_pss_params.hash_alg = map->phash;
+		rsa_pss_params.mgf = map->mgf_id;
+		rsa_pss_params.s_len = spki_params->salt_size;
+
+		mech.mechanism = CKM_RSA_PKCS_PSS;
+		mech.parameter = &rsa_pss_params;
+		mech.parameter_len = sizeof(rsa_pss_params);
+	} else {
+		ret = pk_to_mech(se->pk);
+
+		if (ret == -1)
+			return gnutls_assert_val(GNUTLS_E_UNKNOWN_PK_ALGORITHM);
+
+		mech.mechanism = ret;
+		mech.parameter = NULL;
+		mech.parameter_len = 0;
+	}
 
 	ret = gnutls_mutex_lock(&key->mutex);
 	if (ret != 0)
@@ -309,6 +383,7 @@ _gnutls_pkcs11_privkey_sign_hash(gnutls_pkcs11_privkey_t key,
 	 * earlier. */
 	REPEAT_ON_INVALID_HANDLE(rv = pkcs11_sign_init(sinfo->module, sinfo->pks, &mech, key->ref));
 	if (rv != CKR_OK) {
+		_gnutls_debug_log("p11: %s\n", pkcs11_strerror(rv));
 		gnutls_assert();
 		ret = pkcs11_rv_to_err(rv);
 		goto cleanup;
@@ -404,7 +479,7 @@ _gnutls_pkcs11_privkey_sign_hash(gnutls_pkcs11_privkey_t key,
  *
  * Returns: this function will return non-zero if the token
  * holding the private key is still available (inserted), and zero otherwise.
- * 
+ *
  * Since: 3.1.9
  *
  **/
@@ -413,7 +488,7 @@ unsigned gnutls_pkcs11_privkey_status(gnutls_pkcs11_privkey_t key)
 	ck_rv_t rv;
 	int ret;
 	struct ck_session_info session_info;
-	
+
 	PKCS11_CHECK_INIT_PRIVKEY(key);
 
 	REPEAT_ON_INVALID_HANDLE(rv = (key->sinfo.module)->C_GetSessionInfo(key->sinfo.pks, &session_info));
@@ -547,7 +622,7 @@ gnutls_pkcs11_privkey_import_url(gnutls_pkcs11_privkey_t pkey,
  * @plaintext: will contain the plaintext, allocated with gnutls_malloc()
  *
  * This function will decrypt the given data using the public key algorithm
- * supported by the private key. 
+ * supported by the private key.
  *
  * Returns: On success, %GNUTLS_E_SUCCESS (0) is returned, otherwise a
  *   negative error value.
@@ -702,7 +777,7 @@ gnutls_pkcs11_privkey_generate(const char *url, gnutls_pk_algorithm_t pk,
  * This function will generate a private key in the specified
  * by the @url token. The private key will be generate within
  * the token and will not be exportable. This function will
- * store the DER-encoded public key in the SubjectPublicKeyInfo format 
+ * store the DER-encoded public key in the SubjectPublicKeyInfo format
  * in @pubkey. The @pubkey should be deinitialized using gnutls_free().
  *
  * Note that when generating an elliptic curve key, the curve
diff --git a/lib/pkcs11_write.c b/lib/pkcs11_write.c
index c8da7c094e..c58d6e7617 100644
--- a/lib/pkcs11_write.c
+++ b/lib/pkcs11_write.c
@@ -280,6 +280,7 @@ static int add_pubkey(gnutls_pubkey_t pubkey, struct ck_attribute *a, unsigned *
 	pk = gnutls_pubkey_get_pk_algorithm(pubkey, NULL);
 
 	switch (pk) {
+	case GNUTLS_PK_RSA_PSS:
 	case GNUTLS_PK_RSA: {
 		gnutls_datum_t m, e;
 
@@ -795,6 +796,7 @@ gnutls_pkcs11_copy_x509_privkey2(const char *token_url,
 
 	switch (pk) {
 	case GNUTLS_PK_RSA:
+	case GNUTLS_PK_RSA_PSS:
 		{
 
 			ret = _gnutls_params_get_rsa_raw(&key->params, &m, &e, &d, &p,
@@ -937,6 +939,7 @@ gnutls_pkcs11_copy_x509_privkey2(const char *token_url,
 
       cleanup:
 	switch (pk) {
+	case GNUTLS_PK_RSA_PSS:
 	case GNUTLS_PK_RSA:
 		{
 			gnutls_free(m.data);
diff --git a/lib/privkey.c b/lib/privkey.c
index 9a113ecb38..13d7c53f03 100644
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -307,65 +307,50 @@ _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 				 unsigned flags,
 				 gnutls_x509_spki_st *params)
 {
-	switch (key->type) {
-#ifdef ENABLE_PKCS11
-	case GNUTLS_PRIVKEY_PKCS11:
-		break;
-#endif
-	case GNUTLS_PRIVKEY_EXT:
-		break;
-	case GNUTLS_PRIVKEY_X509: {
-		unsigned salt_size = 0;
-		gnutls_pk_algorithm_t key_pk;
-		unsigned bits;
-
-		if (flags & GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS) {
-			if (!GNUTLS_PK_IS_RSA(pk))
-				return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
-			pk = GNUTLS_PK_RSA_PSS;
-		}
+	unsigned salt_size = 0;
+	unsigned bits = 0;
+	gnutls_pk_algorithm_t key_pk;
 
-		key_pk = gnutls_x509_privkey_get_pk_algorithm2(key->key.x509, &bits);
-		if (!(key_pk == pk ||
-		      (key_pk == GNUTLS_PK_RSA && pk == GNUTLS_PK_RSA_PSS))) {
-			gnutls_assert();
-			return GNUTLS_E_INVALID_REQUEST;
-		}
+	if (flags & GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS) {
+		if (!GNUTLS_PK_IS_RSA(pk))
+			return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+		pk = GNUTLS_PK_RSA_PSS;
+	}
 
-		if (pk == GNUTLS_PK_RSA_PSS) {
-			const mac_entry_st *me;
+	key_pk = gnutls_privkey_get_pk_algorithm(key, &bits);
+	if (!(key_pk == pk ||
+	      (key_pk == GNUTLS_PK_RSA && pk == GNUTLS_PK_RSA_PSS))) {
+		gnutls_assert();
+		return GNUTLS_E_INVALID_REQUEST;
+	}
 
-			me = hash_to_entry(dig);
-			if (unlikely(me == NULL))
-				return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+	if (pk == GNUTLS_PK_RSA_PSS) {
+		const mac_entry_st *me;
 
-			if (params->pk == GNUTLS_PK_RSA)
-				salt_size = 0;
-			else if (params->pk == GNUTLS_PK_RSA_PSS) {
-				if (dig != params->rsa_pss_dig) {
-					gnutls_assert();
-					return GNUTLS_E_INVALID_REQUEST;
-				}
+		me = hash_to_entry(dig);
+		if (unlikely(me == NULL))
+			return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
 
-				salt_size = params->salt_size;
+		if (params->pk == GNUTLS_PK_RSA)
+			salt_size = 0;
+		else if (params->pk == GNUTLS_PK_RSA_PSS) {
+			if (dig != params->rsa_pss_dig) {
+				gnutls_assert();
+				return GNUTLS_E_INVALID_REQUEST;
 			}
 
-			if (!(flags & GNUTLS_PRIVKEY_FLAG_REPRODUCIBLE))
-				salt_size = _gnutls_find_rsa_pss_salt_size(bits, me,
-									   salt_size);
+			salt_size = params->salt_size;
 		}
 
-		params->salt_size = salt_size;
+		if (!(flags & GNUTLS_PRIVKEY_FLAG_REPRODUCIBLE))
+			salt_size = _gnutls_find_rsa_pss_salt_size(bits, me, salt_size);
 
-		break;
-	}
-	default:
-		gnutls_assert();
-		return GNUTLS_E_INVALID_REQUEST;
+		params->salt_size = salt_size;
+		params->rsa_pss_dig = dig;
 	}
 
+
 	params->pk = pk;
-	params->rsa_pss_dig = dig;
 
 	return 0;
 }
@@ -1319,8 +1304,9 @@ privkey_sign_raw_data(gnutls_privkey_t key,
 	switch (key->type) {
 #ifdef ENABLE_PKCS11
 	case GNUTLS_PRIVKEY_PKCS11:
-		return _gnutls_pkcs11_privkey_sign_hash(key->key.pkcs11,
-							data, signature);
+		return _gnutls_pkcs11_privkey_sign(key->key.pkcs11, se,
+						   data, signature,
+						   params);
 #endif
 	case GNUTLS_PRIVKEY_X509:
 		return _gnutls_pk_sign(pk, signature, data,