x509: implement RSA-PSS signature scheme

This patch enables RSA-PSS signature scheme in the X.509 functions and
certtool.

When creating RSA-PSS signature, there are 3 different scenarios:

a. both a private key and a certificate are RSA-PSS
b. the private key is RSA, while the certificate is RSA-PSS
c. both the private key and the certificate are RSA

For (a) and (b), the RSA-PSS parameters are read from the certificate.
Any conflicts in parameters between the private key and the certificate
are reported as an error.

For (c), the sign functions, such as gnutls_x509_crt_privkey_sign() or
gnutls_privkey_sign_data(), shall be instructed to generate an RSA-PSS
signature.  This can be done with the new flag
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

Verification is similar to signing, except for the case (c), use the
flag GNUTLS_VERIFY_USE_RSA_PSS instead of
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

From the command line, certtool has a couple of new options: --rsa-pss
and --rsa-pss-sign.  The --rsa-pss option indicates that the generated
private key or certificate is restricted to RSA-PSS, while the
--rsa-pss-sign option indicates that the generated certificate is signed
with RSA-PSS.

For simplicity, there is no means of choosing arbitrary salt length.
When it is not given by a private key or a certificate, it is
automatically calculated from the underlying hash algorithm and the
RSA modulus bits.

[minor naming changes by nmav]

Signed-off-by: Daiki Ueno <dueno@redhat.com>
Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

1 files changed, 41 insertions, 0 deletions

diff --git a/lib/x509/privkey_pkcs8.c b/lib/x509/privkey_pkcs8.c
index 0f1863d160..827794bec2 100644
--- a/lib/x509/privkey_pkcs8.c
+++ b/lib/x509/privkey_pkcs8.c
@@ -67,6 +67,7 @@ _encode_privkey(gnutls_x509_privkey_t pkey, gnutls_datum_t * raw)
 
 	switch (pkey->pk_algorithm) {
 	case GNUTLS_PK_RSA:
+	case GNUTLS_PK_RSA_PSS:
 	case GNUTLS_PK_EC:
 		ret =
 		    gnutls_x509_privkey_export2(pkey, GNUTLS_X509_FMT_DER,
@@ -939,6 +940,44 @@ _decode_pkcs8_rsa_key(ASN1_TYPE pkcs8_asn, gnutls_x509_privkey_t pkey)
 	return ret;
 }
 
+/* Decodes an RSA-PSS privateKey from a PKCS8 structure.
+ */
+static int
+_decode_pkcs8_rsa_pss_key(ASN1_TYPE pkcs8_asn, gnutls_x509_privkey_t pkey)
+{
+	int ret;
+	gnutls_datum_t tmp;
+	gnutls_x509_spki_st params;
+
+	ret = _gnutls_x509_read_value(pkcs8_asn,
+				      "privateKeyAlgorithm.parameters", &tmp);
+	if (ret < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	ret = _gnutls_x509_read_rsa_pss_params(tmp.data, tmp.size, &params);
+	_gnutls_free_key_datum(&tmp);
+
+	if (ret < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	ret = _decode_pkcs8_rsa_key(pkcs8_asn, pkey);
+	if (ret < 0) {
+		gnutls_assert();
+		goto error;
+	}
+
+	memcpy(&pkey->params.sign, &params, sizeof(gnutls_x509_spki_st));
+
+	ret = 0;
+
+      error:
+	return ret;
+}
+
 /* Decodes an ECC privateKey from a PKCS8 structure.
  */
 static int
@@ -1120,6 +1159,8 @@ decode_private_key_info(const gnutls_datum_t * der,
 
 	if (pkey->pk_algorithm == GNUTLS_PK_RSA)
 		result = _decode_pkcs8_rsa_key(pkcs8_asn, pkey);
+	else if (pkey->pk_algorithm == GNUTLS_PK_RSA_PSS)
+		result = _decode_pkcs8_rsa_pss_key(pkcs8_asn, pkey);
 	else if (pkey->pk_algorithm == GNUTLS_PK_DSA)
 		result = _decode_pkcs8_dsa_key(pkcs8_asn, pkey);
 	else if (pkey->pk_algorithm == GNUTLS_PK_EC)