x509: implement RSA-PSS signature scheme

This patch enables RSA-PSS signature scheme in the X.509 functions and
certtool.

When creating RSA-PSS signature, there are 3 different scenarios:

a. both a private key and a certificate are RSA-PSS
b. the private key is RSA, while the certificate is RSA-PSS
c. both the private key and the certificate are RSA

For (a) and (b), the RSA-PSS parameters are read from the certificate.
Any conflicts in parameters between the private key and the certificate
are reported as an error.

For (c), the sign functions, such as gnutls_x509_crt_privkey_sign() or
gnutls_privkey_sign_data(), shall be instructed to generate an RSA-PSS
signature.  This can be done with the new flag
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

Verification is similar to signing, except for the case (c), use the
flag GNUTLS_VERIFY_USE_RSA_PSS instead of
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

From the command line, certtool has a couple of new options: --rsa-pss
and --rsa-pss-sign.  The --rsa-pss option indicates that the generated
private key or certificate is restricted to RSA-PSS, while the
--rsa-pss-sign option indicates that the generated certificate is signed
with RSA-PSS.

For simplicity, there is no means of choosing arbitrary salt length.
When it is not given by a private key or a certificate, it is
automatically calculated from the underlying hash algorithm and the
RSA modulus bits.

[minor naming changes by nmav]

Signed-off-by: Daiki Ueno <dueno@redhat.com>
Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

4 files changed, 99 insertions, 10 deletions

diff --git a/src/certtool-args.def b/src/certtool-args.def
index f7889ccde6..ba5a7f0eeb 100644
--- a/src/certtool-args.def
+++ b/src/certtool-args.def
@@ -458,6 +458,18 @@ flag = {
 };
 
 flag = {
+    name      = rsa-pss;
+    descrip   = "Generate RSA-PSS key or certificate";
+    doc = "This option can be combined with --generate-privkey or --generate-certificate, to generate private key or certificate restricted to the use with the RSA-PSS padding scheme.";
+};
+
+flag = {
+    name      = rsa-pss-sign;
+    descrip   = "Sign certificate with RSA-PSS";
+    doc = "This option can be combined with --generate-certificate, to sign the certificate with the RSA-PSS padding scheme.";
+};
+
+flag = {
     name      = hash;
     arg-type  = string;
     descrip   = "Hash algorithm to use for signing";
diff --git a/src/certtool-common.c b/src/certtool-common.c
index 9c223ec94d..b48ff743e8 100644
--- a/src/certtool-common.c
+++ b/src/certtool-common.c
@@ -1110,24 +1110,46 @@ static void privkey_info_int(FILE *outfile, common_info_st * cinfo,
 	unsigned int bits = 0;
 	size_t size;
 	const char *cprint;
+	gnutls_x509_spki_t spki;
 
 	/* Public key algorithm
 	 */
 	fprintf(outfile, "Public Key Info:\n");
-	ret = gnutls_x509_privkey_get_pk_algorithm2(key, &bits);
+	ret = gnutls_x509_spki_init(&spki);
+	if (ret < 0) {
+		fprintf(stderr, "spki_init: %s\n", gnutls_strerror(ret));
+	}
+	ret = gnutls_x509_privkey_get_pk_algorithm3(key, spki, &bits);
 	fprintf(outfile, "\tPublic Key Algorithm: ");
 
 	key_type = ret;
 
 	cprint = gnutls_pk_algorithm_get_name(key_type);
 	fprintf(outfile, "%s\n", cprint ? cprint : "Unknown");
+	if (spki && key_type == GNUTLS_PK_RSA_PSS) {
+		ret = gnutls_x509_spki_get_digest_algorithm(spki);
+		if (ret < 0) {
+			fprintf(stderr, "spki_get_digest_algorithm: %s\n",
+				gnutls_strerror(ret));
+		} else {
+			fprintf(outfile, "\t\tHash Algorithm: %s\n",
+				gnutls_digest_get_name(ret));
+		}
+		ret = gnutls_x509_spki_get_salt_size(spki);
+		if (ret < 0) {
+			fprintf(stderr, "spki_get_salt_size: %s\n",
+				gnutls_strerror(ret));
+		} else
+			fprintf(outfile, "\t\tSalt Length: %d\n", ret);
+	}
+	gnutls_x509_spki_deinit(spki);
 	fprintf(outfile, "\tKey Security Level: %s (%u bits)\n\n",
 		gnutls_sec_param_get_name(gnutls_x509_privkey_sec_param
 					  (key)), bits);
 
 	/* Print the raw public and private keys
 	 */
-	if (key_type == GNUTLS_PK_RSA) {
+	if (GNUTLS_PK_IS_RSA(key_type)) {
 		gnutls_datum_t m, e, d, p, q, u, exp1, exp2;
 
 		ret =
diff --git a/src/certtool-common.h b/src/certtool-common.h
index e48396029f..91a4c6d1e9 100644
--- a/src/certtool-common.h
+++ b/src/certtool-common.h
@@ -73,6 +73,8 @@ typedef struct common_info {
 
 	/* set to non zero when no compatibility structs need to be exported */
 	unsigned no_compat;
+
+	unsigned rsa_pss_sign;
 } common_info_st;
 
 /* this must be provided by the app */
@@ -142,4 +144,6 @@ void fix_lbuffer(unsigned long);
 
 void decode_seed(gnutls_datum_t *seed, const char *hex, unsigned hex_size);
 
+#define GNUTLS_PK_IS_RSA(pk) ((pk) == GNUTLS_PK_RSA || (pk) == GNUTLS_PK_RSA_PSS)
+
 #endif
diff --git a/src/certtool.c b/src/certtool.c
index 1a8ccf8a0c..5f08937cc9 100644
--- a/src/certtool.c
+++ b/src/certtool.c
@@ -86,6 +86,7 @@ static void verify_certificate(common_info_st * cinfo);
 
 static void pubkey_keyid(common_info_st * cinfo);
 static void certificate_fpr(common_info_st * cinfo);
+static gnutls_digest_algorithm_t get_dig(gnutls_x509_crt_t crt);
 
 FILE *outfile;
 static const char *outfile_name = NULL; /* to delete on exit */
@@ -165,7 +166,7 @@ generate_private_key_int(common_info_st * cinfo)
 			bits, gnutls_pk_algorithm_get_name(key_type));
 	}
 
-	if (provable && (key_type != GNUTLS_PK_RSA && key_type != GNUTLS_PK_DSA)) {
+	if (provable && (!GNUTLS_PK_IS_RSA(key_type) && key_type != GNUTLS_PK_DSA)) {
 		fprintf(stderr,
 			"The --provable parameter cannot be used with ECDSA keys.\n");
 		app_exit(1);
@@ -175,7 +176,7 @@ generate_private_key_int(common_info_st * cinfo)
 		fprintf(stderr,
 			"Note that DSA keys with size over 1024 may cause incompatibility problems when used with earlier than TLS 1.2 versions.\n\n");
 
-	if ((HAVE_OPT(SEED) || provable) && key_type == GNUTLS_PK_RSA) {
+	if ((HAVE_OPT(SEED) || provable) && GNUTLS_PK_IS_RSA(key_type)) {
 		if (bits != 2048 && bits != 3072) {
 			fprintf(stderr, "Note that the FIPS 186-4 key generation restricts keys to 2048 and 3072 bits\n");
 		}
@@ -188,7 +189,7 @@ generate_private_key_int(common_info_st * cinfo)
 		data.data = (void*)cinfo->seed;
 		data.size = cinfo->seed_size;
 
-		if (key_type == GNUTLS_PK_RSA) {
+		if (GNUTLS_PK_IS_RSA(key_type)) {
 			if ((bits == 3072 && cinfo->seed_size != 32) || (bits == 2048 && cinfo->seed_size != 28)) {
 				fprintf(stderr, "The seed size (%d) doesn't match the size of the request security level; use -d 2 for more information.\n", (int)cinfo->seed_size);
 			}
@@ -200,9 +201,14 @@ generate_private_key_int(common_info_st * cinfo)
 
 		ret = gnutls_x509_privkey_generate2(key, key_type, bits, GNUTLS_PRIVKEY_FLAG_PROVABLE, &data, 1);
 	} else {
+		gnutls_keygen_data_st data;
+
+		data.type = GNUTLS_KEYGEN_DIGEST;
+		data.size = default_dig;
+
 		if (provable)
 			flags |= GNUTLS_PRIVKEY_FLAG_PROVABLE;
-		ret = gnutls_x509_privkey_generate(key, key_type, bits, flags);
+		ret = gnutls_x509_privkey_generate2(key, key_type, bits, flags, &data, 1);
 	}
 	if (ret < 0) {
 		fprintf(stderr, "privkey_generate: %s\n",
@@ -672,6 +678,30 @@ generate_certificate(gnutls_privkey_t * ret_key,
 		app_exit(1);
 	}
 
+	/* Algorithm restriction.
+	 */
+	if (req_key_type == GNUTLS_PK_RSA_PSS) {
+		gnutls_x509_spki_t spki;
+
+		result = gnutls_x509_spki_init(&spki);
+		if (result < 0) {
+			fprintf(stderr, "spki_init: %s\n",
+				gnutls_strerror(result));
+			app_exit(1);
+		}
+
+		gnutls_x509_spki_set_pk_algorithm(spki, GNUTLS_PK_RSA_PSS);
+		gnutls_x509_spki_set_digest_algorithm(spki, get_dig(crt));
+
+		result = gnutls_x509_crt_set_pk_algorithm(crt, spki, 0);
+		gnutls_x509_spki_deinit(spki);
+		if (result < 0) {
+			fprintf(stderr, "error setting signing algorithm: %s\n",
+				gnutls_strerror(result));
+			app_exit(1);
+		}
+	}
+
 	*ret_key = key;
 	return crt;
 
@@ -840,6 +870,7 @@ void generate_self_signed(common_info_st * cinfo)
 	gnutls_privkey_t key;
 	size_t size;
 	int result;
+	unsigned int flags = 0;
 
 	fprintf(stdlog, "Generating a self signed certificate...\n");
 
@@ -854,8 +885,11 @@ void generate_self_signed(common_info_st * cinfo)
 
 	fprintf(stdlog, "\n\nSigning certificate...\n");
 
+	if (cinfo->rsa_pss_sign)
+		flags |= GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS;
+
 	result =
-	    gnutls_x509_crt_privkey_sign(crt, crt, key, get_dig(crt), 0);
+	    gnutls_x509_crt_privkey_sign(crt, crt, key, get_dig(crt), flags);
 	if (result < 0) {
 		fprintf(stderr, "crt_sign: %s\n", gnutls_strerror(result));
 		app_exit(1);
@@ -883,6 +917,7 @@ static void generate_signed_certificate(common_info_st * cinfo)
 	int result;
 	gnutls_privkey_t ca_key;
 	gnutls_x509_crt_t ca_crt;
+	unsigned int flags = 0;
 
 	fprintf(stdlog, "Generating a signed certificate...\n");
 
@@ -901,9 +936,12 @@ static void generate_signed_certificate(common_info_st * cinfo)
 
 	fprintf(stdlog, "\n\nSigning certificate...\n");
 
+	if (cinfo->rsa_pss_sign)
+		flags |= GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS;
+
 	result =
 	    gnutls_x509_crt_privkey_sign(crt, ca_crt, ca_key,
-					 get_dig(ca_crt), 0);
+					 get_dig(ca_crt), flags);
 	if (result < 0) {
 		fprintf(stderr, "crt_sign: %s\n", gnutls_strerror(result));
 		app_exit(1);
@@ -931,6 +969,7 @@ static void generate_proxy_certificate(common_info_st * cinfo)
 	gnutls_privkey_t key, eekey;
 	size_t size;
 	int result;
+	unsigned int flags = 0;
 
 	fprintf(stdlog, "Generating a proxy certificate...\n");
 
@@ -943,9 +982,12 @@ static void generate_proxy_certificate(common_info_st * cinfo)
 
 	fprintf(stdlog, "\n\nSigning certificate...\n");
 
+	if (cinfo->rsa_pss_sign)
+		flags = GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS;
+
 	result =
 	    gnutls_x509_crt_privkey_sign(crt, eecrt, eekey, get_dig(eecrt),
-					 0);
+					 flags);
 	if (result < 0) {
 		fprintf(stderr, "crt_sign: %s\n", gnutls_strerror(result));
 		app_exit(1);
@@ -1005,6 +1047,7 @@ static void update_signed_certificate(common_info_st * cinfo)
 	gnutls_privkey_t ca_key;
 	gnutls_x509_crt_t ca_crt;
 	time_t tim;
+	unsigned int flags = 0;
 
 	fprintf(stdlog, "Generating a signed certificate...\n");
 
@@ -1033,9 +1076,12 @@ static void update_signed_certificate(common_info_st * cinfo)
 
 	fprintf(stderr, "\n\nSigning certificate...\n");
 
+	if (cinfo->rsa_pss_sign)
+		flags = GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS;
+
 	result =
 	    gnutls_x509_crt_privkey_sign(crt, ca_crt, ca_key,
-					 get_dig(ca_crt), 0);
+					 get_dig(ca_crt), flags);
 	if (result < 0) {
 		fprintf(stderr, "crt_sign: %s\n", gnutls_strerror(result));
 		app_exit(1);
@@ -1116,6 +1162,8 @@ static void cmd_parser(int argc, char **argv)
 		req_key_type = GNUTLS_PK_DSA;
 	else if (HAVE_OPT(ECC))
 		req_key_type = GNUTLS_PK_ECC;
+	else if (HAVE_OPT(RSA_PSS))
+		req_key_type = GNUTLS_PK_RSA_PSS;
 	else
 		req_key_type = GNUTLS_PK_RSA;
 
@@ -1279,6 +1327,9 @@ static void cmd_parser(int argc, char **argv)
 		cinfo.password = "";
 	}
 
+	if (HAVE_OPT(RSA_PSS_SIGN))
+		cinfo.rsa_pss_sign = 1;
+
 	if (HAVE_OPT(GENERATE_SELF_SIGNED))
 		generate_self_signed(&cinfo);
 	else if (HAVE_OPT(GENERATE_CERTIFICATE))