x509: implement RSA-PSS signature scheme

This patch enables RSA-PSS signature scheme in the X.509 functions and
certtool.

When creating RSA-PSS signature, there are 3 different scenarios:

a. both a private key and a certificate are RSA-PSS
b. the private key is RSA, while the certificate is RSA-PSS
c. both the private key and the certificate are RSA

For (a) and (b), the RSA-PSS parameters are read from the certificate.
Any conflicts in parameters between the private key and the certificate
are reported as an error.

For (c), the sign functions, such as gnutls_x509_crt_privkey_sign() or
gnutls_privkey_sign_data(), shall be instructed to generate an RSA-PSS
signature.  This can be done with the new flag
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

Verification is similar to signing, except for the case (c), use the
flag GNUTLS_VERIFY_USE_RSA_PSS instead of
GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS.

From the command line, certtool has a couple of new options: --rsa-pss
and --rsa-pss-sign.  The --rsa-pss option indicates that the generated
private key or certificate is restricted to RSA-PSS, while the
--rsa-pss-sign option indicates that the generated certificate is signed
with RSA-PSS.

For simplicity, there is no means of choosing arbitrary salt length.
When it is not given by a private key or a certificate, it is
automatically calculated from the underlying hash algorithm and the
RSA modulus bits.

[minor naming changes by nmav]

Signed-off-by: Daiki Ueno <dueno@redhat.com>
Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

8 files changed, 272 insertions, 8 deletions

diff --git a/tests/cert-tests/Makefile.am b/tests/cert-tests/Makefile.am
index e155fef509..90641679fb 100644
--- a/tests/cert-tests/Makefile.am
+++ b/tests/cert-tests/Makefile.am
@@ -76,7 +76,7 @@ EXTRA_DIST = data/ca-no-pathlen.pem data/no-ca-or-pathlen.pem data/aki-cert.pem
 	data/invalid-date-secs.der data/invalid-date-month.der data/invalid-date-day.der \
 	data/mem-leak.p12 data/alt-chain-new-ca.pem data/alt-chain-old-ca.pem \
 	data/alt-chain.pem data/pkcs7-chain.pem data/pkcs7-chain-root.pem \
-	data/pkcs7-chain-endcert-key.pem
+	data/pkcs7-chain-endcert-key.pem data/cert-rsa-pss.pem
 
 dist_check_SCRIPTS = pathlen aki certtool invalid-sig email \
 	pkcs7 pkcs7-broken-sigs privkey-import name-constraints certtool-long-cn crl provable-privkey \
@@ -104,13 +104,15 @@ dist_check_SCRIPTS += certtool-utf8 crq
 
 if !WINDOWS
 dist_check_SCRIPTS += template-test pem-decoding othername-test krb5-test sha3-test md5-test \
-	tlsfeature-test template-exts-test pkcs1-pad pkcs12-utf8
+	tlsfeature-test template-exts-test pkcs1-pad pkcs12-utf8 rsa-pss-pad
 endif
 
 if ENABLE_DANE
 dist_check_SCRIPTS += dane
 endif
 
+dist_check_SCRIPTS += certtool-rsa-pss
+
 TESTS = $(dist_check_SCRIPTS)
 
 # Set detect_leaks=0 to ASAN. It seems it is detecting many leaks in tools
diff --git a/tests/cert-tests/certtool-rsa-pss b/tests/cert-tests/certtool-rsa-pss
new file mode 100755
index 0000000000..a4c65dc0b7
--- /dev/null
+++ b/tests/cert-tests/certtool-rsa-pss
@@ -0,0 +1,125 @@
+#!/bin/sh
+
+# Copyright (C) 2014 Nikos Mavrogiannopoulos
+#
+# This file is part of GnuTLS.
+#
+# GnuTLS is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at
+# your option) any later version.
+#
+# GnuTLS is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GnuTLS; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+#set -e
+
+srcdir="${srcdir:-.}"
+CERTTOOL="${CERTTOOL:-../../src/certtool${EXEEXT}}"
+DIFF="${DIFF:-diff -b -B}"
+OUTFILE=pss-privkey.tmp
+TMPFILE=pss.$$.tmp
+
+if ! test -x "${CERTTOOL}"; then
+	exit 77
+fi
+
+if ! test -z "${VALGRIND}"; then
+	VALGRIND="${LIBTOOL:-libtool} --mode=execute ${VALGRIND}"
+fi
+
+for i in sha256 sha384 sha512;do
+if test "${GNUTLS_FORCE_FIPS_MODE}" = 1 && test "$i" != sha384;then
+	continue
+fi
+
+# Create an RSA-PSS private key, restricted to the use with RSA-PSS
+${VALGRIND} "${CERTTOOL}" --generate-privkey --pkcs8 --password '' \
+        --rsa-pss --hash $i --outfile "$OUTFILE"
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "Could not generate an RSA-PSS key ($i)"
+	exit 1
+fi
+
+# Create an RSA-PSS certificate from an RSA-PSS private key
+${VALGRIND} "${CERTTOOL}" --generate-self-signed \
+	--pkcs8 --load-privkey "$OUTFILE" --password '' \
+	--template "${srcdir}/templates/template-test.tmpl" \
+	--outfile "${TMPFILE}" --hash $i 2>/dev/null
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "Could not generate an RSA-PSS certificate from an RSA-PSS key ($i)"
+	exit 1
+fi
+
+# Create an RSA-PSS certificate from an RSA-PSS private key, with
+# mismatched parameters
+for j in sha256 sha384 sha512;do
+${VALGRIND} "${CERTTOOL}" --generate-self-signed \
+	--pkcs8 --load-privkey "$OUTFILE" --password '' \
+	--template "${srcdir}/templates/template-test.tmpl" \
+	--outfile "${TMPFILE}" --hash $j 2>/dev/null
+rc=$?
+
+if test "$j" != "$j" && "${rc}" = "0"; then
+	echo "Unexpectedly succeeded to generate an RSA-PSS certificate ($j != $i)"
+	exit 1
+fi
+done
+
+# Create an RSA-PSS certificate from an RSA key
+${VALGRIND} "${CERTTOOL}" --generate-certificate --rsa-pss \
+	    --load-privkey "${srcdir}/../../doc/credentials/x509/key-rsa.pem" \
+	    --load-ca-privkey "${srcdir}/../../doc/credentials/x509/ca-key.pem" \
+	    --load-ca-certificate "${srcdir}/../../doc/credentials/x509/ca.pem" \
+	    --template "${srcdir}/templates/template-test.tmpl" \
+	    --outfile "${TMPFILE}" --hash $i 2>/dev/null
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "Could not generate an RSA-PSS certificate $i"
+	exit 1
+fi
+
+# Create an RSA certificate from an RSA key, and sign it with RSA-PSS
+${VALGRIND} "${CERTTOOL}" --generate-certificate --rsa --rsa-pss-sign \
+	    --load-privkey "${srcdir}/../../doc/credentials/x509/key-rsa.pem" \
+	    --load-ca-privkey "${srcdir}/../../doc/credentials/x509/ca-key.pem" \
+	    --load-ca-certificate "${srcdir}/../../doc/credentials/x509/ca.pem" \
+	    --template "${srcdir}/templates/template-test.tmpl" \
+	    --outfile "${TMPFILE}" --hash $i 2>/dev/null
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "Could not generate an RSA-PSS certificate"
+	exit 1
+fi
+done
+
+export TZ="UTC"
+
+. ${srcdir}/../scripts/common.sh
+
+check_for_datefudge
+
+datefudge "2012-11-22" \
+${VALGRIND} "${CERTTOOL}" --verify --load-ca-certificate "${srcdir}/data/cert-rsa-pss.pem" --infile "${srcdir}/data/cert-rsa-pss.pem"
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "There was an issue verifying the certificate"
+	exit 1
+fi
+
+rm -f "${TMPFILE}"
+
+exit 0
diff --git a/tests/cert-tests/data/cert-rsa-pss.pem b/tests/cert-tests/data/cert-rsa-pss.pem
new file mode 100644
index 0000000000..ffaee16c20
--- /dev/null
+++ b/tests/cert-tests/data/cert-rsa-pss.pem
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE8jCCAyagAwIBAgIBdDBBBgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAQUA
+oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAQUAogMCASAwVzELMAkGA1UEBhMC
+Q1oxFzAVBgNVBAoMDkN6ZWNoIFJlcHVibGljMR0wGwYDVQQLDBRNaW5pc3RyeSBv
+ZiBJbnRlcmlvcjEQMA4GA1UEAwwHQ1NDQV9DWjAeFw0xMTAzMjUwMDAwMDBaFw0y
+NjA2MjUwMDAwMDBaMFcxCzAJBgNVBAYTAkNaMRcwFQYDVQQKDA5DemVjaCBSZXB1
+YmxpYzEdMBsGA1UECwwUTWluaXN0cnkgb2YgSW50ZXJpb3IxEDAOBgNVBAMMB0NT
+Q0FfQ1owggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCdFhq+ScQXepAA
+0kp0IwF/nEv+Md3Wx41Y6WRJkTVVyU6CFPlvr/F6XLGX/ILJtM8QL97CgojDVQbE
+ccNxUsZ+yjtB4n52ttWYLBN5nktJksP2aBVMu1fqoyTqBhaf0JtkpQjWKNVJYbUH
+k2pXkzGUJ+YHG04jOEYIKNclY82f1Ho1Wd7di4gZ1LCBRTzIU9JVPEMOZdmLx1qQ
+InOWf8deZ4Gmkj5UqzXt7vGQF/TbIedhmfnbulvyHN3UEDMZgVgzkF2fIOaBwu5s
+kfo8e08/J155hZIJtUjXk/moLF8U/4ETo7qER1EkoJ9KIRIvoPwPk2QAI4JP0Hd6
+GgUMEsz0dmTxKHcJZAWXcDaKcwTrG7/xhPTAffdOZnTnOpYQPcKmjUvogqY839VQ
+mwYyo+s51tVrAIe2YcdHhIdBY5SLOhHsDNadpRrBYIa8vzUgtfUH50US7dZuwUu6
+WbUiQu5vmDjvGEUa7F6eehCCf9xXkbPJZoE63t/NJYvHSFa1wDcCAwEAAaNhMF8w
+HQYDVR0OBBYEFOuhT488aYrathCbEjUoztRlSghZMA4GA1UdDwEB/wQEAwIBBjAa
+BgNVHSAEEzARMA8GDSqBS7cYAQEBAYnKmXwwEgYDVR0TAQH/BAgwBgEB/wIBADBB
+BgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAQUAoRwwGgYJKoZIhvcNAQEIMA0G
+CWCGSAFlAwQCAQUAogMCASADggGBAEcppLLH0xAkWXbBMXDa1hwqOZfKVdoUVDNT
+G20V3VbDT74R2yFCLWBge9rr7S2LfF/w4Xrl3kyZ2Tn0dYoOmBcqVzH6nCYrniGx
+apPmSkAexI/bjW55BOMe6CVI9qdKayqm99om/2+R+VSAKaopwOKn9IQ+4itNA0VI
+0ZDHbI/pdJClRZ0OPA8DREz7+hZWMNb7S4aAYGdd1fLo0uDKf/siFkUUfkpBtkgr
+4Enb1e8NMU0a5abpI25NgURB+OWhfArLB9jFmnlql6TZADLZfw49bEmr4KOddxyk
+toBmFgu4/uQXcukNWB487iREvEPPIL7A90W+W3ymtlol76SChKKBGexaaNA1JW+O
+sRxoN5FnXTBhyRm96lZOfvF7SjCQHMCaIeYaQ4ajjPJgxB0EbyyJMeSrjh4yJPEH
+pKc3AM9MUCxC0LO/qb5oBio8TQqHYnpyvp2CsvPSezPGh/40hxsKTL8GqTxB5m9G
+g6qpdcDQxM6VtrnTh08nMjaKnfXVJQ==
+-----END CERTIFICATE-----
diff --git a/tests/cert-tests/pkcs7 b/tests/cert-tests/pkcs7
index 8267d2e33d..f32dc46766 100755
--- a/tests/cert-tests/pkcs7
+++ b/tests/cert-tests/pkcs7
@@ -261,6 +261,24 @@ if test "${rc}" != "0"; then
 	exit ${rc}
 fi
 
+FILE="rsa-pss-signing"
+${VALGRIND} "${CERTTOOL}" --p7-sign --load-privkey  "${srcdir}/../../doc/credentials/x509/key-rsa.pem" --load-certificate "${srcdir}/../../doc/credentials/x509/cert-rsa-pss.pem" --infile "${srcdir}/data/pkcs7-detached.txt" >"${OUTFILE}"
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "${FILE}: PKCS7 struct signing failed"
+	exit ${rc}
+fi
+
+FILE="rsa-pss-signing-verify"
+${VALGRIND} "${CERTTOOL}" --p7-verify --load-certificate "${srcdir}/../../doc/credentials/x509/cert-rsa-pss.pem" <"${OUTFILE}"
+rc=$?
+
+if test "${rc}" != "0"; then
+	echo "${FILE}: PKCS7 struct signing failed verification"
+	exit ${rc}
+fi
+
 rm -f "${OUTFILE}"
 rm -f "${OUTFILE2}"
 
diff --git a/tests/cert-tests/rsa-pss-pad b/tests/cert-tests/rsa-pss-pad
new file mode 100755
index 0000000000..8129d9adcb
--- /dev/null
+++ b/tests/cert-tests/rsa-pss-pad
@@ -0,0 +1,74 @@
+#!/bin/sh
+
+# Copyright (C) 2006-2012 Free Software Foundation, Inc.
+#
+# This file is part of GnuTLS.
+#
+# GnuTLS is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at
+# your option) any later version.
+#
+# GnuTLS is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GnuTLS; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+#set -e
+
+srcdir="${srcdir:-.}"
+CERTTOOL="${CERTTOOL:-../../src/certtool${EXEEXT}}"
+DIFF="${DIFF:-diff}"
+TMPFILE=pss.$$.tmp
+
+if ! test -x "${CERTTOOL}"; then
+	exit 77
+fi
+
+export TZ="UTC"
+
+. ${srcdir}/../scripts/common.sh
+
+check_for_datefudge
+
+# Note that in rare cases this test may fail because the
+# time set using datefudge could have changed since the generation
+# (if example the system was busy)
+
+# Test PSS signatures on certificate
+
+for i in sha256 sha384 sha512;do
+datefudge -s "2007-04-22" \
+"${CERTTOOL}" --generate-self-signed --rsa-pss \
+		--load-privkey "${srcdir}/data/template-test.key" \
+		--template "${srcdir}/templates/template-test.tmpl" \
+		--outfile "${TMPFILE}" --hash $i 2>/dev/null
+rc=$?
+
+if test -f "${srcdir}/data/template-rsa-$i.pem";then
+${DIFF} "${srcdir}/data/template-rsa-$i.pem" "${TMPFILE}" >/dev/null 2>&1
+rc=$?
+fi
+
+# We're done.
+if test "${rc}" != "0"; then
+	echo "Test (RSA-PSS-$i) failed"
+	exit ${rc}
+fi
+
+datefudge -s "2007-04-25" \
+	"${CERTTOOL}" --load-ca-certificate "${TMPFILE}" --verify --infile "${TMPFILE}" >/dev/null 2>&1
+rc=$?
+if test "${rc}" != "0"; then
+	echo "Test (verification of RSA-PSS-$i) failed"
+	exit ${rc}
+fi
+done
+
+rm -f "${TMPFILE}"
+
+exit 0
diff --git a/tests/privkey-keygen.c b/tests/privkey-keygen.c
index 72201f4b18..9696392aef 100644
--- a/tests/privkey-keygen.c
+++ b/tests/privkey-keygen.c
@@ -103,9 +103,10 @@ void doit(void)
 		gnutls_global_set_log_level(4711);
 
 	for (i = 0; i < MAX_TRIES; i++) {
-		for (algorithm = GNUTLS_PK_RSA; algorithm <= GNUTLS_PK_EC;
+		for (algorithm = GNUTLS_PK_RSA; algorithm <= GNUTLS_PK_RSA_PSS;
 		     algorithm++) {
-			if (algorithm == GNUTLS_PK_DH)
+			if (algorithm == GNUTLS_PK_DH ||
+			    algorithm == GNUTLS_PK_ECDHX)
 				continue;
 
 			ret = gnutls_x509_privkey_init(&pkey);
diff --git a/tests/privkey-verify-broken.c b/tests/privkey-verify-broken.c
index 7b41e6ffba..7d7c84c7a0 100644
--- a/tests/privkey-verify-broken.c
+++ b/tests/privkey-verify-broken.c
@@ -45,12 +45,13 @@ const gnutls_datum_t raw_data = {
 	11
 };
 
-static int sign_verify_data(gnutls_x509_privkey_t pkey, unsigned algo, unsigned vflags)
+static int sign_verify_data2(gnutls_x509_privkey_t pkey, unsigned algo, unsigned sflags, unsigned vflags)
 {
 	int ret;
 	gnutls_privkey_t privkey;
 	gnutls_pubkey_t pubkey;
 	gnutls_datum_t signature;
+	gnutls_pk_algorithm_t pk;
 
 	/* sign arbitrary data */
 	assert(gnutls_privkey_init(&privkey) >= 0);
@@ -59,7 +60,7 @@ static int sign_verify_data(gnutls_x509_privkey_t pkey, unsigned algo, unsigned
 	if (ret < 0)
 		fail("gnutls_pubkey_import_x509\n");
 
-	ret = gnutls_privkey_sign_data(privkey, algo, 0,
+	ret = gnutls_privkey_sign_data(privkey, algo, sflags,
 					&raw_data, &signature);
 	if (ret < 0) {
 		ret = -1;
@@ -73,7 +74,12 @@ static int sign_verify_data(gnutls_x509_privkey_t pkey, unsigned algo, unsigned
 	if (ret < 0)
 		fail("gnutls_pubkey_import_privkey\n");
 
-	ret = gnutls_pubkey_verify_data2(pubkey, gnutls_pk_to_sign(gnutls_pubkey_get_pk_algorithm(pubkey, NULL),algo),
+	if (sflags & GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS)
+		pk = GNUTLS_PK_RSA_PSS;
+	else
+		pk = gnutls_pubkey_get_pk_algorithm(pubkey, NULL);
+
+	ret = gnutls_pubkey_verify_data2(pubkey, gnutls_pk_to_sign(pk, algo),
 				vflags, &raw_data, &signature);
 	if (ret < 0) {
 		ret = -1;
@@ -89,6 +95,11 @@ static int sign_verify_data(gnutls_x509_privkey_t pkey, unsigned algo, unsigned
 	return ret;
 }
 
+static int sign_verify_data(gnutls_x509_privkey_t pkey, unsigned algo, unsigned vflags)
+{
+	return sign_verify_data2(pkey, algo, 0, vflags);
+}
+
 void doit(void)
 {
 	gnutls_x509_privkey_t pkey;
@@ -123,7 +134,7 @@ void doit(void)
 		fail("failed verification with SHA1 and override flags2!\n");
 
 	if (sign_verify_data(pkey, GNUTLS_DIG_MD5, 0) >= 0)
-		fail("succeeded verification with SHA1!\n");
+		fail("succeeded verification with MD5!\n");
 
 	if (!gnutls_fips140_mode_enabled()) {
 		if (sign_verify_data(pkey, GNUTLS_DIG_MD5, GNUTLS_VERIFY_ALLOW_SIGN_RSA_MD5) < 0)
@@ -142,6 +153,9 @@ void doit(void)
 	if (sign_verify_data(pkey, GNUTLS_DIG_SHA3_256, 0) < 0)
 		fail("failed verification with SHA3-256!\n");
 
+	if (sign_verify_data2(pkey, GNUTLS_DIG_SHA256, GNUTLS_PRIVKEY_SIGN_FLAG_RSA_PSS, GNUTLS_VERIFY_USE_RSA_PSS) < 0)
+		fail("failed verification with SHA256 with PSS!\n");
+
 	gnutls_x509_privkey_deinit(pkey);
 
 	gnutls_global_deinit();
diff --git a/tests/x509sign-verify-rsa.c b/tests/x509sign-verify-rsa.c
index c0f1be1813..f527606e8d 100644
--- a/tests/x509sign-verify-rsa.c
+++ b/tests/x509sign-verify-rsa.c
@@ -62,6 +62,7 @@ void doit(void)
 
 	test_sig(GNUTLS_PK_RSA, GNUTLS_DIG_SHA1, rsa_size1);
 	test_sig(GNUTLS_PK_RSA, GNUTLS_DIG_SHA256, rsa_size2);
+	test_sig(GNUTLS_PK_RSA_PSS, GNUTLS_DIG_SHA256, rsa_size2);
 
 	gnutls_global_deinit();
 }