signature: Clamp PSS salt len to MD len

FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of
the hash function output block (in bytes)."

Introduce a new option RSA_PSS_SALTLEN_AUTO_DIGEST_MAX and make it the
default. The new value will behave like RSA_PSS_SALTLEN_AUTO, but will
not use more than the digest length when signing, so that FIPS 186-4 is
not violated. This value has two advantages when compared with
RSA_PSS_SALTLEN_DIGEST: (1) It will continue to do auto-detection when
verifying signatures for maximum compatibility, where
RSA_PSS_SALTLEN_DIGEST would fail for other digest sizes. (2) It will
work for combinations where the maximum salt length is smaller than the
digest size, which typically happens with large digest sizes (e.g.,
SHA-512) and small RSA keys.

J.-S. Coron shows in "Optimal Security Proofs for PSS and Other
Signature Schemes. Advances in Cryptology – Eurocrypt 2002, volume 2332
of Lecture Notes in Computer Science, pp. 272 – 287. Springer Verlag,
2002." that longer salts than the output size of modern hash functions
do not increase security: "For example,for an application in which at
most one billion signatures will be generated, k0 = 30 bits of random
salt are actually sufficient to guarantee the same level of security as
RSA, and taking a larger salt does not increase the security level."

Signed-off-by: Clemens Lang <cllang@redhat.com>

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19724)

1 files changed, 30 insertions, 10 deletions

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 1c6b515d35..7463efbc0f 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -188,8 +188,8 @@ static void *rsa_newctx(void *provctx, const char *propq)
     prsactx->libctx = PROV_LIBCTX_OF(provctx);
     prsactx->flag_allow_md = 1;
     prsactx->propq = propq_copy;
-    /* Maximum for sign, auto for verify */
-    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
+    /* Maximum up to digest length for sign, auto for verify */
+    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
     prsactx->min_saltlen = -1;
     return prsactx;
 }
@@ -197,13 +197,27 @@ static void *rsa_newctx(void *provctx, const char *propq)
 static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx)
 {
     int saltlen = ctx->saltlen;
-
+    int saltlenMax = -1;
+
+    /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
+     * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
+     * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of
+     * the hash function output block (in bytes)."
+     *
+     * Provide a way to use at most the digest length, so that the default does
+     * not violate FIPS 186-4. */
     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
         saltlen = EVP_MD_get_size(ctx->md);
-    } else if (saltlen == RSA_PSS_SALTLEN_AUTO || saltlen == RSA_PSS_SALTLEN_MAX) {
+    } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
+        saltlen = RSA_PSS_SALTLEN_MAX;
+        saltlenMax = EVP_MD_get_size(ctx->md);
+    }
+    if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
         saltlen = RSA_size(ctx->rsa) - EVP_MD_get_size(ctx->md) - 2;
         if ((RSA_bits(ctx->rsa) & 0x7) == 1)
             saltlen--;
+        if (saltlenMax >= 0 && saltlen > saltlenMax)
+            saltlen = saltlenMax;
     }
     if (saltlen < 0) {
         ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
@@ -407,8 +421,8 @@ static int rsa_signverify_init(void *vprsactx, void *vrsa,
 
     prsactx->operation = operation;
 
-    /* Maximum for sign, auto for verify */
-    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
+    /* Maximize up to digest length for sign, auto for verify */
+    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
     prsactx->min_saltlen = -1;
 
     switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
@@ -1102,6 +1116,9 @@ static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params)
             case RSA_PSS_SALTLEN_AUTO:
                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO;
                 break;
+            case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX:
+                value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX;
+                break;
             default:
                 {
                     int len = BIO_snprintf(p->data, p->data_size, "%d",
@@ -1265,6 +1282,8 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
                 saltlen = RSA_PSS_SALTLEN_MAX;
             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO) == 0)
                 saltlen = RSA_PSS_SALTLEN_AUTO;
+            else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX) == 0)
+                saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
             else
                 saltlen = atoi(p->data);
             break;
@@ -1273,11 +1292,11 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
         }
 
         /*
-         * RSA_PSS_SALTLEN_MAX seems curiously named in this check.
-         * Contrary to what it's name suggests, it's the currently
-         * lowest saltlen number possible.
+         * RSA_PSS_SALTLEN_AUTO_DIGEST_MAX seems curiously named in this check.
+         * Contrary to what it's name suggests, it's the currently lowest
+         * saltlen number possible.
          */
-        if (saltlen < RSA_PSS_SALTLEN_MAX) {
+        if (saltlen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
             return 0;
         }
@@ -1285,6 +1304,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
         if (rsa_pss_restricted(prsactx)) {
             switch (saltlen) {
             case RSA_PSS_SALTLEN_AUTO:
+            case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX:
                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH,
                                    "Cannot use autodetected salt length");