cipher: Support internal hashing for RSA-PSS.

* cipher/pubkey-util.c (_gcry_pk_util_data_to_mpi): Support
"hash-algo" with "value" for internal hashing with RSA-PSS.
* tests/basic.c (check_pubkey_sign): Add test cases.

--

GnuPG-bug-id: 5529
Signed-off-by: NIIBE Yutaka <gniibe@fsij.org>

1 files changed, 160 insertions, 102 deletions

diff --git a/cipher/pubkey-util.c b/cipher/pubkey-util.c
index 02f7909e..e2ea9567 100644
--- a/cipher/pubkey-util.c
+++ b/cipher/pubkey-util.c
@@ -675,7 +675,7 @@ _gcry_pk_util_free_encoding_ctx (struct pk_encoding_ctx *ctx)
    Either the VALUE or the HASH element must be present for use
    with signatures.  VALUE is used for encryption.
 
-   HASH-ALGO is specific to OAEP and EDDSA.
+   HASH-ALGO is specific to OAEP, PSS and EDDSA.
 
    LABEL is specific to OAEP.
 
@@ -1021,122 +1021,180 @@ _gcry_pk_util_data_to_mpi (gcry_sexp_t input, gcry_mpi_t *ret_mpi,
           xfree (random_override);
 	}
     }
-  else if (ctx->encoding == PUBKEY_ENC_PSS && lhash
-	   && ctx->op == PUBKEY_OP_SIGN)
+  else if (ctx->encoding == PUBKEY_ENC_PSS && ctx->op == PUBKEY_OP_SIGN)
     {
-      if (sexp_length (lhash) != 3)
-        rc = GPG_ERR_INV_OBJ;
-      else if ( !(s=sexp_nth_data (lhash, 1, &n)) || !n )
-        rc = GPG_ERR_INV_OBJ;
-      else
-        {
-          const void * value;
-          size_t valuelen;
-          void *random_override = NULL;
+      const void * value;
+      size_t valuelen;
+      gcry_sexp_t list;
+      void *random_override = NULL;
 
-	  ctx->hash_algo = get_hash_algo (s, n);
+      if (lvalue)
+        {
+          /* Get HASH-ALGO. */
+          list = sexp_find_token (ldata, "hash-algo", 0);
+          if (list)
+            {
+              s = sexp_nth_data (list, 1, &n);
+              if (!s)
+                rc = GPG_ERR_NO_OBJ;
+              else
+                {
+                  ctx->hash_algo = get_hash_algo (s, n);
+                  if (!ctx->hash_algo)
+                    rc = GPG_ERR_DIGEST_ALGO;
+                }
+              sexp_release (list);
+              if (rc)
+                goto leave;
+            }
 
-          if (!ctx->hash_algo)
-            rc = GPG_ERR_DIGEST_ALGO;
-          else if ( !(value=sexp_nth_data (lhash, 2, &valuelen))
-                    || !valuelen )
+          if ( !(value=sexp_nth_data (lvalue, 1, &valuelen)) || !valuelen )
+            rc = GPG_ERR_INV_OBJ;
+          ctx->flags |= PUBKEY_FLAG_PREHASH;
+        }
+      else if (lhash)
+        {
+          if (sexp_length (lhash) != 3)
+            rc = GPG_ERR_INV_OBJ;
+          else if ( !(s=sexp_nth_data (lhash, 1, &n)) || !n )
             rc = GPG_ERR_INV_OBJ;
           else
-	    {
-	      gcry_sexp_t list;
+            {
+              ctx->hash_algo = get_hash_algo (s, n);
 
-	      /* Get SALT-LENGTH. */
-	      list = sexp_find_token (ldata, "salt-length", 0);
-	      if (list)
-		{
-		  s = sexp_nth_data (list, 1, &n);
-		  if (!s)
-		    {
-		      rc = GPG_ERR_NO_OBJ;
-		      goto leave;
-		    }
-		  ctx->saltlen = (unsigned int)strtoul (s, NULL, 10);
-		  sexp_release (list);
-		}
+              if (!ctx->hash_algo)
+                rc = GPG_ERR_DIGEST_ALGO;
+              else if ( !(value=sexp_nth_data (lhash, 2, &valuelen))
+                        || !valuelen )
+                rc = GPG_ERR_INV_OBJ;
+            }
+        }
+      else
+        rc = GPG_ERR_CONFLICT;
 
-              /* Get optional RANDOM-OVERRIDE.  */
-              list = sexp_find_token (ldata, "random-override", 0);
-              if (list)
-                {
-                  s = sexp_nth_data (list, 1, &n);
-                  if (!s)
-                    rc = GPG_ERR_NO_OBJ;
-                  else if (n == ctx->saltlen)
-                    {
-                      random_override = xtrymalloc (n);
-                      if (!random_override)
-                        rc = gpg_err_code_from_syserror ();
-                      else
-                        memcpy (random_override, s, n);
-                    }
-                  else
-                    rc = GPG_ERR_INV_ARG;
-                  sexp_release (list);
-                  if (rc)
-                    goto leave;
-                }
+      if (rc)
+        goto leave;
 
-              /* Encode the data.  (NBITS-1 is due to 8.1.1, step 1.) */
-	      rc = _gcry_rsa_pss_encode (ret_mpi, ctx->nbits - 1,
-                                         ctx->hash_algo, ctx->saltlen, 1,
-                                         value, valuelen,
-                                         random_override);
-              xfree (random_override);
-	    }
+      /* Get SALT-LENGTH. */
+      list = sexp_find_token (ldata, "salt-length", 0);
+      if (list)
+        {
+          s = sexp_nth_data (list, 1, &n);
+          if (!s)
+            {
+              rc = GPG_ERR_NO_OBJ;
+              goto leave;
+            }
+          ctx->saltlen = (unsigned int)strtoul (s, NULL, 10);
+          sexp_release (list);
+        }
+
+      /* Get optional RANDOM-OVERRIDE.  */
+      list = sexp_find_token (ldata, "random-override", 0);
+      if (list)
+        {
+          s = sexp_nth_data (list, 1, &n);
+          if (!s)
+            rc = GPG_ERR_NO_OBJ;
+          else if (n == ctx->saltlen)
+            {
+              random_override = xtrymalloc (n);
+              if (!random_override)
+                rc = gpg_err_code_from_syserror ();
+              else
+                memcpy (random_override, s, n);
+            }
+          else
+            rc = GPG_ERR_INV_ARG;
+          sexp_release (list);
+          if (rc)
+            goto leave;
         }
+
+      /* Encode the data.  (NBITS-1 is due to 8.1.1, step 1.) */
+      rc = _gcry_rsa_pss_encode (ret_mpi, ctx->nbits - 1,
+                                 ctx->hash_algo, ctx->saltlen,
+                                 !(ctx->flags & PUBKEY_FLAG_PREHASH),
+                                 value, valuelen,
+                                 random_override);
+      xfree (random_override);
     }
-  else if (ctx->encoding == PUBKEY_ENC_PSS && lhash
-	   && ctx->op == PUBKEY_OP_VERIFY)
+  else if (ctx->encoding == PUBKEY_ENC_PSS && ctx->op == PUBKEY_OP_VERIFY)
     {
-      if (sexp_length (lhash) != 3)
-        rc = GPG_ERR_INV_OBJ;
-      else if ( !(s=sexp_nth_data (lhash, 1, &n)) || !n )
-        rc = GPG_ERR_INV_OBJ;
-      else
+      gcry_sexp_t list;
+
+      if (lvalue)
         {
-	  ctx->hash_algo = get_hash_algo (s, n);
+          /* Get HASH-ALGO. */
+          list = sexp_find_token (ldata, "hash-algo", 0);
+          if (list)
+            {
+              s = sexp_nth_data (list, 1, &n);
+              if (!s)
+                rc = GPG_ERR_NO_OBJ;
+              else
+                {
+                  ctx->hash_algo = get_hash_algo (s, n);
+                  if (!ctx->hash_algo)
+                    rc = GPG_ERR_DIGEST_ALGO;
+                }
+              sexp_release (list);
+              if (rc)
+                goto leave;
+            }
 
-          if (!ctx->hash_algo)
-            rc = GPG_ERR_DIGEST_ALGO;
-	  else
-	    {
-	      gcry_sexp_t list;
-	      /* Get SALT-LENGTH. */
-	      list = sexp_find_token (ldata, "salt-length", 0);
-	      if (list)
-		{
-                  unsigned long ul;
+          *ret_mpi = sexp_nth_mpi (lvalue, 1, GCRYMPI_FMT_OPAQUE);
+          if (!*ret_mpi)
+            rc = GPG_ERR_INV_OBJ;
 
-		  s = sexp_nth_data (list, 1, &n);
-		  if (!s)
-		    {
-		      rc = GPG_ERR_NO_OBJ;
-                      sexp_release (list);
-		      goto leave;
-		    }
-		  ul = strtoul (s, NULL, 10);
-                  if (ul > 16384)
-                    {
-                      rc = GPG_ERR_TOO_LARGE;
-                      sexp_release (list);
-                      goto leave;
-                    }
-                  ctx->saltlen = ul;
-		  sexp_release (list);
-		}
+          ctx->flags |= PUBKEY_FLAG_PREHASH;
+        }
+      else if (lhash)
+        {
+          if (sexp_length (lhash) != 3)
+            rc = GPG_ERR_INV_OBJ;
+          else if ( !(s=sexp_nth_data (lhash, 1, &n)) || !n )
+            rc = GPG_ERR_INV_OBJ;
+          else
+            {
+              ctx->hash_algo = get_hash_algo (s, n);
 
-	      *ret_mpi = sexp_nth_mpi (lhash, 2, GCRYMPI_FMT_OPAQUE);
-	      if (!*ret_mpi)
-		rc = GPG_ERR_INV_OBJ;
-	      ctx->verify_cmp = pss_verify_cmp;
-	      ctx->verify_arg = *ret_mpi;
-	    }
-	}
+              if (!ctx->hash_algo)
+                rc = GPG_ERR_DIGEST_ALGO;
+              else
+                {
+                  *ret_mpi = sexp_nth_mpi (lhash, 2, GCRYMPI_FMT_OPAQUE);
+                  if (!*ret_mpi)
+                    rc = GPG_ERR_INV_OBJ;
+                }
+            }
+        }
+      else
+        rc = GPG_ERR_CONFLICT;
+
+      if (rc)
+        goto leave;
+
+      /* Get SALT-LENGTH. */
+      list = sexp_find_token (ldata, "salt-length", 0);
+      if (list)
+        {
+          s = sexp_nth_data (list, 1, &n);
+          if (!s)
+            {
+              rc = GPG_ERR_NO_OBJ;
+              goto leave;
+            }
+          ctx->saltlen = (unsigned int)strtoul (s, NULL, 10);
+          if (ctx->saltlen > 16384)
+            rc = GPG_ERR_TOO_LARGE;
+          sexp_release (list);
+          if (rc)
+            goto leave;
+        }
+
+      ctx->verify_cmp = pss_verify_cmp;
+      ctx->verify_arg = *ret_mpi;
     }
   else
     rc = GPG_ERR_CONFLICT;