zero pad DHE public key in ServerKeyExchange message for interop

Some versions of the Microsoft TLS stack have problems when the DHE public key
is encoded with fewer bytes than the DHE prime. (Backported from master)

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1350)

1 files changed, 27 insertions, 0 deletions

diff --git a/ssl/s3_srvr.c b/ssl/s3_srvr.c
index aa591eb065..0e57cb3570 100644
--- a/ssl/s3_srvr.c
+++ b/ssl/s3_srvr.c
@@ -1601,6 +1601,9 @@ int ssl3_send_server_key_exchange(SSL *s)
     unsigned int u;
 #endif
 #ifndef OPENSSL_NO_DH
+# ifdef OPENSSL_NO_RSA
+    int j;
+# endif
     DH *dh = NULL, *dhp;
 #endif
 #ifndef OPENSSL_NO_ECDH
@@ -1862,6 +1865,16 @@ int ssl3_send_server_key_exchange(SSL *s)
                 n += 1 + nr[i];
             else
 #endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime, so use the length of the prime here
+             */
+            if ((i == 2) && (type & (SSL_kEDH)))
+                n += 2 + nr[0];
+            else
+#endif
                 n += 2 + nr[i];
         }
 
@@ -1896,6 +1909,20 @@ int ssl3_send_server_key_exchange(SSL *s)
                 p++;
             } else
 #endif
+#ifndef OPENSSL_NO_DH
+            /*
+             * for interoperability with some versions of the Microsoft TLS
+             * stack, we need to zero pad the DHE pub key to the same length
+             * as the prime
+             */
+            if ((i == 2) && (type & (SSL_kEDH))) {
+                s2n(nr[0], p);
+                for (j = 0; j < (nr[0] - nr[2]); ++j) {
+                    *p = 0;
+                    ++p;
+                }
+            } else
+#endif
                 s2n(nr[i], p);
             BN_bn2bin(r[i], p);
             p += nr[i];