Add back check for the DH public key size

This is needed for TLS-1.3.

Also add check for uncompressed point format for ECDHE as
the other formats are not allowed by RFC 8446.

Fixes #17667

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17678)

4 files changed, 27 insertions, 5 deletions

diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index 58a3e9e08a..aee8f9272f 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -811,6 +811,9 @@ int ssl_hmac_final(SSL_HMAC *ctx, unsigned char *md, size_t *len,
 size_t ssl_hmac_size(const SSL_HMAC *ctx);
 
 int ssl_get_EC_curve_nid(const EVP_PKEY *pkey);
+__owur int tls13_set_encoded_pub_key(EVP_PKEY *pkey,
+                                     const unsigned char *enckey,
+                                     size_t enckeylen);
 
 typedef struct tls_group_info_st {
     char *tlsname;           /* Curve Name as in TLS specs */
diff --git a/ssl/statem/extensions_clnt.c b/ssl/statem/extensions_clnt.c
index c5de5ca5ba..0d90f0cc65 100644
--- a/ssl/statem/extensions_clnt.c
+++ b/ssl/statem/extensions_clnt.c
@@ -1856,8 +1856,8 @@ int tls_parse_stoc_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x,
             return 0;
         }
 
-        if (EVP_PKEY_set1_encoded_public_key(skey, PACKET_data(&encoded_pt),
-                                             PACKET_remaining(&encoded_pt)) <= 0) {
+        if (tls13_set_encoded_pub_key(skey, PACKET_data(&encoded_pt),
+                                      PACKET_remaining(&encoded_pt)) <= 0) {
             SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_ECPOINT);
             EVP_PKEY_free(skey);
             return 0;
diff --git a/ssl/statem/extensions_srvr.c b/ssl/statem/extensions_srvr.c
index fa64435a00..c2506879ef 100644
--- a/ssl/statem/extensions_srvr.c
+++ b/ssl/statem/extensions_srvr.c
@@ -665,9 +665,9 @@ int tls_parse_ctos_key_share(SSL *s, PACKET *pkt, unsigned int context, X509 *x,
             return 0;
         }
 
-        if (EVP_PKEY_set1_encoded_public_key(s->s3.peer_tmp,
-                PACKET_data(&encoded_pt),
-                PACKET_remaining(&encoded_pt)) <= 0) {
+        if (tls13_set_encoded_pub_key(s->s3.peer_tmp,
+                                      PACKET_data(&encoded_pt),
+                                      PACKET_remaining(&encoded_pt)) <= 0) {
             SSLfatal(s, SSL_AD_ILLEGAL_PARAMETER, SSL_R_BAD_ECPOINT);
             return 0;
         }
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index ad248c4cdf..218e8a3ae8 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -3532,3 +3532,22 @@ int ssl_get_EC_curve_nid(const EVP_PKEY *pkey)
 
     return NID_undef;
 }
+
+__owur int tls13_set_encoded_pub_key(EVP_PKEY *pkey,
+                                     const unsigned char *enckey,
+                                     size_t enckeylen)
+{
+    if (EVP_PKEY_is_a(pkey, "DH")) {
+        int bits = EVP_PKEY_get_bits(pkey);
+
+        if (bits <= 0 || enckeylen != (size_t)bits / 8)
+            /* the encoded key must be padded to the length of the p */
+            return 0;
+    } else if (EVP_PKEY_is_a(pkey, "EC")) {
+        if (enckeylen < 3 /* point format and at least 1 byte for x and y */
+            || enckey[0] != 0x04)
+            return 0;
+    }
+
+    return EVP_PKEY_set1_encoded_public_key(pkey, enckey, enckeylen);
+}