restrict the validity of handshake messages during handshake process

record: disallow parsing of alert messages prior to session start
handshake: set a maximum number of warning messages that can be received per handshake

That is to avoid DoS due to the assymetry of cost of sending an alert vs the cost
of processing.

4 files changed, 24 insertions, 2 deletions

diff --git a/lib/gnutls_handshake.c b/lib/gnutls_handshake.c
index 3ef763cb7f..c63ec5a40f 100644
--- a/lib/gnutls_handshake.c
+++ b/lib/gnutls_handshake.c
@@ -2653,6 +2653,11 @@ gnutls_handshake (gnutls_session_t session)
   int ret;
   record_parameters_st *params;
 
+  if (STATE == STATE0)
+    { /* first call */
+      session->internals.handshake_in_progress = 1;
+    }
+
   ret = _gnutls_epoch_get (session, session->security_parameters.epoch_next,
                            &params);
   if (ret < 0)
@@ -2713,8 +2718,14 @@ gnutls_handshake (gnutls_session_t session)
 		if (ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED) \
 			return ret; \
                 /* a warning alert might interrupt handshake */ \
-		if (allow_alert != 0 && ret==GNUTLS_E_WARNING_ALERT_RECEIVED) return ret; \
-		gnutls_assert(); \
+		if (ret == GNUTLS_E_GOT_APPLICATION_DATA && session->internals.initial_negotiation_completed != 0) \
+		  return ret; \
+                if (session->internals.handshake_suspicious_loops < 16) { \
+                  if (allow_alert != 0 && ret==GNUTLS_E_WARNING_ALERT_RECEIVED) { \
+                    session->internals.handshake_suspicious_loops++; \
+                    return ret; \
+                  } \
+                } \
 		ERR( str, ret); \
 		if (gnutls_error_is_fatal(ret) == 0) ret = gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR); \
 		session->internals.invalid_connection = 1; \
diff --git a/lib/gnutls_int.h b/lib/gnutls_int.h
index 9fc40ef758..49c72bab65 100644
--- a/lib/gnutls_int.h
+++ b/lib/gnutls_int.h
@@ -723,6 +723,8 @@ typedef struct
 
   unsigned int cb_tls_unique_len;
   unsigned char cb_tls_unique[MAX_VERIFY_DATA_SIZE];
+  unsigned handshake_suspicious_loops;
+  unsigned handshake_in_progress;
 
   /* If you add anything here, check _gnutls_handshake_internal_state_clear().
    */
diff --git a/lib/gnutls_record.c b/lib/gnutls_record.c
index c5ba63314f..0fd49a7854 100644
--- a/lib/gnutls_record.c
+++ b/lib/gnutls_record.c
@@ -702,6 +702,13 @@ record_check_type (gnutls_session_t session,
             ("REC[%p]: Alert[%d|%d] - %s - was received\n", session,
              data[0], data[1], gnutls_alert_get_name ((int) data[1]));
 
+          if (!session->internals.initial_negotiation_completed &&
+              session->internals.handshake_in_progress && STATE == STATE0)
+            { /* handshake hasn't started */
+              gnutls_assert();
+              return GNUTLS_E_UNEXPECTED_PACKET;
+            }
+
           session->internals.last_alert = data[1];
 
           /* if close notify is received and
diff --git a/lib/gnutls_state.c b/lib/gnutls_state.c
index 80769f29f5..4dc6ca769e 100644
--- a/lib/gnutls_state.c
+++ b/lib/gnutls_state.c
@@ -237,6 +237,7 @@ _gnutls_handshake_internal_state_init (gnutls_session_t session)
    */
   session->internals.last_handshake_in = -1;
   session->internals.last_handshake_out = -1;
+  session->internals.handshake_suspicious_loops = 0;
 
   session->internals.resumable = RESUME_TRUE;
 }
@@ -247,6 +248,7 @@ _gnutls_handshake_internal_state_clear (gnutls_session_t session)
   _gnutls_handshake_internal_state_init (session);
 
   _gnutls_free_datum (&session->internals.recv_buffer);
+  session->internals.handshake_in_progress = 0;
 
   deinit_internal_params (session);