schannel: Don't treat encrypted partial record as pending data

- Track when the cached encrypted data contains only a partial record
  that can't be decrypted without more data (SEC_E_INCOMPLETE_MESSAGE).

- Change Curl_schannel_data_pending to return false in such a case.

Other SSL libraries have pending data functions that behave similarly.

Ref: https://github.com/curl/curl/pull/1387

Closes https://github.com/curl/curl/pull/1392

2 files changed, 15 insertions, 2 deletions

diff --git a/lib/urldata.h b/lib/urldata.h
index 34e18ecde..d4a4a2306 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -333,6 +333,11 @@ struct ssl_connect_data {
   size_t encdata_length, decdata_length;
   size_t encdata_offset, decdata_offset;
   unsigned char *encdata_buffer, *decdata_buffer;
+  /* encdata_is_incomplete: if encdata contains only a partial record that
+     can't be decrypted without another Curl_read_plain (that is, status is
+     SEC_E_INCOMPLETE_MESSAGE) then set this true. after Curl_read_plain writes
+     more bytes into encdata then set this back to false. */
+  bool encdata_is_incomplete;
   unsigned long req_flags, ret_flags;
   CURLcode recv_unrecoverable_err; /* schannel_recv had an unrecoverable err */
   bool recv_sspi_close_notify; /* true if connection closed by close_notify */
diff --git a/lib/vtls/schannel.c b/lib/vtls/schannel.c
index c9b513230..d20f30d89 100644
--- a/lib/vtls/schannel.c
+++ b/lib/vtls/schannel.c
@@ -432,6 +432,7 @@ schannel_connect_step1(struct connectdata *conn, int sockindex)
   connssl->recv_unrecoverable_err = CURLE_OK;
   connssl->recv_sspi_close_notify = false;
   connssl->recv_connection_closed = false;
+  connssl->encdata_is_incomplete = false;
 
   /* continue to second handshake step */
   connssl->connecting_state = ssl_connect_2;
@@ -480,6 +481,7 @@ schannel_connect_step2(struct connectdata *conn, int sockindex)
 
   /* buffer to store previously received and encrypted data */
   if(connssl->encdata_buffer == NULL) {
+    connssl->encdata_is_incomplete = false;
     connssl->encdata_offset = 0;
     connssl->encdata_length = CURL_SCHANNEL_BUFFER_INIT_SIZE;
     connssl->encdata_buffer = malloc(connssl->encdata_length);
@@ -532,6 +534,8 @@ schannel_connect_step2(struct connectdata *conn, int sockindex)
 
       /* increase encrypted data buffer offset */
       connssl->encdata_offset += nread;
+      connssl->encdata_is_incomplete = false;
+      infof(data, "schannel: encrypted data got %zd\n", nread);
     }
 
     infof(data, "schannel: encrypted data buffer: offset %zu length %zu\n",
@@ -576,6 +580,7 @@ schannel_connect_step2(struct connectdata *conn, int sockindex)
 
     /* check if the handshake was incomplete */
     if(sspi_status == SEC_E_INCOMPLETE_MESSAGE) {
+      connssl->encdata_is_incomplete = true;
       connssl->connecting_state = ssl_connect_2_reading;
       infof(data, "schannel: received incomplete message, need more data\n");
       return CURLE_OK;
@@ -1177,6 +1182,7 @@ schannel_recv(struct connectdata *conn, int sockindex,
     }
     else if(nread > 0) {
       connssl->encdata_offset += (size_t)nread;
+      connssl->encdata_is_incomplete = false;
       infof(data, "schannel: encrypted data got %zd\n", nread);
     }
   }
@@ -1313,6 +1319,7 @@ schannel_recv(struct connectdata *conn, int sockindex,
       }
     }
     else if(sspi_status == SEC_E_INCOMPLETE_MESSAGE) {
+      connssl->encdata_is_incomplete = true;
       if(!*err)
         *err = CURLE_AGAIN;
       infof(data, "schannel: failed to decrypt data, need more data\n");
@@ -1414,8 +1421,8 @@ bool Curl_schannel_data_pending(const struct connectdata *conn, int sockindex)
   const struct ssl_connect_data *connssl = &conn->ssl[sockindex];
 
   if(connssl->use) /* SSL/TLS is in use */
-    return (connssl->encdata_offset > 0 ||
-            connssl->decdata_offset > 0) ? TRUE : FALSE;
+    return (connssl->decdata_offset > 0 ||
+            (connssl->encdata_offset > 0 && !connssl->encdata_is_incomplete));
   else
     return FALSE;
 }
@@ -1518,6 +1525,7 @@ int Curl_schannel_shutdown(struct connectdata *conn, int sockindex)
     Curl_safefree(connssl->encdata_buffer);
     connssl->encdata_length = 0;
     connssl->encdata_offset = 0;
+    connssl->encdata_is_incomplete = false;
   }
 
   /* free internal buffer for received decrypted data */