Merge of r5004, r5019-r5025: ssl fixes.

*) SSL: speedup loading of configs with many ssl servers.  The patch
   saves one EC_KEY_generate_key() call per server{} block by informing
   OpenSSL about SSL_OP_SINGLE_ECDH_USE we are going to use before
   the SSL_CTX_set_tmp_ecdh() call.

   For a configuration file with 10k simple server{} blocks with SSL
   enabled this change reduces startup time from 18s to 5s on a slow
   test box here.

*) SSL: removed conditions that always hold true.

*) SSL: resetting of flush flag after the data was written.  There is
   no need to flush next chunk of data if it does not contain a buffer
   with the flush or last_buf flags set.

*) SSL: preservation of flush flag for buffered data.  Previously,
   if SSL buffer was not sent we lost information that the data
   must be flushed.

*) SSL: calculation of buffer size moved closer to its usage.
   No functional changes.

*) SSL: avoid calling SSL_write() with zero data size.  According to
   documentation, calling SSL_write() with num=0 bytes to be sent
   results in undefined behavior.

   We don't currently call ngx_ssl_send_chain() with empty chain and
   buffer.  This check handles the case of a chain with total data size
   that is a multiple of NGX_SSL_BUFSIZE, and with the special buffer
   at the end.

   In practice such cases resulted in premature connection close and
   critical error "SSL_write() failed (SSL:)" in the error log.

*) SSL: take into account data in the buffer while limiting output.
   In some rare cases this can result in a more smooth sending rate.

*) SSL: fixed ngx_ssl_handshake() with level-triggered event methods.
   Missing calls to ngx_handle_write_event() and ngx_handle_read_event()
   resulted in a CPU hog during SSL handshake if an level-triggered event
   method (e.g. select) was used.

1 files changed, 27 insertions, 13 deletions

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 5e25b0156..d3663c4f5 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -528,10 +528,10 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name)
         return NGX_ERROR;
     }
 
-    SSL_CTX_set_tmp_ecdh(ssl->ctx, ecdh);
-
     SSL_CTX_set_options(ssl->ctx, SSL_OP_SINGLE_ECDH_USE);
 
+    SSL_CTX_set_tmp_ecdh(ssl->ctx, ecdh);
+
     EC_KEY_free(ecdh);
 #endif
 #endif
@@ -693,6 +693,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
             return NGX_ERROR;
         }
 
+        if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
         return NGX_AGAIN;
     }
 
@@ -701,6 +705,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
         c->read->handler = ngx_ssl_handshake_handler;
         c->write->handler = ngx_ssl_handshake_handler;
 
+        if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
         if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
             return NGX_ERROR;
         }
@@ -1053,8 +1061,8 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         buf->end = buf->start + NGX_SSL_BUFSIZE;
     }
 
-    send = 0;
-    flush = (in == NULL) ? 1 : 0;
+    send = buf->last - buf->pos;
+    flush = (in == NULL) ? 1 : buf->flush;
 
     for ( ;; ) {
 
@@ -1076,7 +1084,6 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
             if (send + size > limit) {
                 size = (ssize_t) (limit - send);
-                flush = 1;
             }
 
             ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
@@ -1093,10 +1100,16 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             }
         }
 
+        if (!flush && send < limit && buf->last < buf->end) {
+            break;
+        }
+
         size = buf->last - buf->pos;
 
-        if (!flush && buf->last < buf->end && c->ssl->buffer) {
-            break;
+        if (size == 0) {
+            buf->flush = 0;
+            c->buffered &= ~NGX_SSL_BUFFERED;
+            return in;
         }
 
         n = ngx_ssl_write(c, buf->pos, size);
@@ -1106,8 +1119,7 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
         }
 
         if (n == NGX_AGAIN) {
-            c->buffered |= NGX_SSL_BUFFERED;
-            return in;
+            break;
         }
 
         buf->pos += n;
@@ -1117,16 +1129,18 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
             break;
         }
 
-        if (buf->pos == buf->last) {
-            buf->pos = buf->start;
-            buf->last = buf->start;
-        }
+        flush = 0;
+
+        buf->pos = buf->start;
+        buf->last = buf->start;
 
         if (in == NULL || send == limit) {
             break;
         }
     }
 
+    buf->flush = flush;
+
     if (buf->pos < buf->last) {
         c->buffered |= NGX_SSL_BUFFERED;