Merge branch 'master' into bagder/http2-easymapbagder/http2-easymap

13 files changed, 130 insertions, 51 deletions

diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index c5b92166c..9c94768d7 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -32,6 +32,7 @@ This release includes the following bugfixes:
  o content_encoding: accept up to 4 unknown trailer bytes after raw deflate data [5]
  o curl: Fix segfault when -H @headerfile is empty [23]
  o curl: use Content-Disposition before the "URL end" for -OJ [29]
+ o curl: warn the user if a given file name looks like an option [56]
  o darwinssl: add support for ALPN negotiation [7]
  o docs/CURLOPT_URL: fix indentation [20]
  o docs/CURLOPT_WRITEFUNCTION: size is always 1 [19]
@@ -43,7 +44,9 @@ This release includes the following bugfixes:
  o formdata: Remove unused macro HTTPPOST_CONTENTTYPE_DEFAULT [40]
  o header output: switch off all styles, not just unbold [8]
  o hostip: fix unused variable warning
+ o http2: check nghttp2_session_set_stream_user_data return code [55]
  o http2: clear the drain counter in Curl_http2_done [27]
+ o http2: make sure to send after RST_STREAM [58]
  o http2: separate easy handle from connections better [12]
  o http: fix for tiny "HTTP/0.9" response [51]
  o http_proxy: Remove unused macro SELECT_TIMEOUT [40]
@@ -79,8 +82,12 @@ This release includes the following bugfixes:
  o travis: build darwinssl on macos 10.12 to fix linker errors [33]
  o travis: execute "set -eo pipefail" for coverage build [45]
  o travis: run a 'make checksrc' too [25]
- o travis: update to GCC [52]
+ o travis: update to GCC-8 [52]
  o travis: verify that man pages can be regenerated [50]
+ o upload: allocate upload buffer on-demand [60]
+ o upload: change default UPLOAD_BUFSIZE to 64KB [60]
+ o urldata: remove unused pipe_broke struct field [57]
+ o vtls: reinstantiate engine on duplicated handles [59]
  o windows: implement send buffer tuning [37]
  o wolfSSL/CyaSSL: Fix memory leak in Curl_cyassl_random [18]
 
@@ -91,16 +98,17 @@ This release includes the following known bugs:
 This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
-  adnn on github, Anderson Toshiyuki Sasaki, Andrei Virtosu, Bas van Schaik,
-  Carie Pointer, Christopher Head, clbr on github, Dan Fandrich,
-  Daniel Jelinski, Daniel Jeliński, Daniel Stenberg, Darío Hereñú,
-  Even Rouault, Harry Sintonen, Jakub Zakrzewski, Jeffrey Walton, Jeroen Ooms,
-  Johannes Schindelin, John Butterfield, Josh Bialkowski, Kamil Dudka,
-  Marcel Raad, Markus Elfring, Michael Kaufmann, Nick Zitzmann,
-  Nikos Mavrogiannopoulos, Patrick Monnerat, Paul Howarth, pszemus on github,
-  Ran Mozes, Ray Satiro, Rikard Falkeborn, Rodger Combs, Ruslan Baratov,
-  Sergei Nikulov, Thomas Klausner, Tobias Blomberg, Viktor Szakats, Zero King,
-  (39 contributors)
+  adnn on github, Anderson Toshiyuki Sasaki, Andrei Virtosu, Anton Gerasimov,
+  Bas van Schaik, Carie Pointer, Christopher Head, clbr on github,
+  Dan Fandrich, Daniel Jelinski, Daniel Jeliński, Daniel Stenberg,
+  Darío Hereñú, Even Rouault, Harry Sintonen, Jakub Zakrzewski, Jeffrey Walton,
+  Jeroen Ooms, Johannes Schindelin, John Butterfield, Josh Bialkowski,
+  Kamil Dudka, Laurent Bonnans, Marcel Raad, Markus Elfring, Michael Kaufmann,
+  Nick Zitzmann, Nikos Mavrogiannopoulos, Patrick Monnerat, Paul Howarth,
+  pszemus on github, Ran Mozes, Ray Satiro, Rikard Falkeborn, Rodger Combs,
+  Ruslan Baratov, Sergei Nikulov, Thomas Klausner, Tobias Blomberg,
+  Viktor Szakats, Zero King,
+  (41 contributors)
 
         Thanks! (and sorry if I forgot to mention someone)
 
@@ -160,3 +168,10 @@ References to bug reports and discussions on issues:
  [52] = https://curl.haxx.se/bug/?i=2869
  [53] = https://curl.haxx.se/bug/?i=2879
  [54] = https://curl.haxx.se/bug/?i=2817
+ [55] = https://curl.haxx.se/bug/?i=2880
+ [56] = https://curl.haxx.se/bug/?i=2885
+ [57] = https://curl.haxx.se/bug/?i=2871
+ [58] = https://curl.haxx.se/bug/?i=2882
+ [59] = https://curl.haxx.se/bug/?i=2829
+ [60] = https://curl.haxx.se/bug/?i=2892
+
diff --git a/docs/SECURITY-PROCESS.md b/docs/SECURITY-PROCESS.md
index 0db6403c6..6ef7757ca 100644
--- a/docs/SECURITY-PROCESS.md
+++ b/docs/SECURITY-PROCESS.md
@@ -109,17 +109,12 @@ Publishing Security Advisories
 1. Write up the security advisory, using markdown syntax. Use the same
    subtitles as last time to maintain consistency.
 
-2. Name the advisory file (and ultimately the URL to be used when the flaw
-   gets published), using a randomized component so that third parties that
-   are involved in the process for each individual flaw will not be given
-   insights about possible *other* flaws worked on in parallel.
-   `adv_YEAR_RANDOM.md` has been used before.
+2. Name the advisory file after the allocated CVE id.
 
 3. Add a line on the top of the array in `curl-www/docs/vuln.pm'.
 
 4. Put the new advisory markdown file in the curl-www/docs/ directory. Add it
-   to the git repo.  Update the Makefile in the same directory to build the
-   HTML representation.
+   to the git repo.
 
 5. Run `make` in your local web checkout and verify that things look fine.
 
diff --git a/docs/SSLCERTS.md b/docs/SSLCERTS.md
index 3fcd345b0..2c5be68e6 100644
--- a/docs/SSLCERTS.md
+++ b/docs/SSLCERTS.md
@@ -92,8 +92,8 @@ server, do one of the following:
     If you use the 'openssl' tool, this is one way to get extract the CA cert
     for a particular server:
 
-     - `openssl s_client -connect xxxxx.com:443 |tee logfile`
-     - type "QUIT", followed by the "ENTER" key
+     - `openssl s_client -showcerts -servername server -connect server:443 > cacert.pem`
+     - type "quit", followed by the "ENTER" key
      - The certificate will have "BEGIN CERTIFICATE" and "END CERTIFICATE"
        markers.
      - If you want to see the data in the certificate, you can do: "openssl
diff --git a/lib/easy.c b/lib/easy.c
index 5af90e326..027d0bef3 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -958,6 +958,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
     outcurl->change.referer_alloc = TRUE;
   }
 
+  /* Reinitialize an SSL engine for the new handle
+   * note: the engine name has already been copied by dupset */
+  if(outcurl->set.str[STRING_SSL_ENGINE]) {
+    if(Curl_ssl_set_engine(outcurl, outcurl->set.str[STRING_SSL_ENGINE]))
+      goto fail;
+  }
+
   /* Clone the resolver handle, if present, for the new handle */
   if(Curl_resolver_duphandle(&outcurl->state.resolver,
                              data->state.resolver))
diff --git a/lib/http.c b/lib/http.c
index 2e9f40719..e727ed823 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1123,7 +1123,8 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
   CURLcode result;
   char *ptr;
   size_t size;
-  struct HTTP *http = conn->data->req.protop;
+  struct Curl_easy *data = conn->data;
+  struct HTTP *http = data->req.protop;
   size_t sendsize;
   curl_socket_t sockfd;
   size_t headersize;
@@ -1143,7 +1144,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
 
   DEBUGASSERT(size > included_body_bytes);
 
-  result = Curl_convert_to_network(conn->data, ptr, headersize);
+  result = Curl_convert_to_network(data, ptr, headersize);
   /* Curl_convert_to_network calls failf if unsuccessful */
   if(result) {
     /* conversion failed, free memory and return to the caller */
@@ -1168,8 +1169,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
        must copy the data to the uploadbuffer first, since that is the buffer
        we will be using if this send is retried later.
     */
-    memcpy(conn->data->state.uploadbuffer, ptr, sendsize);
-    ptr = conn->data->state.uploadbuffer;
+    result = Curl_get_upload_buffer(data);
+    if(result) {
+      /* malloc failed, free memory and return to the caller */
+      Curl_add_buffer_free(in);
+      return result;
+    }
+    memcpy(data->state.ulbuf, ptr, sendsize);
+    ptr = data->state.ulbuf;
   }
   else
     sendsize = size;
@@ -1186,13 +1193,13 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
     size_t headlen = (size_t)amount>headersize ? headersize : (size_t)amount;
     size_t bodylen = amount - headlen;
 
-    if(conn->data->set.verbose) {
+    if(data->set.verbose) {
       /* this data _may_ contain binary stuff */
-      Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, headlen);
+      Curl_debug(data, CURLINFO_HEADER_OUT, ptr, headlen);
       if(bodylen) {
         /* there was body data sent beyond the initial header part, pass that
            on to the debug callback too */
-        Curl_debug(conn->data, CURLINFO_DATA_OUT,
+        Curl_debug(data, CURLINFO_DATA_OUT,
                    ptr + headlen, bodylen);
       }
     }
@@ -1217,14 +1224,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
         ptr = in->buffer + amount;
 
         /* backup the currently set pointers */
-        http->backup.fread_func = conn->data->state.fread_func;
-        http->backup.fread_in = conn->data->state.in;
+        http->backup.fread_func = data->state.fread_func;
+        http->backup.fread_in = data->state.in;
         http->backup.postdata = http->postdata;
         http->backup.postsize = http->postsize;
 
         /* set the new pointers for the request-sending */
-        conn->data->state.fread_func = (curl_read_callback)readmoredata;
-        conn->data->state.in = (void *)conn;
+        data->state.fread_func = (curl_read_callback)readmoredata;
+        data->state.in = (void *)conn;
         http->postdata = ptr;
         http->postsize = (curl_off_t)size;
 
diff --git a/lib/http2.c b/lib/http2.c
index ea13ec935..702a71731 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -1147,6 +1147,9 @@ void Curl_http2_done(struct connectdata *conn, bool premature)
   struct HTTP *http = data->req.protop;
   struct http_conn *httpc = &conn->proto.httpc;
 
+  if(!httpc->h2) /* not HTTP/2 ? */
+    return;
+
   if(data->state.drain)
     drained_transfer(data, httpc);
 
@@ -1167,8 +1170,10 @@ void Curl_http2_done(struct connectdata *conn, bool premature)
 
   if(premature) {
     /* RST_STREAM */
-    nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE, http->stream_id,
-                              NGHTTP2_STREAM_CLOSED);
+    if(!nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE,
+                                  http->stream_id, NGHTTP2_STREAM_CLOSED))
+      (void)nghttp2_session_send(httpc->h2);
+
     if(http->stream_id == httpc->pause_stream_id) {
       infof(data, "stopped the pause stream!\n");
       httpc->pause_stream_id = 0;
@@ -1177,6 +1182,17 @@ void Curl_http2_done(struct connectdata *conn, bool premature)
   if(http->emap)
     /* remove stream <=> easy association */
     disassociate_easymap(http->emap);
+  /* -1 means unassigned and 0 means cleared */
+  if(http->stream_id > 0) {
+    int rv = nghttp2_session_set_stream_user_data(httpc->h2,
+                                                  http->stream_id, 0);
+    if(rv) {
+      infof(data, "http/2: failed to clear user_data for stream %u!\n",
+            http->stream_id);
+      DEBUGASSERT(0);
+    }
+    http->stream_id = 0;
+  }
 }
 
 /*
diff --git a/lib/multi.c b/lib/multi.c
index c2c5e1c27..0caf94322 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -592,6 +592,7 @@ static CURLcode multi_done(struct connectdata **connp,
     conn->dns_entry = NULL;
   }
   Curl_hostcache_prune(data);
+  Curl_safefree(data->state.ulbuf);
 
   /* if the transfer was completed in a paused state there can be buffered
      data left to free */
@@ -1575,6 +1576,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           multistate(data, CURLM_STATE_SENDPROTOCONNECT);
         }
       }
+      else if(result)
+        stream_error = TRUE;
       break;
 #endif
 
diff --git a/lib/setopt.c b/lib/setopt.c
index 5ecf5b97f..5c5f4b381 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -1603,14 +1603,19 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option,
      * String that holds the SSL crypto engine.
      */
     argptr = va_arg(param, char *);
-    if(argptr && argptr[0])
-      result = Curl_ssl_set_engine(data, argptr);
+    if(argptr && argptr[0]) {
+      result = Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], argptr);
+      if(!result) {
+        result = Curl_ssl_set_engine(data, argptr);
+      }
+    }
     break;
 
   case CURLOPT_SSLENGINE_DEFAULT:
     /*
      * flag to set engine as default.
      */
+    Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], NULL);
     result = Curl_ssl_set_engine_default(data);
     break;
   case CURLOPT_CRLF:
diff --git a/lib/smb.c b/lib/smb.c
index 09aa8efc7..32f0ac889 100644
--- a/lib/smb.c
+++ b/lib/smb.c
@@ -367,7 +367,7 @@ static CURLcode smb_send(struct connectdata *conn, ssize_t len,
   ssize_t bytes_written;
   CURLcode result;
 
-  result = Curl_write(conn, FIRSTSOCKET, conn->data->state.uploadbuffer,
+  result = Curl_write(conn, FIRSTSOCKET, conn->data->state.ulbuf,
                       len, &bytes_written);
   if(result)
     return result;
@@ -393,7 +393,7 @@ static CURLcode smb_flush(struct connectdata *conn)
     return CURLE_OK;
 
   result = Curl_write(conn, FIRSTSOCKET,
-                      conn->data->state.uploadbuffer + smbc->sent,
+                      conn->data->state.ulbuf + smbc->sent,
                       len, &bytes_written);
   if(result)
     return result;
@@ -409,9 +409,12 @@ static CURLcode smb_flush(struct connectdata *conn)
 static CURLcode smb_send_message(struct connectdata *conn, unsigned char cmd,
                                  const void *msg, size_t msg_len)
 {
-  smb_format_message(conn, (struct smb_header *)conn->data->state.uploadbuffer,
+  CURLcode result = Curl_get_upload_buffer(conn->data);
+  if(result)
+    return result;
+  smb_format_message(conn, (struct smb_header *)conn->data->state.ulbuf,
                      cmd, msg_len);
-  memcpy(conn->data->state.uploadbuffer + sizeof(struct smb_header),
+  memcpy(conn->data->state.ulbuf + sizeof(struct smb_header),
          msg, msg_len);
 
   return smb_send(conn, sizeof(struct smb_header) + msg_len, 0);
@@ -572,11 +575,15 @@ static CURLcode smb_send_read(struct connectdata *conn)
 
 static CURLcode smb_send_write(struct connectdata *conn)
 {
-  struct smb_write *msg = (struct smb_write *)conn->data->state.uploadbuffer;
+  struct smb_write *msg;
   struct smb_request *req = conn->data->req.protop;
   curl_off_t offset = conn->data->req.offset;
-
   curl_off_t upload_size = conn->data->req.size - conn->data->req.bytecount;
+  CURLcode result = Curl_get_upload_buffer(conn->data);
+  if(result)
+    return result;
+  msg = (struct smb_write *)conn->data->state.ulbuf;
+
   if(upload_size >= MAX_PAYLOAD_SIZE - 1) /* There is one byte of padding */
     upload_size = MAX_PAYLOAD_SIZE - 1;
 
@@ -605,7 +612,7 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg)
   if(!smbc->send_size && smbc->upload_size) {
     int nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE :
       (int) smbc->upload_size;
-    conn->data->req.upload_fromhere = conn->data->state.uploadbuffer;
+    conn->data->req.upload_fromhere = conn->data->state.ulbuf;
     result = Curl_fillreadbuffer(conn, nread, &nread);
     if(result && result != CURLE_AGAIN)
       return result;
diff --git a/lib/transfer.c b/lib/transfer.c
index 49f32568f..ab9094adc 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -106,6 +106,16 @@ char *Curl_checkheaders(const struct connectdata *conn,
 }
 #endif
 
+CURLcode Curl_get_upload_buffer(struct Curl_easy *data)
+{
+  if(!data->state.ulbuf) {
+    data->state.ulbuf = malloc(data->set.upload_buffer_size);
+    if(!data->state.ulbuf)
+      return CURLE_OUT_OF_MEMORY;
+  }
+  return CURLE_OK;
+}
+
 /*
  * This function will call the read callback to fill our buffer with data
  * to upload.
@@ -914,8 +924,11 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
     /* only read more data if there's no upload data already
        present in the upload buffer */
     if(0 == k->upload_present) {
+      result = Curl_get_upload_buffer(data);
+      if(result)
+        return result;
       /* init the "upload from here" pointer */
-      k->upload_fromhere = data->state.uploadbuffer;
+      k->upload_fromhere = data->state.ulbuf;
 
       if(!k->upload_done) {
         /* HTTP pollution, this should be written nicer to become more
@@ -979,7 +992,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
          (data->set.crlf))) {
         /* Do we need to allocate a scratch buffer? */
         if(!data->state.scratch) {
-          data->state.scratch = malloc(2 * data->set.buffer_size);
+          data->state.scratch = malloc(2 * UPLOAD_BUFSIZE);
           if(!data->state.scratch) {
             failf(data, "Failed to alloc scratch buffer!");
 
@@ -1071,7 +1084,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
     }
     else {
       /* we've uploaded that buffer now */
-      k->upload_fromhere = data->state.uploadbuffer;
+      result = Curl_get_upload_buffer(data);
+      if(result)
+        return result;
+      k->upload_fromhere = data->state.ulbuf;
       k->upload_present = 0; /* no more bytes left */
 
       if(k->upload_done) {
diff --git a/lib/transfer.h b/lib/transfer.h
index 9ba398d27..df75f9a97 100644
--- a/lib/transfer.h
+++ b/lib/transfer.h
@@ -54,6 +54,7 @@ CURLcode Curl_readrewind(struct connectdata *conn);
 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp);
 CURLcode Curl_retry_request(struct connectdata *conn, char **url);
 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc);
+CURLcode Curl_get_upload_buffer(struct Curl_easy *data);
 
 /* This sets up a forthcoming transfer */
 void
diff --git a/lib/url.c b/lib/url.c
index dcb7fcf6c..12f0f8ae7 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -367,11 +367,9 @@ CURLcode Curl_close(struct Curl_easy *data)
 
   Curl_safefree(data->state.buffer);
   Curl_safefree(data->state.headerbuff);
-
+  Curl_safefree(data->state.ulbuf);
   Curl_flush_cookies(data, 1);
-
   Curl_digest_cleanup(data);
-
   Curl_safefree(data->info.contenttype);
   Curl_safefree(data->info.wouldredirect);
 
@@ -534,6 +532,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
   set->expect_100_timeout = 1000L; /* Wait for a second by default. */
   set->sep_headers = TRUE; /* separated header lists by default */
   set->buffer_size = READBUFFER_SIZE;
+  set->upload_buffer_size = UPLOAD_BUFSIZE;
   set->happy_eyeballs_timeout = CURL_HET_DEFAULT;
 
   Curl_http2_init_userset(set);
diff --git a/lib/urldata.h b/lib/urldata.h
index 2a0540fa9..67db3b2fc 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -143,8 +143,13 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */
 #endif /* HAVE_LIBSSH2_H */
 
 /* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as
-   it needs to hold a full buffer as could be sent in a write callback */
-#define UPLOAD_BUFSIZE CURL_MAX_WRITE_SIZE
+   it needs to hold a full buffer as could be sent in a write callback.
+
+   The size was 16KB for many years but was bumped to 64KB because it makes
+   libcurl able to do significantly faster uploads in some circumstances. Even
+   larger buffers can help further, but this is deemed a fair memory/speed
+   compromise. */
+#define UPLOAD_BUFSIZE 65536
 
 /* The "master buffer" is for HTTP pipelining */
 #define MASTERBUF_SIZE 16384
@@ -1220,7 +1225,7 @@ struct UrlState {
   size_t headersize;   /* size of the allocation */
 
   char *buffer; /* download buffer */
-  char uploadbuffer[UPLOAD_BUFSIZE + 1]; /* upload buffer */
+  char *ulbuf; /* alloced upload buffer or NULL */
   curl_off_t current_speed;  /* the ProgressShow() function sets this,
                                 bytes / second */
   bool this_is_a_follow; /* this is a followed Location: request */
@@ -1404,6 +1409,7 @@ enum dupstring {
   STRING_SSL_CRLFILE_PROXY, /* crl file to check certificate */
   STRING_SSL_ISSUERCERT_ORIG, /* issuer cert file to check certificate */
   STRING_SSL_ISSUERCERT_PROXY, /* issuer cert file to check certificate */
+  STRING_SSL_ENGINE,      /* name of ssl engine */
   STRING_USERNAME,        /* <username>, if used */
   STRING_PASSWORD,        /* <password>, if used */
   STRING_OPTIONS,         /* <options>, if used */
@@ -1556,6 +1562,8 @@ struct UserDefined {
   curl_proxytype proxytype; /* what kind of proxy that is in use */
   long dns_cache_timeout; /* DNS cache timeout */
   long buffer_size;      /* size of receive buffer to use */
+  long upload_buffer_size; /* size of upload buffer to use,
+                              keep it >= CURL_MAX_WRITE_SIZE */
   void *private_data; /* application-private data */
 
   struct curl_slist *http200aliases; /* linked list of aliases for http200 */