multi: assign IDs to all timers and make each timer singleton

 A) reduces the timeout lists drastically

 B) prevents a lot of superfluous loops for timers that expires "in vain"
    when it has actually already been extended to fire later on

10 files changed, 98 insertions, 39 deletions

diff --git a/lib/asyn-ares.c b/lib/asyn-ares.c
index 281fb03c8..a8396ea52 100644
--- a/lib/asyn-ares.c
+++ b/lib/asyn-ares.c
@@ -232,7 +232,7 @@ int Curl_resolver_getsock(struct connectdata *conn,
   milli = (timeout->tv_sec * 1000) + (timeout->tv_usec/1000);
   if(milli == 0)
     milli += 10;
-  Curl_expire_latest(conn->data, milli);
+  Curl_expire_latest(conn->data, milli, EXPIRE_ARES);
 
   return max;
 }
diff --git a/lib/connect.c b/lib/connect.c
index 63ec50fdc..de84daa4f 100644
--- a/lib/connect.c
+++ b/lib/connect.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -1070,7 +1070,7 @@ static CURLcode singleipconnect(struct connectdata *conn,
 
   conn->connecttime = Curl_tvnow();
   if(conn->num_addr > 1)
-    Curl_expire_latest(data, conn->timeoutms_per_addr);
+    Curl_expire_latest(data, conn->timeoutms_per_addr, EXPIRE_DNS_PER_NAME);
 
   /* Connect TCP sockets, bind UDP */
   if(!isconnected && (conn->socktype == SOCK_STREAM)) {
@@ -1169,7 +1169,7 @@ CURLcode Curl_connecthost(struct connectdata *conn,  /* context */
   conn->tempaddr[1] = NULL;
   conn->tempsock[0] = CURL_SOCKET_BAD;
   conn->tempsock[1] = CURL_SOCKET_BAD;
-  Curl_expire(conn->data, HAPPY_EYEBALLS_TIMEOUT);
+  Curl_expire(conn->data, HAPPY_EYEBALLS_TIMEOUT, EXPIRE_HAPPY_EYEBALLS);
 
   /* Max time for the next connection attempt */
   conn->timeoutms_per_addr =
diff --git a/lib/easy.c b/lib/easy.c
index 531f18b43..d4add46cc 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -1044,7 +1044,7 @@ CURLcode curl_easy_pause(struct Curl_easy *data, int action)
   if(!result &&
      ((newstate&(KEEP_RECV_PAUSE|KEEP_SEND_PAUSE)) !=
       (KEEP_RECV_PAUSE|KEEP_SEND_PAUSE)) )
-    Curl_expire(data, 0); /* get this handle going again */
+    Curl_expire(data, 0, EXPIRE_UNPAUSE); /* get this handle going again */
 
   return result;
 }
diff --git a/lib/ftp.c b/lib/ftp.c
index f6ce03a2d..5edec3761 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -580,10 +580,8 @@ static CURLcode AllowServerConnect(struct connectdata *conn, bool *connected)
   else {
     /* Add timeout to multi handle and break out of the loop */
     if(!result && *connected == FALSE) {
-      if(data->set.accepttimeout > 0)
-        Curl_expire(data, data->set.accepttimeout);
-      else
-        Curl_expire(data, DEFAULT_ACCEPT_TIMEOUT);
+      Curl_expire(data, data->set.accepttimeout > 0 ?
+                  data->set.accepttimeout: DEFAULT_ACCEPT_TIMEOUT, 0);
     }
   }
 
diff --git a/lib/http2.c b/lib/http2.c
index 46919c2ee..994e53deb 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -569,7 +569,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame,
 
       /* if we receive data for another handle, wake that up */
       if(conn_s->data != data_s)
-        Curl_expire(data_s, 0);
+        Curl_expire(data_s, 0, EXPIRE_H2DATA);
     }
     break;
   case NGHTTP2_PUSH_PROMISE:
@@ -646,7 +646,7 @@ static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags,
 
   /* if we receive data for another handle, wake that up */
   if(conn->data != data_s)
-    Curl_expire(data_s, 0);
+    Curl_expire(data_s, 0, EXPIRE_H2DATA);
 
   DEBUGF(infof(data_s, "%zu data received for stream %u "
                "(%zu left in buffer %p, total %zu)\n",
@@ -909,7 +909,7 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame,
     Curl_add_buffer(stream->header_recvbuf, " \r\n", 3);
     /* if we receive data for another handle, wake that up */
     if(conn->data != data_s)
-      Curl_expire(data_s, 0);
+      Curl_expire(data_s, 0, EXPIRE_H2DATA);
 
     DEBUGF(infof(data_s, "h2 status: HTTP/2 %03d (easy %p)\n",
                  stream->status_code, data_s));
@@ -925,7 +925,7 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame,
   Curl_add_buffer(stream->header_recvbuf, "\r\n", 2);
   /* if we receive data for another handle, wake that up */
   if(conn->data != data_s)
-    Curl_expire(data_s, 0);
+    Curl_expire(data_s, 0, EXPIRE_H2DATA);
 
   DEBUGF(infof(data_s, "h2 header: %.*s: %.*s\n", namelen, name, valuelen,
                value));
diff --git a/lib/multi.c b/lib/multi.c
index 26d5f1bd6..cfcc8d6c5 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -430,7 +430,7 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi,
      sockets that time-out or have actions will be dealt with. Since this
      handle has no action yet, we make sure it times out to get things to
      happen. */
-  Curl_expire(data, 0);
+  Curl_expire(data, 0, EXPIRE_ADD_HANDLE);
 
   /* increase the node-counter */
   multi->num_easy++;
@@ -1844,9 +1844,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         if(send_timeout_ms <= 0 && recv_timeout_ms <= 0)
           multistate(data, CURLM_STATE_PERFORM);
         else if(send_timeout_ms >= recv_timeout_ms)
-          Curl_expire_latest(data, send_timeout_ms);
+          Curl_expire_latest(data, send_timeout_ms, EXPIRE_TOOFAST);
         else
-          Curl_expire_latest(data, recv_timeout_ms);
+          Curl_expire_latest(data, recv_timeout_ms, EXPIRE_TOOFAST);
       }
       break;
 
@@ -1877,9 +1877,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       if(send_timeout_ms > 0 || recv_timeout_ms > 0) {
         multistate(data, CURLM_STATE_TOOFAST);
         if(send_timeout_ms >= recv_timeout_ms)
-          Curl_expire_latest(data, send_timeout_ms);
+          Curl_expire_latest(data, send_timeout_ms, EXPIRE_TOOFAST);
         else
-          Curl_expire_latest(data, recv_timeout_ms);
+          Curl_expire_latest(data, recv_timeout_ms, EXPIRE_TOOFAST);
         break;
       }
 
@@ -1940,7 +1940,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 
         /* expire the new receiving pipeline head */
         if(data->easy_conn->recv_pipe.head)
-          Curl_expire_latest(data->easy_conn->recv_pipe.head->ptr, 0);
+          Curl_expire_latest(data->easy_conn->recv_pipe.head->ptr, 0,
+                             EXPIRE_PIPELINE_READ);
 
         /* Check if we can move pending requests to send pipe */
         Curl_multi_process_pending_handles(multi);
@@ -2484,6 +2485,7 @@ void Curl_multi_closed(struct connectdata *conn, curl_socket_t s)
 struct time_node {
   struct curl_llist_element list;
   struct timeval time;
+  expire_id id;
 };
 
 /*
@@ -2872,6 +2874,27 @@ static void multi_freetimeout(void *user, void *entryptr)
 }
 
 /*
+ * multi_deltimeout()
+ *
+ * Remove a given timestamp from the list of timeouts.
+ */
+static void
+multi_deltimeout(struct Curl_easy *data, expire_id id)
+{
+  struct curl_llist_element *e;
+  struct curl_llist *timeoutlist = &data->state.timeoutlist;
+
+  /* find and remove the node(s) from the list */
+  for(e = timeoutlist->head; e; e = e->next) {
+    struct time_node *node = (struct time_node *)e->ptr;
+    if(node->id == id) {
+      Curl_llist_remove(timeoutlist, e, NULL);
+      return;
+    }
+  }
+}
+
+/*
  * multi_addtimeout()
  *
  * Add a timestamp to the list of timeouts. Keep the list sorted so that head
@@ -2879,21 +2902,27 @@ static void multi_freetimeout(void *user, void *entryptr)
  *
  */
 static CURLMcode
-multi_addtimeout(struct curl_llist *timeoutlist,
-                 struct timeval *stamp)
+multi_addtimeout(struct Curl_easy *data,
+                 struct timeval *stamp,
+                 int id)
 {
   struct curl_llist_element *e;
   struct time_node *node;
   struct curl_llist_element *prev = NULL;
+  size_t n;
+  struct curl_llist *timeoutlist = &data->state.timeoutlist;
 
   node = malloc(sizeof(struct time_node));
   if(!node)
     return CURLM_OUT_OF_MEMORY;
 
-  /* copy the timestamp */
+  /* copy the timestamp and id */
   memcpy(&node->time, stamp, sizeof(*stamp));
+  node->id = id;
 
-  if(Curl_llist_count(timeoutlist)) {
+  n = Curl_llist_count(timeoutlist);
+  infof(data, "TIMEOUTS %zd\n", n);
+  if(n) {
     /* find the correct spot in the list */
     for(e = timeoutlist->head; e; e = e->next) {
       struct time_node *check = (struct time_node *)e->ptr;
@@ -2919,8 +2948,12 @@ multi_addtimeout(struct curl_llist *timeoutlist,
  *
  * The timeout will be added to a queue of timeouts if it defines a moment in
  * time that is later than the current head of queue.
+ *
+ * If 'id' is given (non-zero), expire will replace a former timeout using the
+ * same id. id is also a good way to keep track of the purpose of each
+ * timeout.
  */
-void Curl_expire(struct Curl_easy *data, time_t milli)
+void Curl_expire(struct Curl_easy *data, time_t milli, expire_id id)
 {
   struct Curl_multi *multi = data->multi;
   struct timeval *nowp = &data->state.expiretime;
@@ -2932,6 +2965,10 @@ void Curl_expire(struct Curl_easy *data, time_t milli)
   if(!multi)
     return;
 
+  DEBUGASSERT(id < EXPIRE_LAST);
+
+  infof(data, "EXPIRE in %d, id %d\n", (int)milli, id);
+
   set = Curl_tvnow();
   set.tv_sec += (long)(milli/1000);
   set.tv_usec += (long)(milli%1000)*1000;
@@ -2946,16 +2983,20 @@ void Curl_expire(struct Curl_easy *data, time_t milli)
        Compare if the new time is earlier, and only remove-old/add-new if it
        is. */
     time_t diff = curlx_tvdiff(set, *nowp);
+
+    /* remove the previous timer first, if there */
+    multi_deltimeout(data, id);
+
     if(diff > 0) {
       /* the new expire time was later so just add it to the queue
          and get out */
-      multi_addtimeout(&data->state.timeoutlist, &set);
+      multi_addtimeout(data, &set, id);
       return;
     }
 
     /* the new time is newer than the presently set one, so add the current
        to the queue and update the head */
-    multi_addtimeout(&data->state.timeoutlist, nowp);
+    multi_addtimeout(data, nowp, id);
 
     /* Since this is an updated time, we must remove the previous entry from
        the splay tree first and then re-add the new value */
@@ -2983,7 +3024,7 @@ void Curl_expire(struct Curl_easy *data, time_t milli)
  * time-out period to expire.
  *
  */
-void Curl_expire_latest(struct Curl_easy *data, time_t milli)
+void Curl_expire_latest(struct Curl_easy *data, time_t milli, expire_id id)
 {
   struct timeval *expire = &data->state.expiretime;
 
@@ -3012,7 +3053,7 @@ void Curl_expire_latest(struct Curl_easy *data, time_t milli)
   }
 
   /* Just add the timeout like normal */
-  Curl_expire(data, milli);
+  Curl_expire(data, milli, id);
 }
 
 
@@ -3119,7 +3160,7 @@ void Curl_multi_process_pending_handles(struct Curl_multi *multi)
       Curl_llist_remove(&multi->pending, e, NULL);
 
       /* Make sure that the handle will be processed soonish. */
-      Curl_expire_latest(data, 0);
+      Curl_expire_latest(data, 0, EXPIRE_MULTI_PENDING);
     }
 
     e = next; /* operate on next handle */
diff --git a/lib/multiif.h b/lib/multiif.h
index e5de1fc49..a9a7a5577 100644
--- a/lib/multiif.h
+++ b/lib/multiif.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -25,9 +25,29 @@
 /*
  * Prototypes for library-wide functions provided by multi.c
  */
-void Curl_expire(struct Curl_easy *data, time_t milli);
+
+/* Timers */
+typedef enum {
+  EXPIRE_SPEEDCHECK,
+  EXPIRE_H2DATA,
+  EXPIRE_PIPELINE_SEND,
+  EXPIRE_PIPELINE_READ,
+  EXPIRE_ADD_HANDLE,
+  EXPIRE_TOOFAST,
+  EXPIRE_UNPAUSE,
+  EXPIRE_ARES,
+  EXPIRE_MULTI_PENDING,
+  EXPIRE_DNS_PER_NAME,
+  EXPIRE_HAPPY_EYEBALLS,
+  EXPIRE_100_TIMEOUT,
+  EXPIRE_TIMEOUT,
+  EXPIRE_CONNECTTIMEOUT,
+  EXPIRE_LAST /* not an actual timer, used as a marker only */
+} expire_id;
+
+void Curl_expire(struct Curl_easy *data, time_t milli, expire_id);
 void Curl_expire_clear(struct Curl_easy *data);
-void Curl_expire_latest(struct Curl_easy *data, time_t milli);
+void Curl_expire_latest(struct Curl_easy *data, time_t milli, expire_id);
 bool Curl_pipeline_wanted(const struct Curl_multi* multi, int bits);
 void Curl_multi_handlePipeBreak(struct Curl_easy *data);
 
diff --git a/lib/pipeline.c b/lib/pipeline.c
index 72b9fb843..729e69c27 100644
--- a/lib/pipeline.c
+++ b/lib/pipeline.c
@@ -113,7 +113,7 @@ CURLcode Curl_add_handle_to_pipeline(struct Curl_easy *handle,
   if(pipeline == &conn->send_pipe && sendhead != conn->send_pipe.head) {
     /* this is a new one as head, expire it */
     Curl_pipeline_leave_write(conn); /* not in use yet */
-    Curl_expire(conn->send_pipe.head->ptr, 0);
+    Curl_expire(conn->send_pipe.head->ptr, 0, EXPIRE_PIPELINE_SEND);
   }
 
 #if 0 /* enable for pipeline debugging */
@@ -148,7 +148,7 @@ void Curl_move_handle_from_send_to_recv_pipe(struct Curl_easy *handle,
         infof(conn->data, "%p is at send pipe head B!\n",
               (void *)conn->send_pipe.head->ptr);
 #endif
-        Curl_expire(conn->send_pipe.head->ptr, 0);
+        Curl_expire(conn->send_pipe.head->ptr, 0, EXPIRE_PIPELINE_READ);
       }
 
       /* The receiver's list is not really interesting here since either this
diff --git a/lib/speedcheck.c b/lib/speedcheck.c
index f0daf82c0..694d7f695 100644
--- a/lib/speedcheck.c
+++ b/lib/speedcheck.c
@@ -67,7 +67,7 @@ CURLcode Curl_speedcheck(struct Curl_easy *data,
   if(data->set.low_speed_limit)
     /* if low speed limit is enabled, set the expire timer to make this
        connection's speed get checked again in a second */
-    Curl_expire_latest(data, 1000);
+    Curl_expire_latest(data, 1000, EXPIRE_SPEEDCHECK);
 
   return CURLE_OK;
 }
diff --git a/lib/transfer.c b/lib/transfer.c
index 0bba3f529..b7435f9ae 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -889,7 +889,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
           *didwhat &= ~KEEP_SEND;  /* we didn't write anything actually */
 
           /* set a timeout for the multi interface */
-          Curl_expire(data, data->set.expect_100_timeout);
+          Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
           break;
         }
 
@@ -1338,10 +1338,10 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
     Curl_pgrsStartNow(data);
 
     if(data->set.timeout)
-      Curl_expire(data, data->set.timeout);
+      Curl_expire(data, data->set.timeout, EXPIRE_TIMEOUT);
 
     if(data->set.connecttimeout)
-      Curl_expire(data, data->set.connecttimeout);
+      Curl_expire(data, data->set.connecttimeout, EXPIRE_CONNECTTIMEOUT);
 
     /* In case the handle is re-used and an authentication method was picked
        in the session we need to make sure we only use the one(s) we now
@@ -1942,7 +1942,7 @@ Curl_setup_transfer(
 
         /* Set a timeout for the multi interface. Add the inaccuracy margin so
            that we don't fire slightly too early and get denied to run. */
-        Curl_expire(data, data->set.expect_100_timeout);
+        Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
       }
       else {
         if(data->state.expect100header)