multi: retry pending transfers less aggressively [WIP]

When a transfer is requested to get done and it is put in the pending
queue when limited by number of connections, total or per-host, libcurl
would previously very aggressively retry *ALL* pending transfers to get
them transferring. That was very time consuming.

By reducing the aggressiveness in how pending are being retried, we
waste MUCH less time on putting transfers back into pending again.

Reported-by: Cyril B
Fixes #2369

2 files changed, 22 insertions, 30 deletions

diff --git a/lib/multi.c b/lib/multi.c
index 98e5fca2a..6fe80f7b3 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -77,6 +77,7 @@ static CURLMcode add_next_timeout(struct curltime now,
                                   struct Curl_easy *d);
 static CURLMcode multi_timeout(struct Curl_multi *multi,
                                long *timeout_ms);
+static void process_pending_handles(struct Curl_multi *multi);
 
 #ifdef DEBUGBUILD
 static const char * const statename[]={
@@ -538,6 +539,8 @@ static CURLcode multi_done(struct connectdata **connp,
       result = CURLE_ABORTED_BY_CALLBACK;
   }
 
+  process_pending_handles(data->multi); /* connection / multiplex */
+
   if(conn->send_pipe.size || conn->recv_pipe.size) {
     /* Stop if pipeline is not empty . */
     data->easy_conn = NULL;
@@ -653,10 +656,6 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
     /* this handle is "alive" so we need to count down the total number of
        alive connections when this is removed */
     multi->num_alive--;
-
-    /* When this handle gets removed, other handles may be able to get the
-       connection */
-    Curl_multi_process_pending_handles(multi);
   }
 
   if(data->easy_conn &&
@@ -1339,7 +1338,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 
     if(multi_ischanged(multi, TRUE)) {
       DEBUGF(infof(data, "multi changed, check CONNECT_PEND queue!\n"));
-      Curl_multi_process_pending_handles(multi);
+      process_pending_handles(multi); /* pipelined/multiplexed */
     }
 
     if(data->easy_conn && data->mstate > CURLM_STATE_CONNECT &&
@@ -1785,8 +1784,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
     case CURLM_STATE_DO_DONE:
       /* Move ourselves from the send to recv pipeline */
       Curl_move_handle_from_send_to_recv_pipe(data, data->easy_conn);
-      /* Check if we can move pending requests to send pipe */
-      Curl_multi_process_pending_handles(multi);
+
+      if(data->easy_conn->bits.multiplex || data->easy_conn->send_pipe.size)
+        /* Check if we can move pending requests to send pipe */
+        process_pending_handles(multi); /*  pipelined/multiplexed */
 
       /* Only perform the transfer if there's a good socket to work with.
          Having both BAD is a signal to skip immediately to DONE */
@@ -1938,9 +1939,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         if(data->easy_conn->recv_pipe.head)
           Curl_expire(data->easy_conn->recv_pipe.head->ptr, 0, EXPIRE_RUN_NOW);
 
-        /* Check if we can move pending requests to send pipe */
-        Curl_multi_process_pending_handles(multi);
-
         /* When we follow redirects or is set to retry the connection, we must
            to go back to the CONNECT state */
         if(data->req.newurl || retry) {
@@ -1997,8 +1995,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 
         /* Remove ourselves from the receive pipeline, if we are there. */
         Curl_removeHandleFromPipeline(data, &data->easy_conn->recv_pipe);
-        /* Check if we can move pending requests to send pipe */
-        Curl_multi_process_pending_handles(multi);
+
+        if(data->easy_conn->bits.multiplex || data->easy_conn->send_pipe.size)
+          /* Check if we can move pending requests to connection */
+          process_pending_handles(multi); /* pipelined/multiplexing */
 
         /* post-transfer command */
         res = multi_done(&data->easy_conn, result, FALSE);
@@ -2066,7 +2066,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         data->state.pipe_broke = FALSE;
 
         /* Check if we can move pending requests to send pipe */
-        Curl_multi_process_pending_handles(multi);
+        process_pending_handles(multi); /* connection */
 
         if(data->easy_conn) {
           /* if this has a connection, unsubscribe from the pipelines */
@@ -3068,26 +3068,20 @@ struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi)
   return &multi->pipelining_server_bl;
 }
 
-void Curl_multi_process_pending_handles(struct Curl_multi *multi)
+static void process_pending_handles(struct Curl_multi *multi)
 {
   struct curl_llist_element *e = multi->pending.head;
+  struct Curl_easy *data = e->ptr;
 
-  while(e) {
-    struct Curl_easy *data = e->ptr;
-    struct curl_llist_element *next = e->next;
-
-    if(data->mstate == CURLM_STATE_CONNECT_PEND) {
-      multistate(data, CURLM_STATE_CONNECT);
+  DEBUGASSERT(data->mstate == CURLM_STATE_CONNECT_PEND);
 
-      /* Remove this node from the list */
-      Curl_llist_remove(&multi->pending, e, NULL);
+  multistate(data, CURLM_STATE_CONNECT);
 
-      /* Make sure that the handle will be processed soonish. */
-      Curl_expire(data, 0, EXPIRE_RUN_NOW);
-    }
+  /* Remove this node from the list */
+  Curl_llist_remove(&multi->pending, e, NULL);
 
-    e = next; /* operate on next handle */
-  }
+  /* Make sure that the handle will be processed soonish. */
+  Curl_expire(data, 0, EXPIRE_RUN_NOW);
 }
 
 void Curl_set_in_callback(struct Curl_easy *easy, bool value)
diff --git a/lib/multiif.h b/lib/multiif.h
index a988bfd4a..e6eb591e4 100644
--- a/lib/multiif.h
+++ b/lib/multiif.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -58,8 +58,6 @@ struct Curl_multi *Curl_multi_handle(int hashsize, int chashsize);
 void Curl_multi_dump(struct Curl_multi *multi);
 #endif
 
-void Curl_multi_process_pending_handles(struct Curl_multi *multi);
-
 /* Return the value of the CURLMOPT_MAX_HOST_CONNECTIONS option */
 size_t Curl_multi_max_host_connections(struct Curl_multi *multi);