Add SoupMessage flag SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS

It ensures that a new connection will be created for the message if
there aren't any idle connections available and the current number of
connections have reached any of the limits. When a new dedicated
connection is created, it's automatically removed when the message is
unqueued, without switching to idle state to make sure it can't be
reused.

https://bugzilla.gnome.org/show_bug.cgi?id=744720

6 files changed, 84 insertions, 22 deletions

diff --git a/libsoup/soup-cache.c b/libsoup/soup-cache.c
index fe1ed898..90fce5a4 100644
--- a/libsoup/soup-cache.c
+++ b/libsoup/soup-cache.c
@@ -1383,6 +1383,7 @@ soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original
 	/* Copy the data we need from the original message */
 	uri = soup_message_get_uri (original);
 	msg = soup_message_new_from_uri (original->method, uri);
+	soup_message_set_flags (msg, soup_message_get_flags (original));
 	soup_message_disable_feature (msg, SOUP_TYPE_CACHE);
 
 	soup_message_headers_foreach (original->request_headers,
diff --git a/libsoup/soup-message-queue.h b/libsoup/soup-message-queue.h
index d2dfda43..85012cc4 100644
--- a/libsoup/soup-message-queue.h
+++ b/libsoup/soup-message-queue.h
@@ -48,6 +48,7 @@ struct _SoupMessageQueueItem {
 	guint new_api           : 1;
 	guint io_started        : 1;
 	guint async             : 1;
+	guint conn_is_dedicated : 1;
 	guint priority          : 3;
 	guint resend_count      : 25;
 
diff --git a/libsoup/soup-message.c b/libsoup/soup-message.c
index e28ce7fd..7ad0afff 100644
--- a/libsoup/soup-message.c
+++ b/libsoup/soup-message.c
@@ -1445,6 +1445,12 @@ soup_message_cleanup_response (SoupMessage *msg)
  *   regardless its #SoupMessage:method, and allows reuse of existing
  *   idle connections, instead of always requiring a new one, unless
  *   #SOUP_MESSAGE_NEW_CONNECTION is set.
+ * @SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS: Request that a new connection is
+ *   created for the message if there aren't idle connections available
+ *   and it's not possible to create new connections due to any of the
+ *   connection limits has been reached. If a dedicated connection is
+ *   eventually created for this message, it will be dropped when the
+ *   message finishes. Since 2.50
  *
  * Various flags that can be set on a #SoupMessage to alter its
  * behavior.
diff --git a/libsoup/soup-message.h b/libsoup/soup-message.h
index 79261130..1dc2258c 100644
--- a/libsoup/soup-message.h
+++ b/libsoup/soup-message.h
@@ -118,15 +118,16 @@ void             soup_message_set_first_party     (SoupMessage       *msg,
 						   SoupURI           *first_party);
 
 typedef enum {
-	SOUP_MESSAGE_NO_REDIRECT          = (1 << 1),
-	SOUP_MESSAGE_CAN_REBUILD          = (1 << 2),
+	SOUP_MESSAGE_NO_REDIRECT              = (1 << 1),
+	SOUP_MESSAGE_CAN_REBUILD              = (1 << 2),
 #ifndef SOUP_DISABLE_DEPRECATED
-	SOUP_MESSAGE_OVERWRITE_CHUNKS     = (1 << 3),
+	SOUP_MESSAGE_OVERWRITE_CHUNKS         = (1 << 3),
 #endif
-	SOUP_MESSAGE_CONTENT_DECODED      = (1 << 4),
-	SOUP_MESSAGE_CERTIFICATE_TRUSTED  = (1 << 5),
-	SOUP_MESSAGE_NEW_CONNECTION       = (1 << 6),
-	SOUP_MESSAGE_IDEMPOTENT           = (1 << 7)
+	SOUP_MESSAGE_CONTENT_DECODED          = (1 << 4),
+	SOUP_MESSAGE_CERTIFICATE_TRUSTED      = (1 << 5),
+	SOUP_MESSAGE_NEW_CONNECTION           = (1 << 6),
+	SOUP_MESSAGE_IDEMPOTENT               = (1 << 7),
+	SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS = (1 << 8)
 } SoupMessageFlags;
 
 void             soup_message_set_flags           (SoupMessage           *msg,
diff --git a/libsoup/soup-session.c b/libsoup/soup-session.c
index 4ef2b689..ed5d2abb 100644
--- a/libsoup/soup-session.c
+++ b/libsoup/soup-session.c
@@ -1253,6 +1253,7 @@ soup_session_set_item_connection (SoupSession          *session,
 	}
 
 	item->conn = conn;
+	item->conn_is_dedicated = FALSE;
 	soup_message_set_connection (item->msg, conn);
 
 	if (item->conn) {
@@ -1496,10 +1497,13 @@ soup_session_unqueue_item (SoupSession          *session,
 {
 	SoupSessionPrivate *priv = SOUP_SESSION_GET_PRIVATE (session);
 	SoupSessionHost *host;
+	SoupConnection *dedicated_conn = NULL;
 
 	if (item->conn) {
-		if (item->msg->method != SOUP_METHOD_CONNECT ||
-		    !SOUP_STATUS_IS_SUCCESSFUL (item->msg->status_code))
+		if (item->conn_is_dedicated)
+			dedicated_conn = g_object_ref (item->conn);
+		else if (item->msg->method != SOUP_METHOD_CONNECT ||
+			 !SOUP_STATUS_IS_SUCCESSFUL (item->msg->status_code))
 			soup_connection_set_state (item->conn, SOUP_CONNECTION_IDLE);
 		soup_session_set_item_connection (session, item, NULL);
 	}
@@ -1514,9 +1518,21 @@ soup_session_unqueue_item (SoupSession          *session,
 	g_mutex_lock (&priv->conn_lock);
 	host = get_host_for_message (session, item->msg);
 	host->num_messages--;
+	if (dedicated_conn) {
+		/* FIXME: Do not drop the connection if current number of connections
+		 * is no longer over the limits, just mark it as IDLE so it can be reused.
+		 */
+		g_hash_table_remove (priv->conns, dedicated_conn);
+		drop_connection (session, host, dedicated_conn);
+	}
 	g_cond_broadcast (&priv->conn_cond);
 	g_mutex_unlock (&priv->conn_lock);
 
+	if (dedicated_conn) {
+		soup_connection_disconnect (dedicated_conn);
+		g_object_unref (dedicated_conn);
+	}
+
 	/* g_signal_handlers_disconnect_by_func doesn't work if you
 	 * have a metamarshal, meaning it doesn't work with
 	 * soup_message_add_header_handler()
@@ -1806,7 +1822,9 @@ get_connection_for_host (SoupSession *session,
 			 SoupMessageQueueItem *item,
 			 SoupSessionHost *host,
 			 gboolean need_new_connection,
-			 gboolean *try_cleanup)
+			 gboolean ignore_connection_limits,
+			 gboolean *try_cleanup,
+			 gboolean *is_dedicated_connection)
 {
 	SoupSessionPrivate *priv = SOUP_SESSION_GET_PRIVATE (session);
 	SoupConnection *conn;
@@ -1834,18 +1852,30 @@ get_connection_for_host (SoupSession *session,
 	/* Limit the number of pending connections; num_messages / 2
 	 * is somewhat arbitrary...
 	 */
-	if (num_pending > host->num_messages / 2)
-		return NULL;
+	if (num_pending > host->num_messages / 2) {
+		if (!ignore_connection_limits)
+			return NULL;
+
+		*is_dedicated_connection = TRUE;
+	}
 
 	if (host->num_conns >= priv->max_conns_per_host) {
-		if (need_new_connection)
-			*try_cleanup = TRUE;
-		return NULL;
+		if (!ignore_connection_limits) {
+			if (need_new_connection)
+				*try_cleanup = TRUE;
+			return NULL;
+		}
+
+		*is_dedicated_connection = TRUE;
 	}
 
 	if (priv->num_conns >= priv->max_conns) {
-		*try_cleanup = TRUE;
-		return NULL;
+		if (!ignore_connection_limits) {
+			*try_cleanup = TRUE;
+			return NULL;
+		}
+
+		*is_dedicated_connection = TRUE;
 	}
 
 	ensure_socket_props (session);
@@ -1892,6 +1922,8 @@ get_connection (SoupMessageQueueItem *item, gboolean *should_cleanup)
 	SoupConnection *conn = NULL;
 	gboolean my_should_cleanup = FALSE;
 	gboolean need_new_connection;
+	gboolean ignore_connection_limits;
+	gboolean is_dedicated_connection = FALSE;
 
 	soup_session_cleanup_connections (session, FALSE);
 
@@ -1899,13 +1931,17 @@ get_connection (SoupMessageQueueItem *item, gboolean *should_cleanup)
 		(soup_message_get_flags (item->msg) & SOUP_MESSAGE_NEW_CONNECTION) ||
 		(!(soup_message_get_flags (item->msg) & SOUP_MESSAGE_IDEMPOTENT) &&
 		 !SOUP_METHOD_IS_IDEMPOTENT (item->msg->method));
+	ignore_connection_limits =
+		(soup_message_get_flags (item->msg) & SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS);
 
 	g_mutex_lock (&priv->conn_lock);
 	host = get_host_for_message (session, item->msg);
 	while (TRUE) {
 		conn = get_connection_for_host (session, item, host,
 						need_new_connection,
-						&my_should_cleanup);
+						ignore_connection_limits,
+						&my_should_cleanup,
+						&is_dedicated_connection);
 		if (conn || item->async)
 			break;
 
@@ -1929,6 +1965,7 @@ get_connection (SoupMessageQueueItem *item, gboolean *should_cleanup)
 	}
 
 	soup_session_set_item_connection (session, item, conn);
+	item->conn_is_dedicated = is_dedicated_connection;
 
 	if (soup_connection_get_state (item->conn) != SOUP_CONNECTION_NEW) {
 		item->state = SOUP_MESSAGE_READY;
diff --git a/tests/connection-test.c b/tests/connection-test.c
index 90233ed8..f460b557 100644
--- a/tests/connection-test.c
+++ b/tests/connection-test.c
@@ -404,7 +404,7 @@ static GMainLoop *max_conns_loop;
 static int msgs_done;
 static guint quit_loop_timeout;
 #define MAX_CONNS 2
-#define TEST_CONNS (MAX_CONNS * 2)
+#define TEST_CONNS (MAX_CONNS * 2) + 1
 
 static gboolean
 idle_start_server (gpointer data)
@@ -425,7 +425,7 @@ static void
 max_conns_request_started (SoupSession *session, SoupMessage *msg,
 			   SoupSocket *socket, gpointer user_data)
 {
-	if (++msgs_done == MAX_CONNS) {
+	if (++msgs_done >= MAX_CONNS) {
 		if (quit_loop_timeout)
 			g_source_remove (quit_loop_timeout);
 		quit_loop_timeout = g_timeout_add (100, quit_loop, NULL);
@@ -442,7 +442,8 @@ max_conns_message_complete (SoupSession *session, SoupMessage *msg, gpointer use
 static void
 do_max_conns_test_for_session (SoupSession *session)
 {
-	SoupMessage *msgs[TEST_CONNS];
+	SoupMessage *msgs[TEST_CONNS + 1];
+	SoupMessageFlags flags;
 	int i;
 
 	max_conns_loop = g_main_loop_new (NULL, TRUE);
@@ -452,7 +453,7 @@ do_max_conns_test_for_session (SoupSession *session)
 	g_signal_connect (session, "request-started",
 			  G_CALLBACK (max_conns_request_started), NULL);
 	msgs_done = 0;
-	for (i = 0; i < TEST_CONNS; i++) {
+	for (i = 0; i < TEST_CONNS - 1; i++) {
 		msgs[i] = soup_message_new_from_uri ("GET", base_uri);
 		g_object_ref (msgs[i]);
 		soup_session_queue_message (session, msgs[i],
@@ -461,6 +462,21 @@ do_max_conns_test_for_session (SoupSession *session)
 
 	g_main_loop_run (max_conns_loop);
 	g_assert_cmpint (msgs_done, ==, MAX_CONNS);
+
+	if (quit_loop_timeout)
+		g_source_remove (quit_loop_timeout);
+	quit_loop_timeout = g_timeout_add (1000, quit_loop, NULL);
+
+	/* Message with SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS should start */
+	msgs[i] = soup_message_new_from_uri ("GET", base_uri);
+	flags = soup_message_get_flags (msgs[i]);
+	soup_message_set_flags (msgs[i], flags | SOUP_MESSAGE_IGNORE_CONNECTION_LIMITS);
+	g_object_ref (msgs[i]);
+	soup_session_queue_message (session, msgs[i],
+				    max_conns_message_complete, NULL);
+
+	g_main_loop_run (max_conns_loop);
+	g_assert_cmpint (msgs_done, ==, MAX_CONNS + 1);
 	g_signal_handlers_disconnect_by_func (session, max_conns_request_started, NULL);
 
 	msgs_done = 0;