On the trunk:

mod_proxy_http2: Reliability of reconnect handling improved.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1790826 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 63 insertions, 47 deletions

diff --git a/modules/http2/mod_proxy_http2.c b/modules/http2/mod_proxy_http2.c
index 6f9fbecf1b..86a8c3dc5e 100644
--- a/modules/http2/mod_proxy_http2.c
+++ b/modules/http2/mod_proxy_http2.c
@@ -26,6 +26,8 @@
 #include "h2_version.h"
 #include "h2_proxy_session.h"
 
+#define H2MIN(x,y) ((x) < (y) ? (x) : (y))
+
 static void register_hook(apr_pool_t *p);
 
 AP_DECLARE_MODULE(proxy_http2) = {
@@ -65,7 +67,7 @@ typedef struct h2_proxy_ctx {
     const char *engine_type;
     apr_pool_t *engine_pool;    
     apr_size_t req_buffer_size;
-    request_rec *next;
+    h2_proxy_fifo *requests;
     int capacity;
     
     unsigned standalone : 1;
@@ -230,7 +232,7 @@ static apr_status_t proxy_engine_init(h2_req_engine *engine,
     ctx->engine_type = type;
     ctx->engine_pool = pool;
     ctx->req_buffer_size = req_buffer_size;
-    ctx->capacity = 100;
+    ctx->capacity = H2MIN(100, h2_proxy_fifo_capacity(ctx->requests));
     
     *pconsumed = out_consumed;
     *pctx = ctx;
@@ -257,10 +259,9 @@ static apr_status_t add_request(h2_proxy_session *session, request_rec *r)
     return status;
 }
 
-static void request_done(h2_proxy_session *session, request_rec *r,
+static void request_done(h2_proxy_ctx *ctx, request_rec *r,
                          apr_status_t status, int touched)
 {   
-    h2_proxy_ctx *ctx = session->user_data;
     const char *task_id = apr_table_get(r->connection->notes, H2_TASK_ID_NOTE);
 
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, r->connection, 
@@ -269,35 +270,26 @@ static void request_done(h2_proxy_session *session, request_rec *r,
     if (status != APR_SUCCESS) {
         if (!touched) {
             /* untouched request, need rescheduling */
-            if (req_engine_push && is_h2 && is_h2(ctx->owner)) {
-                if (req_engine_push(ctx->engine_type, r, NULL) == APR_SUCCESS) {
-                    /* push to engine */
-                    ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, r->connection, 
-                                  APLOGNO(03369)
-                                  "h2_proxy_session(%s): rescheduled request %s",
-                                  ctx->engine_id, task_id);
-                    return;
-                }
-            }
-            else if (!ctx->next) {
-                ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, r->connection, 
-                              "h2_proxy_session(%s): retry untouched request",
-                              ctx->engine_id);
-                ctx->next = r;
-            }
+            status = h2_proxy_fifo_push(ctx->requests, r);
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, r->connection, 
+                          APLOGNO(03369)
+                          "h2_proxy_session(%s): rescheduled request %s",
+                          ctx->engine_id, task_id);
+            return;
         }
         else {
             const char *uri;
             uri = apr_uri_unparse(r->pool, &r->parsed_uri, 0);
             ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, r->connection, 
                           APLOGNO(03471) "h2_proxy_session(%s): request %s -> %s "
-                          "not complete, was touched",
+                          "not complete, cannot repeat", 
                           ctx->engine_id, task_id, uri);
         }
     }
     
     if (r == ctx->rbase) {
-        ctx->r_status = (status == APR_SUCCESS)? APR_SUCCESS : HTTP_SERVICE_UNAVAILABLE;
+        ctx->r_status = ((status == APR_SUCCESS)? APR_SUCCESS
+                         : HTTP_SERVICE_UNAVAILABLE);
     }
     
     if (req_engine_done && ctx->engine) {
@@ -309,21 +301,32 @@ static void request_done(h2_proxy_session *session, request_rec *r,
     }
 }    
 
+static void session_req_done(h2_proxy_session *session, request_rec *r,
+                             apr_status_t status, int touched)
+{
+    request_done(session->user_data, r, status, touched);
+}
+
 static apr_status_t next_request(h2_proxy_ctx *ctx, int before_leave)
 {
-    if (ctx->next) {
+    if (h2_proxy_fifo_count(ctx->requests) > 0) {
         return APR_SUCCESS;
     }
     else if (req_engine_pull && ctx->engine) {
         apr_status_t status;
+        request_rec *r = NULL;
+        
         status = req_engine_pull(ctx->engine, before_leave? 
                                  APR_BLOCK_READ: APR_NONBLOCK_READ, 
-                                 ctx->capacity, &ctx->next);
-        ap_log_cerror(APLOG_MARK, APLOG_TRACE3, status, ctx->owner, 
-                      "h2_proxy_engine(%s): pulled request (%s) %s", 
-                      ctx->engine_id, 
-                      before_leave? "before leave" : "regular", 
-                      (ctx->next? ctx->next->the_request : "NULL"));
+                                 ctx->capacity, &r);
+        if (status == APR_SUCCESS && r) {
+            ap_log_cerror(APLOG_MARK, APLOG_TRACE3, status, ctx->owner, 
+                          "h2_proxy_engine(%s): pulled request (%s) %s", 
+                          ctx->engine_id, 
+                          before_leave? "before leave" : "regular", 
+                          r->the_request);
+            h2_proxy_fifo_push(ctx->requests, r);
+        }
         return APR_STATUS_IS_EAGAIN(status)? APR_SUCCESS : status;
     }
     return APR_EOF;
@@ -332,6 +335,7 @@ static apr_status_t next_request(h2_proxy_ctx *ctx, int before_leave)
 static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
     apr_status_t status = OK;
     int h2_front;
+    request_rec *r;
     
     /* Step Four: Send the Request in a new HTTP/2 stream and
      * loop until we got the response or encounter errors.
@@ -342,7 +346,7 @@ static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
     ctx->session = h2_proxy_session_setup(ctx->engine_id, ctx->p_conn, ctx->conf,
                                           h2_front, 30, 
                                           h2_proxy_log2((int)ctx->req_buffer_size), 
-                                          request_done);
+                                          session_req_done);
     if (!ctx->session) {
         ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, 
                       APLOGNO(03372) "session unavailable");
@@ -353,10 +357,9 @@ static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
                   "eng(%s): run session %s", ctx->engine_id, ctx->session->id);
     ctx->session->user_data = ctx;
     
-    while (1) {
-        if (ctx->next) {
-            add_request(ctx->session, ctx->next);
-            ctx->next = NULL;
+    while (!ctx->owner->aborted) {
+        if (APR_SUCCESS == h2_proxy_fifo_try_pull(ctx->requests, (void**)&r)) {
+            add_request(ctx->session, r);
         }
         
         status = h2_proxy_session_process(ctx->session);
@@ -366,7 +369,8 @@ static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
             /* ongoing processing, call again */
             if (ctx->session->remote_max_concurrent > 0
                 && ctx->session->remote_max_concurrent != ctx->capacity) {
-                ctx->capacity = (int)ctx->session->remote_max_concurrent;
+                ctx->capacity = H2MIN((int)ctx->session->remote_max_concurrent, 
+                                      h2_proxy_fifo_capacity(ctx->requests));
             }
             s2 = next_request(ctx, 0);
             if (s2 == APR_ECONNABORTED) {
@@ -382,7 +386,8 @@ static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
                 status = ctx->r_status = APR_SUCCESS;
                 break;
             }
-            if (!ctx->next && h2_proxy_ihash_empty(ctx->session->streams)) {
+            if ((h2_proxy_fifo_count(ctx->requests) == 0) 
+                && h2_proxy_ihash_empty(ctx->session->streams)) {
                 break;
             }
         }
@@ -396,7 +401,7 @@ static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) {
              * a) be reopened on the new session iff safe to do so
              * b) reported as done (failed) otherwise
              */
-            h2_proxy_session_cleanup(ctx->session, request_done);
+            h2_proxy_session_cleanup(ctx->session, session_req_done);
             break;
         }
     }
@@ -446,7 +451,8 @@ static apr_status_t push_request_somewhere(h2_proxy_ctx *ctx, request_rec *r)
         ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, 
                       "H2: hosting engine %s", ctx->engine_id);
     }
-    return APR_SUCCESS;
+
+    return h2_proxy_fifo_push(ctx->requests, r);
 }
 
 static int proxy_http2_handler(request_rec *r, 
@@ -463,7 +469,7 @@ static int proxy_http2_handler(request_rec *r,
     apr_status_t status;
     h2_proxy_ctx *ctx;
     apr_uri_t uri;
-    int reconnected = 0;
+    int reconnects = 0;
     
     /* find the scheme */
     if ((url[0] != 'h' && url[0] != 'H') || url[1] != '2') {
@@ -500,8 +506,8 @@ static int proxy_http2_handler(request_rec *r,
     ctx->conf       = conf;
     ctx->flushall   = apr_table_get(r->subprocess_env, "proxy-flushall")? 1 : 0;
     ctx->r_status   = HTTP_SERVICE_UNAVAILABLE;
-    ctx->next       = r;
-    r = NULL;
+    
+    h2_proxy_fifo_set_create(&ctx->requests, ctx->pool, 100);
     
     ap_set_module_config(ctx->owner->conn_config, &proxy_http2_module, ctx);
 
@@ -534,11 +540,11 @@ run_connect:
     
     /* If we are not already hosting an engine, try to push the request 
      * to an already existing engine or host a new engine here. */
-    if (!ctx->engine) {
-        ctx->r_status = push_request_somewhere(ctx, ctx->next);
+    if (r && !ctx->engine) {
+        ctx->r_status = push_request_somewhere(ctx, r);
+        r = NULL;
         if (ctx->r_status == SUSPENDED) {
             /* request was pushed to another thread, leave processing here */
-            ctx->next = NULL;
             goto cleanup;
         }
     }
@@ -599,7 +605,7 @@ run_session:
     }
 
 reconnect:
-    if (!reconnected && next_request(ctx, 1) == APR_SUCCESS) {
+    if (next_request(ctx, 1) == APR_SUCCESS) {
         /* Still more to do, tear down old conn and start over */
         if (ctx->p_conn) {
             ctx->p_conn->close = 1;
@@ -607,8 +613,13 @@ reconnect:
             ap_proxy_release_connection(ctx->proxy_func, ctx->p_conn, ctx->server);
             ctx->p_conn = NULL;
         }
-        reconnected = 1; /* we do this only once, then fail */
-        goto run_connect;
+        ++reconnects;
+        if (reconnects < 5 && !ctx->owner->aborted) {
+            goto run_connect;
+        } 
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, APLOGNO(10023)
+                      "giving up after %d reconnects, %d requests todo",
+                      reconnects, h2_proxy_fifo_count(ctx->requests));
     }
     
 cleanup:
@@ -622,6 +633,11 @@ cleanup:
         ctx->p_conn = NULL;
     }
 
+    /* Any requests will still have need to fail */
+    while (APR_SUCCESS == h2_proxy_fifo_try_pull(ctx->requests, (void**)&r)) {
+        request_done(ctx, r, HTTP_SERVICE_UNAVAILABLE, true);
+    }
+    
     ap_set_module_config(ctx->owner->conn_config, &proxy_http2_module, NULL);
     ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, 
                   APLOGNO(03377) "leaving handler");