msh3: update to v0.6

Closes #10192

4 files changed, 137 insertions, 86 deletions

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index b3a9daca6..33c890fe3 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -199,7 +199,7 @@ jobs:
 
     - if: ${{ contains(matrix.build.install_steps, 'msh3') }}
       run: |
-        git clone -b v0.5.0 --depth=1 --recursive https://github.com/nibanks/msh3
+        git clone -b v0.6.0 --depth=1 --recursive https://github.com/nibanks/msh3
         cd msh3 && mkdir build && cd build
         cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=$HOME/msh3 ..
         cmake --build .
diff --git a/docs/HTTP3.md b/docs/HTTP3.md
index eb20cc51b..8e83490dd 100644
--- a/docs/HTTP3.md
+++ b/docs/HTTP3.md
@@ -189,7 +189,7 @@ Build curl:
 
 Build msh3:
 
-     % git clone -b v0.5.0 --depth 1 --recursive https://github.com/nibanks/msh3
+     % git clone -b v0.6.0 --depth 1 --recursive https://github.com/nibanks/msh3
      % cd msh3 && mkdir build && cd build
      % cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
      % cmake --build .
diff --git a/lib/http.h b/lib/http.h
index 9da4694fa..ba7ad030e 100644
--- a/lib/http.h
+++ b/lib/http.h
@@ -278,6 +278,7 @@ struct HTTP {
   /* Receive Buffer (Headers and Data) */
   uint8_t* recv_buf;
   size_t recv_buf_alloc;
+  size_t recv_buf_max;
   /* Receive Headers */
   size_t recv_header_len;
   bool recv_header_complete;
diff --git a/lib/vquic/curl_msh3.c b/lib/vquic/curl_msh3.c
index 8f323ddd3..6f7497e9d 100644
--- a/lib/vquic/curl_msh3.c
+++ b/lib/vquic/curl_msh3.c
@@ -50,7 +50,8 @@
 #define CF_DEBUGF(x) do { } while(0)
 #endif
 
-#define MSH3_REQ_INIT_BUF_LEN 8192
+#define MSH3_REQ_INIT_BUF_LEN 16384
+#define MSH3_REQ_MAX_BUF_LEN 0x100000
 
 #ifdef _WIN32
 #define msh3_lock CRITICAL_SECTION
@@ -74,17 +75,25 @@
 #endif /* _WIN32 */
 
 
+static void MSH3_CALL msh3_conn_connected(MSH3_CONNECTION *Connection,
+                                          void *IfContext);
+static void MSH3_CALL msh3_conn_shutdown_complete(MSH3_CONNECTION *Connection,
+                                          void *IfContext);
+static void MSH3_CALL msh3_conn_new_request(MSH3_CONNECTION *Connection,
+                                          void *IfContext,
+                                          MSH3_REQUEST *Request);
 static void MSH3_CALL msh3_header_received(MSH3_REQUEST *Request,
                                            void *IfContext,
                                            const MSH3_HEADER *Header);
-static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
-                                        void *IfContext, uint32_t Length,
+static bool MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
+                                        void *IfContext, uint32_t *Length,
                                         const uint8_t *Data);
 static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
                                     bool Aborted, uint64_t AbortError);
-static void MSH3_CALL msh3_shutdown(MSH3_REQUEST *Request, void *IfContext);
-static void MSH3_CALL msh3_send_complete(MSH3_REQUEST *Request,
-                                         void *IfContext, void *SendContext);
+static void MSH3_CALL msh3_shutdown_complete(MSH3_REQUEST *Request,
+                                             void *IfContext);
+static void MSH3_CALL msh3_data_sent(MSH3_REQUEST *Request,
+                                     void *IfContext, void *SendContext);
 
 
 void Curl_msh3_ver(char *p, size_t len)
@@ -104,15 +113,59 @@ struct cf_msh3_ctx {
   curl_socket_t sock[2]; /* fake socket pair until we get support in msh3 */
   char l_ip[MAX_IPADR_LEN];          /* local IP as string */
   int l_port;                        /* local port number */
+  /* Flags written by msh3/msquic thread */
+  bool handshake_complete;
+  bool handshake_succeeded;
+  bool connected;
+  /* Flags written by curl thread */
+  BIT(verbose);
   BIT(active);
 };
 
+static const MSH3_CONNECTION_IF msh3_conn_if = {
+  msh3_conn_connected,
+  msh3_conn_shutdown_complete,
+  msh3_conn_new_request
+};
+
+static void MSH3_CALL msh3_conn_connected(MSH3_CONNECTION *Connection,
+                                          void *IfContext)
+{
+  struct cf_msh3_ctx *ctx = IfContext;
+  (void)Connection;
+  if(ctx->verbose)
+    CF_DEBUGF(fprintf(stderr, "* [MSH3] evt: connected\n"));
+  ctx->handshake_succeeded = true;
+  ctx->connected = true;
+  ctx->handshake_complete = true;
+}
+
+static void MSH3_CALL msh3_conn_shutdown_complete(MSH3_CONNECTION *Connection,
+                                          void *IfContext)
+{
+  struct cf_msh3_ctx *ctx = IfContext;
+  (void)Connection;
+  if(ctx->verbose)
+    CF_DEBUGF(fprintf(stderr, "* [MSH3] evt: shutdown complete\n"));
+  ctx->connected = false;
+  ctx->handshake_complete = true;
+}
+
+static void MSH3_CALL msh3_conn_new_request(MSH3_CONNECTION *Connection,
+                                          void *IfContext,
+                                          MSH3_REQUEST *Request)
+{
+  (void)Connection;
+  (void)IfContext;
+  (void)Request;
+}
+
 static const MSH3_REQUEST_IF msh3_request_if = {
   msh3_header_received,
   msh3_data_received,
   msh3_complete,
-  msh3_shutdown,
-  msh3_send_complete
+  msh3_shutdown_complete,
+  msh3_data_sent
 };
 
 static CURLcode msh3_data_setup(struct Curl_cfilter *cf,
@@ -123,7 +176,7 @@ static CURLcode msh3_data_setup(struct Curl_cfilter *cf,
 
   DEBUGASSERT(stream);
   if(!stream->recv_buf) {
-    CF_DEBUGF(infof(data, CFMSG(cf, "msh3_data_setup")));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: setup")));
     stream->recv_buf = malloc(MSH3_REQ_INIT_BUF_LEN);
     if(!stream->recv_buf) {
       return CURLE_OUT_OF_MEMORY;
@@ -131,6 +184,7 @@ static CURLcode msh3_data_setup(struct Curl_cfilter *cf,
     stream->req = ZERO_NULL;
     msh3_lock_initialize(&stream->recv_lock);
     stream->recv_buf_alloc = MSH3_REQ_INIT_BUF_LEN;
+    stream->recv_buf_max = MSH3_REQ_MAX_BUF_LEN;
     stream->recv_header_len = 0;
     stream->recv_header_complete = false;
     stream->recv_data_len = 0;
@@ -220,8 +274,8 @@ release_lock:
   msh3_lock_release(&stream->recv_lock);
 }
 
-static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
-                                         void *IfContext, uint32_t Length,
+static bool MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
+                                         void *IfContext, uint32_t *Length,
                                          const uint8_t *Data)
 {
   struct Curl_easy *data = IfContext;
@@ -229,13 +283,19 @@ static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
   size_t cur_recv_len = stream->recv_header_len + stream->recv_data_len;
 
   (void)Request;
-  CF_DEBUGF(fprintf(stderr, "* msh3_data_received %u. %zu buffered, "
-            "%zu allocated\n",
-            Length, cur_recv_len, stream->recv_buf_alloc));
+  if(data && data->set.verbose)
+    CF_DEBUGF(fprintf(stderr, "* [MSH3] req: evt: received %u. %zu buffered, "
+              "%zu allocated\n",
+              *Length, cur_recv_len, stream->recv_buf_alloc));
+  /* TODO - Update this code to limit data bufferring by `stream->recv_buf_max`
+     and return `false` when we reach that limit. Then, when curl drains some
+     of the buffer, making room, call MsH3RequestSetReceiveEnabled to enable
+     receive callbacks again. */
   msh3_lock_acquire(&stream->recv_lock);
 
   if(!stream->recv_header_complete) {
-    CF_DEBUGF(fprintf(stderr, "* Headers complete!\n"));
+    if(data && data->set.verbose)
+      CF_DEBUGF(fprintf(stderr, "* [MSH3] req: Headers complete!\n"));
     if(!msh3request_ensure_room(stream, 2)) {
       stream->recv_error = CURLE_OUT_OF_MEMORY;
       goto release_lock;
@@ -245,16 +305,17 @@ static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
     stream->recv_header_complete = true;
     cur_recv_len += 2;
   }
-  if(!msh3request_ensure_room(stream, Length)) {
+  if(!msh3request_ensure_room(stream, *Length)) {
     stream->recv_error = CURLE_OUT_OF_MEMORY;
     goto release_lock;
   }
-  memcpy(stream->recv_buf + cur_recv_len, Data, Length);
-  stream->recv_data_len += (size_t)Length;
+  memcpy(stream->recv_buf + cur_recv_len, Data, *Length);
+  stream->recv_data_len += (size_t)*Length;
   data->state.drain = 1;
 
 release_lock:
   msh3_lock_release(&stream->recv_lock);
+  return true;
 }
 
 static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
@@ -265,8 +326,9 @@ static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
 
   (void)Request;
   (void)AbortError;
-  CF_DEBUGF(fprintf(stderr, "* msh3_complete, aborted=%s\n",
-            Aborted ? "true" : "false"));
+  if(data && data->set.verbose)
+    CF_DEBUGF(fprintf(stderr, "* [MSH3] req: evt: complete, aborted=%s\n",
+              Aborted ? "true" : "false"));
   msh3_lock_acquire(&stream->recv_lock);
   if(Aborted) {
     stream->recv_error = CURLE_HTTP3; /* TODO - how do we pass AbortError? */
@@ -276,7 +338,8 @@ static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
   msh3_lock_release(&stream->recv_lock);
 }
 
-static void MSH3_CALL msh3_shutdown(MSH3_REQUEST *Request, void *IfContext)
+static void MSH3_CALL msh3_shutdown_complete(MSH3_REQUEST *Request,
+                                             void *IfContext)
 {
   struct Curl_easy *data = IfContext;
   struct HTTP *stream = data->req.p.http;
@@ -284,8 +347,8 @@ static void MSH3_CALL msh3_shutdown(MSH3_REQUEST *Request, void *IfContext)
   (void)stream;
 }
 
-static void MSH3_CALL msh3_send_complete(MSH3_REQUEST *Request,
-                                         void *IfContext, void *SendContext)
+static void MSH3_CALL msh3_data_sent(MSH3_REQUEST *Request,
+                                     void *IfContext, void *SendContext)
 {
   struct Curl_easy *data = IfContext;
   struct HTTP *stream = data->req.p.http;
@@ -301,7 +364,8 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
   size_t outsize = 0;
 
   (void)cf;
-  CF_DEBUGF(infof(data, CFMSG(cf, "recv(len=%zu)"), len));
+  CF_DEBUGF(infof(data, CFMSG(cf, "req: recv with %zu byte buffer"),
+            len));
 
   if(stream->recv_error) {
     failf(data, "request aborted");
@@ -324,7 +388,7 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
               stream->recv_header_len + stream->recv_data_len - outsize);
     }
     stream->recv_header_len -= outsize;
-    CF_DEBUGF(infof(data, CFMSG(cf, "returned %zu bytes of headers"),
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: returned %zu bytes of header"),
               outsize));
   }
   else if(stream->recv_data_len) {
@@ -338,16 +402,17 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
               stream->recv_data_len - outsize);
     }
     stream->recv_data_len -= outsize;
-    CF_DEBUGF(infof(data, CFMSG(cf, "returned %zu bytes of data"), outsize));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: returned %zu bytes of data"),
+              outsize));
     if(stream->recv_data_len == 0 && stream->recv_data_complete)
       data->state.drain = 1;
   }
   else if(stream->recv_data_complete) {
-    CF_DEBUGF(infof(data, CFMSG(cf, "receive complete")));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: receive complete")));
     data->state.drain = 0;
   }
   else {
-    CF_DEBUGF(infof(data, CFMSG(cf, "nothing here, call again")));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: nothing here, call again")));
     *err = CURLE_AGAIN;
     outsize = -1;
   }
@@ -368,6 +433,7 @@ static ssize_t cf_msh3_send(struct Curl_cfilter *cf, struct Curl_easy *data,
 
   /* Sizes must match for cast below to work" */
   DEBUGASSERT(sizeof(MSH3_HEADER) == sizeof(struct h2h3pseudo));
+  CF_DEBUGF(infof(data, CFMSG(cf, "req: send %zu bytes"), len));
 
   if(!stream->req) {
     /* The first send on the request contains the headers and possibly some
@@ -386,8 +452,8 @@ static ssize_t cf_msh3_send(struct Curl_cfilter *cf, struct Curl_easy *data,
       return -1;
     }
 
-    CF_DEBUGF(infof(data, CFMSG(cf, "opening request(%zu headers, len=%zu)"),
-              hreq->entries, len - hdrlen));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: send %zu headers"),
+              hreq->entries));
     stream->req = MsH3RequestOpen(ctx->qconn, &msh3_request_if, data,
                                   (MSH3_HEADER*)hreq->header, hreq->entries,
                                   hdrlen == len ? MSH3_REQUEST_FLAG_FIN :
@@ -402,7 +468,7 @@ static ssize_t cf_msh3_send(struct Curl_cfilter *cf, struct Curl_easy *data,
     return len;
   }
 
-  CF_DEBUGF(infof(data, CFMSG(cf, "send body(len=%zu)"), len));
+  CF_DEBUGF(infof(data, CFMSG(cf, "req: send %zd body bytes"), len));
   if(len > 0xFFFFFFFF) {
     /* msh3 doesn't support size_t sends currently. */
     *err = CURLE_SEND_ERROR;
@@ -442,7 +508,7 @@ static int cf_msh3_get_select_socks(struct Curl_cfilter *cf,
       data->state.drain = 1;
     }
   }
-  CF_DEBUGF(infof(data, CFMSG(cf, "get_select_socks %u -> %d"),
+  CF_DEBUGF(infof(data, CFMSG(cf, "select_sock %u -> %d"),
             (uint32_t)data->state.drain, bitmap));
 
   return bitmap;
@@ -454,7 +520,8 @@ static bool cf_msh3_data_pending(struct Curl_cfilter *cf,
   struct HTTP *stream = data->req.p.http;
 
   (void)cf;
-  CF_DEBUGF(infof((struct Curl_easy *)data, CFMSG(cf, "data_pending")));
+  CF_DEBUGF(infof((struct Curl_easy *)data, CFMSG(cf, "data pending = %hhu"),
+            (bool)(stream->recv_header_len || stream->recv_data_len)));
   return stream->recv_header_len || stream->recv_data_len;
 }
 
@@ -490,7 +557,7 @@ static CURLcode cf_msh3_data_event(struct Curl_cfilter *cf,
     break;
 
   case CF_CTRL_DATA_DONE:
-    CF_DEBUGF(infof(data, CFMSG(cf, "Curl_quic_done")));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: done")));
     if(stream) {
       if(stream->recv_buf) {
         Curl_safefree(stream->recv_buf);
@@ -504,11 +571,12 @@ static CURLcode cf_msh3_data_event(struct Curl_cfilter *cf,
     break;
 
   case CF_CTRL_DATA_DONE_SEND:
-    CF_DEBUGF(infof(data, CFMSG(cf, "Curl_quic_done_sending")));
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: send done")));
     stream->upload_done = TRUE;
     break;
 
   case CF_CTRL_CONN_INFO_UPDATE:
+    CF_DEBUGF(infof(data, CFMSG(cf, "req: update")));
     cf_msh3_active(cf, data);
     break;
 
@@ -523,17 +591,21 @@ static CURLcode cf_connect_start(struct Curl_cfilter *cf,
 {
   struct cf_msh3_ctx *ctx = cf->ctx;
   bool verify = !!cf->conn->ssl_config.verifypeer;
+  MSH3_ADDR addr = {0};
+  memcpy(&addr, &ctx->addr.sa_addr, ctx->addr.addrlen);
+  MSH3_SET_PORT(&addr, (uint16_t)cf->conn->remote_port);
+  ctx->verbose = (data && data->set.verbose);
 
   if(verify && (cf->conn->ssl_config.CAfile || cf->conn->ssl_config.CApath)) {
-    /* TODO: find out if there is a way to provide trust anchors to MSH3 */
+    /* TODO: need a way to provide trust anchors to MSH3 */
 #ifdef DEBUGBUILD
     /* we need this for our test cases to run */
-    CF_DEBUGF(infof(data, CFMSG(cf, "non-standard CA not supported"
+    CF_DEBUGF(infof(data, CFMSG(cf, "non-standard CA not supported, "
               "switching off verifypeer in DEBUG mode")));
     verify = 0;
 #else
-    failf(data, "non-standard CA not supported via MsH3");
-    return CURLE_FAILED_INIT;
+    CF_DEBUGF(infof(data, CFMSG(cf, "non-standard CA not supported, "
+              "attempting with built-in verification")));
 #endif
   }
 
@@ -547,8 +619,10 @@ static CURLcode cf_connect_start(struct Curl_cfilter *cf,
   }
 
   ctx->qconn = MsH3ConnectionOpen(ctx->api,
+                                  &msh3_conn_if,
+                                  ctx,
                                   cf->conn->host.name,
-                                  (uint16_t)cf->conn->remote_port,
+                                  &addr,
                                   !verify);
   if(!ctx->qconn) {
     failf(data, "can't create msh3 connection");
@@ -567,7 +641,6 @@ static CURLcode cf_msh3_connect(struct Curl_cfilter *cf,
                                 bool blocking, bool *done)
 {
   struct cf_msh3_ctx *ctx = cf->ctx;
-  MSH3_CONNECTION_STATE state;
   CURLcode result = CURLE_OK;
 
   (void)blocking;
@@ -591,41 +664,33 @@ static CURLcode cf_msh3_connect(struct Curl_cfilter *cf,
       goto out;
   }
 
-  state = MsH3ConnectionGetState(ctx->qconn, FALSE);
-  if(state == MSH3_CONN_DISCONNECTED) {
-    failf(data, "failed to connect, MsH3 reports `DISCONNECTED`");
-    result = CURLE_COULDNT_CONNECT;
-    goto out;
-  }
-  if(state == MSH3_CONN_HANDSHAKE_FAILED) {
-    failf(data, "failed to connect, handshake failed");
-    result = CURLE_COULDNT_CONNECT;
-    goto out;
-  }
-
-  if(state == MSH3_CONN_CONNECTED) {
-    DEBUGF(infof(data, "msh3 established connection"));
-    cf->conn->bits.multiplex = TRUE; /* at least potentially multiplexed */
-    cf->conn->httpversion = 30;
-    cf->conn->bundle->multiuse = BUNDLE_MULTIPLEX;
-
-    cf->connected = TRUE;
-    cf->conn->alpn = CURL_HTTP_VERSION_3;
-    *done = TRUE;
-    connkeep(cf->conn, "HTTP/3 default");
+  if(ctx->handshake_complete) {
+    if(ctx->handshake_succeeded) {
+      cf->conn->bits.multiplex = TRUE; /* at least potentially multiplexed */
+      cf->conn->httpversion = 30;
+      cf->conn->bundle->multiuse = BUNDLE_MULTIPLEX;
+      cf->connected = TRUE;
+      cf->conn->alpn = CURL_HTTP_VERSION_3;
+      *done = TRUE;
+      connkeep(cf->conn, "HTTP/3 default");
+    }
+    else {
+      failf(data, "failed to connect, handshake failed");
+      result = CURLE_COULDNT_CONNECT;
+    }
   }
 
 out:
   return result;
 }
 
-static void cf_msh3_ctx_clear(struct Curl_cfilter *cf, struct Curl_easy *data)
+static void cf_msh3_close(struct Curl_cfilter *cf, struct Curl_easy *data)
 {
   struct cf_msh3_ctx *ctx = cf->ctx;
 
   (void)data;
-  CF_DEBUGF(infof(data, CFMSG(cf, "clearing ctx")));
   if(ctx) {
+    CF_DEBUGF(infof(data, CFMSG(cf, "destroying")));
     if(ctx->qconn)
       MsH3ConnectionClose(ctx->qconn);
     if(ctx->api)
@@ -661,23 +726,10 @@ static void cf_msh3_ctx_clear(struct Curl_cfilter *cf, struct Curl_easy *data)
   }
 }
 
-static void cf_msh3_close(struct Curl_cfilter *cf, struct Curl_easy *data)
-{
-  struct cf_msh3_ctx *ctx = cf->ctx;
-
-  (void)data;
-  if(ctx) {
-    cf_msh3_ctx_clear(cf, data);
-  }
-}
-
 static void cf_msh3_destroy(struct Curl_cfilter *cf, struct Curl_easy *data)
 {
-  struct cf_msh3_ctx *ctx = cf->ctx;
-
-  (void)data;
-  cf_msh3_ctx_clear(cf, data);
-  free(ctx);
+  cf_msh3_close(cf, data);
+  free(cf->ctx);
   cf->ctx = NULL;
 }
 
@@ -708,10 +760,8 @@ static bool cf_msh3_conn_is_alive(struct Curl_cfilter *cf,
   struct cf_msh3_ctx *ctx = cf->ctx;
 
   (void)data;
-  if(!ctx || ctx->sock[SP_LOCAL] == CURL_SOCKET_BAD || !ctx->qconn)
-    return FALSE;
-
-  return MSH3_CONN_CONNECTED == MsH3ConnectionGetState(ctx->qconn, FALSE);
+  return ctx && ctx->sock[SP_LOCAL] != CURL_SOCKET_BAD && ctx->qconn &&
+         ctx->connected;
 }
 
 static const struct Curl_cftype cft_msh3 = {