msh3: run again in its cfilter

- test 2500, single GET works
- test 2501, single POST stalls
- test 2502, multiple, sequential GETs each use a new connection since
  MsH3ConnectionGetState(qconn) no longer reports CONNECTED after one
  GET.

Closes #10204

3 files changed, 258 insertions, 76 deletions

diff --git a/lib/cf-socket.c b/lib/cf-socket.c
index 83e3b5f3c..df292d7b1 100644
--- a/lib/cf-socket.c
+++ b/lib/cf-socket.c
@@ -79,7 +79,7 @@
 #include "memdebug.h"
 
 
-#define DEBUG_CF 0
+#define DEBUG_CF 1
 
 #if DEBUG_CF
 #define CF_DEBUGF(x) x
@@ -203,9 +203,9 @@ tcpkeepalive(struct Curl_easy *data,
   }
 }
 
-static void assign_sockaddr_ex(struct Curl_sockaddr_ex *dest,
-                               const struct Curl_addrinfo *ai,
-                               int transport)
+void Curl_sock_assign_addr(struct Curl_sockaddr_ex *dest,
+                           const struct Curl_addrinfo *ai,
+                           int transport)
 {
   /*
    * The Curl_sockaddr_ex structure is basically libcurl's external API
@@ -296,7 +296,7 @@ CURLcode Curl_socket_open(struct Curl_easy *data,
     /* if the caller doesn't want info back, use a local temp copy */
     addr = &dummy;
 
-  assign_sockaddr_ex(addr, ai, transport);
+  Curl_sock_assign_addr(addr, ai, transport);
   return socket_open(data, addr, sockfd);
 }
 
@@ -786,6 +786,8 @@ static void cf_socket_close(struct Curl_cfilter *cf, struct Curl_easy *data)
         CF_DEBUGF(infof(data, CFMSG(cf, "cf_socket_close(%d) no longer at "
                   "conn->sock[%d], discarding"), (int)ctx->sock));
       }
+      if(cf->sockindex == FIRSTSOCKET)
+        cf->conn->remote_addr = NULL;
     }
     else {
       /* this is our local socket, we did never publish it */
@@ -797,8 +799,6 @@ static void cf_socket_close(struct Curl_cfilter *cf, struct Curl_easy *data)
     ctx->active = FALSE;
   }
 
-  if(cf->sockindex == FIRSTSOCKET)
-    cf->conn->remote_addr = NULL;
   cf->connected = FALSE;
 }
 
@@ -1297,7 +1297,7 @@ CURLcode Curl_cf_tcp_create(struct Curl_cfilter **pcf,
     goto out;
   }
   ctx->transport = TRNSPRT_TCP;
-  assign_sockaddr_ex(&ctx->addr, ai, ctx->transport);
+  Curl_sock_assign_addr(&ctx->addr, ai, ctx->transport);
   ctx->sock = CURL_SOCKET_BAD;
 
   result = Curl_cf_create(&cf, &cft_tcp, ctx);
@@ -1376,7 +1376,7 @@ CURLcode Curl_cf_udp_create(struct Curl_cfilter **pcf,
     goto out;
   }
   ctx->transport = TRNSPRT_UDP;
-  assign_sockaddr_ex(&ctx->addr, ai, ctx->transport);
+  Curl_sock_assign_addr(&ctx->addr, ai, ctx->transport);
   ctx->sock = CURL_SOCKET_BAD;
 
   result = Curl_cf_create(&cf, &cft_udp, ctx);
@@ -1426,7 +1426,7 @@ CURLcode Curl_cf_unix_create(struct Curl_cfilter **pcf,
     goto out;
   }
   ctx->transport = TRNSPRT_UNIX;
-  assign_sockaddr_ex(&ctx->addr, ai, ctx->transport);
+  Curl_sock_assign_addr(&ctx->addr, ai, ctx->transport);
   ctx->sock = CURL_SOCKET_BAD;
 
   result = Curl_cf_create(&cf, &cft_unix, ctx);
diff --git a/lib/cf-socket.h b/lib/cf-socket.h
index b770b2ae2..8e30d2862 100644
--- a/lib/cf-socket.h
+++ b/lib/cf-socket.h
@@ -99,6 +99,14 @@ void Curl_sndbufset(curl_socket_t sockfd);
 #endif
 
 /**
+ * Assign the address `ai` to the Curl_sockaddr_ex `dest` and
+ * set the transport used.
+ */
+void Curl_sock_assign_addr(struct Curl_sockaddr_ex *dest,
+                           const struct Curl_addrinfo *ai,
+                           int transport);
+
+/**
  * Creates a cfilter that opens a TCP socket to the given address
  * when calling its `connect` implementation.
  * The filter will not touch any connection/data flags and can be
diff --git a/lib/vquic/curl_msh3.c b/lib/vquic/curl_msh3.c
index 15d0bab5b..8f323ddd3 100644
--- a/lib/vquic/curl_msh3.c
+++ b/lib/vquic/curl_msh3.c
@@ -31,20 +31,23 @@
 #include "multiif.h"
 #include "sendf.h"
 #include "cfilters.h"
+#include "cf-socket.h"
 #include "connect.h"
 #include "h2h3.h"
 #include "curl_msh3.h"
+#include "socketpair.h"
 
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
 #include "memdebug.h"
 
-/* #define DEBUG_HTTP3 1 */
-#ifdef DEBUG_HTTP3
-#define H3BUGF(x) x
+#define DEBUG_CF 1
+
+#if DEBUG_CF
+#define CF_DEBUGF(x) x
 #else
-#define H3BUGF(x) do { } while(0)
+#define CF_DEBUGF(x) do { } while(0)
 #endif
 
 #define MSH3_REQ_INIT_BUF_LEN 8192
@@ -91,9 +94,17 @@ void Curl_msh3_ver(char *p, size_t len)
   (void)msnprintf(p, len, "msh3/%d.%d.%d.%d", v[0], v[1], v[2], v[3]);
 }
 
+#define SP_LOCAL   0
+#define SP_REMOTE  1
+
 struct cf_msh3_ctx {
   MSH3_API *api;
   MSH3_CONNECTION *qconn;
+  struct Curl_sockaddr_ex addr;
+  curl_socket_t sock[2]; /* fake socket pair until we get support in msh3 */
+  char l_ip[MAX_IPADR_LEN];          /* local IP as string */
+  int l_port;                        /* local port number */
+  BIT(active);
 };
 
 static const MSH3_REQUEST_IF msh3_request_if = {
@@ -110,19 +121,22 @@ static CURLcode msh3_data_setup(struct Curl_cfilter *cf,
   struct HTTP *stream = data->req.p.http;
   (void)cf;
 
-  H3BUGF(infof(data, "msh3_data_setup"));
-  stream->recv_buf = malloc(MSH3_REQ_INIT_BUF_LEN);
+  DEBUGASSERT(stream);
   if(!stream->recv_buf) {
-    return CURLE_OUT_OF_MEMORY;
-  }
-  stream->req = ZERO_NULL;
-  msh3_lock_initialize(&stream->recv_lock);
-  stream->recv_buf_alloc = MSH3_REQ_INIT_BUF_LEN;
-  stream->recv_header_len = 0;
-  stream->recv_header_complete = false;
-  stream->recv_data_len = 0;
-  stream->recv_data_complete = false;
-  stream->recv_error = CURLE_OK;
+    CF_DEBUGF(infof(data, CFMSG(cf, "msh3_data_setup")));
+    stream->recv_buf = malloc(MSH3_REQ_INIT_BUF_LEN);
+    if(!stream->recv_buf) {
+      return CURLE_OUT_OF_MEMORY;
+    }
+    stream->req = ZERO_NULL;
+    msh3_lock_initialize(&stream->recv_lock);
+    stream->recv_buf_alloc = MSH3_REQ_INIT_BUF_LEN;
+    stream->recv_header_len = 0;
+    stream->recv_header_complete = false;
+    stream->recv_data_len = 0;
+    stream->recv_data_complete = false;
+    stream->recv_error = CURLE_OK;
+  }
   return CURLE_OK;
 }
 
@@ -131,13 +145,18 @@ static bool msh3request_ensure_room(struct HTTP *stream, size_t len)
 {
   uint8_t *new_recv_buf;
   const size_t cur_recv_len = stream->recv_header_len + stream->recv_data_len;
+
   if(cur_recv_len + len > stream->recv_buf_alloc) {
     size_t new_recv_buf_alloc_len = stream->recv_buf_alloc;
     do {
       new_recv_buf_alloc_len <<= 1; /* TODO - handle overflow */
     } while(cur_recv_len + len > new_recv_buf_alloc_len);
+    CF_DEBUGF(fprintf(stderr, "* enlarging buffer to %zu\n",
+              new_recv_buf_alloc_len));
     new_recv_buf = malloc(new_recv_buf_alloc_len);
     if(!new_recv_buf) {
+      CF_DEBUGF(fprintf(stderr, "* FAILED: enlarging buffer to %zu\n",
+                new_recv_buf_alloc_len));
       return false;
     }
     if(cur_recv_len) {
@@ -154,12 +173,13 @@ static void MSH3_CALL msh3_header_received(MSH3_REQUEST *Request,
                                            void *IfContext,
                                            const MSH3_HEADER *Header)
 {
-  struct HTTP *stream = IfContext;
+  struct Curl_easy *data = IfContext;
+  struct HTTP *stream = data->req.p.http;
   size_t total_len;
   (void)Request;
 
   if(stream->recv_header_complete) {
-    H3BUGF(printf("* ignoring header after data\n"));
+    CF_DEBUGF(fprintf(stderr, "* ignoring header after data\n"));
     return;
   }
 
@@ -167,9 +187,11 @@ static void MSH3_CALL msh3_header_received(MSH3_REQUEST *Request,
 
   if((Header->NameLength == 7) &&
      !strncmp(H2H3_PSEUDO_STATUS, (char *)Header->Name, 7)) {
-     total_len = 9 + Header->ValueLength;
+    total_len = 10 + Header->ValueLength;
     if(!msh3request_ensure_room(stream, total_len)) {
-      /* TODO - handle error */
+      CF_DEBUGF(fprintf(stderr, "* ERROR: unable to buffer: %.*s\n",
+                (int)Header->NameLength, Header->Name));
+      stream->recv_error = CURLE_OUT_OF_MEMORY;
       goto release_lock;
     }
     msnprintf((char *)stream->recv_buf + stream->recv_header_len,
@@ -177,9 +199,11 @@ static void MSH3_CALL msh3_header_received(MSH3_REQUEST *Request,
               "HTTP/3 %.*s \r\n", (int)Header->ValueLength, Header->Value);
   }
   else {
-    total_len = Header->NameLength + 4 + Header->ValueLength;
+    total_len = 4 + Header->NameLength + Header->ValueLength;
     if(!msh3request_ensure_room(stream, total_len)) {
-      /* TODO - handle error */
+      CF_DEBUGF(fprintf(stderr, "* ERROR: unable to buffer: %.*s\n",
+                (int)Header->NameLength, Header->Name));
+      stream->recv_error = CURLE_OUT_OF_MEMORY;
       goto release_lock;
     }
     msnprintf((char *)stream->recv_buf + stream->recv_header_len,
@@ -189,7 +213,8 @@ static void MSH3_CALL msh3_header_received(MSH3_REQUEST *Request,
               (int)Header->ValueLength, Header->Value);
   }
 
-  stream->recv_header_len += total_len - 1; /* don't include null-terminator */
+  stream->recv_header_len += total_len;
+  data->state.drain = 1;
 
 release_lock:
   msh3_lock_release(&stream->recv_lock);
@@ -199,16 +224,20 @@ static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
                                          void *IfContext, uint32_t Length,
                                          const uint8_t *Data)
 {
-  struct HTTP *stream = IfContext;
+  struct Curl_easy *data = IfContext;
+  struct HTTP *stream = data->req.p.http;
   size_t cur_recv_len = stream->recv_header_len + stream->recv_data_len;
+
   (void)Request;
-  H3BUGF(printf("* msh3_data_received %u. %zu buffered, %zu allocated\n",
-                Length, cur_recv_len, stream->recv_buf_alloc));
+  CF_DEBUGF(fprintf(stderr, "* msh3_data_received %u. %zu buffered, "
+            "%zu allocated\n",
+            Length, cur_recv_len, stream->recv_buf_alloc));
   msh3_lock_acquire(&stream->recv_lock);
+
   if(!stream->recv_header_complete) {
-    H3BUGF(printf("* Headers complete!\n"));
+    CF_DEBUGF(fprintf(stderr, "* Headers complete!\n"));
     if(!msh3request_ensure_room(stream, 2)) {
-      /* TODO - handle error */
+      stream->recv_error = CURLE_OUT_OF_MEMORY;
       goto release_lock;
     }
     stream->recv_buf[stream->recv_header_len++] = '\r';
@@ -217,11 +246,13 @@ static void MSH3_CALL msh3_data_received(MSH3_REQUEST *Request,
     cur_recv_len += 2;
   }
   if(!msh3request_ensure_room(stream, Length)) {
-    /* TODO - handle error */
+    stream->recv_error = CURLE_OUT_OF_MEMORY;
     goto release_lock;
   }
   memcpy(stream->recv_buf + cur_recv_len, Data, Length);
   stream->recv_data_len += (size_t)Length;
+  data->state.drain = 1;
+
 release_lock:
   msh3_lock_release(&stream->recv_lock);
 }
@@ -229,10 +260,13 @@ release_lock:
 static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
                                     bool Aborted, uint64_t AbortError)
 {
-  struct HTTP *stream = IfContext;
+  struct Curl_easy *data = IfContext;
+  struct HTTP *stream = data->req.p.http;
+
   (void)Request;
   (void)AbortError;
-  H3BUGF(printf("* msh3_complete, aborted=%s\n", Aborted ? "true" : "false"));
+  CF_DEBUGF(fprintf(stderr, "* msh3_complete, aborted=%s\n",
+            Aborted ? "true" : "false"));
   msh3_lock_acquire(&stream->recv_lock);
   if(Aborted) {
     stream->recv_error = CURLE_HTTP3; /* TODO - how do we pass AbortError? */
@@ -244,7 +278,8 @@ static void MSH3_CALL msh3_complete(MSH3_REQUEST *Request, void *IfContext,
 
 static void MSH3_CALL msh3_shutdown(MSH3_REQUEST *Request, void *IfContext)
 {
-  struct HTTP *stream = IfContext;
+  struct Curl_easy *data = IfContext;
+  struct HTTP *stream = data->req.p.http;
   (void)Request;
   (void)stream;
 }
@@ -252,7 +287,8 @@ static void MSH3_CALL msh3_shutdown(MSH3_REQUEST *Request, void *IfContext)
 static void MSH3_CALL msh3_send_complete(MSH3_REQUEST *Request,
                                          void *IfContext, void *SendContext)
 {
-  struct HTTP *stream = IfContext;
+  struct Curl_easy *data = IfContext;
+  struct HTTP *stream = data->req.p.http;
   (void)Request;
   (void)stream;
   (void)SendContext;
@@ -265,14 +301,16 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
   size_t outsize = 0;
 
   (void)cf;
-  H3BUGF(infof(data, "msh3_stream_recv %zu", len));
+  CF_DEBUGF(infof(data, CFMSG(cf, "recv(len=%zu)"), len));
 
   if(stream->recv_error) {
     failf(data, "request aborted");
+    data->state.drain = 0;
     *err = stream->recv_error;
     return -1;
   }
 
+  *err = CURLE_OK;
   msh3_lock_acquire(&stream->recv_lock);
 
   if(stream->recv_header_len) {
@@ -286,7 +324,8 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
               stream->recv_header_len + stream->recv_data_len - outsize);
     }
     stream->recv_header_len -= outsize;
-    H3BUGF(infof(data, "returned %zu bytes of headers", outsize));
+    CF_DEBUGF(infof(data, CFMSG(cf, "returned %zu bytes of headers"),
+              outsize));
   }
   else if(stream->recv_data_len) {
     outsize = len;
@@ -299,10 +338,18 @@ static ssize_t cf_msh3_recv(struct Curl_cfilter *cf, struct Curl_easy *data,
               stream->recv_data_len - outsize);
     }
     stream->recv_data_len -= outsize;
-    H3BUGF(infof(data, "returned %zu bytes of data", outsize));
+    CF_DEBUGF(infof(data, CFMSG(cf, "returned %zu bytes of data"), outsize));
+    if(stream->recv_data_len == 0 && stream->recv_data_complete)
+      data->state.drain = 1;
   }
   else if(stream->recv_data_complete) {
-    H3BUGF(infof(data, "receive complete"));
+    CF_DEBUGF(infof(data, CFMSG(cf, "receive complete")));
+    data->state.drain = 0;
+  }
+  else {
+    CF_DEBUGF(infof(data, CFMSG(cf, "nothing here, call again")));
+    *err = CURLE_AGAIN;
+    outsize = -1;
   }
 
   msh3_lock_release(&stream->recv_lock);
@@ -322,20 +369,26 @@ static ssize_t cf_msh3_send(struct Curl_cfilter *cf, struct Curl_easy *data,
   /* Sizes must match for cast below to work" */
   DEBUGASSERT(sizeof(MSH3_HEADER) == sizeof(struct h2h3pseudo));
 
-  H3BUGF(infof(data, "msh3_stream_send %zu", len));
-
   if(!stream->req) {
     /* The first send on the request contains the headers and possibly some
        data. Parse out the headers and create the request, then if there is
        any data left over go ahead and send it too. */
+
+    *err = msh3_data_setup(cf, data);
+    if(*err) {
+      failf(data, "could not setup data");
+      return -1;
+    }
+
     *err = Curl_pseudo_headers(data, buf, len, &hdrlen, &hreq);
     if(*err) {
       failf(data, "Curl_pseudo_headers failed");
       return -1;
     }
 
-    H3BUGF(infof(data, "starting request with %zu headers", hreq->entries));
-    stream->req = MsH3RequestOpen(ctx->qconn, &msh3_request_if, stream,
+    CF_DEBUGF(infof(data, CFMSG(cf, "opening request(%zu headers, len=%zu)"),
+              hreq->entries, len - hdrlen));
+    stream->req = MsH3RequestOpen(ctx->qconn, &msh3_request_if, data,
                                   (MSH3_HEADER*)hreq->header, hreq->entries,
                                   hdrlen == len ? MSH3_REQUEST_FLAG_FIN :
                                   MSH3_REQUEST_FLAG_NONE);
@@ -348,8 +401,8 @@ static ssize_t cf_msh3_send(struct Curl_cfilter *cf, struct Curl_easy *data,
     *err = CURLE_OK;
     return len;
   }
-  H3BUGF(infof(data, "send %zd body bytes on request %p", len,
-               (void *)stream->req));
+
+  CF_DEBUGF(infof(data, CFMSG(cf, "send body(len=%zu)"), len));
   if(len > 0xFFFFFFFF) {
     /* msh3 doesn't support size_t sends currently. */
     *err = CURLE_SEND_ERROR;
@@ -377,17 +430,20 @@ static int cf_msh3_get_select_socks(struct Curl_cfilter *cf,
   struct HTTP *stream = data->req.p.http;
   int bitmap = GETSOCK_BLANK;
 
-  socks[0] = cf->conn->sock[FIRSTSOCKET];
+  if(stream && cf->conn->sock[FIRSTSOCKET] != CURL_SOCKET_BAD) {
+    socks[0] = cf->conn->sock[FIRSTSOCKET];
 
-  if(stream->recv_error) {
-    bitmap |= GETSOCK_READSOCK(0);
-    data->state.drain++;
-  }
-  else if(stream->recv_header_len || stream->recv_data_len) {
-    bitmap |= GETSOCK_READSOCK(0);
-    data->state.drain++;
+    if(stream->recv_error) {
+      bitmap |= GETSOCK_READSOCK(0);
+      data->state.drain = 1;
+    }
+    else if(stream->recv_header_len || stream->recv_data_len) {
+      bitmap |= GETSOCK_READSOCK(0);
+      data->state.drain = 1;
+    }
   }
-  H3BUGF(infof(data, "msh3_getsock %u", (uint32_t)data->state.drain));
+  CF_DEBUGF(infof(data, CFMSG(cf, "get_select_socks %u -> %d"),
+            (uint32_t)data->state.drain, bitmap));
 
   return bitmap;
 }
@@ -397,12 +453,28 @@ static bool cf_msh3_data_pending(struct Curl_cfilter *cf,
 {
   struct HTTP *stream = data->req.p.http;
 
-  (void)data;
   (void)cf;
-  H3BUGF(infof((struct Curl_easy *)data, "Curl_quic_data_pending"));
+  CF_DEBUGF(infof((struct Curl_easy *)data, CFMSG(cf, "data_pending")));
   return stream->recv_header_len || stream->recv_data_len;
 }
 
+static void cf_msh3_active(struct Curl_cfilter *cf, struct Curl_easy *data)
+{
+  struct cf_msh3_ctx *ctx = cf->ctx;
+
+  /* use this socket from now on */
+  cf->conn->sock[cf->sockindex] = ctx->sock[SP_LOCAL];
+  /* the first socket info gets set at conn and data */
+  if(cf->sockindex == FIRSTSOCKET) {
+    cf->conn->remote_addr = &ctx->addr;
+  #ifdef ENABLE_IPV6
+    cf->conn->bits.ipv6 = (ctx->addr.family == AF_INET6)? TRUE : FALSE;
+  #endif
+    Curl_persistconninfo(data, cf->conn, ctx->l_ip, ctx->l_port);
+  }
+  ctx->active = TRUE;
+}
+
 static CURLcode cf_msh3_data_event(struct Curl_cfilter *cf,
                                    struct Curl_easy *data,
                                    int event, int arg1, void *arg2)
@@ -418,7 +490,7 @@ static CURLcode cf_msh3_data_event(struct Curl_cfilter *cf,
     break;
 
   case CF_CTRL_DATA_DONE:
-    H3BUGF(infof(data, "Curl_quic_done"));
+    CF_DEBUGF(infof(data, CFMSG(cf, "Curl_quic_done")));
     if(stream) {
       if(stream->recv_buf) {
         Curl_safefree(stream->recv_buf);
@@ -432,10 +504,14 @@ static CURLcode cf_msh3_data_event(struct Curl_cfilter *cf,
     break;
 
   case CF_CTRL_DATA_DONE_SEND:
-    H3BUGF(infof(data, "Curl_quic_done_sending"));
+    CF_DEBUGF(infof(data, CFMSG(cf, "Curl_quic_done_sending")));
     stream->upload_done = TRUE;
     break;
 
+  case CF_CTRL_CONN_INFO_UPDATE:
+    cf_msh3_active(cf, data);
+    break;
+
   default:
     break;
   }
@@ -446,9 +522,23 @@ static CURLcode cf_connect_start(struct Curl_cfilter *cf,
                                  struct Curl_easy *data)
 {
   struct cf_msh3_ctx *ctx = cf->ctx;
-  bool insecure = !cf->conn->ssl_config.verifypeer;
+  bool verify = !!cf->conn->ssl_config.verifypeer;
+
+  if(verify && (cf->conn->ssl_config.CAfile || cf->conn->ssl_config.CApath)) {
+    /* TODO: find out if there is a way to provide trust anchors to MSH3 */
+#ifdef DEBUGBUILD
+    /* we need this for our test cases to run */
+    CF_DEBUGF(infof(data, CFMSG(cf, "non-standard CA not supported"
+              "switching off verifypeer in DEBUG mode")));
+    verify = 0;
+#else
+    failf(data, "non-standard CA not supported via MsH3");
+    return CURLE_FAILED_INIT;
+#endif
+  }
 
-  H3BUGF(infof(data, "creating new api/connection"));
+  CF_DEBUGF(infof(data, CFMSG(cf, "connecting to %s:%d (verify=%d)"),
+            cf->conn->host.name, (int)cf->conn->remote_port, verify));
 
   ctx->api = MsH3ApiOpen();
   if(!ctx->api) {
@@ -459,7 +549,7 @@ static CURLcode cf_connect_start(struct Curl_cfilter *cf,
   ctx->qconn = MsH3ConnectionOpen(ctx->api,
                                   cf->conn->host.name,
                                   (uint16_t)cf->conn->remote_port,
-                                  insecure);
+                                  !verify);
   if(!ctx->qconn) {
     failf(data, "can't create msh3 connection");
     if(ctx->api) {
@@ -486,6 +576,14 @@ static CURLcode cf_msh3_connect(struct Curl_cfilter *cf,
     return CURLE_OK;
   }
 
+  if(ctx->sock[SP_LOCAL] == CURL_SOCKET_BAD) {
+    if(Curl_socketpair(AF_UNIX, SOCK_STREAM, 0, &ctx->sock[0]) < 0) {
+      ctx->sock[SP_LOCAL] = CURL_SOCKET_BAD;
+      ctx->sock[SP_REMOTE] = CURL_SOCKET_BAD;
+      return CURLE_COULDNT_CONNECT;
+    }
+  }
+
   *done = FALSE;
   if(!ctx->qconn) {
     result = cf_connect_start(cf, data);
@@ -494,14 +592,23 @@ static CURLcode cf_msh3_connect(struct Curl_cfilter *cf,
   }
 
   state = MsH3ConnectionGetState(ctx->qconn, FALSE);
-  if(state == MSH3_CONN_HANDSHAKE_FAILED || state == MSH3_CONN_DISCONNECTED) {
-    failf(data, "failed to connect, state=%u", (uint32_t)state);
+  if(state == MSH3_CONN_DISCONNECTED) {
+    failf(data, "failed to connect, MsH3 reports `DISCONNECTED`");
+    result = CURLE_COULDNT_CONNECT;
+    goto out;
+  }
+  if(state == MSH3_CONN_HANDSHAKE_FAILED) {
+    failf(data, "failed to connect, handshake failed");
     result = CURLE_COULDNT_CONNECT;
     goto out;
   }
 
   if(state == MSH3_CONN_CONNECTED) {
     DEBUGF(infof(data, "msh3 established connection"));
+    cf->conn->bits.multiplex = TRUE; /* at least potentially multiplexed */
+    cf->conn->httpversion = 30;
+    cf->conn->bundle->multiuse = BUNDLE_MULTIPLEX;
+
     cf->connected = TRUE;
     cf->conn->alpn = CURL_HTTP_VERSION_3;
     *done = TRUE;
@@ -512,14 +619,45 @@ out:
   return result;
 }
 
-static void cf_msh3_ctx_clear(struct cf_msh3_ctx *ctx)
+static void cf_msh3_ctx_clear(struct Curl_cfilter *cf, struct Curl_easy *data)
 {
+  struct cf_msh3_ctx *ctx = cf->ctx;
+
+  (void)data;
+  CF_DEBUGF(infof(data, CFMSG(cf, "clearing ctx")));
   if(ctx) {
     if(ctx->qconn)
       MsH3ConnectionClose(ctx->qconn);
     if(ctx->api)
       MsH3ApiClose(ctx->api);
+
+    if(ctx->active) {
+      /* We share our socket at cf->conn->sock[cf->sockindex] when active.
+       * If it is no longer there, someone has stolen (and hopefully
+       * closed it) and we just forget about it.
+       */
+      if(ctx->sock[SP_LOCAL] == cf->conn->sock[cf->sockindex]) {
+        CF_DEBUGF(infof(data, CFMSG(cf, "cf_msh3_close(%d) active"),
+                 (int)ctx->sock[SP_LOCAL]));
+        cf->conn->sock[cf->sockindex] = CURL_SOCKET_BAD;
+      }
+      else {
+        CF_DEBUGF(infof(data, CFMSG(cf, "cf_socket_close(%d) no longer at "
+                  "conn->sock[%d], discarding"), (int)ctx->sock[SP_LOCAL]));
+        ctx->sock[SP_LOCAL] = CURL_SOCKET_BAD;
+      }
+      if(cf->sockindex == FIRSTSOCKET)
+        cf->conn->remote_addr = NULL;
+    }
+    if(ctx->sock[SP_LOCAL] != CURL_SOCKET_BAD) {
+      sclose(ctx->sock[SP_LOCAL]);
+    }
+    if(ctx->sock[SP_REMOTE] != CURL_SOCKET_BAD) {
+      sclose(ctx->sock[SP_REMOTE]);
+    }
     memset(ctx, 0, sizeof(*ctx));
+    ctx->sock[SP_LOCAL] = CURL_SOCKET_BAD;
+    ctx->sock[SP_REMOTE] = CURL_SOCKET_BAD;
   }
 }
 
@@ -529,7 +667,7 @@ static void cf_msh3_close(struct Curl_cfilter *cf, struct Curl_easy *data)
 
   (void)data;
   if(ctx) {
-    cf_msh3_ctx_clear(ctx);
+    cf_msh3_ctx_clear(cf, data);
   }
 }
 
@@ -538,11 +676,44 @@ static void cf_msh3_destroy(struct Curl_cfilter *cf, struct Curl_easy *data)
   struct cf_msh3_ctx *ctx = cf->ctx;
 
   (void)data;
-  cf_msh3_ctx_clear(ctx);
+  cf_msh3_ctx_clear(cf, data);
   free(ctx);
   cf->ctx = NULL;
 }
 
+static CURLcode cf_msh3_query(struct Curl_cfilter *cf,
+                              struct Curl_easy *data,
+                              int query, int *pres1, void **pres2)
+{
+  struct cf_msh3_ctx *ctx = cf->ctx;
+
+  switch(query) {
+  case CF_QUERY_MAX_CONCURRENT: {
+    /* TODO: we do not have access to this so far, fake it */
+    (void)ctx;
+    *pres1 = 100;
+    return CURLE_OK;
+  }
+  default:
+    break;
+  }
+  return cf->next?
+    cf->next->cft->query(cf->next, data, query, pres1, pres2) :
+    CURLE_UNKNOWN_OPTION;
+}
+
+static bool cf_msh3_conn_is_alive(struct Curl_cfilter *cf,
+                                  struct Curl_easy *data)
+{
+  struct cf_msh3_ctx *ctx = cf->ctx;
+
+  (void)data;
+  if(!ctx || ctx->sock[SP_LOCAL] == CURL_SOCKET_BAD || !ctx->qconn)
+    return FALSE;
+
+  return MSH3_CONN_CONNECTED == MsH3ConnectionGetState(ctx->qconn, FALSE);
+}
+
 static const struct Curl_cftype cft_msh3 = {
   "HTTP/3-MSH3",
   CF_TYPE_IP_CONNECT | CF_TYPE_SSL | CF_TYPE_MULTIPLEX,
@@ -555,9 +726,9 @@ static const struct Curl_cftype cft_msh3 = {
   cf_msh3_send,
   cf_msh3_recv,
   cf_msh3_data_event,
-  Curl_cf_def_conn_is_alive,
+  cf_msh3_conn_is_alive,
   Curl_cf_def_conn_keep_alive,
-  Curl_cf_def_query,
+  cf_msh3_query,
 };
 
 CURLcode Curl_cf_msh3_create(struct Curl_cfilter **pcf,
@@ -577,6 +748,9 @@ CURLcode Curl_cf_msh3_create(struct Curl_cfilter **pcf,
     result = CURLE_OUT_OF_MEMORY;
     goto out;
   }
+  Curl_sock_assign_addr(&ctx->addr, ai, TRNSPRT_QUIC);
+  ctx->sock[SP_LOCAL] = CURL_SOCKET_BAD;
+  ctx->sock[SP_REMOTE] = CURL_SOCKET_BAD;
 
   result = Curl_cf_create(&cf, &cft_msh3, ctx);