proxy: Support HTTPS proxy and SOCKS+HTTP(s)

* HTTPS proxies:

An HTTPS proxy receives all transactions over an SSL/TLS connection.
Once a secure connection with the proxy is established, the user agent
uses the proxy as usual, including sending CONNECT requests to instruct
the proxy to establish a [usually secure] TCP tunnel with an origin
server. HTTPS proxies protect nearly all aspects of user-proxy
communications as opposed to HTTP proxies that receive all requests
(including CONNECT requests) in vulnerable clear text.

With HTTPS proxies, it is possible to have two concurrent _nested_
SSL/TLS sessions: the "outer" one between the user agent and the proxy
and the "inner" one between the user agent and the origin server
(through the proxy). This change adds supports for such nested sessions
as well.

A secure connection with a proxy requires its own set of the usual SSL
options (their actual descriptions differ and need polishing, see TODO):

  --proxy-cacert FILE        CA certificate to verify peer against
  --proxy-capath DIR         CA directory to verify peer against
  --proxy-cert CERT[:PASSWD] Client certificate file and password
  --proxy-cert-type TYPE     Certificate file type (DER/PEM/ENG)
  --proxy-ciphers LIST       SSL ciphers to use
  --proxy-crlfile FILE       Get a CRL list in PEM format from the file
  --proxy-insecure           Allow connections to proxies with bad certs
  --proxy-key KEY            Private key file name
  --proxy-key-type TYPE      Private key file type (DER/PEM/ENG)
  --proxy-pass PASS          Pass phrase for the private key
  --proxy-ssl-allow-beast    Allow security flaw to improve interop
  --proxy-sslv2              Use SSLv2
  --proxy-sslv3              Use SSLv3
  --proxy-tlsv1              Use TLSv1
  --proxy-tlsuser USER       TLS username
  --proxy-tlspassword STRING TLS password
  --proxy-tlsauthtype STRING TLS authentication type (default SRP)

All --proxy-foo options are independent from their --foo counterparts,
except --proxy-crlfile which defaults to --crlfile and --proxy-capath
which defaults to --capath.

Curl now also supports %{proxy_ssl_verify_result} --write-out variable,
similar to the existing %{ssl_verify_result} variable.

Supported backends: OpenSSL, GnuTLS, and NSS.

* A SOCKS proxy + HTTP/HTTPS proxy combination:

If both --socks* and --proxy options are given, Curl first connects to
the SOCKS proxy and then connects (through SOCKS) to the HTTP or HTTPS
proxy.

TODO: Update documentation for the new APIs and --proxy-* options.
Look for "Added in 7.XXX" marks.

1 files changed, 31 insertions, 97 deletions

diff --git a/lib/ftp.c b/lib/ftp.c
index 04670b625..1e61fc551 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -499,7 +499,7 @@ static CURLcode InitiateTransfer(struct connectdata *conn)
   struct FTP *ftp = data->req.protop;
   CURLcode result = CURLE_OK;
 
-  if(conn->ssl[SECONDARYSOCKET].use) {
+  if(conn->bits.ftp_use_data_ssl) {
     /* since we only have a plaintext TCP connection here, we must now
      * do the TLS stuff */
     infof(data, "Doing the SSL/TLS handshake on the data stream\n");
@@ -740,7 +740,7 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
        * wait for more data anyway.
        */
     }
-    else {
+    else if(!Curl_ssl_data_pending(conn, FIRSTSOCKET)) {
       switch (SOCKET_READABLE(sockfd, interval_ms)) {
       case -1: /* select() error, stop reading */
         failf(data, "FTP response aborted due to select/poll error: %d",
@@ -1850,84 +1850,6 @@ static CURLcode ftp_epsv_disable(struct connectdata *conn)
   return result;
 }
 
-/*
- * Perform the necessary magic that needs to be done once the TCP connection
- * to the proxy has completed.
- */
-static CURLcode proxy_magic(struct connectdata *conn,
-                            char *newhost, unsigned short newport,
-                            bool *magicdone)
-{
-  CURLcode result = CURLE_OK;
-  struct Curl_easy *data = conn->data;
-
-#if defined(CURL_DISABLE_PROXY)
-  (void) newhost;
-  (void) newport;
-#endif
-
-  *magicdone = FALSE;
-
-  switch(conn->proxytype) {
-  case CURLPROXY_SOCKS5:
-  case CURLPROXY_SOCKS5_HOSTNAME:
-    result = Curl_SOCKS5(conn->proxyuser, conn->proxypasswd, newhost,
-                         newport, SECONDARYSOCKET, conn);
-    *magicdone = TRUE;
-    break;
-  case CURLPROXY_SOCKS4:
-    result = Curl_SOCKS4(conn->proxyuser, newhost, newport,
-                         SECONDARYSOCKET, conn, FALSE);
-    *magicdone = TRUE;
-    break;
-  case CURLPROXY_SOCKS4A:
-    result = Curl_SOCKS4(conn->proxyuser, newhost, newport,
-                         SECONDARYSOCKET, conn, TRUE);
-    *magicdone = TRUE;
-    break;
-  case CURLPROXY_HTTP:
-  case CURLPROXY_HTTP_1_0:
-    /* do nothing here. handled later. */
-    break;
-  default:
-    failf(data, "unknown proxytype option given");
-    result = CURLE_COULDNT_CONNECT;
-    break;
-  }
-
-  if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
-    /* BLOCKING */
-    /* We want "seamless" FTP operations through HTTP proxy tunnel */
-
-    /* Curl_proxyCONNECT is based on a pointer to a struct HTTP at the
-     * member conn->proto.http; we want FTP through HTTP and we have to
-     * change the member temporarily for connecting to the HTTP proxy. After
-     * Curl_proxyCONNECT we have to set back the member to the original
-     * struct FTP pointer
-     */
-    struct HTTP http_proxy;
-    struct FTP *ftp_save = data->req.protop;
-    memset(&http_proxy, 0, sizeof(http_proxy));
-    data->req.protop = &http_proxy;
-
-    result = Curl_proxyCONNECT(conn, SECONDARYSOCKET, newhost, newport, TRUE);
-
-    data->req.protop = ftp_save;
-
-    if(result)
-      return result;
-
-    if(conn->tunnel_state[SECONDARYSOCKET] != TUNNEL_COMPLETE) {
-      /* the CONNECT procedure is not complete, the tunnel is not yet up */
-      state(conn, FTP_STOP); /* this phase is completed */
-      return result;
-    }
-    else
-      *magicdone = TRUE;
-  }
-
-  return result;
-}
 
 static char *control_address(struct connectdata *conn)
 {
@@ -1935,11 +1857,7 @@ static char *control_address(struct connectdata *conn)
      If a proxy tunnel is used, returns the original host name instead, because
      the effective control connection address is the proxy address,
      not the ftp host. */
-  if(conn->bits.tunnel_proxy ||
-     conn->proxytype == CURLPROXY_SOCKS5 ||
-     conn->proxytype == CURLPROXY_SOCKS5_HOSTNAME ||
-     conn->proxytype == CURLPROXY_SOCKS4 ||
-     conn->proxytype == CURLPROXY_SOCKS4A)
+  if(conn->bits.tunnel_proxy || conn->bits.socksproxy)
     return conn->host.name;
 
   return conn->ip_addr_str;
@@ -2063,7 +1981,9 @@ static CURLcode ftp_state_pasv_resp(struct connectdata *conn,
      * here. We don't want to rely on a former host lookup that might've
      * expired now, instead we remake the lookup here and now!
      */
-    rc = Curl_resolv(conn, conn->proxy.name, (int)conn->port, &addr);
+    const char * const host_name = conn->bits.socksproxy ?
+      conn->socks_proxy.host.name : conn->http_proxy.host.name;
+    rc = Curl_resolv(conn, host_name, (int)conn->port, &addr);
     if(rc == CURLRESOLV_PENDING)
       /* BLOCKING, ignores the return code but 'addr' will be NULL in
          case of failure */
@@ -2073,8 +1993,7 @@ static CURLcode ftp_state_pasv_resp(struct connectdata *conn,
       (unsigned short)conn->port; /* we connect to the proxy's port */
 
     if(!addr) {
-      failf(data, "Can't resolve proxy host %s:%hu",
-            conn->proxy.name, connectport);
+      failf(data, "Can't resolve proxy host %s:%hu", host_name, connectport);
       return CURLE_FTP_CANT_GET_HOST;
     }
   }
@@ -2115,6 +2034,10 @@ static CURLcode ftp_state_pasv_resp(struct connectdata *conn,
     /* this just dumps information about this second connection */
     ftp_pasv_verbose(conn, addr->addr, ftpc->newhost, connectport);
 
+  Curl_safefree(conn->secondaryhostname);
+  conn->secondaryhostname = strdup(ftpc->newhost);
+  conn->secondary_port = ftpc->newport;
+
   Curl_resolv_unlock(data, addr); /* we're done using this address */
   conn->bits.do_more = TRUE;
   state(conn, FTP_STOP); /* this phase is completed */
@@ -2763,7 +2686,10 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
       }
 #endif
 
-      if(data->set.use_ssl && !conn->ssl[FIRSTSOCKET].use) {
+      if(data->set.use_ssl &&
+         (!conn->ssl[FIRSTSOCKET].use ||
+          (conn->bits.proxy_ssl_connected[FIRSTSOCKET] &&
+           !conn->proxy_ssl[FIRSTSOCKET].use))) {
         /* We don't have a SSL/TLS connection yet, but FTPS is
            requested. Try a FTPS connection now */
 
@@ -2808,7 +2734,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
         /* Curl_ssl_connect is BLOCKING */
         result = Curl_ssl_connect(conn, FIRSTSOCKET);
         if(!result) {
-          conn->ssl[SECONDARYSOCKET].use = FALSE; /* clear-text data */
+          conn->bits.ftp_use_data_ssl = FALSE; /* clear-text data */
           result = ftp_state_user(conn);
         }
       }
@@ -2850,7 +2776,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
     case FTP_PROT:
       if(ftpcode/100 == 2)
         /* We have enabled SSL for the data connection! */
-        conn->ssl[SECONDARYSOCKET].use =
+        conn->bits.ftp_use_data_ssl =
           (data->set.use_ssl != CURLUSESSL_CONTROL) ? TRUE : FALSE;
       /* FTP servers typically responds with 500 if they decide to reject
          our 'P' request */
@@ -3666,10 +3592,6 @@ static CURLcode ftp_do_more(struct connectdata *conn, int *completep)
     /* Ready to do more? */
     if(connected) {
       DEBUGF(infof(data, "DO-MORE connected phase starts\n"));
-      if(conn->bits.proxy) {
-        infof(data, "Connection to proxy confirmed\n");
-        result = proxy_magic(conn, ftpc->newhost, ftpc->newport, &connected);
-      }
     }
     else {
       if(result && (ftpc->count1 == 0)) {
@@ -3681,6 +3603,18 @@ static CURLcode ftp_do_more(struct connectdata *conn, int *completep)
     }
   }
 
+  result = Curl_proxy_connect(conn, SECONDARYSOCKET);
+  if(result)
+    return result;
+
+  if(CONNECT_SECONDARYSOCKET_PROXY_SSL())
+    return result;
+
+  if(conn->bits.tunnel_proxy && conn->bits.httpproxy &&
+     conn->tunnel_state[SECONDARYSOCKET] != TUNNEL_COMPLETE)
+    return result;
+
+
   if(ftpc->state) {
     /* already in a state so skip the intial commands.
        They are only done to kickstart the do_more state */
@@ -4246,8 +4180,8 @@ CURLcode ftp_parse_url_path(struct connectdata *conn)
   const char *cur_pos;
   const char *filename = NULL;
 
-  cur_pos = path_to_use; /* current position in path. point at the begin
-                            of next path component */
+  cur_pos = path_to_use; /* current position in path. point at the begin of
+                            next path component */
 
   ftpc->ctl_valid = FALSE;
   ftpc->cwdfail = FALSE;