Add support for HTTPS connections to proxies. (#1679)

* Add support to talk HTTPS to proxies.

Currently there's no way to validate identify for the proxy you might be
connecting. Proxies supporting HTTPS endpoints are becoming more common
and we need to extend the support for them.

When an HTTPS proxy is provided, instead of doing the HTTP CONNECT,
we'll forward any requests directly to the proxy and ultimately to the
destination.

* Fix proxy_headers missing on HTTPS proxy connections.

* blackfmt missing files.

* Prevent usage of HTTPS proxies when fetching HTTPS resources.

- Will be supported by default when we can do TLS within TLS.

* Update proxy documentation with more information.

* Renamed flag for HTTPS websites through HTTPS proxies.
* Added myself to contributors.

* Documentation and contributors fixes.

* Removed mention that TLS in TLS is being developed as requested.
* Space in between my name and the github page.

* Add flag to enable HTTPS proxy support.

Now that we're adding support for HTTPS proxies we want to avoid
a breaking change with clients that had an improper proxy configuration.

For now, we're adding a warning an defaulting to the previous behavior.
In the future we'll change the behavior to enable HTTPS proxies by
default.

* Remove guard flag, error out on HTTPS/HTTPS.

As requested in the last revision for the PR:

- Removed the _enable_https_proxies flag. Instead the feature will be
  enabled and will error out on invalid configurations. (HTTPS + HTTPS)
- Other comments: rename a method, parentheses to clarify order of
  operations.

2 files changed, 82 insertions, 3 deletions

diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py
index 51adc4b8..fb947dae 100644
--- a/test/test_proxymanager.py
+++ b/test/test_proxymanager.py
@@ -1,12 +1,15 @@
 import pytest
 
 from urllib3.poolmanager import ProxyManager
+from urllib3.util.url import parse_url
 
 
 class TestProxyManager(object):
-    def test_proxy_headers(self):
+    @pytest.mark.parametrize("proxy_scheme", ["http", "https"])
+    def test_proxy_headers(self, proxy_scheme):
         url = "http://pypi.org/project/urllib3/"
-        with ProxyManager("http://something:1234") as p:
+        proxy_url = "{}://something:1234".format(proxy_scheme)
+        with ProxyManager(proxy_url) as p:
             # Verify default headers
             default_headers = {"Accept": "*/*", "Host": "pypi.org"}
             headers = p._set_proxy_headers(url)
@@ -43,3 +46,14 @@ class TestProxyManager(object):
             ProxyManager("invalid://host/p")
         with pytest.raises(ValueError):
             ProxyManager("invalid://host/p")
+
+    def test_proxy_tunnel(self):
+        http_url = parse_url("http://example.com")
+        https_url = parse_url("https://example.com")
+        with ProxyManager("http://proxy:8080") as p:
+            assert p._proxy_requires_url_absolute_form(http_url)
+            assert p._proxy_requires_url_absolute_form(https_url) is False
+
+        with ProxyManager("https://proxy:8080") as p:
+            assert p._proxy_requires_url_absolute_form(http_url)
+            assert p._proxy_requires_url_absolute_form(https_url)
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
index 063ab360..feface09 100644
--- a/test/with_dummyserver/test_proxy_poolmanager.py
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -14,7 +14,13 @@ from .. import TARPIT_HOST, requires_network
 
 from urllib3._collections import HTTPHeaderDict
 from urllib3.poolmanager import proxy_from_url, ProxyManager
-from urllib3.exceptions import MaxRetryError, SSLError, ProxyError, ConnectTimeoutError
+from urllib3.exceptions import (
+    MaxRetryError,
+    SSLError,
+    ProxyError,
+    ConnectTimeoutError,
+    ProxySchemeUnsupported,
+)
 from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection
 
 from test import SHORT_TIMEOUT, LONG_TIMEOUT
@@ -32,6 +38,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
         cls.https_url = "https://%s:%d" % (cls.https_host, cls.https_port)
         cls.https_url_alt = "https://%s:%d" % (cls.https_host_alt, cls.https_port)
         cls.proxy_url = "http://%s:%d" % (cls.proxy_host, cls.proxy_port)
+        cls.https_proxy_url = "https://%s:%d" % (cls.proxy_host, cls.https_proxy_port,)
 
         # Generate another CA to test verification failure
         cls.certs_dir = tempfile.mkdtemp()
@@ -53,6 +60,23 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
             r = http.request("GET", "%s/" % self.https_url)
             assert r.status == 200
 
+    def test_https_proxy(self):
+        with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+            r = https.request("GET", "%s/" % self.http_url)
+            assert r.status == 200
+
+            with pytest.raises(ProxySchemeUnsupported):
+                https.request("GET", "%s/" % self.https_url)
+
+        with proxy_from_url(
+            self.https_proxy_url,
+            ca_certs=DEFAULT_CA,
+            _allow_https_proxy_to_see_traffic=True,
+        ) as https:
+            r = https.request("GET", "%s/" % self.http_url)
+            https.request("GET", "%s/" % self.https_url)
+            assert r.status == 200
+
     def test_nagle_proxy(self):
         """ Test that proxy connections do not have TCP_NODELAY turned on """
         with ProxyManager(self.proxy_url) as http:
@@ -275,6 +299,47 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
                 self.https_port,
             )
 
+    def test_https_headers(self):
+        with proxy_from_url(
+            self.https_proxy_url,
+            headers={"Foo": "bar"},
+            proxy_headers={"Hickory": "dickory"},
+            ca_certs=DEFAULT_CA,
+        ) as http:
+
+            r = http.request_encode_url("GET", "%s/headers" % self.http_url)
+            returned_headers = json.loads(r.data.decode())
+            assert returned_headers.get("Foo") == "bar"
+            assert returned_headers.get("Hickory") == "dickory"
+            assert returned_headers.get("Host") == "%s:%s" % (
+                self.http_host,
+                self.http_port,
+            )
+
+            r = http.request_encode_url("GET", "%s/headers" % self.http_url_alt)
+            returned_headers = json.loads(r.data.decode())
+            assert returned_headers.get("Foo") == "bar"
+            assert returned_headers.get("Hickory") == "dickory"
+            assert returned_headers.get("Host") == "%s:%s" % (
+                self.http_host_alt,
+                self.http_port,
+            )
+
+            with pytest.raises(ProxySchemeUnsupported):
+                http.request_encode_url("GET", "%s/headers" % self.https_url)
+
+            r = http.request_encode_url(
+                "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"}
+            )
+            returned_headers = json.loads(r.data.decode())
+            assert returned_headers.get("Foo") is None
+            assert returned_headers.get("Baz") == "quux"
+            assert returned_headers.get("Hickory") == "dickory"
+            assert returned_headers.get("Host") == "%s:%s" % (
+                self.http_host,
+                self.http_port,
+            )
+
     def test_headerdict(self):
         default_headers = HTTPHeaderDict(a="b")
         proxy_headers = HTTPHeaderDict()