Merge pull request #795 from Lukasa/issue/791

Enable PyOpenSSL testing.

12 files changed, 170 insertions, 45 deletions

diff --git a/.travis.yml b/.travis.yml
index 6beeb57b..2a334142 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,6 @@
 language: python
-script: tox
+script:
+  - ./_travis/travis-run.sh
 before_install:
   - openssl version
 cache:
diff --git a/_travis/travis-install.sh b/_travis/travis-install.sh
index 4e3b2d94..ba44c612 100755
--- a/_travis/travis-install.sh
+++ b/_travis/travis-install.sh
@@ -4,6 +4,18 @@ set -ev
 
 pip install tox==2.1.1
 
+# Workaround Travis' old PyPy release. If Travis updates, we can remove this
+# code.
+if [[ "${TOXENV}" == pypy* ]]; then
+    git clone https://github.com/yyuu/pyenv.git ~/.pyenv
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+    pyenv install pypy-4.0.1
+    pyenv global pypy-4.0.1
+    pyenv rehash
+fi
+
 if [[ "${TOXENV}" == "gae" && ! -d ${GAE_PYTHONPATH} ]]; then
     python _travis/fetch_gae_sdk.py `dirname ${GAE_PYTHONPATH}`
 fi
diff --git a/_travis/travis-run.sh b/_travis/travis-run.sh
new file mode 100755
index 00000000..c044a8d4
--- /dev/null
+++ b/_travis/travis-run.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ev
+
+if [[ "${TOXENV}" == "pypy" ]]; then
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+fi
+
+tox
\ No newline at end of file
diff --git a/dummyserver/certs/server.crt b/dummyserver/certs/server.crt
index 29aea389..c8302bf2 100644
--- a/dummyserver/certs/server.crt
+++ b/dummyserver/certs/server.crt
@@ -1,22 +1,21 @@
 -----BEGIN CERTIFICATE-----
-MIIDqDCCAxGgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgTELMAkGA1UEBhMCRkkx
-DjAMBgNVBAgTBWR1bW15MQ4wDAYDVQQHEwVkdW1teTEOMAwGA1UEChMFZHVtbXkx
-DjAMBgNVBAsTBWR1bW15MREwDwYDVQQDEwhTbmFrZU9pbDEfMB0GCSqGSIb3DQEJ
+MIIDczCCAtygAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgTELMAkGA1UEBhMCRkkx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQHDAVkdW1teTEOMAwGA1UECgwFZHVtbXkx
+DjAMBgNVBAsMBWR1bW15MREwDwYDVQQDDAhTbmFrZU9pbDEfMB0GCSqGSIb3DQEJ
 ARYQZHVtbXlAdGVzdC5sb2NhbDAeFw0xMTEyMjIwNzU4NDBaFw0yMTEyMTgwNzU4
-NDBaMGExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt
-bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTESMBAGA1UEAxMJbG9j
+NDBaMGExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UEBwwFZHVt
+bXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVkdW1teTESMBAGA1UEAwwJbG9j
 YWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXe3FqmCWvP8XPxqtT
 +0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB
 0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN
-3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo4IBTTCCAUkwCQYDVR0TBAIwADARBglg
-hkgBhvhCAQEEBAMCBkAwKwYJYIZIAYb4QgENBB4WHFRpbnlDQSBHZW5lcmF0ZWQg
-Q2VydGlmaWNhdGUwHQYDVR0OBBYEFBvnSuVKLNPEFMAFqHw292vGHGJSMIG2BgNV
-HSMEga4wgauAFBl3fyNiYkJZRft1ncdzcgS7MwotoYGHpIGEMIGBMQswCQYDVQQG
-EwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQKEwVk
-dW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJKoZI
-hvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsggkAs+uxyi/hv+MwCQYDVR0SBAIwADAZ
-BgNVHREEEjAQgQ5yb290QGxvY2FsaG9zdDANBgkqhkiG9w0BAQUFAAOBgQBXdedG
-XHLPmOVBeKWjTmaekcaQi44snhYqE1uXRoIQXQsyw+Ya5+n/uRxPKZO/C78EESL0
-8rnLTdZXm4GBYyHYmMy0AdWR7y030viOzAkWWRRRbuecsaUzFCI+F9jTV5LHuRzz
-V8fUKwiEE9swzkWgMpfVTPFuPgzxwG9gMbrBfg==
+3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo4IBGDCCARQwCQYDVR0TBAIwADAdBgNV
+HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwgbYGA1UdIwSBrjCBq4AUGXd/I2Ji
+QllF+3Wdx3NyBLszCi2hgYekgYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVk
+dW1teTEOMAwGA1UEBwwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVk
+dW1teTERMA8GA1UEAwwIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRl
+c3QubG9jYWyCCQCz67HKL+G/4zAJBgNVHRIEAjAAMCQGA1UdEQQdMBuBDnJvb3RA
+bG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZIhvcNAQEFBQADgYEAgcW6X1ZUyufm
+TFEqEAdpKXdL0rxDwcsM/qqqsXbkz17otH6ujPhBEagzdKtgeNKfy0aXz6rWZugk
+lF0IqyC4mcI+vvfgGR5Iy4KdXMrIX98MbrvGJBfbdKhGW2b84wDV42DIDiD2ZGGe
+6YZQQIo9LxjuOTf9jsvf+PIkbI4H0To=
 -----END CERTIFICATE-----
diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py
index 5d575277..ab304f83 100644
--- a/test/contrib/test_pyopenssl.py
+++ b/test/contrib/test_pyopenssl.py
@@ -1,9 +1,6 @@
 from nose.plugins.skip import SkipTest
 from urllib3.packages import six
 
-if six.PY3:
-    raise SkipTest('Testing of PyOpenSSL disabled on PY3')
-
 try:
     from urllib3.contrib.pyopenssl import (inject_into_urllib3,
                                            extract_from_urllib3)
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
index 7319d7ed..2c5f035e 100644
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -38,7 +38,7 @@ from urllib3.exceptions import (
 )
 from urllib3.packages import six
 from urllib3.util.timeout import Timeout
-from urllib3.util.ssl_ import HAS_SNI
+import urllib3.util as util
 
 
 ResourceWarning = getattr(
@@ -77,11 +77,11 @@ class TestHTTPS(HTTPSDummyServerTestCase):
             r = https_pool.request('GET', '/')
             self.assertEqual(r.status, 200)
 
-            if sys.version_info >= (2, 7, 9):
+            if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
                 self.assertFalse(warn.called, warn.call_args_list)
             else:
                 self.assertTrue(warn.called)
-                if HAS_SNI:
+                if util.HAS_SNI:
                     call = warn.call_args_list[0]
                 else:
                     call = warn.call_args_list[1]
@@ -181,9 +181,9 @@ class TestHTTPS(HTTPSDummyServerTestCase):
             self.assertTrue(warn.called)
 
             calls = warn.call_args_list
-            if sys.version_info >= (2, 7, 9):
+            if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL:
                 category = calls[0][0][1]
-            elif HAS_SNI:
+            elif util.HAS_SNI:
                 category = calls[1][0][1]
             else:
                 category = calls[2][0][1]
@@ -237,8 +237,9 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          cert_reqs='CERT_REQUIRED',
                                          ca_certs=DEFAULT_CA)
 
-        https_pool.assert_fingerprint = 'CA:84:E1:AD0E5a:ef:2f:C3:09' \
-                                        ':E7:30:F8:CD:C8:5B'
+        https_pool.assert_fingerprint = 'F2:06:5A:42:10:3F:45:1C:17:FE:E6:' \
+                                        '07:1E:8A:86:E5'
+
         https_pool.request('GET', '/')
 
     def test_assert_fingerprint_sha1(self):
@@ -246,8 +247,8 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          cert_reqs='CERT_REQUIRED',
                                          ca_certs=DEFAULT_CA)
 
-        https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
-                                        '7A:F2:8A:D7:1E:07:33:67:DE'
+        https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
+                                        'BF:93:CF:F9:71:CC:07:7D:0A'
         https_pool.request('GET', '/')
 
     def test_assert_fingerprint_sha256(self):
@@ -255,9 +256,9 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          cert_reqs='CERT_REQUIRED',
                                          ca_certs=DEFAULT_CA)
 
-        https_pool.assert_fingerprint = ('9A:29:9D:4F:47:85:1C:51:23:F5:9A:A3:'
-                                         '0F:5A:EF:96:F9:2E:3C:22:2E:FC:E8:BC:'
-                                         '0E:73:90:37:ED:3B:AA:AB')
+        https_pool.assert_fingerprint = ('C5:4D:0B:83:84:89:2E:AE:B4:58:BB:12:'
+                                         'F7:A6:C4:76:05:03:88:D8:57:65:51:F3:'
+                                         '1E:60:B0:8B:70:18:64:E6')
         https_pool.request('GET', '/')
 
     def test_assert_invalid_fingerprint(self):
@@ -294,8 +295,8 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          cert_reqs='CERT_NONE',
                                          ca_certs=DEFAULT_CA_BAD)
 
-        https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
-                                        '7A:F2:8A:D7:1E:07:33:67:DE'
+        https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
+                                        'BF:93:CF:F9:71:CC:07:7D:0A'
         https_pool.request('GET', '/')
 
     def test_good_fingerprint_and_hostname_mismatch(self):
@@ -303,8 +304,8 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          cert_reqs='CERT_REQUIRED',
                                          ca_certs=DEFAULT_CA)
 
-        https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
-                                        '7A:F2:8A:D7:1E:07:33:67:DE'
+        https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
+                                        'BF:93:CF:F9:71:CC:07:7D:0A'
         https_pool.request('GET', '/')
 
     @requires_network
@@ -325,8 +326,8 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                                          timeout=timeout, retries=False,
                                          cert_reqs='CERT_REQUIRED')
         https_pool.ca_certs = DEFAULT_CA
-        https_pool.assert_fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:' \
-                                        '7A:F2:8A:D7:1E:07:33:67:DE'
+        https_pool.assert_fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:' \
+                                        'BF:93:CF:F9:71:CC:07:7D:0A'
 
         timeout = Timeout(total=None)
         https_pool = HTTPSConnectionPool(self.host, self.port, timeout=timeout,
@@ -385,7 +386,7 @@ class TestHTTPS(HTTPSDummyServerTestCase):
                           timeout=Timeout(total=None, connect=0.001))
 
     def test_enhanced_ssl_connection(self):
-        fingerprint = 'CC:45:6A:90:82:F7FF:C0:8218:8e:7A:F2:8A:D7:1E:07:33:67:DE'
+        fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
 
         conn = VerifiedHTTPSConnection(self.host, self.port)
         https_pool = HTTPSConnectionPool(self.host, self.port,
diff --git a/tox.ini b/tox.ini
index 843d0ea4..29fc9fd8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@ envlist = flake8-py3, py26, py27, py33, py34, py35, pypy
 [testenv]
 deps= -r{toxinidir}/dev-requirements.txt
 commands=
-    pip install .[socks]
+    pip install .[socks,secure]
     nosetests []
 setenv =
     PYTHONWARNINGS=always::DeprecationWarning
diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py
index aabd96f2..1d71368c 100644
--- a/urllib3/contrib/pyopenssl.py
+++ b/urllib3/contrib/pyopenssl.py
@@ -54,9 +54,17 @@ except SyntaxError as e:
 import OpenSSL.SSL
 from pyasn1.codec.der import decoder as der_decoder
 from pyasn1.type import univ, constraint
-from socket import _fileobject, timeout, error as SocketError
+from socket import timeout, error as SocketError
+
+try:  # Platform-specific: Python 2
+    from socket import _fileobject
+except ImportError:  # Platform-specific: Python 3
+    _fileobject = None
+    from urllib3.packages.backports.makefile import backport_makefile
+
 import ssl
 import select
+import six
 
 from .. import connection
 from .. import util
@@ -104,6 +112,7 @@ def inject_into_urllib3():
 
     connection.ssl_wrap_socket = ssl_wrap_socket
     util.HAS_SNI = HAS_SNI
+    util.IS_PYOPENSSL = True
 
 
 def extract_from_urllib3():
@@ -111,6 +120,7 @@ def extract_from_urllib3():
 
     connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket
     util.HAS_SNI = orig_util_HAS_SNI
+    util.IS_PYOPENSSL = False
 
 
 # Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
@@ -135,7 +145,7 @@ def get_subj_alt_name(peer_cert):
     for i in range(peer_cert.get_extension_count()):
         ext = peer_cert.get_extension(i)
         ext_name = ext.get_short_name()
-        if ext_name != 'subjectAltName':
+        if ext_name != b'subjectAltName':
             continue
 
         # PyOpenSSL returns extension data in ASN.1 encoded form
@@ -167,13 +177,17 @@ class WrappedSocket(object):
         self.socket = socket
         self.suppress_ragged_eofs = suppress_ragged_eofs
         self._makefile_refs = 0
+        self._closed = False
 
     def fileno(self):
         return self.socket.fileno()
 
-    def makefile(self, mode, bufsize=-1):
-        self._makefile_refs += 1
-        return _fileobject(self, mode, bufsize, close=True)
+    # Copy-pasted from Python 3.5 source code
+    def _decref_socketios(self):
+        if self._makefile_refs > 0:
+            self._makefile_refs -= 1
+        if self._closed:
+            self.close()
 
     def recv(self, *args, **kwargs):
         try:
@@ -198,6 +212,27 @@ class WrappedSocket(object):
         else:
             return data
 
+    def recv_into(self, *args, **kwargs):
+        try:
+            return self.connection.recv_into(*args, **kwargs)
+        except OpenSSL.SSL.SysCallError as e:
+            if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
+                return 0
+            else:
+                raise SocketError(str(e))
+        except OpenSSL.SSL.ZeroReturnError as e:
+            if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+                return 0
+            else:
+                raise
+        except OpenSSL.SSL.WantReadError:
+            rd, wd, ed = select.select(
+                [self.socket], [], [], self.socket.gettimeout())
+            if not rd:
+                raise timeout('The read operation timed out')
+            else:
+                return self.recv_into(*args, **kwargs)
+
     def settimeout(self, timeout):
         return self.socket.settimeout(timeout)
 
@@ -225,6 +260,7 @@ class WrappedSocket(object):
     def close(self):
         if self._makefile_refs < 1:
             try:
+                self._closed = True
                 return self.connection.close()
             except OpenSSL.SSL.Error:
                 return
@@ -262,6 +298,16 @@ class WrappedSocket(object):
             self._makefile_refs -= 1
 
 
+if _fileobject:  # Platform-specific: Python 2
+    def makefile(self, mode, bufsize=-1):
+        self._makefile_refs += 1
+        return _fileobject(self, mode, bufsize, close=True)
+else:  # Platform-specific: Python 3
+    makefile = backport_makefile
+
+WrappedSocket.makefile = makefile
+
+
 def _verify_callback(cnx, x509, err_no, err_depth, return_code):
     return err_no == 0
 
@@ -293,6 +339,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
     ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST)
 
     cnx = OpenSSL.SSL.Connection(ctx, sock)
+    if isinstance(server_hostname, six.text_type):  # Platform-specific: Python 3
+        server_hostname = server_hostname.encode('utf-8')
     cnx.set_tlsext_host_name(server_hostname)
     cnx.set_connect_state()
     while True:
diff --git a/urllib3/packages/backports/__init__.py b/urllib3/packages/backports/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/urllib3/packages/backports/__init__.py
diff --git a/urllib3/packages/backports/makefile.py b/urllib3/packages/backports/makefile.py
new file mode 100644
index 00000000..75b80dcf
--- /dev/null
+++ b/urllib3/packages/backports/makefile.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+backports.makefile
+~~~~~~~~~~~~~~~~~~
+
+Backports the Python 3 ``socket.makefile`` method for use with anything that
+wants to create a "fake" socket object.
+"""
+import io
+
+from socket import SocketIO
+
+
+def backport_makefile(self, mode="r", buffering=None, encoding=None,
+                      errors=None, newline=None):
+    """
+    Backport of ``socket.makefile`` from Python 3.5.
+    """
+    if not set(mode) <= set(["r", "w", "b"]):
+        raise ValueError(
+            "invalid mode %r (only r, w, b allowed)" % (mode,)
+        )
+    writing = "w" in mode
+    reading = "r" in mode or not writing
+    assert reading or writing
+    binary = "b" in mode
+    rawmode = ""
+    if reading:
+        rawmode += "r"
+    if writing:
+        rawmode += "w"
+    raw = SocketIO(self, rawmode)
+    self._makefile_refs += 1
+    if buffering is None:
+        buffering = -1
+    if buffering < 0:
+        buffering = io.DEFAULT_BUFFER_SIZE
+    if buffering == 0:
+        if not binary:
+            raise ValueError("unbuffered streams must be binary")
+        return raw
+    if reading and writing:
+        buffer = io.BufferedRWPair(raw, raw, buffering)
+    elif reading:
+        buffer = io.BufferedReader(raw, buffering)
+    else:
+        assert writing
+        buffer = io.BufferedWriter(raw, buffering)
+    if binary:
+        return buffer
+    text = io.TextIOWrapper(buffer, encoding, errors, newline)
+    text.mode = mode
+    return text
diff --git a/urllib3/util/__init__.py b/urllib3/util/__init__.py
index c6c6243c..4778cf99 100644
--- a/urllib3/util/__init__.py
+++ b/urllib3/util/__init__.py
@@ -6,6 +6,7 @@ from .response import is_fp_closed
 from .ssl_ import (
     SSLContext,
     HAS_SNI,
+    IS_PYOPENSSL,
     assert_fingerprint,
     resolve_cert_reqs,
     resolve_ssl_version,
@@ -26,6 +27,7 @@ from .url import (
 
 __all__ = (
     'HAS_SNI',
+    'IS_PYOPENSSL',
     'SSLContext',
     'Retry',
     'Timeout',
diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py
index e24a9508..e8d9e7d2 100644
--- a/urllib3/util/ssl_.py
+++ b/urllib3/util/ssl_.py
@@ -12,6 +12,7 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
 SSLContext = None
 HAS_SNI = False
 create_default_context = None
+IS_PYOPENSSL = False
 
 # Maps the length of a digest to a possible hash function producing this digest
 HASHFUNC_MAP = {