diff options
| -rw-r--r-- | .coveragerc | 4 | ||||
| -rw-r--r-- | .travis.yml | 5 | ||||
| -rw-r--r-- | CHANGES.rst | 9 | ||||
| -rw-r--r-- | CONTRIBUTORS.txt | 3 | ||||
| -rw-r--r-- | docs/contrib.rst | 60 | ||||
| -rw-r--r-- | docs/index.rst | 2 | ||||
| -rw-r--r-- | docs/security.rst | 8 | ||||
| -rwxr-xr-x | dummyserver/server.py | 4 | ||||
| -rw-r--r-- | dummyserver/testcase.py | 15 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rw-r--r-- | test/__init__.py | 2 | ||||
| -rw-r--r-- | test/contrib/test_socks.py | 599 | ||||
| -rw-r--r-- | tox.ini | 5 | ||||
| -rw-r--r-- | urllib3/__init__.py | 4 | ||||
| -rw-r--r-- | urllib3/connectionpool.py | 23 | ||||
| -rw-r--r-- | urllib3/contrib/appengine.py | 12 | ||||
| -rw-r--r-- | urllib3/contrib/ntlmpool.py | 20 | ||||
| -rw-r--r-- | urllib3/contrib/socks.py | 172 | ||||
| -rw-r--r-- | urllib3/exceptions.py | 8 | ||||
| -rw-r--r-- | urllib3/poolmanager.py | 17 | ||||
| -rw-r--r-- | urllib3/util/response.py | 2 | ||||
| -rw-r--r-- | urllib3/util/retry.py | 4 | ||||
| -rw-r--r-- | urllib3/util/ssl_.py | 3 |
23 files changed, 935 insertions, 49 deletions
diff --git a/.coveragerc b/.coveragerc index 6017701a..c21c72e1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,9 @@ [run] omit = urllib3/packages/* - urllib3/contrib/* + urllib3/contrib/appengine.py + urllib3/contrib/ntlmpool.py + urllib3/contrib/pyopenssl.py [report] exclude_lines = diff --git a/.travis.yml b/.travis.yml index 4b52f531..459d7562 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,4 +22,9 @@ env: - TOXENV=py34 - TOXENV=pypy - TOXENV=gae +# https://github.com/travis-ci/travis-ci/issues/4794 +matrix: + include: + - python: 3.5 + env: TOXENV=py35 sudo: false diff --git a/CHANGES.rst b/CHANGES.rst index 674aa35b..b2b10e86 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes ======= +1.14 (2015-12-29) ++++++++++++++++++ + +* contrib: SOCKS proxy support! (Issue #762) + +* Fixed AppEngine handling of transfer-encoding header and bug + in Timeout defaults checking. (Issue #763) + + 1.13.1 (2015-12-18) +++++++++++++++++++ diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 448a4ab8..cbea9cea 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -184,5 +184,8 @@ In chronological order: * Andy Caldwell <andy.m.caldwell@googlemail.com> * Bugfix related to reusing connections in indeterminate states. +* Ville Skyttä <ville.skytta@iki.fi> + * Logging efficiency improvements, spelling fixes, Travis config. + * [Your name or handle] <[email or website]> * [Brief summary of your changes] diff --git a/docs/contrib.rst b/docs/contrib.rst index 5a88f8e9..03cc63bf 100644 --- a/docs/contrib.rst +++ b/docs/contrib.rst @@ -4,7 +4,7 @@ Contrib Modules =============== These modules implement various extra features, that may not be ready for -prime time. +prime time or that require optional third-party dependencies. .. _contrib-pyopenssl: @@ -16,7 +16,7 @@ SNI-support for Python 2 .. _gae: -Google App Engine +Google App Engine ----------------- The :mod:`urllib3.contrib.appengine` module provides a pool manager that @@ -45,8 +45,62 @@ There are `limitations <https://cloud.google.com/appengine/docs/python/urlfetch/ 1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is cost-effective in many circumstances as long as your usage is within the limitations. 2. You can use a normal :class:`PoolManager` by enabling sockets. Sockets also have `limitations and restrictions <https://cloud.google.com/appengine/docs/python/sockets/#limitations-and-restrictions>`_ and have a lower free quota than URLFetch. To use sockets, be sure to specify the following in your ``app.yaml``:: - + env_variables: GAE_USE_SOCKETS_HTTPLIB : 'true' 3. If you are using `Managed VMs <https://cloud.google.com/appengine/docs/managed-vms/>`_, you can use the standard :class:`PoolManager` without any configuration or special environment variables. + + +SOCKS Proxies +------------- + +.. versionadded:: 1.14.0 + +The :mod:`urllib3.contrib.socks` module enables urllib3 to work with proxies +that use either the SOCKS4 or SOCKS5 protocols. These proxies are common in +environments that want to allow generic TCP/UDP traffic through their borders, +but don't want unrestricted traffic flows. + +To use it, either install ``PySocks`` or install urllib3 with the ``socks`` +extra, like so: + +.. code-block:: bash + + $ pip install urllib3[socks] + +If you have already got urllib3 1.14.0 or later installed, run: + +.. code-block:: bash + + $ pip install -U urllib3[socks] + +The SOCKS module provides a +:class:`SOCKSProxyManager <urllib3.contrib.socks.SOCKSProxyManager>` that can +be used when SOCKS support is required. This class behaves very much like a +standard :class:`ProxyManager <urllib3.poolmanager.ProxyManager>`, but allows +the use of a SOCKS proxy instead. + +Using it is simple. For example, with a SOCKS5 proxy running on the local +machine, listening on port 8889: + +.. code-block:: python + + from urllib3.contrib.socks import SOCKSProxyManager + + http = SOCKSProxyManager('socks5://localhost:8889/') + r = http.request('GET', 'https://www.google.com/') + +The SOCKS implementation supports the full range of urllib3 features. It also +supports the following SOCKS features: + +- SOCKS4 +- SOCKS4a +- SOCKS5 +- Usernames and passwords for the SOCKS proxy + +The SOCKS module does have the following limitations: + +- No support for contacting a SOCKS proxy via IPv6. +- No support for reaching websites via a literal IPv6 address: domain names + must be used. diff --git a/docs/index.rst b/docs/index.rst index 29e7ad7a..b2fb96e2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -327,7 +327,7 @@ Contrib Modules --------------- These modules implement various extra features, that may not be ready for -prime time. +prime time or that require optional third-party dependencies. * :ref:`contrib-modules` diff --git a/docs/security.rst b/docs/security.rst index 48de053e..6ebc9413 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -199,9 +199,11 @@ have restrictions in their ``ssl`` module that limit the configuration that succeed on more featureful platforms to fail, and can cause certain security features to be unavailable. -If you encounter this warning, it is strongly recommended you upgrade to a -newer Python version, or that you use pyOpenSSL as described in the -:ref:`pyopenssl` section. +If you encounter this warning, it is strongly recommended you: + +- upgrade to a newer Python version +- upgrade ``ndg-httpsclient`` with ``pip install --upgrade ndg-httpsclient`` +- use pyOpenSSL as described in the :ref:`pyopenssl` section For info about disabling warnings, see `Disabling Warnings`_. diff --git a/dummyserver/server.py b/dummyserver/server.py index ef053a73..3c410b35 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -90,6 +90,8 @@ class SocketServerThread(threading.Thread): :param ready_event: Event which gets set when the socket handler is ready to receive requests. """ + USE_IPV6 = HAS_IPV6_AND_DNS + def __init__(self, socket_handler, host='localhost', port=8081, ready_event=None): threading.Thread.__init__(self) @@ -100,7 +102,7 @@ class SocketServerThread(threading.Thread): self.ready_event = ready_event def _start_server(self): - if HAS_IPV6_AND_DNS: + if self.USE_IPV6: sock = socket.socket(socket.AF_INET6) else: warnings.warn("No IPv6 support. Falling back to IPv4.", diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index f5588a0e..e45fedf0 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -71,6 +71,21 @@ class SocketDummyServerTestCase(unittest.TestCase): cls.server_thread.join(0.1) +class IPV4SocketDummyServerTestCase(SocketDummyServerTestCase): + @classmethod + def _start_server(cls, socket_handler): + ready_event = threading.Event() + cls.server_thread = SocketServerThread(socket_handler=socket_handler, + ready_event=ready_event, + host=cls.host) + cls.server_thread.USE_IPV6 = False + cls.server_thread.start() + ready_event.wait(5) + if not ready_event.is_set(): + raise Exception("most likely failed to start server") + cls.port = cls.server_thread.port + + class HTTPDummyServerTestCase(unittest.TestCase): """ A simple HTTP server that runs when your test class runs @@ -61,5 +61,8 @@ setup(name='urllib3', 'pyasn1', 'certifi', ], + 'socks': [ + 'PySocks>=1.5.6,<2.0', + ] }, ) diff --git a/test/__init__.py b/test/__init__.py index f7c4a7a8..05ecea69 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -57,7 +57,7 @@ def onlyPy27OrNewer(test): return wrapper def onlyPy279OrNewer(test): - """Skips this test unless you are onl Python 2.7.9 or later.""" + """Skips this test unless you are on Python 2.7.9 or later.""" @functools.wraps(test) def wrapper(*args, **kwargs): diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py new file mode 100644 index 00000000..34b6c187 --- /dev/null +++ b/test/contrib/test_socks.py @@ -0,0 +1,599 @@ +import threading +import socket + +from urllib3.contrib import socks +from urllib3.exceptions import ConnectTimeoutError, NewConnectionError + +from dummyserver.server import DEFAULT_CERTS +from dummyserver.testcase import IPV4SocketDummyServerTestCase + +from nose.plugins.skip import SkipTest + +try: + import ssl + from urllib3.util import ssl_ as better_ssl + HAS_SSL = True +except ImportError: + ssl = None + better_ssl = None + HAS_SSL = False + + +SOCKS_NEGOTIATION_NONE = b'\x00' +SOCKS_NEGOTIATION_PASSWORD = b'\x02' + +SOCKS_VERSION_SOCKS4 = b'\x04' +SOCKS_VERSION_SOCKS5 = b'\x05' + + +def _get_free_port(host): + """ + Gets a free port by opening a socket, binding it, checking the assigned + port, and then closing it. + """ + s = socket.socket() + s.bind((host, 0)) + port = s.getsockname()[1] + s.close() + return port + + +def _read_exactly(sock, amt): + """ + Read *exactly* ``amt`` bytes from the socket ``sock``. + """ + data = b'' + + while amt > 0: + chunk = sock.recv(amt) + data += chunk + amt -= len(chunk) + + return data + + +def _read_until(sock, char): + """ + Read from the socket until the character is received. + """ + chunks = [] + while True: + chunk = sock.recv(1) + chunks.append(chunk) + if chunk == char: + break + + return b''.join(chunks) + + +def _address_from_socket(sock): + """ + Returns the address from the SOCKS socket + """ + addr_type = sock.recv(1) + + if addr_type == b'\x01': + ipv4_addr = _read_exactly(sock, 4) + return socket.inet_ntoa(ipv4_addr) + elif addr_type == b'\x04': + ipv6_addr = _read_exactly(sock, 16) + return socket.inet_ntop(socket.AF_INET6, ipv6_addr) + elif addr_type == b'\x03': + addr_len = ord(sock.recv(1)) + return _read_exactly(sock, addr_len) + else: + raise RuntimeError("Unexpected addr type: %r" % addr_type) + + +def handle_socks5_negotiation(sock, negotiate, username=None, + password=None): + """ + Handle the SOCKS5 handshake. + + Returns a generator object that allows us to break the handshake into + steps so that the test code can intervene at certain useful points. + """ + received_version = sock.recv(1) + assert received_version == SOCKS_VERSION_SOCKS5 + nmethods = ord(sock.recv(1)) + methods = _read_exactly(sock, nmethods) + + if negotiate: + assert SOCKS_NEGOTIATION_PASSWORD in methods + send_data = SOCKS_VERSION_SOCKS5 + SOCKS_NEGOTIATION_PASSWORD + sock.sendall(send_data) + + # This is the password negotiation. + negotiation_version = sock.recv(1) + assert negotiation_version == b'\x01' + ulen = ord(sock.recv(1)) + provided_username = _read_exactly(sock, ulen) + plen = ord(sock.recv(1)) + provided_password = _read_exactly(sock, plen) + + if username == provided_username and password == provided_password: + sock.sendall(b'\x01\x00') + else: + sock.sendall(b'\x01\x01') + sock.close() + yield False + return + else: + assert SOCKS_NEGOTIATION_NONE in methods + send_data = SOCKS_VERSION_SOCKS5 + SOCKS_NEGOTIATION_NONE + sock.sendall(send_data) + + # Client sends where they want to go. + received_version = sock.recv(1) + command = sock.recv(1) + reserved = sock.recv(1) + addr = _address_from_socket(sock) + port = _read_exactly(sock, 2) + port = (ord(port[0:1]) << 8) + (ord(port[1:2])) + + # Check some basic stuff. + assert received_version == SOCKS_VERSION_SOCKS5 + assert command == b'\x01' # Only support connect, not bind. + assert reserved == b'\x00' + + # Yield the address port tuple. + succeed = yield addr, port + + if succeed: + # Hard-coded response for now. + response = ( + SOCKS_VERSION_SOCKS5 + b'\x00\x00\x01\x7f\x00\x00\x01\xea\x60' + ) + else: + # Hard-coded response for now. + response = SOCKS_VERSION_SOCKS5 + b'\x01\00' + + sock.sendall(response) + yield True # Avoid StopIteration exceptions getting fired. + + +def handle_socks4_negotiation(sock, username=None): + """ + Handle the SOCKS4 handshake. + + Returns a generator object that allows us to break the handshake into + steps so that the test code can intervene at certain useful points. + """ + received_version = sock.recv(1) + command = sock.recv(1) + port = _read_exactly(sock, 2) + port = (ord(port[0:1]) << 8) + (ord(port[1:2])) + addr = _read_exactly(sock, 4) + provided_username = _read_until(sock, b'\x00')[:-1] # Strip trailing null. + + if addr == b'\x00\x00\x00\x01': + # Magic string: means DNS name. + addr = _read_until(sock, b'\x00')[:-1] # Strip trailing null. + else: + addr = socket.inet_ntoa(addr) + + # Check some basic stuff. + assert received_version == SOCKS_VERSION_SOCKS4 + assert command == b'\x01' # Only support connect, not bind. + + if username is not None and username != provided_username: + sock.sendall(b'\x00\x5d\x00\x00\x00\x00\x00\x00') + sock.close() + yield False + return + + # Yield the address port tuple. + succeed = yield addr, port + + if succeed: + response = b'\x00\x5a\xea\x60\x7f\x00\x00\x01' + else: + response = b'\x00\x5b\x00\x00\x00\x00\x00\x00' + + sock.sendall(response) + yield True # Avoid StopIteration exceptions getting fired. + + +class TestSocks5Proxy(IPV4SocketDummyServerTestCase): + """ + Test the SOCKS proxy in SOCKS5 mode. + """ + def test_basic_request(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation(sock, negotiate=False) + addr, port = next(handler) + + self.assertEqual(addr, '16.17.18.19') + self.assertTrue(port, 80) + handler.send(True) + + while True: + buf = sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + response = pm.request('GET', 'http://16.17.18.19') + + self.assertEqual(response.status, 200) + self.assertEqual(response.data, b'') + self.assertEqual(response.headers['Server'], 'SocksTestServer') + + def test_correct_header_line(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation(sock, negotiate=False) + addr, port = next(handler) + + self.assertEqual(addr, b'example.com') + self.assertTrue(port, 80) + handler.send(True) + + buf = b'' + while True: + buf += sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + self.assertTrue(buf.startswith(b'GET / HTTP/1.1')) + self.assertTrue(b'Host: example.com' in buf) + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + response = pm.request('GET', 'http://example.com') + self.assertEqual(response.status, 200) + + def test_connection_timeouts(self): + event = threading.Event() + + def request_handler(listener): + event.wait() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + + self.assertRaises( + ConnectTimeoutError, pm.request, 'GET', 'http://example.com', + timeout=0.001, retries=False + ) + event.set() + + def test_connection_failure(self): + event = threading.Event() + + def request_handler(listener): + listener.close() + event.set() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + + event.wait() + self.assertRaises( + NewConnectionError, pm.request, 'GET', 'http://example.com', + retries=False + ) + + def test_proxy_rejection(self): + evt = threading.Event() + + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation(sock, negotiate=False) + addr, port = next(handler) + handler.send(False) + + evt.wait() + sock.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + + self.assertRaises( + NewConnectionError, pm.request, 'GET', 'http://example.com', + retries=False + ) + evt.set() + + def test_socks_with_password(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation( + sock, negotiate=True, username=b'user', password=b'pass' + ) + addr, port = next(handler) + + self.assertEqual(addr, '16.17.18.19') + self.assertTrue(port, 80) + handler.send(True) + + while True: + buf = sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url, username='user', + password='pass') + response = pm.request('GET', 'http://16.17.18.19') + + self.assertEqual(response.status, 200) + self.assertEqual(response.data, b'') + self.assertEqual(response.headers['Server'], 'SocksTestServer') + + def test_socks_with_invalid_password(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation( + sock, negotiate=True, username=b'user', password=b'pass' + ) + next(handler) + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url, username='user', + password='badpass') + + try: + pm.request('GET', 'http://example.com', retries=False) + except NewConnectionError as e: + self.assertTrue("SOCKS5 authentication failed" in str(e)) + else: + self.fail("Did not raise") + + def test_source_address_works(self): + expected_port = _get_free_port(self.host) + + def request_handler(listener): + sock = listener.accept()[0] + self.assertEqual(sock.getpeername()[0], '127.0.0.1') + self.assertEqual(sock.getpeername()[1], expected_port) + + handler = handle_socks5_negotiation(sock, negotiate=False) + addr, port = next(handler) + + self.assertEqual(addr, '16.17.18.19') + self.assertTrue(port, 80) + handler.send(True) + + while True: + buf = sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager( + proxy_url, source_address=('127.0.0.1', expected_port) + ) + response = pm.request('GET', 'http://16.17.18.19') + self.assertEqual(response.status, 200) + + +class TestSOCKS4Proxy(IPV4SocketDummyServerTestCase): + """ + Test the SOCKS proxy in SOCKS4 mode. + + Has relatively fewer tests than the SOCKS5 case, mostly because once the + negotiation is done the two cases behave identically. + """ + def test_basic_request(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks4_negotiation(sock) + addr, port = next(handler) + + self.assertEqual(addr, '16.17.18.19') + self.assertTrue(port, 80) + handler.send(True) + + while True: + buf = sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks4://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + response = pm.request('GET', 'http://16.17.18.19') + + self.assertEqual(response.status, 200) + self.assertEqual(response.headers['Server'], 'SocksTestServer') + self.assertEqual(response.data, b'') + + def test_correct_header_line(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks4_negotiation(sock) + addr, port = next(handler) + + self.assertEqual(addr, b'example.com') + self.assertTrue(port, 80) + handler.send(True) + + buf = b'' + while True: + buf += sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + self.assertTrue(buf.startswith(b'GET / HTTP/1.1')) + self.assertTrue(b'Host: example.com' in buf) + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks4://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + response = pm.request('GET', 'http://example.com') + self.assertEqual(response.status, 200) + + def test_proxy_rejection(self): + evt = threading.Event() + + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks4_negotiation(sock) + addr, port = next(handler) + handler.send(False) + + evt.wait() + sock.close() + + self._start_server(request_handler) + proxy_url = "socks4://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + + self.assertRaises( + NewConnectionError, pm.request, 'GET', 'http://example.com', + retries=False + ) + evt.set() + + def test_socks4_with_username(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks4_negotiation(sock, username=b'user') + addr, port = next(handler) + + self.assertEqual(addr, '16.17.18.19') + self.assertTrue(port, 80) + handler.send(True) + + while True: + buf = sock.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + sock.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + sock.close() + + self._start_server(request_handler) + proxy_url = "socks4://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url, username='user') + response = pm.request('GET', 'http://16.17.18.19') + + self.assertEqual(response.status, 200) + self.assertEqual(response.data, b'') + self.assertEqual(response.headers['Server'], 'SocksTestServer') + + def test_socks_with_invalid_username(self): + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks4_negotiation(sock, username=b'user') + next(handler) + + self._start_server(request_handler) + proxy_url = "socks4://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url, username='baduser') + + try: + pm.request('GET', 'http://example.com', retries=False) + except NewConnectionError as e: + self.assertTrue("different user-ids" in str(e)) + else: + self.fail("Did not raise") + + +class TestSOCKSWithTLS(IPV4SocketDummyServerTestCase): + """ + Test that TLS behaves properly for SOCKS proxies. + """ + def test_basic_request(self): + if not HAS_SSL: + raise SkipTest("No TLS available") + + def request_handler(listener): + sock = listener.accept()[0] + + handler = handle_socks5_negotiation(sock, negotiate=False) + addr, port = next(handler) + + self.assertEqual(addr, b'localhost') + self.assertTrue(port, 443) + handler.send(True) + + # Wrap in TLS + context = better_ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain( + DEFAULT_CERTS['certfile'], DEFAULT_CERTS['keyfile'] + ) + tls = context.wrap_socket(sock, server_side=True) + buf = b'' + + while True: + buf += tls.recv(65535) + if buf.endswith(b'\r\n\r\n'): + break + + self.assertTrue(buf.startswith(b'GET / HTTP/1.1\r\n')) + + tls.sendall(b'HTTP/1.1 200 OK\r\n' + b'Server: SocksTestServer\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + tls.close() + + self._start_server(request_handler) + proxy_url = "socks5://%s:%s" % (self.host, self.port) + pm = socks.SOCKSProxyManager(proxy_url) + response = pm.request('GET', 'https://localhost') + + self.assertEqual(response.status, 200) + self.assertEqual(response.data, b'') + self.assertEqual(response.headers['Server'], 'SocksTestServer') @@ -1,9 +1,10 @@ [tox] -envlist = flake8-py3, py26, py27, py32, py33, py34, pypy +envlist = flake8-py3, py26, py27, py32, py33, py34, py35, pypy [testenv] deps= -r{toxinidir}/dev-requirements.txt commands= + pip install .[socks] nosetests [] setenv = PYTHONWARNINGS=always::DeprecationWarning @@ -18,7 +19,7 @@ commands= -c {toxinidir}/test/appengine/nose.cfg \ test/appengine \ [] - # For now, this test is ran seperately because the sandbox activation in + # For now, this test is ran separately because the sandbox activation in # the app engine tests will blow up everything. nosetests \ -c {toxinidir}/test/appengine/nose.cfg \ diff --git a/urllib3/__init__.py b/urllib3/__init__.py index e43991a9..b744315b 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.13.1' +__version__ = '1.14' __all__ = ( 'HTTPConnectionPool', @@ -68,7 +68,7 @@ def add_stderr_logger(level=logging.DEBUG): handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s', __name__) return handler # ... Clean up. diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index 995b4167..01afd616 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -203,8 +203,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s" % - (self.num_connections, self.host)) + log.info("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -239,7 +239,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s" % self.host) + log.info("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -272,7 +272,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except Full: # This should never happen if self.block == True log.warning( - "Connection pool is full, discarding connection: %s" % + "Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. @@ -382,9 +382,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, - httplib_response.status, - httplib_response.length)) + log.debug("\"%s %s %s\" %s %s", method, url, http_version, + httplib_response.status, httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -622,7 +621,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again log.warning("Retrying (%r) after connection " - "broken by '%r': %s" % (retries, err, url)) + "broken by '%r': %s", retries, err, url) return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, @@ -644,7 +643,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -656,7 +655,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if retries.is_forced_retry(method, status_code=response.status): retries = retries.increment(method, url, response=response, _pool=self) retries.sleep() - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -754,8 +753,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s" - % (self.num_connections, self.host)) + log.info("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " diff --git a/urllib3/contrib/appengine.py b/urllib3/contrib/appengine.py index 884cdb22..f4289c0f 100644 --- a/urllib3/contrib/appengine.py +++ b/urllib3/contrib/appengine.py @@ -144,7 +144,7 @@ class AppEngineManager(RequestMethods): if retries.is_forced_retry(method, status_code=http_response.status): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) retries.sleep() return self.urlopen( method, url, @@ -164,6 +164,14 @@ class AppEngineManager(RequestMethods): if content_encoding == 'deflate': del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. @@ -177,7 +185,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if timeout.read is not timeout.connect: + if timeout._read is not timeout._connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) diff --git a/urllib3/contrib/ntlmpool.py b/urllib3/contrib/ntlmpool.py index c136a238..11d0b5c3 100644 --- a/urllib3/contrib/ntlmpool.py +++ b/urllib3/contrib/ntlmpool.py @@ -43,8 +43,8 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % - (self.num_connections, self.host, self.authurl)) + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', + self.num_connections, self.host, self.authurl) headers = {} headers['Connection'] = 'Keep-Alive' @@ -56,13 +56,13 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % reshdr) - log.debug('Response data: %s [...]' % res.read(100)) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', reshdr) + log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) @@ -87,12 +87,12 @@ class NTLMConnectionPool(HTTPSConnectionPool): self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % dict(res.getheaders())) - log.debug('Response data: %s [...]' % res.read()[:100]) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', dict(res.getheaders())) + log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' diff --git a/urllib3/contrib/socks.py b/urllib3/contrib/socks.py new file mode 100644 index 00000000..885776c3 --- /dev/null +++ b/urllib3/contrib/socks.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +SOCKS support for urllib3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contrib module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from urllib3.exceptions import DependencyWarning + + warnings.warn(( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from urllib3.connection import ( + HTTPConnection, HTTPSConnection +) +from urllib3.connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool +) +from urllib3.exceptions import ConnectTimeoutError, NewConnectionError +from urllib3.poolmanager import PoolManager +from urllib3.util.url import parse_url + +try: + import ssl +except ImportError: + ssl = None + + +class SOCKSConnection(HTTPConnection): + """ + A plain-text HTTP connection that connects via a SOCKS proxy. + """ + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ + Establish a new connection via the SOCKS proxy. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = socks.create_connection( + (self.host, self.port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + timeout=self.timeout, + **extra_kw + ) + + except SocketTimeout as e: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout) + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + + return conn + + +# We don't need to duplicate the Verified/Unverified distinction from +# urllib3/connection.py here because the HTTPSConnection will already have been +# correctly set to either the Verified or Unverified form by that module. This +# means the SOCKSHTTPSConnection will automatically be the correct type. +class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): + pass + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSHTTPSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, + 'https': SOCKSHTTPSConnectionPool, + } + + def __init__(self, proxy_url, username=None, password=None, + num_pools=10, headers=None, **connection_pool_kw): + parsed = parse_url(proxy_url) + + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + } + connection_pool_kw['_socks_options'] = socks_options + + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py index 8e07eb61..f2e65917 100644 --- a/urllib3/exceptions.py +++ b/urllib3/exceptions.py @@ -180,6 +180,14 @@ class SNIMissingWarning(HTTPWarning): pass +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py index f13e673d..1023dcba 100644 --- a/urllib3/poolmanager.py +++ b/urllib3/poolmanager.py @@ -18,16 +18,16 @@ from .util.retry import Retry __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} - log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + class PoolManager(RequestMethods): """ @@ -65,6 +65,9 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + # Locally set the pool classes so other PoolManagers can override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + def __enter__(self): return self @@ -81,7 +84,7 @@ class PoolManager(RequestMethods): by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = pool_classes_by_scheme[scheme] + pool_cls = self.pool_classes_by_scheme[scheme] kwargs = self.connection_pool_kw if scheme == 'http': kwargs = self.connection_pool_kw.copy() @@ -186,7 +189,7 @@ class PoolManager(RequestMethods): kw['retries'] = retries kw['redirect'] = redirect - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) diff --git a/urllib3/util/response.py b/urllib3/util/response.py index bc723272..0b5c75c1 100644 --- a/urllib3/util/response.py +++ b/urllib3/util/response.py @@ -61,7 +61,7 @@ def assert_header_parsing(headers): def is_response_to_head(response): """ - Checks, wether a the request of a response has been a HEAD-request. + Checks whether the request of a response has been a HEAD-request. Handles the quirks of AppEngine. :param conn: diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py index 03a01249..862174ab 100644 --- a/urllib3/util/retry.py +++ b/urllib3/util/retry.py @@ -153,7 +153,7 @@ class Retry(object): redirect = bool(redirect) and None new_retries = cls(retries, redirect=redirect) - log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries def get_backoff_time(self): @@ -272,7 +272,7 @@ class Retry(object): if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) - log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py index 67f83441..313d3a7e 100644 --- a/urllib3/util/ssl_.py +++ b/urllib3/util/ssl_.py @@ -110,7 +110,7 @@ except ImportError: ) self.ciphers = cipher_suite - def wrap_socket(self, socket, server_hostname=None): + def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' @@ -125,6 +125,7 @@ except ImportError: 'ca_certs': self.ca_certs, 'cert_reqs': self.verify_mode, 'ssl_version': self.protocol, + 'server_side': server_side, } if self.supports_set_ciphers: # Platform-specific: Python 2.7+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs) |