Merging new release version: 1.10.41.10.4

16 files changed, 321 insertions, 114 deletions

diff --git a/.travis.yml b/.travis.yml
index abb27b44..fc88d9d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ python:
   - "3.4"
   - "pypy"
 script: make test
+before_install:
+  - openssl version
 install:
   - make
 notifications:
diff --git a/CHANGES.rst b/CHANGES.rst
index 9b060e11..8d922a42 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,21 +1,37 @@
 Changes
 =======
 
+1.10.4 (2015-05-03)
++++++++++++++++++++
+
+* Migrate tests to Tornado 4. (Issue #594)
+
+* Append default warning configuration rather than overwrite.
+  (Issue #603)
+
+* Fix streaming decoding regression. (Issue #595)
+
+* Fix chunked requests losing state across keep-alive connections.
+  (Issue #599)
+
+* Fix hanging when chunked HEAD response has no body. (Issue #605)
+
+
 1.10.3 (2015-04-21)
 +++++++++++++++++++
 
 * Emit ``InsecurePlatformWarning`` when SSLContext object is missing.
   (Issue #558)
-  
+
 * Fix regression of duplicate header keys being discarded.
   (Issue #563)
-  
+
 * ``Response.stream()`` returns a generator for chunked responses.
   (Issue #560)
-  
+
 * Set upper-bound timeout when waiting for a socket in PyOpenSSL.
   (Issue #585)
-  
+
 * Work on platforms without `ssl` module for plain HTTP requests.
   (Issue #587)
 
@@ -89,7 +105,7 @@ Changes
 
 * Fixed packaging issues of some development-related files not
   getting included. (Issue #440)
-  
+
 * Allow performing *only* fingerprint verification. (Issue #444)
 
 * Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445)
diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index 4a7f70a2..58073075 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -145,6 +145,9 @@ In chronological order:
 * Tomas Tomecek <ttomecek@redhat.com>
   * Implemented generator for getting chunks from chunked responses.
 
+* tlynn <https://github.com/tlynn>
+  * Respect the warning preferences at import.
+
 * [Your name or handle] <[email or website]>
   * [Brief summary of your changes]
 
diff --git a/Makefile b/Makefile
index b692b120..c038b08a 100644
--- a/Makefile
+++ b/Makefile
@@ -31,6 +31,7 @@ clean:
 	find . -name "__pycache__" -delete
 	rm -f $(REQUIREMENTS_OUT)
 	rm -rf docs/_build
+	rm -rf build/
 
 test: requirements
 	nosetests
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 2eb58751..9ea36913 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -4,6 +4,4 @@ coverage==3.7.1
 tox==1.7.1
 twine==1.3.1
 wheel==0.24.0
-
-# Tornado 3.2.2 makes our tests flaky, so we stick with 3.1
-tornado==3.1.1
+tornado==4.1
diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py
index 72faa1aa..53fbe4a8 100644
--- a/dummyserver/handlers.py
+++ b/dummyserver/handlers.py
@@ -9,7 +9,7 @@ import time
 import zlib
 
 from io import BytesIO
-from tornado.wsgi import HTTPRequest
+from tornado.web import RequestHandler
 
 try:
     from urllib.parse import urlsplit
@@ -21,23 +21,34 @@ log = logging.getLogger(__name__)
 
 class Response(object):
     def __init__(self, body='', status='200 OK', headers=None):
-        if not isinstance(body, bytes):
-            body = body.encode('utf8')
-
         self.body = body
         self.status = status
         self.headers = headers or [("Content-type", "text/plain")]
 
-    def __call__(self, environ, start_response):
-        start_response(self.status, self.headers)
-        return [self.body]
+    def __call__(self, request_handler):
+        status, reason = self.status.split(' ', 1)
+        request_handler.set_status(int(status), reason)
+        for header,value in self.headers:
+            request_handler.add_header(header,value)
+
+        # chunked
+        if isinstance(self.body, list):
+            for item in self.body:
+                if not isinstance(item, bytes):
+                    item = item.encode('utf8')
+                request_handler.write(item)
+                request_handler.flush()
+        else:
+            body = self.body
+            if not isinstance(body, bytes):
+                body = body.encode('utf8')
 
+            request_handler.write(body)
 
-class WSGIHandler(object):
-    pass
 
+RETRY_TEST_NAMES = collections.defaultdict(int)
 
-class TestingApp(WSGIHandler):
+class TestingApp(RequestHandler):
     """
     Simple app that performs various operations, useful for testing an HTTP
     library.
@@ -46,10 +57,25 @@ class TestingApp(WSGIHandler):
     it exists. Status code 200 indicates success, 400 indicates failure. Each
     method has its own conditions for success/failure.
     """
-    def __call__(self, environ, start_response):
-        """ Call the correct method in this class based on the incoming URI """
-        req = HTTPRequest(environ)
+    def get(self):
+        """ Handle GET requests """
+        self._call_method()
+
+    def post(self):
+        """ Handle POST requests """
+        self._call_method()
 
+    def put(self):
+        """ Handle PUT requests """
+        self._call_method()
+
+    def options(self):
+        """ Handle OPTIONS requests """
+        self._call_method()
+
+    def _call_method(self):
+        """ Call the correct method in this class based on the incoming URI """
+        req = self.request
         req.params = {}
         for k, v in req.arguments.items():
             req.params[k] = next(iter(v))
@@ -60,13 +86,14 @@ class TestingApp(WSGIHandler):
 
         target = path[1:].replace('/', '_')
         method = getattr(self, target, self.index)
+
         resp = method(req)
 
         if dict(resp.headers).get('Connection') == 'close':
             # FIXME: Can we kill the connection somehow?
             pass
 
-        return resp(environ, start_response)
+        resp(self)
 
     def index(self, _request):
         "Render simple message"
@@ -184,15 +211,27 @@ class TestingApp(WSGIHandler):
             return Response("test-name header not set",
                             status="400 Bad Request")
 
-        if not hasattr(self, 'retry_test_names'):
-            self.retry_test_names = collections.defaultdict(int)
-        self.retry_test_names[test_name] += 1
+        RETRY_TEST_NAMES[test_name] += 1
 
-        if self.retry_test_names[test_name] >= 2:
+        if RETRY_TEST_NAMES[test_name] >= 2:
             return Response("Retry successful!")
         else:
             return Response("need to keep retrying!", status="418 I'm A Teapot")
 
+    def chunked(self, request):
+        return Response(['123'] * 4)
+
+    def chunked_gzip(self, request):
+        chunks = []
+        compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+
+        for uncompressed in [b'123'] * 4:
+            chunks.append(compressor.compress(uncompressed))
+
+        chunks.append(compressor.flush())
+
+        return Response(chunks, headers=[('Content-Encoding', 'gzip')])
+
     def shutdown(self, request):
         sys.exit()
 
diff --git a/dummyserver/server.py b/dummyserver/server.py
index 73f84372..63124d35 100755
--- a/dummyserver/server.py
+++ b/dummyserver/server.py
@@ -18,7 +18,6 @@ import warnings
 from urllib3.exceptions import HTTPWarning
 
 from tornado.platform.auto import set_close_exec
-import tornado.wsgi
 import tornado.httpserver
 import tornado.ioloop
 import tornado.web
@@ -206,7 +205,7 @@ if __name__ == '__main__':
     host = '127.0.0.1'
 
     io_loop = tornado.ioloop.IOLoop()
-    app = tornado.wsgi.WSGIContainer(TestingApp())
+    app = tornado.web.Application([(r".*", TestingApp)])
     server, port = run_tornado_app(app, io_loop, None,
                                    'http', host)
     server_thread = run_loop_in_thread(io_loop)
diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py
index 335b2f2d..67e62cfe 100644
--- a/dummyserver/testcase.py
+++ b/dummyserver/testcase.py
@@ -2,7 +2,7 @@ import unittest
 import socket
 import threading
 from nose.plugins.skip import SkipTest
-from tornado import ioloop, web, wsgi
+from tornado import ioloop, web
 
 from dummyserver.server import (
     SocketServerThread,
@@ -30,7 +30,9 @@ class SocketDummyServerTestCase(unittest.TestCase):
                                                ready_event=ready_event,
                                                host=cls.host)
         cls.server_thread.start()
-        ready_event.wait()
+        ready_event.wait(5)
+        if not ready_event.is_set():
+            raise Exception("most likely failed to start server")
         cls.port = cls.server_thread.port
 
     @classmethod
@@ -55,7 +57,7 @@ class HTTPDummyServerTestCase(unittest.TestCase):
     @classmethod
     def _start_server(cls):
         cls.io_loop = ioloop.IOLoop()
-        app = wsgi.WSGIContainer(TestingApp())
+        app = web.Application([(r".*", TestingApp)])
         cls.server, cls.port = run_tornado_app(app, cls.io_loop, cls.certs,
                                                cls.scheme, cls.host)
         cls.server_thread = run_loop_in_thread(cls.io_loop)
@@ -97,11 +99,11 @@ class HTTPDummyProxyTestCase(unittest.TestCase):
     def setUpClass(cls):
         cls.io_loop = ioloop.IOLoop()
 
-        app = wsgi.WSGIContainer(TestingApp())
+        app = web.Application([(r'.*', TestingApp)])
         cls.http_server, cls.http_port = run_tornado_app(
             app, cls.io_loop, None, 'http', cls.http_host)
 
-        app = wsgi.WSGIContainer(TestingApp())
+        app = web.Application([(r'.*', TestingApp)])
         cls.https_server, cls.https_port = run_tornado_app(
             app, cls.io_loop, cls.https_certs, 'https', cls.http_host)
 
diff --git a/test/test_response.py b/test/test_response.py
index e74fd5d3..2e2be0ec 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -418,6 +418,29 @@ class TestResponse(unittest.TestCase):
             self.assertEqual(c, stream[i])
             i += 1
 
+    def test_mock_gzipped_transfer_encoding_chunked_decoded(self):
+        """Show that we can decode the gizpped and chunked body."""
+        def stream():
+            # Set up a generator to chunk the gzipped body
+            import zlib
+            compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+            data = compress.compress(b'foobar')
+            data += compress.flush()
+            for i in range(0, len(data), 2):
+                yield data[i:i+2]
+
+        fp = MockChunkedEncodingResponse(list(stream()))
+        r = httplib.HTTPResponse(MockSock)
+        r.fp = fp
+        headers = {'transfer-encoding': 'chunked', 'content-encoding': 'gzip'}
+        resp = HTTPResponse(r, preload_content=False, headers=headers)
+
+        data = b''
+        for c in resp.stream(decode_content=True):
+            data += c
+
+        self.assertEqual(b'foobar', data)
+
     def test_mock_transfer_encoding_chunked_custom_read(self):
         stream = [b"foooo", b"bbbbaaaaar"]
         fp = MockChunkedEncodingResponse(stream)
@@ -517,7 +540,9 @@ class MockChunkedEncodingResponse(object):
 
     @staticmethod
     def _encode_chunk(chunk):
-        return '%X\r\n%s\r\n' % (len(chunk), chunk.decode())
+        # In the general case, we can't decode the chunk to unicode
+        length = '%X\r\n' % len(chunk)
+        return length.encode() + chunk + b'\r\n'
 
     def _pop_new_chunk(self):
         if self.chunks_exhausted:
@@ -529,8 +554,10 @@ class MockChunkedEncodingResponse(object):
             self.chunks_exhausted = True
         else:
             self.index += 1
-        encoded_chunk = self._encode_chunk(chunk)
-        return encoded_chunk.encode()
+        chunk = self._encode_chunk(chunk)
+        if not isinstance(chunk, bytes):
+            chunk = chunk.encode()
+        return chunk
 
     def pop_current_chunk(self, amt=-1, till_crlf=False):
         if amt > 0 and till_crlf:
diff --git a/test/test_util.py b/test/test_util.py
index ba26af5a..19ba57e3 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -97,6 +97,7 @@ class TestUtil(unittest.TestCase):
     parse_url_host_map = {
         'http://google.com/mail': Url('http', host='google.com', path='/mail'),
         'http://google.com/mail/': Url('http', host='google.com', path='/mail/'),
+        'http://google.com/mail': Url('http', host='google.com', path='mail'),
         'google.com/mail': Url(host='google.com', path='/mail'),
         'http://google.com/': Url('http', host='google.com', path='/'),
         'http://google.com': Url('http', host='google.com'),
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py
index a789e9bd..d6cb1622 100644
--- a/test/with_dummyserver/test_connectionpool.py
+++ b/test/with_dummyserver/test_connectionpool.py
@@ -618,13 +618,34 @@ class TestConnectionPool(HTTPDummyServerTestCase):
             self.assertRaises(ProtocolError,
                     pool.request, 'GET', '/source_address')
 
-    @onlyPy3
-    def test_httplib_headers_case_insensitive(self):
-        HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain',
-                    'Server': 'TornadoServer/%s' % tornado.version}
-        r = self.pool.request('GET', '/specific_method',
-                               fields={'method': 'GET'})
-        self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
+    def test_stream_keepalive(self):
+        x = 2
+
+        for _ in range(x):
+            response = self.pool.request(
+                    'GET',
+                    '/chunked',
+                    headers={
+                        'Connection': 'keep-alive',
+                        },
+                    preload_content=False,
+                    retries=False,
+                    )
+            for chunk in response.stream():
+                self.assertEqual(chunk, b'123')
+
+        self.assertEqual(self.pool.num_connections, 1)
+        self.assertEqual(self.pool.num_requests, x)
+
+    def test_chunked_gzip(self):
+        response = self.pool.request(
+                'GET',
+                '/chunked_gzip',
+                preload_content=False,
+                decode_content=True,
+                )
+
+        self.assertEqual(b'123' * 4, response.read())
 
 
 class TestRetry(HTTPDummyServerTestCase):
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
index f3e00acf..992b8ef1 100644
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -32,9 +32,15 @@ from urllib3.exceptions import (
     SystemTimeWarning,
     InsecurePlatformWarning,
 )
+from urllib3.packages import six
 from urllib3.util.timeout import Timeout
 
 
+ResourceWarning = getattr(
+        six.moves.builtins,
+        'ResourceWarning', type('ResourceWarning', (), {}))
+
+
 log = logging.getLogger('urllib3.connectionpool')
 log.setLevel(logging.NOTSET)
 log.addHandler(logging.StreamHandler(sys.stdout))
@@ -362,10 +368,8 @@ class TestHTTPS(HTTPSDummyServerTestCase):
     def test_ssl_correct_system_time(self):
         self._pool.cert_reqs = 'CERT_REQUIRED'
         self._pool.ca_certs = DEFAULT_CA
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self._pool.request('GET', '/')
 
+        w = self._request_without_resource_warnings('GET', '/')
         self.assertEqual([], w)
 
     def test_ssl_wrong_system_time(self):
@@ -374,9 +378,7 @@ class TestHTTPS(HTTPSDummyServerTestCase):
         with mock.patch('urllib3.connection.datetime') as mock_date:
             mock_date.date.today.return_value = datetime.date(1970, 1, 1)
 
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter('always')
-                self._pool.request('GET', '/')
+            w = self._request_without_resource_warnings('GET', '/')
 
             self.assertEqual(len(w), 1)
             warning = w[0]
@@ -384,6 +386,13 @@ class TestHTTPS(HTTPSDummyServerTestCase):
             self.assertEqual(SystemTimeWarning, warning.category)
             self.assertTrue(str(RECENT_DATE) in warning.message.args[0])
 
+    def _request_without_resource_warnings(self, method, url):
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always')
+            self._pool.request(method, url)
+
+        return [x for x in w if not isinstance(x.message, ResourceWarning)]
+
 
 class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
     certs = DEFAULT_CERTS.copy()
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
index db481d30..6c996538 100644
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -18,6 +18,8 @@ from dummyserver.testcase import SocketDummyServerTestCase
 from dummyserver.server import (
     DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address)
 
+from .. import onlyPy3
+
 from nose.plugins.skip import SkipTest
 from threading import Event
 import socket
@@ -598,3 +600,49 @@ class TestErrorWrapping(SocketDummyServerTestCase):
         self._start_server(handler)
         pool = HTTPConnectionPool(self.host, self.port, retries=False)
         self.assertRaises(ProtocolError, pool.request, 'GET', '/')
+
+class TestHeaders(SocketDummyServerTestCase):
+
+    @onlyPy3
+    def test_httplib_headers_case_insensitive(self):
+        handler = create_response_handler(
+           b'HTTP/1.1 200 OK\r\n'
+           b'Content-Length: 0\r\n'
+           b'Content-type: text/plain\r\n'
+           b'\r\n'
+        )
+        self._start_server(handler)
+        pool = HTTPConnectionPool(self.host, self.port, retries=False)
+        HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain'}
+        r = pool.request('GET', '/')
+        self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
+
+
+class TestHEAD(SocketDummyServerTestCase):
+    def test_chunked_head_response_does_not_hang(self):
+        handler = create_response_handler(
+           b'HTTP/1.1 200 OK\r\n'
+           b'Transfer-Encoding: chunked\r\n'
+           b'Content-type: text/plain\r\n'
+           b'\r\n'
+        )
+        self._start_server(handler)
+        pool = HTTPConnectionPool(self.host, self.port, retries=False)
+        r = pool.request('HEAD', '/', timeout=1, preload_content=False)
+
+        # stream will use the read_chunked method here.
+        self.assertEqual([], list(r.stream()))
+
+    def test_empty_head_response_does_not_hang(self):
+        handler = create_response_handler(
+           b'HTTP/1.1 200 OK\r\n'
+           b'Content-Length: 256\r\n'
+           b'Content-type: text/plain\r\n'
+           b'\r\n'
+        )
+        self._start_server(handler)
+        pool = HTTPConnectionPool(self.host, self.port, retries=False)
+        r = pool.request('HEAD', '/', timeout=1, preload_content=False)
+
+        # stream will use the read method here.
+        self.assertEqual([], list(r.stream()))
diff --git a/urllib3/__init__.py b/urllib3/__init__.py
index 333060c2..f48ac4af 100644
--- a/urllib3/__init__.py
+++ b/urllib3/__init__.py
@@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using.
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.10.3'
+__version__ = '1.10.4'
 
 
 from .connectionpool import (
@@ -57,9 +57,10 @@ del NullHandler
 
 import warnings
 # SecurityWarning's always go off by default.
-warnings.simplefilter('always', exceptions.SecurityWarning)
+warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
 # InsecurePlatformWarning's don't vary between requests, so we keep it default.
-warnings.simplefilter('default', exceptions.InsecurePlatformWarning)
+warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
+                      append=True)
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """
diff --git a/urllib3/response.py b/urllib3/response.py
index 7e08fffe..24140c4c 100644
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -126,14 +126,15 @@ class HTTPResponse(io.IOBase):
         # Are we using the chunked-style of transfer encoding?
         self.chunked = False
         self.chunk_left = None
-        tr_enc = self.headers.get('transfer-encoding', '')
-        if tr_enc.lower() == "chunked":
+        tr_enc = self.headers.get('transfer-encoding', '').lower()
+        # Don't incur the penalty of creating a list and then discarding it
+        encodings = (enc.strip() for enc in tr_enc.split(","))
+        if "chunked" in encodings:
             self.chunked = True
 
         # We certainly don't want to preload content when the response is chunked.
-        if not self.chunked:
-            if preload_content and not self._body:
-                self._body = self.read(decode_content=decode_content)
+        if not self.chunked and preload_content and not self._body:
+            self._body = self.read(decode_content=decode_content)
 
     def get_redirect_location(self):
         """
@@ -172,6 +173,35 @@ class HTTPResponse(io.IOBase):
         """
         return self._fp_bytes_read
 
+    def _init_decoder(self):
+        """
+        Set-up the _decoder attribute if necessar.
+        """
+        # Note: content-encoding value should be case-insensitive, per RFC 7230
+        # Section 3.2
+        content_encoding = self.headers.get('content-encoding', '').lower()
+        if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
+            self._decoder = _get_decoder(content_encoding)
+
+    def _decode(self, data, decode_content, flush_decoder):
+        """
+        Decode the data passed in and potentially flush the decoder.
+        """
+        try:
+            if decode_content and self._decoder:
+                data = self._decoder.decompress(data)
+        except (IOError, zlib.error) as e:
+            content_encoding = self.headers.get('content-encoding', '').lower()
+            raise DecodeError(
+                "Received response with content-encoding: %s, but "
+                "failed to decode it." % content_encoding, e)
+
+        if flush_decoder and decode_content and self._decoder:
+            buf = self._decoder.decompress(binary_type())
+            data += buf + self._decoder.flush()
+
+        return data
+
     def read(self, amt=None, decode_content=None, cache_content=False):
         """
         Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
@@ -193,12 +223,7 @@ class HTTPResponse(io.IOBase):
             after having ``.read()`` the file object. (Overridden if ``amt`` is
             set.)
         """
-        # Note: content-encoding value should be case-insensitive, per RFC 7230
-        # Section 3.2
-        content_encoding = self.headers.get('content-encoding', '').lower()
-        if self._decoder is None:
-            if content_encoding in self.CONTENT_DECODERS:
-                self._decoder = _get_decoder(content_encoding)
+        self._init_decoder()
         if decode_content is None:
             decode_content = self.decode_content
 
@@ -247,17 +272,7 @@ class HTTPResponse(io.IOBase):
 
             self._fp_bytes_read += len(data)
 
-            try:
-                if decode_content and self._decoder:
-                    data = self._decoder.decompress(data)
-            except (IOError, zlib.error) as e:
-                raise DecodeError(
-                    "Received response with content-encoding: %s, but "
-                    "failed to decode it." % content_encoding, e)
-
-            if flush_decoder and decode_content and self._decoder:
-                buf = self._decoder.decompress(binary_type())
-                data += buf + self._decoder.flush()
+            data = self._decode(data, decode_content, flush_decoder)
 
             if cache_content:
                 self._body = data
@@ -285,7 +300,7 @@ class HTTPResponse(io.IOBase):
             'content-encoding' header.
         """
         if self.chunked:
-            for line in self.read_chunked(amt):
+            for line in self.read_chunked(amt, decode_content=decode_content):
                 yield line
         else:
             while not is_fp_closed(self._fp):
@@ -371,48 +386,70 @@ class HTTPResponse(io.IOBase):
             b[:len(temp)] = temp
             return len(temp)
 
-    def read_chunked(self, amt=None):
-        # FIXME: Rewrite this method and make it a class with
-        #        a better structured logic.
+    def _update_chunk_length(self):
+        # First, we'll figure out length of a chunk and then
+        # we'll try to read it from socket.
+        if self.chunk_left is not None:
+            return
+        line = self._fp.fp.readline()
+        line = line.split(b';', 1)[0]
+        try:
+            self.chunk_left = int(line, 16)
+        except ValueError:
+            # Invalid chunked protocol response, abort.
+            self.close()
+            raise httplib.IncompleteRead(line)
+
+    def _handle_chunk(self, amt):
+        returned_chunk = None
+        if amt is None:
+            chunk = self._fp._safe_read(self.chunk_left)
+            returned_chunk = chunk
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+        elif amt < self.chunk_left:
+            value = self._fp._safe_read(amt)
+            self.chunk_left = self.chunk_left - amt
+            returned_chunk = value
+        elif amt == self.chunk_left:
+            value = self._fp._safe_read(amt)
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+            returned_chunk = value
+        else:  # amt > self.chunk_left
+            returned_chunk = self._fp._safe_read(self.chunk_left)
+            self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
+            self.chunk_left = None
+        return returned_chunk
+
+    def read_chunked(self, amt=None, decode_content=None):
+        """
+        Similar to :meth:`HTTPResponse.read`, but with an additional
+        parameter: ``decode_content``.
+
+        :param decode_content:
+            If True, will attempt to decode the body based on the
+            'content-encoding' header.
+        """
+        self._init_decoder()
+        # FIXME: Rewrite this method and make it a class with a better structured logic.
         if not self.chunked:
             raise ResponseNotChunked("Response is not chunked. "
                 "Header 'transfer-encoding: chunked' is missing.")
+
+        if self._original_response and self._original_response._method.upper() == 'HEAD':
+            # Don't bother reading the body of a HEAD request.
+            # FIXME: Can we do this somehow without accessing private httplib _method?
+            self._original_response.close()
+            return
+
         while True:
-            # First, we'll figure out length of a chunk and then
-            # we'll try to read it from socket.
-            if self.chunk_left is None:
-                line = self._fp.fp.readline()
-                line = line.decode()
-                # See RFC 7230: Chunked Transfer Coding.
-                i = line.find(';')
-                if i >= 0:
-                    line = line[:i]  # Strip chunk-extensions.
-                try:
-                    self.chunk_left = int(line, 16)
-                except ValueError:
-                    # Invalid chunked protocol response, abort.
-                    self.close()
-                    raise httplib.IncompleteRead(''.join(line))
-                if self.chunk_left == 0:
-                    break
-            if amt is None:
-                chunk = self._fp._safe_read(self.chunk_left)
-                yield chunk
-                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
-                self.chunk_left = None
-            elif amt < self.chunk_left:
-                value = self._fp._safe_read(amt)
-                self.chunk_left = self.chunk_left - amt
-                yield value
-            elif amt == self.chunk_left:
-                value = self._fp._safe_read(amt)
-                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
-                self.chunk_left = None
-                yield value
-            else:  # amt > self.chunk_left
-                yield self._fp._safe_read(self.chunk_left)
-                self._fp._safe_read(2)  # Toss the CRLF at the end of the chunk.
-                self.chunk_left = None
+            self._update_chunk_length()
+            if self.chunk_left == 0:
+                break
+            chunk = self._handle_chunk(amt)
+            yield self._decode(chunk, decode_content=decode_content,
+                               flush_decoder=True)
 
         # Chunk content ends with \r\n: discard it.
         while True:
@@ -424,5 +461,6 @@ class HTTPResponse(io.IOBase):
                 break
 
         # We read everything; close the "file".
+        if self._original_response:
+            self._original_response.close()
         self.release_conn()
-
diff --git a/urllib3/util/url.py b/urllib3/util/url.py
index b2ec834f..e58050cd 100644
--- a/urllib3/util/url.py
+++ b/urllib3/util/url.py
@@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)):
 
     def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
                 query=None, fragment=None):
+        if path and not path.startswith('/'):
+            path = '/' + path
         return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
                                        query, fragment)