Merge pull request #1888 from hodbn/fix-1516

Fix 1516

5 files changed, 76 insertions, 9 deletions

diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py
index 880f1ba2..957fb966 100644
--- a/src/urllib3/exceptions.py
+++ b/src/urllib3/exceptions.py
@@ -231,6 +231,23 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
         )
 
 
+class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
+    """Invalid chunk length in a chunked response."""
+
+    def __init__(self, response, length):
+        super(InvalidChunkLength, self).__init__(
+            response.tell(), response.length_remaining
+        )
+        self.response = response
+        self.length = length
+
+    def __repr__(self):
+        return "InvalidChunkLength(got length %r, %i bytes read)" % (
+            self.length,
+            self.partial,
+        )
+
+
 class InvalidHeader(HTTPError):
     "The header provided was somehow invalid."
     pass
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
index 7522f9a6..fcf9d7f9 100644
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -19,11 +19,11 @@ from .exceptions import (
     ReadTimeoutError,
     ResponseNotChunked,
     IncompleteRead,
+    InvalidChunkLength,
     InvalidHeader,
     HTTPError,
 )
 from .packages.six import string_types as basestring, PY3
-from .packages.six.moves import http_client as httplib
 from .connection import HTTPException, BaseSSLError
 from .util.response import is_fp_closed, is_response_to_head
 
@@ -697,7 +697,7 @@ class HTTPResponse(io.IOBase):
         except ValueError:
             # Invalid chunked protocol response, abort.
             self.close()
-            raise httplib.IncompleteRead(line)
+            raise InvalidChunkLength(self, line)
 
     def _handle_chunk(self, amt):
         returned_chunk = None
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
index 615fdfc0..193ba91d 100644
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -10,8 +10,9 @@ from urllib3.connectionpool import (
     HTTPConnectionPool,
     HTTPSConnectionPool,
 )
-from urllib3.response import httplib, HTTPResponse
+from urllib3.response import HTTPResponse
 from urllib3.util.timeout import Timeout
+from urllib3.packages.six.moves import http_client as httplib
 from urllib3.packages.six.moves.http_client import HTTPException
 from urllib3.packages.six.moves.queue import Empty
 from urllib3.packages.ssl_match_hostname import CertificateError
diff --git a/test/test_response.py b/test/test_response.py
index ef84e44b..a476f23b 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -16,6 +16,9 @@ from urllib3.exceptions import (
     ResponseNotChunked,
     ProtocolError,
     InvalidHeader,
+    httplib_IncompleteRead,
+    IncompleteRead,
+    InvalidChunkLength,
 )
 from urllib3.packages.six.moves import http_client as httplib
 from urllib3.util.retry import Retry, RequestHistory
@@ -758,9 +761,44 @@ class TestResponse(object):
         with pytest.raises(ResponseNotChunked):
             next(r)
 
-    def test_invalid_chunks(self):
+    def test_buggy_incomplete_read(self):
+        # Simulate buggy versions of Python (<2.7.4)
+        # See http://bugs.python.org/issue16298
+        content_length = 1337
+        fp = BytesIO(b"")
+        resp = HTTPResponse(
+            fp,
+            headers={"content-length": str(content_length)},
+            preload_content=False,
+            enforce_content_length=True,
+        )
+        with pytest.raises(ProtocolError) as ctx:
+            resp.read(3)
+
+        orig_ex = ctx.value.args[1]
+        assert isinstance(orig_ex, IncompleteRead)
+        assert orig_ex.partial == 0
+        assert orig_ex.expected == content_length
+
+    def test_incomplete_chunk(self):
+        stream = [b"foooo", b"bbbbaaaaar"]
+        fp = MockChunkedIncompleteRead(stream)
+        r = httplib.HTTPResponse(MockSock)
+        r.fp = fp
+        r.chunked = True
+        r.chunk_left = None
+        resp = HTTPResponse(
+            r, preload_content=False, headers={"transfer-encoding": "chunked"}
+        )
+        with pytest.raises(ProtocolError) as ctx:
+            next(resp.read_chunked())
+
+        orig_ex = ctx.value.args[1]
+        assert isinstance(orig_ex, httplib_IncompleteRead)
+
+    def test_invalid_chunk_length(self):
         stream = [b"foooo", b"bbbbaaaaar"]
-        fp = MockChunkedInvalidEncoding(stream)
+        fp = MockChunkedInvalidChunkLength(stream)
         r = httplib.HTTPResponse(MockSock)
         r.fp = fp
         r.chunked = True
@@ -768,9 +806,13 @@ class TestResponse(object):
         resp = HTTPResponse(
             r, preload_content=False, headers={"transfer-encoding": "chunked"}
         )
-        with pytest.raises(ProtocolError):
+        with pytest.raises(ProtocolError) as ctx:
             next(resp.read_chunked())
 
+        orig_ex = ctx.value.args[1]
+        assert isinstance(orig_ex, InvalidChunkLength)
+        assert orig_ex.length == six.b(fp.BAD_LENGTH_LINE)
+
     def test_chunked_response_without_crlf_on_end(self):
         stream = [b"foo", b"bar", b"baz"]
         fp = MockChunkedEncodingWithoutCRLFOnEnd(stream)
@@ -971,9 +1013,16 @@ class MockChunkedEncodingResponse(object):
         self.closed = True
 
 
-class MockChunkedInvalidEncoding(MockChunkedEncodingResponse):
+class MockChunkedIncompleteRead(MockChunkedEncodingResponse):
+    def _encode_chunk(self, chunk):
+        return "9999\r\n%s\r\n" % chunk.decode()
+
+
+class MockChunkedInvalidChunkLength(MockChunkedEncodingResponse):
+    BAD_LENGTH_LINE = "ZZZ\r\n"
+
     def _encode_chunk(self, chunk):
-        return "ZZZ\r\n%s\r\n" % chunk.decode()
+        return "%s%s\r\n" % (self.BAD_LENGTH_LINE, chunk.decode())
 
 
 class MockChunkedEncodingWithoutCRLFOnEnd(MockChunkedEncodingResponse):
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
index 2c3ba47d..ec4fc8db 100644
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -9,7 +9,7 @@ from urllib3.exceptions import (
     SSLError,
     ProtocolError,
 )
-from urllib3.response import httplib
+from urllib3.packages.six.moves import http_client as httplib
 from urllib3.util import ssl_wrap_socket
 from urllib3.util.ssl_ import HAS_SNI
 from urllib3.util import ssl_