From 10b7a0fefa6596f47a9a6afc80f1f4d1ae950b66 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 21 Apr 2015 16:07:20 -0400 Subject: Nuke build on clean. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b692b120..c038b08a 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ clean: find . -name "__pycache__" -delete rm -f $(REQUIREMENTS_OUT) rm -rf docs/_build + rm -rf build/ test: requirements nosetests -- cgit v1.2.1 From 01994d6e604677e19aebe06058f2a1e772070293 Mon Sep 17 00:00:00 2001 From: matejcik Date: Thu, 23 Apr 2015 15:29:33 +0200 Subject: use Tornado directly instead of WSGI compatibility layer --- dummyserver/handlers.py | 47 +++++++++++++++++++++++++++++++---------------- dummyserver/server.py | 3 +-- dummyserver/testcase.py | 8 ++++---- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 72faa1aa..5694593e 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -9,7 +9,7 @@ import time import zlib from io import BytesIO -from tornado.wsgi import HTTPRequest +from tornado.web import RequestHandler try: from urllib.parse import urlsplit @@ -28,16 +28,17 @@ class Response(object): self.status = status self.headers = headers or [("Content-type", "text/plain")] - def __call__(self, environ, start_response): - start_response(self.status, self.headers) - return [self.body] + def __call__(self, request_handler): + status, reason = self.status.split(' ', 1) + request_handler.set_status(int(status), reason) + for header,value in self.headers: + request_handler.add_header(header,value.decode('utf8')) + request_handler.write(self.body) -class WSGIHandler(object): - pass +RETRY_TEST_NAMES = collections.defaultdict(int) - -class TestingApp(WSGIHandler): +class TestingApp(RequestHandler): """ Simple app that performs various operations, useful for testing an HTTP library. @@ -46,10 +47,25 @@ class TestingApp(WSGIHandler): it exists. Status code 200 indicates success, 400 indicates failure. Each method has its own conditions for success/failure. """ - def __call__(self, environ, start_response): - """ Call the correct method in this class based on the incoming URI """ - req = HTTPRequest(environ) + def get(self): + """ Handle GET requests """ + self._call_method() + + def post(self): + """ Handle POST requests """ + self._call_method() + + def put(self): + """ Handle PUT requests """ + self._call_method() + def options(self): + """ Handle OPTIONS requests """ + self._call_method() + + def _call_method(self): + """ Call the correct method in this class based on the incoming URI """ + req = self.request req.params = {} for k, v in req.arguments.items(): req.params[k] = next(iter(v)) @@ -60,13 +76,14 @@ class TestingApp(WSGIHandler): target = path[1:].replace('/', '_') method = getattr(self, target, self.index) + resp = method(req) if dict(resp.headers).get('Connection') == 'close': # FIXME: Can we kill the connection somehow? pass - return resp(environ, start_response) + resp(self) def index(self, _request): "Render simple message" @@ -184,11 +201,9 @@ class TestingApp(WSGIHandler): return Response("test-name header not set", status="400 Bad Request") - if not hasattr(self, 'retry_test_names'): - self.retry_test_names = collections.defaultdict(int) - self.retry_test_names[test_name] += 1 + RETRY_TEST_NAMES[test_name] += 1 - if self.retry_test_names[test_name] >= 2: + if RETRY_TEST_NAMES[test_name] >= 2: return Response("Retry successful!") else: return Response("need to keep retrying!", status="418 I'm A Teapot") diff --git a/dummyserver/server.py b/dummyserver/server.py index 73f84372..63124d35 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -18,7 +18,6 @@ import warnings from urllib3.exceptions import HTTPWarning from tornado.platform.auto import set_close_exec -import tornado.wsgi import tornado.httpserver import tornado.ioloop import tornado.web @@ -206,7 +205,7 @@ if __name__ == '__main__': host = '127.0.0.1' io_loop = tornado.ioloop.IOLoop() - app = tornado.wsgi.WSGIContainer(TestingApp()) + app = tornado.web.Application([(r".*", TestingApp)]) server, port = run_tornado_app(app, io_loop, None, 'http', host) server_thread = run_loop_in_thread(io_loop) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 335b2f2d..f988f926 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -2,7 +2,7 @@ import unittest import socket import threading from nose.plugins.skip import SkipTest -from tornado import ioloop, web, wsgi +from tornado import ioloop, web from dummyserver.server import ( SocketServerThread, @@ -55,7 +55,7 @@ class HTTPDummyServerTestCase(unittest.TestCase): @classmethod def _start_server(cls): cls.io_loop = ioloop.IOLoop() - app = wsgi.WSGIContainer(TestingApp()) + app = web.Application([(r".*", TestingApp)]) cls.server, cls.port = run_tornado_app(app, cls.io_loop, cls.certs, cls.scheme, cls.host) cls.server_thread = run_loop_in_thread(cls.io_loop) @@ -97,11 +97,11 @@ class HTTPDummyProxyTestCase(unittest.TestCase): def setUpClass(cls): cls.io_loop = ioloop.IOLoop() - app = wsgi.WSGIContainer(TestingApp()) + app = web.Application([(r'.*', TestingApp)]) cls.http_server, cls.http_port = run_tornado_app( app, cls.io_loop, None, 'http', cls.http_host) - app = wsgi.WSGIContainer(TestingApp()) + app = web.Application([(r'.*', TestingApp)]) cls.https_server, cls.https_port = run_tornado_app( app, cls.io_loop, cls.https_certs, 'https', cls.http_host) -- cgit v1.2.1 From 5416e7e936bb4317741c93c77b4ae2d2d37c938c Mon Sep 17 00:00:00 2001 From: matejcik Date: Thu, 23 Apr 2015 15:30:22 +0200 Subject: make socket test not hang if server fails to start --- dummyserver/testcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index f988f926..710cf149 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -30,7 +30,8 @@ class SocketDummyServerTestCase(unittest.TestCase): ready_event=ready_event, host=cls.host) cls.server_thread.start() - ready_event.wait() + if not ready_event.wait(5): + raise Exception("most likely failed to start server") cls.port = cls.server_thread.port @classmethod -- cgit v1.2.1 From 6ca5cae3359604ad070db8b5765fab6880c7c084 Mon Sep 17 00:00:00 2001 From: matejcik Date: Thu, 23 Apr 2015 15:43:15 +0200 Subject: let tornado take care of encoding (fixes py3 error) --- dummyserver/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 5694593e..80e951ec 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -32,7 +32,7 @@ class Response(object): status, reason = self.status.split(' ', 1) request_handler.set_status(int(status), reason) for header,value in self.headers: - request_handler.add_header(header,value.decode('utf8')) + request_handler.add_header(header,value) request_handler.write(self.body) -- cgit v1.2.1 From 2cfc21644ae868e0ac8b5f24a27eabda8a1ebc1e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 08:57:53 -0500 Subject: Decode information received from read_chunked in stream Previously, when we read chunked data we always decoded it because it was handled by the read method on the HTTPResponse. Now that it's a separate method, we need to handle it. This is a quick fix for #593 to address a regression in behaviour. Fixes #593 --- urllib3/response.py | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 7e08fffe..f1ea9bb5 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -172,6 +172,36 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessar. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, e) + + if flush_decoder and decode_content and self._decoder: + buf = self._decoder.decompress(binary_type()) + data += buf + self._decoder.flush() + + return data + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional @@ -193,12 +223,7 @@ class HTTPResponse(io.IOBase): after having ``.read()`` the file object. (Overridden if ``amt`` is set.) """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + self._init_decoder() if decode_content is None: decode_content = self.decode_content @@ -247,17 +272,7 @@ class HTTPResponse(io.IOBase): self._fp_bytes_read += len(data) - try: - if decode_content and self._decoder: - data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data @@ -284,9 +299,10 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ + self._init_decoder() if self.chunked: for line in self.read_chunked(amt): - yield line + yield self._decode(line, decode_content, True) else: while not is_fp_closed(self._fp): data = self.read(amt=amt, decode_content=decode_content) -- cgit v1.2.1 From 466e5a53be04688899e3376ff136ae49eefa43fc Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 10:13:29 -0500 Subject: Prevent futher regressions in gzipped chunk handling Here we add a quick test where we compress the body of a request and then make a response where the body is also chunked. We show that we properly decode the content after handling the chunked encoding. This should prevent regressions related to #593. --- test/test_response.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/test/test_response.py b/test/test_response.py index e74fd5d3..2e2be0ec 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -418,6 +418,29 @@ class TestResponse(unittest.TestCase): self.assertEqual(c, stream[i]) i += 1 + def test_mock_gzipped_transfer_encoding_chunked_decoded(self): + """Show that we can decode the gizpped and chunked body.""" + def stream(): + # Set up a generator to chunk the gzipped body + import zlib + compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + data = compress.compress(b'foobar') + data += compress.flush() + for i in range(0, len(data), 2): + yield data[i:i+2] + + fp = MockChunkedEncodingResponse(list(stream())) + r = httplib.HTTPResponse(MockSock) + r.fp = fp + headers = {'transfer-encoding': 'chunked', 'content-encoding': 'gzip'} + resp = HTTPResponse(r, preload_content=False, headers=headers) + + data = b'' + for c in resp.stream(decode_content=True): + data += c + + self.assertEqual(b'foobar', data) + def test_mock_transfer_encoding_chunked_custom_read(self): stream = [b"foooo", b"bbbbaaaaar"] fp = MockChunkedEncodingResponse(stream) @@ -517,7 +540,9 @@ class MockChunkedEncodingResponse(object): @staticmethod def _encode_chunk(chunk): - return '%X\r\n%s\r\n' % (len(chunk), chunk.decode()) + # In the general case, we can't decode the chunk to unicode + length = '%X\r\n' % len(chunk) + return length.encode() + chunk + b'\r\n' def _pop_new_chunk(self): if self.chunks_exhausted: @@ -529,8 +554,10 @@ class MockChunkedEncodingResponse(object): self.chunks_exhausted = True else: self.index += 1 - encoded_chunk = self._encode_chunk(chunk) - return encoded_chunk.encode() + chunk = self._encode_chunk(chunk) + if not isinstance(chunk, bytes): + chunk = chunk.encode() + return chunk def pop_current_chunk(self, amt=-1, till_crlf=False): if amt > 0 and till_crlf: -- cgit v1.2.1 From 7d5206f2d9df64ad6910cdcf084bc7c0d8b4fe44 Mon Sep 17 00:00:00 2001 From: matejcik Date: Thu, 23 Apr 2015 18:48:49 +0200 Subject: py2.6 threading fix --- dummyserver/testcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 710cf149..67e62cfe 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -30,7 +30,8 @@ class SocketDummyServerTestCase(unittest.TestCase): ready_event=ready_event, host=cls.host) cls.server_thread.start() - if not ready_event.wait(5): + ready_event.wait(5) + if not ready_event.is_set(): raise Exception("most likely failed to start server") cls.port = cls.server_thread.port -- cgit v1.2.1 From 54138d34616421753fc159d701d1eb054d5131ce Mon Sep 17 00:00:00 2001 From: matejcik Date: Thu, 23 Apr 2015 19:12:52 +0200 Subject: move header-case-sensitivity test to socket-based --- test/with_dummyserver/test_connectionpool.py | 8 -------- test/with_dummyserver/test_socketlevel.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index a789e9bd..e76fcb7f 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -618,14 +618,6 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertRaises(ProtocolError, pool.request, 'GET', '/source_address') - @onlyPy3 - def test_httplib_headers_case_insensitive(self): - HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain', - 'Server': 'TornadoServer/%s' % tornado.version} - r = self.pool.request('GET', '/specific_method', - fields={'method': 'GET'}) - self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity - class TestRetry(HTTPDummyServerTestCase): def setUp(self): diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index db481d30..9872249e 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -18,6 +18,8 @@ from dummyserver.testcase import SocketDummyServerTestCase from dummyserver.server import ( DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address) +from .. import onlyPy3 + from nose.plugins.skip import SkipTest from threading import Event import socket @@ -598,3 +600,19 @@ class TestErrorWrapping(SocketDummyServerTestCase): self._start_server(handler) pool = HTTPConnectionPool(self.host, self.port, retries=False) self.assertRaises(ProtocolError, pool.request, 'GET', '/') + +class TestHeaders(SocketDummyServerTestCase): + + @onlyPy3 + def test_httplib_headers_case_insensitive(self): + handler = create_response_handler( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + ) + self._start_server(handler) + pool = HTTPConnectionPool(self.host, self.port, retries=False) + HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain'} + r = pool.request('GET', '/') + self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity -- cgit v1.2.1 From 69d6d377f23bf2644c40f6628f26c80b33e8bcd1 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Thu, 23 Apr 2015 14:42:51 -0400 Subject: Tornado 4 --- dev-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2eb58751..9ea36913 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,6 +4,4 @@ coverage==3.7.1 tox==1.7.1 twine==1.3.1 wheel==0.24.0 - -# Tornado 3.2.2 makes our tests flaky, so we stick with 3.1 -tornado==3.1.1 +tornado==4.1 -- cgit v1.2.1 From 6ecc5f392a0ed66be3bef519d8a50f6395ff5a77 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Thu, 23 Apr 2015 14:52:36 -0400 Subject: CHANGES for #594 and #595. --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 33e802ab..4c198694 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,10 @@ Changes dev (master) ++++++++++++ +* Fix streaming decoding regression. (Issue #595) + +* Migrate tests to Tornado 4. (Issue #594) + * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From f3c01ff5a9a6f78b6bee44ecbf26b92fa330f28b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 21:56:37 -0500 Subject: Refactor fix for 593 This refactors both the work in 2cfc21644ae868e0ac8b5f24a27eabda8a1ebc1e and the work that originally caused the bug. This splits out some of the responsibility for handling chunks sent over the wire to two other methods. We move the compression handling to the read_chunked method since some users may actually wish to use that. By adding the optional decode_content parameter, we allow stream to simply pass the value through to read_chunked. --- urllib3/response.py | 97 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index f1ea9bb5..eb30bd10 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -299,10 +299,9 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - self._init_decoder() if self.chunked: - for line in self.read_chunked(amt): - yield self._decode(line, decode_content, True) + for line in self.read_chunked(amt, decode_content=decode_content): + yield line else: while not is_fp_closed(self._fp): data = self.read(amt=amt, decode_content=decode_content) @@ -387,48 +386,67 @@ class HTTPResponse(io.IOBase): b[:len(temp)] = temp return len(temp) - def read_chunked(self, amt=None): + def _get_next_chunk(self): + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is None: + line = self._fp.fp.readline() + line = line.decode() + # See RFC 7230: Chunked Transfer Coding. + i = line.find(';') + if i >= 0: + line = line[:i] # Strip chunk-extensions. + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(''.join(line)) + + def _handle_chunk(self, amt): + returned_chunk = None + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + returned_chunk = chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + returned_chunk = value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + returned_chunk = value + else: # amt > self.chunk_left + returned_chunk = self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + return returned_chunk + + def read_chunked(self, amt=None, decode_content=None): + """ + Similar to :meth:`HTTPResponse.read`, but with an additional + parameter: ``decode_content``. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + self._init_decoder() # FIXME: Rewrite this method and make it a class with # a better structured logic. if not self.chunked: raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") while True: - # First, we'll figure out length of a chunk and then - # we'll try to read it from socket. - if self.chunk_left is None: - line = self._fp.fp.readline() - line = line.decode() - # See RFC 7230: Chunked Transfer Coding. - i = line.find(';') - if i >= 0: - line = line[:i] # Strip chunk-extensions. - try: - self.chunk_left = int(line, 16) - except ValueError: - # Invalid chunked protocol response, abort. - self.close() - raise httplib.IncompleteRead(''.join(line)) - if self.chunk_left == 0: - break - if amt is None: - chunk = self._fp._safe_read(self.chunk_left) - yield chunk - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - elif amt < self.chunk_left: - value = self._fp._safe_read(amt) - self.chunk_left = self.chunk_left - amt - yield value - elif amt == self.chunk_left: - value = self._fp._safe_read(amt) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - yield value - else: # amt > self.chunk_left - yield self._fp._safe_read(self.chunk_left) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None + self._get_next_chunk() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) + yield self._decode(chunk, decode_content=decode_content, + flush_decoder=True) # Chunk content ends with \r\n: discard it. while True: @@ -441,4 +459,3 @@ class HTTPResponse(io.IOBase): # We read everything; close the "file". self.release_conn() - -- cgit v1.2.1 From 2a6d4e2b570d7f809baea4ea4cb007aed5202767 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 22:15:26 -0500 Subject: Fix check for chunked transfer-encoding The Transfer-Encoding header can contain many values separated by a comma, e.g., Transfer-Encoding: chunked, gzip If we want to check for chunked transfer encodings we're better off not looking for equality since the above header value also indicates a chunked body. Instead, we need to split on ", " to ensure we can accurately determine if a response is chunked. --- urllib3/response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index eb30bd10..24400b38 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -126,8 +126,8 @@ class HTTPResponse(io.IOBase): # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '') - if tr_enc.lower() == "chunked": + tr_enc = self.headers.get('transfer-encoding', '').lower() + if "chunked" in set(tr_enc.split(", ")): self.chunked = True # We certainly don't want to preload content when the response is chunked. -- cgit v1.2.1 From 23060d67836f71a838e68d6f3889baacf6a530c9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 24 Apr 2015 08:49:10 -0500 Subject: Do not call join with a bytes object line is no longer stored as a list. Since it will now always be a bytes object we cannot call `''.join(line)` since it will iterate over the bytes object and try to concatenate integers which are not valid to be passed to str.join. --- urllib3/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/response.py b/urllib3/response.py index 24400b38..191dc558 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -401,7 +401,7 @@ class HTTPResponse(io.IOBase): except ValueError: # Invalid chunked protocol response, abort. self.close() - raise httplib.IncompleteRead(''.join(line)) + raise httplib.IncompleteRead(line) def _handle_chunk(self, amt): returned_chunk = None -- cgit v1.2.1 From bdabaafdb0ad707aff42a79c03842b6a090da485 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 24 Apr 2015 12:12:56 -0500 Subject: Reorder logic of _update_chunk_length By bailing out of the method early if the chunk_size is not None, we can make the rest of the method easier to read. --- urllib3/response.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 191dc558..88d823dc 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -389,19 +389,16 @@ class HTTPResponse(io.IOBase): def _get_next_chunk(self): # First, we'll figure out length of a chunk and then # we'll try to read it from socket. - if self.chunk_left is None: - line = self._fp.fp.readline() - line = line.decode() - # See RFC 7230: Chunked Transfer Coding. - i = line.find(';') - if i >= 0: - line = line[:i] # Strip chunk-extensions. - try: - self.chunk_left = int(line, 16) - except ValueError: - # Invalid chunked protocol response, abort. - self.close() - raise httplib.IncompleteRead(line) + if self.chunk_left is not None: + return + line = self._fp.fp.readline() + line = line.split(b';', 1)[0] + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(line) def _handle_chunk(self, amt): returned_chunk = None -- cgit v1.2.1 From 3f36b3f8ce593fc16d5cd14d3cc5389f0e30d489 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 24 Apr 2015 12:15:56 -0500 Subject: Do not convert the transfer-encodings to a set --- urllib3/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/response.py b/urllib3/response.py index 88d823dc..25d3d555 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -127,7 +127,7 @@ class HTTPResponse(io.IOBase): self.chunked = False self.chunk_left = None tr_enc = self.headers.get('transfer-encoding', '').lower() - if "chunked" in set(tr_enc.split(", ")): + if "chunked" in tr_enc.split(", "): self.chunked = True # We certainly don't want to preload content when the response is chunked. -- cgit v1.2.1 From 6c1dcdb8b2ff216409f7c654f22155c001c0fb61 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 24 Apr 2015 18:31:07 -0500 Subject: Rename _get_next_chunk to _update_chunk_length --- urllib3/response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 25d3d555..0f619cbc 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -386,7 +386,7 @@ class HTTPResponse(io.IOBase): b[:len(temp)] = temp return len(temp) - def _get_next_chunk(self): + def _update_chunk_length(self): # First, we'll figure out length of a chunk and then # we'll try to read it from socket. if self.chunk_left is not None: @@ -438,7 +438,7 @@ class HTTPResponse(io.IOBase): raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") while True: - self._get_next_chunk() + self._update_chunk_length() if self.chunk_left == 0: break chunk = self._handle_chunk(amt) -- cgit v1.2.1 From 563f0e8b7a3e32f034cbf41474ad25cff2fccbbd Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 25 Apr 2015 08:10:06 -0500 Subject: Split then strip the transfer-encoding list It is more reliable to split on "," and then strip the whitespace around each part of the Transfer-Encoding than it is to split on ", ". Further, since we don't have to create a list, we just setup a generator to allow us to inspect the Transfer-Encodings the server returns. --- urllib3/response.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/urllib3/response.py b/urllib3/response.py index 0f619cbc..9b262efd 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -127,7 +127,9 @@ class HTTPResponse(io.IOBase): self.chunked = False self.chunk_left = None tr_enc = self.headers.get('transfer-encoding', '').lower() - if "chunked" in tr_enc.split(", "): + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: self.chunked = True # We certainly don't want to preload content when the response is chunked. -- cgit v1.2.1 From b06e1b72643255250c752286f502e93efee80e7c Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 25 Apr 2015 08:43:13 -0500 Subject: Simplify a couple conditions There were two conditions added that could be simplified into additive conditionals. This is more cosmetic than anything else. --- urllib3/response.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 9b262efd..79a5e789 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -133,9 +133,8 @@ class HTTPResponse(io.IOBase): self.chunked = True # We certainly don't want to preload content when the response is chunked. - if not self.chunked: - if preload_content and not self._body: - self._body = self.read(decode_content=decode_content) + if not self.chunked and preload_content and not self._body: + self._body = self.read(decode_content=decode_content) def get_redirect_location(self): """ @@ -181,9 +180,8 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ -- cgit v1.2.1 From c92fe455117c2de5224d150bf89d3fbca0c17f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Sun, 26 Apr 2015 10:03:50 +0000 Subject: test keep-alive chunked requests see #598 --- dummyserver/handlers.py | 21 +++++++++++++++++---- test/with_dummyserver/test_connectionpool.py | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 80e951ec..755a380d 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -21,9 +21,6 @@ log = logging.getLogger(__name__) class Response(object): def __init__(self, body='', status='200 OK', headers=None): - if not isinstance(body, bytes): - body = body.encode('utf8') - self.body = body self.status = status self.headers = headers or [("Content-type", "text/plain")] @@ -34,7 +31,20 @@ class Response(object): for header,value in self.headers: request_handler.add_header(header,value) - request_handler.write(self.body) + # chunked + if isinstance(self.body, list): + for item in self.body: + if not isinstance(item, bytes): + item = item.encode('utf8') + request_handler.write(item) + request_handler.flush() + else: + body = self.body + if not isinstance(body, bytes): + body = body.encode('utf8') + + request_handler.write(body) + RETRY_TEST_NAMES = collections.defaultdict(int) @@ -208,6 +218,9 @@ class TestingApp(RequestHandler): else: return Response("need to keep retrying!", status="418 I'm A Teapot") + def chunked(self, request): + return Response(['123'] * 4) + def shutdown(self, request): sys.exit() diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index e76fcb7f..ecf209be 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -618,6 +618,25 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertRaises(ProtocolError, pool.request, 'GET', '/source_address') + def test_stream_keepalive(self): + x = 2 + + for _ in range(x): + response = self.pool.request( + 'GET', + '/chunked', + headers={ + 'Connection': 'keep-alive', + }, + preload_content=False, + retries=0, + ) + for chunk in response.stream(): + self.assertEqual(chunk, b'123') + + self.assertEqual(self.pool.num_connections, 1) + self.assertEqual(self.pool.num_requests, x) + class TestRetry(HTTPDummyServerTestCase): def setUp(self): -- cgit v1.2.1 From dfb5d499722312274ff69cd67c6617b594d68503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Sun, 26 Apr 2015 10:06:12 +0000 Subject: fix #598 --- urllib3/response.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/urllib3/response.py b/urllib3/response.py index 79a5e789..e0236e5d 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -455,4 +455,6 @@ class HTTPResponse(io.IOBase): break # We read everything; close the "file". + if self._original_response: + self._original_response.close() self.release_conn() -- cgit v1.2.1 From 42b22baf848d4c7844108aeada63d7f7861aa06c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 27 Apr 2015 06:18:28 +0000 Subject: [dummyserver] emit all chunks --- dummyserver/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 755a380d..358f06e8 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -36,8 +36,8 @@ class Response(object): for item in self.body: if not isinstance(item, bytes): item = item.encode('utf8') - request_handler.write(item) - request_handler.flush() + request_handler.write(item) + request_handler.flush() else: body = self.body if not isinstance(body, bytes): -- cgit v1.2.1 From a984fdfea70b4460b9ce4ca93b588a4a436ce3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 27 Apr 2015 16:32:15 +0000 Subject: style.. --- test/with_dummyserver/test_connectionpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index ecf209be..a867c50b 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -629,7 +629,7 @@ class TestConnectionPool(HTTPDummyServerTestCase): 'Connection': 'keep-alive', }, preload_content=False, - retries=0, + retries=False, ) for chunk in response.stream(): self.assertEqual(chunk, b'123') -- cgit v1.2.1 From 98a4af2223cdcca0c9e491ac0fa94c33d127706a Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Mon, 27 Apr 2015 12:58:56 -0400 Subject: CHANGES for #599 --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4c198694..df1311d0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ dev (master) * Fix streaming decoding regression. (Issue #595) +* Fix chunked requests losing state across keep-alive connections. + (Issue #599) + * Migrate tests to Tornado 4. (Issue #594) * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From f9d426b65e8d2d8a663a844801b9574dcba7cd01 Mon Sep 17 00:00:00 2001 From: tlynn Date: Tue, 28 Apr 2015 15:59:07 +0100 Subject: Fix #602: don't override existing warning config at import --- CONTRIBUTORS.txt | 3 +++ urllib3/__init__.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 4a7f70a2..58073075 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -145,6 +145,9 @@ In chronological order: * Tomas Tomecek * Implemented generator for getting chunks from chunked responses. +* tlynn + * Respect the warning preferences at import. + * [Your name or handle] <[email or website]> * [Brief summary of your changes] diff --git a/urllib3/__init__.py b/urllib3/__init__.py index 8cd51444..cf83d7dd 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -57,9 +57,10 @@ del NullHandler import warnings # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning) +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning) +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) def disable_warnings(category=exceptions.HTTPWarning): """ -- cgit v1.2.1 From 7906b5f166da93c61ddd41608cd64d920953a9cd Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 28 Apr 2015 12:08:25 -0400 Subject: CHANGES for #603 --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index df1311d0..3fca6763 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,9 @@ dev (master) * Migrate tests to Tornado 4. (Issue #594) +* Append default warning configuration rather than overwrite. + (Issue #603) + * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From 4917250ab6d5647d505e0f3ea7bcf9f995722392 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 28 Apr 2015 11:16:16 -0500 Subject: Learn from httplib how to handle chunked HEAD requests When we perform a HEAD request and get a response with a chunked transfer-encoding, there's no point in looking for a body at all. We should just close the response immediately. Fixes #601 --- urllib3/response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/urllib3/response.py b/urllib3/response.py index e0236e5d..8c8db76a 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -437,6 +437,12 @@ class HTTPResponse(io.IOBase): if not self.chunked: raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") + + if (self._original_response and + self._original_response._method.upper() == 'HEAD'): + self._original_response.close() + return + while True: self._update_chunk_length() if self.chunk_left == 0: -- cgit v1.2.1 From 1bc3d2003ffbe3464a96a47a6a2ce3e18e8cc593 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 28 Apr 2015 12:23:05 -0400 Subject: Style tweaks. --- urllib3/response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 8c8db76a..24140c4c 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -432,14 +432,14 @@ class HTTPResponse(io.IOBase): 'content-encoding' header. """ self._init_decoder() - # FIXME: Rewrite this method and make it a class with - # a better structured logic. + # FIXME: Rewrite this method and make it a class with a better structured logic. if not self.chunked: raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") - if (self._original_response and - self._original_response._method.upper() == 'HEAD'): + if self._original_response and self._original_response._method.upper() == 'HEAD': + # Don't bother reading the body of a HEAD request. + # FIXME: Can we do this somehow without accessing private httplib _method? self._original_response.close() return -- cgit v1.2.1 From 697362076d23983744602d8682f6fc6b10188e80 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 28 Apr 2015 18:17:34 +0100 Subject: Add tests for HEAD methods --- test/with_dummyserver/test_socketlevel.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 9872249e..6c996538 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -616,3 +616,33 @@ class TestHeaders(SocketDummyServerTestCase): HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain'} r = pool.request('GET', '/') self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity + + +class TestHEAD(SocketDummyServerTestCase): + def test_chunked_head_response_does_not_hang(self): + handler = create_response_handler( + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + ) + self._start_server(handler) + pool = HTTPConnectionPool(self.host, self.port, retries=False) + r = pool.request('HEAD', '/', timeout=1, preload_content=False) + + # stream will use the read_chunked method here. + self.assertEqual([], list(r.stream())) + + def test_empty_head_response_does_not_hang(self): + handler = create_response_handler( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 256\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + ) + self._start_server(handler) + pool = HTTPConnectionPool(self.host, self.port, retries=False) + r = pool.request('HEAD', '/', timeout=1, preload_content=False) + + # stream will use the read method here. + self.assertEqual([], list(r.stream())) -- cgit v1.2.1 From 425a80e5bc9f49a488f007f0579be7b8567f14e7 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 28 Apr 2015 14:26:08 -0400 Subject: CHANGES for #605 --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 3fca6763..cadf9e9b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,8 @@ dev (master) * Append default warning configuration rather than overwrite. (Issue #603) + +* Fix hanging when HEAD response has no body. (Issue #605) * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From 585983ab3f7fb7a0e0b223ebdab1b45471dbefe4 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 28 Apr 2015 14:28:09 -0400 Subject: Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cadf9e9b..1a99f1b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,7 +14,7 @@ dev (master) * Append default warning configuration rather than overwrite. (Issue #603) -* Fix hanging when HEAD response has no body. (Issue #605) +* Fix hanging when chunked HEAD response has no body. (Issue #605) * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From 3f54764e66784e2311e2ba28f8b13d8c8083ecc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 29 Apr 2015 22:03:10 +0000 Subject: test chunked response with gzip encoding duplicates #441 --- dummyserver/handlers.py | 11 +++++++++++ test/with_dummyserver/test_connectionpool.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 358f06e8..53fbe4a8 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -221,6 +221,17 @@ class TestingApp(RequestHandler): def chunked(self, request): return Response(['123'] * 4) + def chunked_gzip(self, request): + chunks = [] + compressor = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) + + for uncompressed in [b'123'] * 4: + chunks.append(compressor.compress(uncompressed)) + + chunks.append(compressor.flush()) + + return Response(chunks, headers=[('Content-Encoding', 'gzip')]) + def shutdown(self, request): sys.exit() diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index a867c50b..d6cb1622 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -637,6 +637,16 @@ class TestConnectionPool(HTTPDummyServerTestCase): self.assertEqual(self.pool.num_connections, 1) self.assertEqual(self.pool.num_requests, x) + def test_chunked_gzip(self): + response = self.pool.request( + 'GET', + '/chunked_gzip', + preload_content=False, + decode_content=True, + ) + + self.assertEqual(b'123' * 4, response.read()) + class TestRetry(HTTPDummyServerTestCase): def setUp(self): -- cgit v1.2.1 From e4c89df894548648624699286fc2dbd196ec21fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 29 Apr 2015 22:47:17 +0000 Subject: suppress spurious resource warnings while testing --- test/with_dummyserver/test_https.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index f3e00acf..992b8ef1 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -32,9 +32,15 @@ from urllib3.exceptions import ( SystemTimeWarning, InsecurePlatformWarning, ) +from urllib3.packages import six from urllib3.util.timeout import Timeout +ResourceWarning = getattr( + six.moves.builtins, + 'ResourceWarning', type('ResourceWarning', (), {})) + + log = logging.getLogger('urllib3.connectionpool') log.setLevel(logging.NOTSET) log.addHandler(logging.StreamHandler(sys.stdout)) @@ -362,10 +368,8 @@ class TestHTTPS(HTTPSDummyServerTestCase): def test_ssl_correct_system_time(self): self._pool.cert_reqs = 'CERT_REQUIRED' self._pool.ca_certs = DEFAULT_CA - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self._pool.request('GET', '/') + w = self._request_without_resource_warnings('GET', '/') self.assertEqual([], w) def test_ssl_wrong_system_time(self): @@ -374,9 +378,7 @@ class TestHTTPS(HTTPSDummyServerTestCase): with mock.patch('urllib3.connection.datetime') as mock_date: mock_date.date.today.return_value = datetime.date(1970, 1, 1) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self._pool.request('GET', '/') + w = self._request_without_resource_warnings('GET', '/') self.assertEqual(len(w), 1) warning = w[0] @@ -384,6 +386,13 @@ class TestHTTPS(HTTPSDummyServerTestCase): self.assertEqual(SystemTimeWarning, warning.category) self.assertTrue(str(RECENT_DATE) in warning.message.args[0]) + def _request_without_resource_warnings(self, method, url): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self._pool.request(method, url) + + return [x for x in w if not isinstance(x.message, ResourceWarning)] + class TestHTTPS_TLSv1(HTTPSDummyServerTestCase): certs = DEFAULT_CERTS.copy() -- cgit v1.2.1 From 7fc5e87c04889eaa1f298af10aa2dc1c82b8429e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 29 Apr 2015 22:53:22 +0000 Subject: show openssl version in travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index abb27b44..fc88d9d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ python: - "3.4" - "pypy" script: make test +before_install: + - openssl version install: - make notifications: -- cgit v1.2.1 From cedab2b040d19e8cbf88482252f4bd0c25a58488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Thu, 30 Apr 2015 20:57:51 +0000 Subject: don't allow path in urls which do not begin with / It was possible to create such an object, but it would not serialize -> parse roundtrip. The alternative would be to reject paths without a leading / --- test/test_util.py | 1 + urllib3/util/url.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/test/test_util.py b/test/test_util.py index ba26af5a..19ba57e3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -97,6 +97,7 @@ class TestUtil(unittest.TestCase): parse_url_host_map = { 'http://google.com/mail': Url('http', host='google.com', path='/mail'), 'http://google.com/mail/': Url('http', host='google.com', path='/mail/'), + 'http://google.com/mail': Url('http', host='google.com', path='mail'), 'google.com/mail': Url(host='google.com', path='/mail'), 'http://google.com/': Url('http', host='google.com', path='/'), 'http://google.com': Url('http', host='google.com'), diff --git a/urllib3/util/url.py b/urllib3/util/url.py index b2ec834f..e58050cd 100644 --- a/urllib3/util/url.py +++ b/urllib3/util/url.py @@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)): def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): + if path and not path.startswith('/'): + path = '/' + path return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) -- cgit v1.2.1 From 42141fb0df0188d0eb8ec22e2f5d7a69f61322b3 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 3 May 2015 10:09:08 -0400 Subject: Sorted CHANGES. --- CHANGES.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1a99f1b6..343c343b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,16 +4,16 @@ Changes dev (master) ++++++++++++ -* Fix streaming decoding regression. (Issue #595) - -* Fix chunked requests losing state across keep-alive connections. - (Issue #599) - * Migrate tests to Tornado 4. (Issue #594) * Append default warning configuration rather than overwrite. (Issue #603) +* Fix streaming decoding regression. (Issue #595) + +* Fix chunked requests losing state across keep-alive connections. + (Issue #599) + * Fix hanging when chunked HEAD response has no body. (Issue #605) * ... [Short description of non-trivial change.] (Issue #) -- cgit v1.2.1 From a91975b77a2e28394859487fe5ebbf4a3a74e634 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 3 May 2015 10:12:16 -0400 Subject: CHANGES for v1.10.4 --- CHANGES.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 343c343b..eca2049a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,19 +4,23 @@ Changes dev (master) ++++++++++++ +* ... [Short description of non-trivial change.] (Issue #) + + +1.10.4 (2015-05-03) ++++++++++++++++++++ + * Migrate tests to Tornado 4. (Issue #594) * Append default warning configuration rather than overwrite. (Issue #603) - + * Fix streaming decoding regression. (Issue #595) * Fix chunked requests losing state across keep-alive connections. (Issue #599) - -* Fix hanging when chunked HEAD response has no body. (Issue #605) -* ... [Short description of non-trivial change.] (Issue #) +* Fix hanging when chunked HEAD response has no body. (Issue #605) 1.10.3 (2015-04-21) @@ -24,16 +28,16 @@ dev (master) * Emit ``InsecurePlatformWarning`` when SSLContext object is missing. (Issue #558) - + * Fix regression of duplicate header keys being discarded. (Issue #563) - + * ``Response.stream()`` returns a generator for chunked responses. (Issue #560) - + * Set upper-bound timeout when waiting for a socket in PyOpenSSL. (Issue #585) - + * Work on platforms without `ssl` module for plain HTTP requests. (Issue #587) @@ -107,7 +111,7 @@ dev (master) * Fixed packaging issues of some development-related files not getting included. (Issue #440) - + * Allow performing *only* fingerprint verification. (Issue #444) * Emit ``SecurityWarning`` if system clock is waaay off. (Issue #445) -- cgit v1.2.1