diff options
Diffstat (limited to 'mercurial/httpclient')
| -rw-r--r-- | mercurial/httpclient/__init__.py | 214 | ||||
| -rw-r--r-- | mercurial/httpclient/_readers.py | 195 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/__init__.py | 1 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/simple_http_test.py | 386 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/test_bogus_responses.py | 68 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/test_chunked_transfer.py | 137 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/test_proxy_support.py | 135 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/test_ssl.py | 93 | ||||
| -rw-r--r-- | mercurial/httpclient/tests/util.py | 195 |
9 files changed, 1134 insertions, 290 deletions
diff --git a/mercurial/httpclient/__init__.py b/mercurial/httpclient/__init__.py index f5c3baf..227d60b 100644 --- a/mercurial/httpclient/__init__.py +++ b/mercurial/httpclient/__init__.py @@ -45,7 +45,6 @@ import rfc822 import select import socket -import _readers import socketutil logger = logging.getLogger(__name__) @@ -55,6 +54,8 @@ __all__ = ['HTTPConnection', 'HTTPResponse'] HTTP_VER_1_0 = 'HTTP/1.0' HTTP_VER_1_1 = 'HTTP/1.1' +_LEN_CLOSE_IS_END = -1 + OUTGOING_BUFFER_SIZE = 1 << 15 INCOMING_BUFFER_SIZE = 1 << 20 @@ -82,19 +83,23 @@ class HTTPResponse(object): The response will continue to load as available. If you need the complete response before continuing, check the .complete() method. """ - def __init__(self, sock, timeout, method): + def __init__(self, sock, timeout): self.sock = sock - self.method = method self.raw_response = '' + self._body = None self._headers_len = 0 + self._content_len = 0 self.headers = None self.will_close = False self.status_line = '' self.status = None - self.continued = False self.http_version = None self.reason = None - self._reader = None + self._chunked = False + self._chunked_done = False + self._chunked_until_next = 0 + self._chunked_skip_bytes = 0 + self._chunked_preloaded_block = None self._read_location = 0 self._eol = EOL @@ -112,12 +117,11 @@ class HTTPResponse(object): socket is closed, this will nearly always return False, even in cases where all the data has actually been loaded. """ - if self._reader: - return self._reader.done() - - def _close(self): - if self._reader is not None: - self._reader._close() + if self._chunked: + return self._chunked_done + if self._content_len == _LEN_CLOSE_IS_END: + return False + return self._body is not None and len(self._body) >= self._content_len def readline(self): """Read a single line from the response body. @@ -125,34 +129,30 @@ class HTTPResponse(object): This may block until either a line ending is found or the response is complete. """ - # TODO: move this into the reader interface where it can be - # smarter (and probably avoid copies) - bytes = [] - while not bytes: - try: - bytes = [self._reader.read(1)] - except _readers.ReadNotReady: - self._select() - while bytes[-1] != '\n' and not self.complete(): + eol = self._body.find('\n', self._read_location) + while eol == -1 and not self.complete(): self._select() - bytes.append(self._reader.read(1)) - if bytes[-1] != '\n': - next = self._reader.read(1) - while next and next != '\n': - bytes.append(next) - next = self._reader.read(1) - bytes.append(next) - return ''.join(bytes) + eol = self._body.find('\n', self._read_location) + if eol != -1: + eol += 1 + else: + eol = len(self._body) + data = self._body[self._read_location:eol] + self._read_location = eol + return data def read(self, length=None): # if length is None, unbounded read while (not self.complete() # never select on a finished read and (not length # unbounded, so we wait for complete() - or length > self._reader.available_data)): + or (self._read_location + length) > len(self._body))): self._select() if not length: - length = self._reader.available_data - r = self._reader.read(length) + length = len(self._body) - self._read_location + elif len(self._body) < (self._read_location + length): + length = len(self._body) - self._read_location + r = self._body[self._read_location:self._read_location + length] + self._read_location += len(r) if self.complete() and self.will_close: self.sock.close() return r @@ -160,11 +160,15 @@ class HTTPResponse(object): def _select(self): r, _, _ = select.select([self.sock], [], [], self._timeout) if not r: - # socket was not readable. If the response is not - # complete, raise a timeout. - if not self.complete(): + # socket was not readable. If the response is not complete + # and we're not a _LEN_CLOSE_IS_END response, raise a timeout. + # If we are a _LEN_CLOSE_IS_END response and we have no data, + # raise a timeout. + if not (self.complete() or + (self._content_len == _LEN_CLOSE_IS_END and self._body)): logger.info('timed out with timeout of %s', self._timeout) raise HTTPTimeoutException('timeout reading data') + logger.info('cl: %r body: %r', self._content_len, self._body) try: data = self.sock.recv(INCOMING_BUFFER_SIZE) except socket.sslerror, e: @@ -173,22 +177,68 @@ class HTTPResponse(object): logger.debug('SSL_WANT_READ in _select, should retry later') return True logger.debug('response read %d data during _select', len(data)) - # If the socket was readable and no data was read, that means - # the socket was closed. Inform the reader (if any) so it can - # raise an exception if this is an invalid situation. if not data: - if self._reader: - self._reader._close() + if self.headers and self._content_len == _LEN_CLOSE_IS_END: + self._content_len = len(self._body) return False else: self._load_response(data) return True + def _chunked_parsedata(self, data): + if self._chunked_preloaded_block: + data = self._chunked_preloaded_block + data + self._chunked_preloaded_block = None + while data: + logger.debug('looping with %d data remaining', len(data)) + # Slice out anything we should skip + if self._chunked_skip_bytes: + if len(data) <= self._chunked_skip_bytes: + self._chunked_skip_bytes -= len(data) + data = '' + break + else: + data = data[self._chunked_skip_bytes:] + self._chunked_skip_bytes = 0 + + # determine how much is until the next chunk + if self._chunked_until_next: + amt = self._chunked_until_next + logger.debug('reading remaining %d of existing chunk', amt) + self._chunked_until_next = 0 + body = data + else: + try: + amt, body = data.split(self._eol, 1) + except ValueError: + self._chunked_preloaded_block = data + logger.debug('saving %r as a preloaded block for chunked', + self._chunked_preloaded_block) + return + amt = int(amt, base=16) + logger.debug('reading chunk of length %d', amt) + if amt == 0: + self._chunked_done = True + + # read through end of what we have or the chunk + self._body += body[:amt] + if len(body) >= amt: + data = body[amt:] + self._chunked_skip_bytes = len(self._eol) + else: + self._chunked_until_next = amt - len(body) + self._chunked_skip_bytes = 0 + data = '' + def _load_response(self, data): - # Being here implies we're not at the end of the headers yet, - # since at the end of this method if headers were completely - # loaded we replace this method with the load() method of the - # reader we created. + if self._chunked: + self._chunked_parsedata(data) + return + elif self._body is not None: + self._body += data + return + + # We haven't seen end of headers yet self.raw_response += data # This is a bogus server with bad line endings if self._eol not in self.raw_response: @@ -212,7 +262,6 @@ class HTTPResponse(object): http_ver, status = hdrs.split(' ', 1) if status.startswith('100'): self.raw_response = body - self.continued = True logger.debug('continue seen, setting body to %r', body) return @@ -232,46 +281,23 @@ class HTTPResponse(object): if self._eol != EOL: hdrs = hdrs.replace(self._eol, '\r\n') headers = rfc822.Message(cStringIO.StringIO(hdrs)) - content_len = None if HDR_CONTENT_LENGTH in headers: - content_len = int(headers[HDR_CONTENT_LENGTH]) + self._content_len = int(headers[HDR_CONTENT_LENGTH]) if self.http_version == HTTP_VER_1_0: self.will_close = True elif HDR_CONNECTION_CTRL in headers: self.will_close = ( headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE) + if self._content_len == 0: + self._content_len = _LEN_CLOSE_IS_END if (HDR_XFER_ENCODING in headers and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED): - self._reader = _readers.ChunkedReader(self._eol) - logger.debug('using a chunked reader') - else: - # HEAD responses are forbidden from returning a body, and - # it's implausible for a CONNECT response to use - # close-is-end logic for an OK response. - if (self.method == 'HEAD' or - (self.method == 'CONNECT' and content_len is None)): - content_len = 0 - if content_len is not None: - logger.debug('using a content-length reader with length %d', - content_len) - self._reader = _readers.ContentLengthReader(content_len) - else: - # Response body had no length specified and is not - # chunked, so the end of the body will only be - # identifiable by the termination of the socket by the - # server. My interpretation of the spec means that we - # are correct in hitting this case if - # transfer-encoding, content-length, and - # connection-control were left unspecified. - self._reader = _readers.CloseIsEndReader() - logger.debug('using a close-is-end reader') - self.will_close = True - - if body: - self._reader._load(body) - logger.debug('headers complete') + self._body = '' + self._chunked_parsedata(body) + self._chunked = True + if self._body is None: + self._body = body self.headers = headers - self._load_response = self._reader._load class HTTPConnection(object): @@ -348,14 +374,13 @@ class HTTPConnection(object): {}, HTTP_VER_1_0) sock.send(data) sock.setblocking(0) - r = self.response_class(sock, self.timeout, 'CONNECT') + r = self.response_class(sock, self.timeout) timeout_exc = HTTPTimeoutException( 'Timed out waiting for CONNECT response from proxy') while not r.complete(): try: if not r._select(): - if not r.complete(): - raise timeout_exc + raise timeout_exc except HTTPTimeoutException: # This raise/except pattern looks goofy, but # _select can raise the timeout as well as the @@ -372,10 +397,6 @@ class HTTPConnection(object): else: sock = socketutil.create_connection((self.host, self.port)) if self.ssl: - # This is the default, but in the case of proxied SSL - # requests the proxy logic above will have cleared - # blocking mode, so reenable it just to be safe. - sock.setblocking(1) logger.debug('wrapping socket for ssl with options %r', self.ssl_opts) sock = socketutil.wrap_socket(sock, **self.ssl_opts) @@ -498,7 +519,7 @@ class HTTPConnection(object): out = outgoing_headers or body blocking_on_continue = False if expect_continue and not outgoing_headers and not ( - response and (response.headers or response.continued)): + response and response.headers): logger.info( 'waiting up to %s seconds for' ' continue response from server', @@ -521,6 +542,11 @@ class HTTPConnection(object): 'server, optimistically sending request body') else: raise HTTPTimeoutException('timeout sending data') + # TODO exceptional conditions with select? (what are those be?) + # TODO if the response is loading, must we finish sending at all? + # + # Certainly not if it's going to close the connection and/or + # the response is already done...I think. was_first = first # incoming data @@ -538,11 +564,11 @@ class HTTPConnection(object): logger.info('socket appears closed in read') self.sock = None self._current_response = None - if response is not None: - response._close() # This if/elif ladder is a bit subtle, # comments in each branch should help. - if response is not None and response.complete(): + if response is not None and ( + response.complete() or + response._content_len == _LEN_CLOSE_IS_END): # Server responded completely and then # closed the socket. We should just shut # things down and let the caller get their @@ -571,7 +597,7 @@ class HTTPConnection(object): 'response was missing or incomplete!') logger.debug('read %d bytes in request()', len(data)) if response is None: - response = self.response_class(r[0], self.timeout, method) + response = self.response_class(r[0], self.timeout) response._load_response(data) # Jump to the next select() call so we load more # data if the server is still sending us content. @@ -579,6 +605,10 @@ class HTTPConnection(object): except socket.error, e: if e[0] != errno.EPIPE and not was_first: raise + if (response._content_len + and response._content_len != _LEN_CLOSE_IS_END): + outgoing_headers = sent_data + outgoing_headers + reconnect('read') # outgoing data if w and out: @@ -623,7 +653,7 @@ class HTTPConnection(object): # close if the server response said to or responded before eating # the whole request if response is None: - response = self.response_class(self.sock, self.timeout, method) + response = self.response_class(self.sock, self.timeout) complete = response.complete() data_left = bool(outgoing_headers or body) if data_left: @@ -641,8 +671,7 @@ class HTTPConnection(object): raise httplib.ResponseNotReady() r = self._current_response while r.headers is None: - if not r._select() and not r.complete(): - raise _readers.HTTPRemoteClosedError() + r._select() if r.will_close: self.sock = None self._current_response = None @@ -664,11 +693,6 @@ class BadRequestData(httplib.HTTPException): class HTTPProxyConnectFailedException(httplib.HTTPException): """Connecting to the HTTP proxy failed.""" - class HTTPStateError(httplib.HTTPException): """Invalid internal state encountered.""" - -# Forward this exception type from _readers since it needs to be part -# of the public API. -HTTPRemoteClosedError = _readers.HTTPRemoteClosedError # no-check-code diff --git a/mercurial/httpclient/_readers.py b/mercurial/httpclient/_readers.py deleted file mode 100644 index 0beb551..0000000 --- a/mercurial/httpclient/_readers.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Reader objects to abstract out different body response types. - -This module is package-private. It is not expected that these will -have any clients outside of httpplus. -""" - -import httplib -import itertools -import logging - -logger = logging.getLogger(__name__) - - -class ReadNotReady(Exception): - """Raised when read() is attempted but not enough data is loaded.""" - - -class HTTPRemoteClosedError(httplib.HTTPException): - """The server closed the remote socket in the middle of a response.""" - - -class AbstractReader(object): - """Abstract base class for response readers. - - Subclasses must implement _load, and should implement _close if - it's not an error for the server to close their socket without - some termination condition being detected during _load. - """ - def __init__(self): - self._finished = False - self._done_chunks = [] - - @property - def available_data(self): - return sum(map(len, self._done_chunks)) - - def done(self): - return self._finished - - def read(self, amt): - if self.available_data < amt and not self._finished: - raise ReadNotReady() - need = [amt] - def pred(s): - needed = need[0] > 0 - need[0] -= len(s) - return needed - blocks = list(itertools.takewhile(pred, self._done_chunks)) - self._done_chunks = self._done_chunks[len(blocks):] - over_read = sum(map(len, blocks)) - amt - if over_read > 0 and blocks: - logger.debug('need to reinsert %d data into done chunks', over_read) - last = blocks[-1] - blocks[-1], reinsert = last[:-over_read], last[-over_read:] - self._done_chunks.insert(0, reinsert) - result = ''.join(blocks) - assert len(result) == amt or (self._finished and len(result) < amt) - return result - - def _load(self, data): # pragma: no cover - """Subclasses must implement this. - - As data is available to be read out of this object, it should - be placed into the _done_chunks list. Subclasses should not - rely on data remaining in _done_chunks forever, as it may be - reaped if the client is parsing data as it comes in. - """ - raise NotImplementedError - - def _close(self): - """Default implementation of close. - - The default implementation assumes that the reader will mark - the response as finished on the _finished attribute once the - entire response body has been read. In the event that this is - not true, the subclass should override the implementation of - close (for example, close-is-end responses have to set - self._finished in the close handler.) - """ - if not self._finished: - raise HTTPRemoteClosedError( - 'server appears to have closed the socket mid-response') - - -class AbstractSimpleReader(AbstractReader): - """Abstract base class for simple readers that require no response decoding. - - Examples of such responses are Connection: Close (close-is-end) - and responses that specify a content length. - """ - def _load(self, data): - if data: - assert not self._finished, ( - 'tried to add data (%r) to a closed reader!' % data) - logger.debug('%s read an addtional %d data', self.name, len(data)) - self._done_chunks.append(data) - - -class CloseIsEndReader(AbstractSimpleReader): - """Reader for responses that specify Connection: Close for length.""" - name = 'close-is-end' - - def _close(self): - logger.info('Marking close-is-end reader as closed.') - self._finished = True - - -class ContentLengthReader(AbstractSimpleReader): - """Reader for responses that specify an exact content length.""" - name = 'content-length' - - def __init__(self, amount): - AbstractReader.__init__(self) - self._amount = amount - if amount == 0: - self._finished = True - self._amount_seen = 0 - - def _load(self, data): - AbstractSimpleReader._load(self, data) - self._amount_seen += len(data) - if self._amount_seen >= self._amount: - self._finished = True - logger.debug('content-length read complete') - - -class ChunkedReader(AbstractReader): - """Reader for chunked transfer encoding responses.""" - def __init__(self, eol): - AbstractReader.__init__(self) - self._eol = eol - self._leftover_skip_amt = 0 - self._leftover_data = '' - - def _load(self, data): - assert not self._finished, 'tried to add data to a closed reader!' - logger.debug('chunked read an addtional %d data', len(data)) - position = 0 - if self._leftover_data: - logger.debug('chunked reader trying to finish block from leftover data') - # TODO: avoid this string concatenation if possible - data = self._leftover_data + data - position = self._leftover_skip_amt - self._leftover_data = '' - self._leftover_skip_amt = 0 - datalen = len(data) - while position < datalen: - split = data.find(self._eol, position) - if split == -1: - self._leftover_data = data - self._leftover_skip_amt = position - return - amt = int(data[position:split], base=16) - block_start = split + len(self._eol) - # If the whole data chunk plus the eol trailer hasn't - # loaded, we'll wait for the next load. - if block_start + amt + len(self._eol) > len(data): - self._leftover_data = data - self._leftover_skip_amt = position - return - if amt == 0: - self._finished = True - logger.debug('closing chunked redaer due to chunk of length 0') - return - self._done_chunks.append(data[block_start:block_start + amt]) - position = block_start + amt + len(self._eol) -# no-check-code diff --git a/mercurial/httpclient/tests/__init__.py b/mercurial/httpclient/tests/__init__.py new file mode 100644 index 0000000..84b3a07 --- /dev/null +++ b/mercurial/httpclient/tests/__init__.py @@ -0,0 +1 @@ +# no-check-code diff --git a/mercurial/httpclient/tests/simple_http_test.py b/mercurial/httpclient/tests/simple_http_test.py new file mode 100644 index 0000000..dba0188 --- /dev/null +++ b/mercurial/httpclient/tests/simple_http_test.py @@ -0,0 +1,386 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import socket +import unittest + +import http + +# relative import to ease embedding the library +import util + + +class SimpleHttpTest(util.HttpTestBase, unittest.TestCase): + + def _run_simple_test(self, host, server_data, expected_req, expected_data): + con = http.HTTPConnection(host) + con._connect() + con.sock.data = server_data + con.request('GET', '/') + + self.assertStringEqual(expected_req, con.sock.sent) + self.assertEqual(expected_data, con.getresponse().read()) + + def test_broken_data_obj(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + self.assertRaises(http.BadRequestData, + con.request, 'POST', '/', body=1) + + def test_no_keepalive_http_1_0(self): + expected_request_one = """GET /remote/.hg/requires HTTP/1.1 +Host: localhost:9999 +range: bytes=0- +accept-encoding: identity +accept: application/mercurial-0.1 +user-agent: mercurial/proto-1.0 + +""".replace('\n', '\r\n') + expected_response_headers = """HTTP/1.0 200 OK +Server: SimpleHTTP/0.6 Python/2.6.1 +Date: Sun, 01 May 2011 13:56:57 GMT +Content-type: application/octet-stream +Content-Length: 33 +Last-Modified: Sun, 01 May 2011 13:56:56 GMT + +""".replace('\n', '\r\n') + expected_response_body = """revlogv1 +store +fncache +dotencode +""" + con = http.HTTPConnection('localhost:9999') + con._connect() + con.sock.data = [expected_response_headers, expected_response_body] + con.request('GET', '/remote/.hg/requires', + headers={'accept-encoding': 'identity', + 'range': 'bytes=0-', + 'accept': 'application/mercurial-0.1', + 'user-agent': 'mercurial/proto-1.0', + }) + self.assertStringEqual(expected_request_one, con.sock.sent) + self.assertEqual(con.sock.closed, False) + self.assertNotEqual(con.sock.data, []) + self.assert_(con.busy()) + resp = con.getresponse() + self.assertStringEqual(resp.read(), expected_response_body) + self.failIf(con.busy()) + self.assertEqual(con.sock, None) + self.assertEqual(resp.sock.data, []) + self.assert_(resp.sock.closed) + + def test_multiline_header(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Multiline: Value\r\n', + ' Rest of value\r\n', + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + resp = con.getresponse() + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['Value\n Rest of value'], + resp.headers.getheaders('multiline')) + # Socket should not be closed + self.assertEqual(resp.sock.closed, False) + self.assertEqual(con.sock.closed, False) + + def testSimpleRequest(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + resp = con.getresponse() + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['Value', 'Other Value', 'One More!'], + resp.headers.getheaders('multiheader')) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) + + def testHeaderlessResponse(self): + con = http.HTTPConnection('1.2.3.4', use_ssl=False) + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + '\r\n' + '1234567890' + ] + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + resp = con.getresponse() + self.assertEqual('1234567890', resp.read()) + self.assertEqual({}, dict(resp.headers)) + self.assertEqual(resp.status, 200) + + def testReadline(self): + con = http.HTTPConnection('1.2.3.4') + con._connect() + # make sure it trickles in one byte at a time + # so that we touch all the cases in readline + con.sock.data = list(''.join( + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Connection: Close\r\n', + '\r\n' + '1\n2\nabcdefg\n4\n5'])) + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + con.request('GET', '/') + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + r = con.getresponse() + for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']: + actual = r.readline() + self.assertEqual(expected, actual, + 'Expected %r, got %r' % (expected, actual)) + + def testIPv6(self): + self._run_simple_test('[::1]:8221', + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10', + '\r\n\r\n' + '1234567890'], + ('GET / HTTP/1.1\r\n' + 'Host: [::1]:8221\r\n' + 'accept-encoding: identity\r\n\r\n'), + '1234567890') + self._run_simple_test('::2', + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10', + '\r\n\r\n' + '1234567890'], + ('GET / HTTP/1.1\r\n' + 'Host: ::2\r\n' + 'accept-encoding: identity\r\n\r\n'), + '1234567890') + self._run_simple_test('[::3]:443', + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10', + '\r\n\r\n' + '1234567890'], + ('GET / HTTP/1.1\r\n' + 'Host: ::3\r\n' + 'accept-encoding: identity\r\n\r\n'), + '1234567890') + + def testEarlyContinueResponse(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 403 Forbidden\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 18', + '\r\n\r\n' + "You can't do that."] + expected_req = self.doPost(con, expect_body=False) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertStringEqual(expected_req, sock.sent) + self.assertEqual("You can't do that.", con.getresponse().read()) + self.assertEqual(sock.closed, True) + + def testDeniedAfterContinueTimeoutExpires(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 403 Forbidden\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 18\r\n', + 'Connection: close', + '\r\n\r\n' + "You can't do that."] + sock.read_wait_sentinel = 'Dear server, send response!' + sock.close_on_empty = True + # send enough data out that we'll chunk it into multiple + # blocks and the socket will close before we can send the + # whole request. + post_body = ('This is some POST data\n' * 1024 * 32 + + 'Dear server, send response!\n' + + 'This is some POST data\n' * 1024 * 32) + expected_req = self.doPost(con, expect_body=False, + body_to_send=post_body) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assert_('POST data\n' in sock.sent) + self.assert_('Dear server, send response!\n' in sock.sent) + # We expect not all of our data was sent. + self.assertNotEqual(sock.sent, expected_req) + self.assertEqual("You can't do that.", con.getresponse().read()) + self.assertEqual(sock.closed, True) + + def testPostData(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.read_wait_sentinel = 'POST data' + sock.early_data = ['HTTP/1.1 100 Co', 'ntinue\r\n\r\n'] + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 16', + '\r\n\r\n', + "You can do that."] + expected_req = self.doPost(con, expect_body=True) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertEqual(expected_req, sock.sent) + self.assertEqual("You can do that.", con.getresponse().read()) + self.assertEqual(sock.closed, False) + + def testServerWithoutContinue(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.read_wait_sentinel = 'POST data' + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 16', + '\r\n\r\n', + "You can do that."] + expected_req = self.doPost(con, expect_body=True) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertEqual(expected_req, sock.sent) + self.assertEqual("You can do that.", con.getresponse().read()) + self.assertEqual(sock.closed, False) + + def testServerWithSlowContinue(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.read_wait_sentinel = 'POST data' + sock.data = ['HTTP/1.1 100 ', 'Continue\r\n\r\n', + 'HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 16', + '\r\n\r\n', + "You can do that."] + expected_req = self.doPost(con, expect_body=True) + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertEqual(expected_req, sock.sent) + resp = con.getresponse() + self.assertEqual("You can do that.", resp.read()) + self.assertEqual(200, resp.status) + self.assertEqual(sock.closed, False) + + def testSlowConnection(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + # simulate one byte arriving at a time, to check for various + # corner cases + con.sock.data = list('HTTP/1.1 200 OK\r\n' + 'Server: BogusServer 1.0\r\n' + 'Content-Length: 10' + '\r\n\r\n' + '1234567890') + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + self.assertEqual('1234567890', con.getresponse().read()) + + def testTimeout(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.sock.data = [] + con.request('GET', '/') + self.assertRaises(http.HTTPTimeoutException, + con.getresponse) + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + + def test_conn_keep_alive_but_server_close_anyway(self): + sockets = [] + def closingsocket(*args, **kwargs): + s = util.MockSocket(*args, **kwargs) + sockets.append(s) + s.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Connection: Keep-Alive\r\n', + 'Content-Length: 16', + '\r\n\r\n', + 'You can do that.'] + s.close_on_empty = True + return s + + socket.socket = closingsocket + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.request('GET', '/') + r1 = con.getresponse() + r1.read() + self.assertFalse(con.sock.closed) + self.assert_(con.sock.remote_closed) + con.request('GET', '/') + self.assertEqual(2, len(sockets)) + + def test_no_response_raises_response_not_ready(self): + con = http.HTTPConnection('foo') + self.assertRaises(http.httplib.ResponseNotReady, con.getresponse) +# no-check-code diff --git a/mercurial/httpclient/tests/test_bogus_responses.py b/mercurial/httpclient/tests/test_bogus_responses.py new file mode 100644 index 0000000..486e770 --- /dev/null +++ b/mercurial/httpclient/tests/test_bogus_responses.py @@ -0,0 +1,68 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Tests against malformed responses. + +Server implementations that respond with only LF instead of CRLF have +been observed. Checking against ones that use only CR is a hedge +against that potential insanit.y +""" +import unittest + +import http + +# relative import to ease embedding the library +import util + + +class SimpleHttpTest(util.HttpTestBase, unittest.TestCase): + + def bogusEOL(self, eol): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + con.sock.data = ['HTTP/1.1 200 OK%s' % eol, + 'Server: BogusServer 1.0%s' % eol, + 'Content-Length: 10', + eol * 2, + '1234567890'] + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 80), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + self.assertEqual('1234567890', con.getresponse().read()) + + def testOnlyLinefeed(self): + self.bogusEOL('\n') + + def testOnlyCarriageReturn(self): + self.bogusEOL('\r') +# no-check-code diff --git a/mercurial/httpclient/tests/test_chunked_transfer.py b/mercurial/httpclient/tests/test_chunked_transfer.py new file mode 100644 index 0000000..87153e3 --- /dev/null +++ b/mercurial/httpclient/tests/test_chunked_transfer.py @@ -0,0 +1,137 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import cStringIO +import unittest + +import http + +# relative import to ease embedding the library +import util + + +def chunkedblock(x, eol='\r\n'): + r"""Make a chunked transfer-encoding block. + + >>> chunkedblock('hi') + '2\r\nhi\r\n' + >>> chunkedblock('hi' * 10) + '14\r\nhihihihihihihihihihi\r\n' + >>> chunkedblock('hi', eol='\n') + '2\nhi\n' + """ + return ''.join((hex(len(x))[2:], eol, x, eol)) + + +class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase): + def testChunkedUpload(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.read_wait_sentinel = '0\r\n\r\n' + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 6', + '\r\n\r\n', + "Thanks"] + + zz = 'zz\n' + con.request('POST', '/', body=cStringIO.StringIO( + (zz * (0x8010 / 3)) + 'end-of-body')) + expected_req = ('POST / HTTP/1.1\r\n' + 'transfer-encoding: chunked\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + expected_req += chunkedblock('zz\n' * (0x8000 / 3) + 'zz') + expected_req += chunkedblock( + '\n' + 'zz\n' * ((0x1b - len('end-of-body')) / 3) + 'end-of-body') + expected_req += '0\r\n\r\n' + self.assertEqual(('1.2.3.4', 80), sock.sa) + self.assertStringEqual(expected_req, sock.sent) + self.assertEqual("Thanks", con.getresponse().read()) + self.assertEqual(sock.closed, False) + + def testChunkedDownload(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'transfer-encoding: chunked', + '\r\n\r\n', + chunkedblock('hi '), + chunkedblock('there'), + chunkedblock(''), + ] + con.request('GET', '/') + self.assertStringEqual('hi there', con.getresponse().read()) + + def testChunkedDownloadBadEOL(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 200 OK\n', + 'Server: BogusServer 1.0\n', + 'transfer-encoding: chunked', + '\n\n', + chunkedblock('hi ', eol='\n'), + chunkedblock('there', eol='\n'), + chunkedblock('', eol='\n'), + ] + con.request('GET', '/') + self.assertStringEqual('hi there', con.getresponse().read()) + + def testChunkedDownloadPartialChunkBadEOL(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 200 OK\n', + 'Server: BogusServer 1.0\n', + 'transfer-encoding: chunked', + '\n\n', + chunkedblock('hi ', eol='\n'), + ] + list(chunkedblock('there\n' * 5, eol='\n')) + [ + chunkedblock('', eol='\n')] + con.request('GET', '/') + self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n', + con.getresponse().read()) + + def testChunkedDownloadPartialChunk(self): + con = http.HTTPConnection('1.2.3.4:80') + con._connect() + sock = con.sock + sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'transfer-encoding: chunked', + '\r\n\r\n', + chunkedblock('hi '), + ] + list(chunkedblock('there\n' * 5)) + [chunkedblock('')] + con.request('GET', '/') + self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n', + con.getresponse().read()) +# no-check-code diff --git a/mercurial/httpclient/tests/test_proxy_support.py b/mercurial/httpclient/tests/test_proxy_support.py new file mode 100644 index 0000000..1526a9a --- /dev/null +++ b/mercurial/httpclient/tests/test_proxy_support.py @@ -0,0 +1,135 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import unittest +import socket + +import http + +# relative import to ease embedding the library +import util + + +def make_preloaded_socket(data): + """Make a socket pre-loaded with data so it can be read during connect. + + Useful for https proxy tests because we have to read from the + socket during _connect rather than later on. + """ + def s(*args, **kwargs): + sock = util.MockSocket(*args, **kwargs) + sock.early_data = data[:] + return sock + return s + + +class ProxyHttpTest(util.HttpTestBase, unittest.TestCase): + + def _run_simple_test(self, host, server_data, expected_req, expected_data): + con = http.HTTPConnection(host) + con._connect() + con.sock.data = server_data + con.request('GET', '/') + + self.assertEqual(expected_req, con.sock.sent) + self.assertEqual(expected_data, con.getresponse().read()) + + def testSimpleRequest(self): + con = http.HTTPConnection('1.2.3.4:80', + proxy_hostport=('magicproxy', 4242)) + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + con.request('GET', '/') + + expected_req = ('GET http://1.2.3.4/ HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('127.0.0.42', 4242), con.sock.sa) + self.assertStringEqual(expected_req, con.sock.sent) + resp = con.getresponse() + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['Value', 'Other Value', 'One More!'], + resp.headers.getheaders('multiheader')) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) + + def testSSLRequest(self): + con = http.HTTPConnection('1.2.3.4:443', + proxy_hostport=('magicproxy', 4242)) + socket.socket = make_preloaded_socket( + ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10\r\n', + '\r\n' + '1234567890']) + con._connect() + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + connect_sent = con.sock.sent + con.sock.sent = '' + con.request('GET', '/') + + expected_connect = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n' + '\r\n') + expected_request = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('127.0.0.42', 4242), con.sock.sa) + self.assertStringEqual(expected_connect, connect_sent) + self.assertStringEqual(expected_request, con.sock.sent) + resp = con.getresponse() + self.assertEqual(resp.status, 200) + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) + + def testSSLProxyFailure(self): + con = http.HTTPConnection('1.2.3.4:443', + proxy_hostport=('magicproxy', 4242)) + socket.socket = make_preloaded_socket( + ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n']) + self.assertRaises(http.HTTPProxyConnectFailedException, con._connect) + self.assertRaises(http.HTTPProxyConnectFailedException, + con.request, 'GET', '/') +# no-check-code diff --git a/mercurial/httpclient/tests/test_ssl.py b/mercurial/httpclient/tests/test_ssl.py new file mode 100644 index 0000000..5799a8f --- /dev/null +++ b/mercurial/httpclient/tests/test_ssl.py @@ -0,0 +1,93 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import unittest + +import http + +# relative import to ease embedding the library +import util + + + +class HttpSslTest(util.HttpTestBase, unittest.TestCase): + def testSslRereadRequired(self): + con = http.HTTPConnection('1.2.3.4:443') + con._connect() + # extend the list instead of assign because of how + # MockSSLSocket works. + con.sock.data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + con.request('GET', '/') + + expected_req = ('GET / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'accept-encoding: identity\r\n\r\n') + + self.assertEqual(('1.2.3.4', 443), con.sock.sa) + self.assertEqual(expected_req, con.sock.sent) + resp = con.getresponse() + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['Value', 'Other Value', 'One More!'], + resp.headers.getheaders('multiheader')) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) + + def testSslRereadInEarlyResponse(self): + con = http.HTTPConnection('1.2.3.4:443') + con._connect() + con.sock.early_data = ['HTTP/1.1 200 OK\r\n', + 'Server: BogusServer 1.0\r\n', + 'MultiHeader: Value\r\n' + 'MultiHeader: Other Value\r\n' + 'MultiHeader: One More!\r\n' + 'Content-Length: 10\r\n', + '\r\n' + '1234567890' + ] + + expected_req = self.doPost(con, False) + self.assertEqual(None, con.sock, + 'Connection should have disowned socket') + + resp = con.getresponse() + self.assertEqual(('1.2.3.4', 443), resp.sock.sa) + self.assertEqual(expected_req, resp.sock.sent) + self.assertEqual('1234567890', resp.read()) + self.assertEqual(['Value', 'Other Value', 'One More!'], + resp.headers.getheaders('multiheader')) + self.assertEqual(['BogusServer 1.0'], + resp.headers.getheaders('server')) +# no-check-code diff --git a/mercurial/httpclient/tests/util.py b/mercurial/httpclient/tests/util.py new file mode 100644 index 0000000..bbc3d87 --- /dev/null +++ b/mercurial/httpclient/tests/util.py @@ -0,0 +1,195 @@ +# Copyright 2010, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import difflib +import socket + +import http + + +class MockSocket(object): + """Mock non-blocking socket object. + + This is ONLY capable of mocking a nonblocking socket. + + Attributes: + early_data: data to always send as soon as end of headers is seen + data: a list of strings to return on recv(), with the + assumption that the socket would block between each + string in the list. + read_wait_sentinel: data that must be written to the socket before + beginning the response. + close_on_empty: If true, close the socket when it runs out of data + for the client. + """ + def __init__(self, af, socktype, proto): + self.af = af + self.socktype = socktype + self.proto = proto + + self.early_data = [] + self.data = [] + self.remote_closed = self.closed = False + self.close_on_empty = False + self.sent = '' + self.read_wait_sentinel = http._END_HEADERS + + def close(self): + self.closed = True + + def connect(self, sa): + self.sa = sa + + def setblocking(self, timeout): + assert timeout == 0 + + def recv(self, amt=-1): + if self.early_data: + datalist = self.early_data + elif not self.data: + return '' + else: + datalist = self.data + if amt == -1: + return datalist.pop(0) + data = datalist.pop(0) + if len(data) > amt: + datalist.insert(0, data[amt:]) + if not self.data and not self.early_data and self.close_on_empty: + self.remote_closed = True + return data[:amt] + + @property + def ready_for_read(self): + return ((self.early_data and http._END_HEADERS in self.sent) + or (self.read_wait_sentinel in self.sent and self.data) + or self.closed or self.remote_closed) + + def send(self, data): + # this is a horrible mock, but nothing needs us to raise the + # correct exception yet + assert not self.closed, 'attempted to write to a closed socket' + assert not self.remote_closed, ('attempted to write to a' + ' socket closed by the server') + if len(data) > 8192: + data = data[:8192] + self.sent += data + return len(data) + + +def mockselect(r, w, x, timeout=0): + """Simple mock for select() + """ + readable = filter(lambda s: s.ready_for_read, r) + return readable, w[:], [] + + +class MockSSLSocket(object): + def __init__(self, sock): + self._sock = sock + self._fail_recv = True + + def __getattr__(self, key): + return getattr(self._sock, key) + + def __setattr__(self, key, value): + if key not in ('_sock', '_fail_recv'): + return setattr(self._sock, key, value) + return object.__setattr__(self, key, value) + + def recv(self, amt=-1): + try: + if self._fail_recv: + raise socket.sslerror(socket.SSL_ERROR_WANT_READ) + return self._sock.recv(amt=amt) + finally: + self._fail_recv = not self._fail_recv + + +def mocksslwrap(sock, keyfile=None, certfile=None, + server_side=False, cert_reqs=http.socketutil.CERT_NONE, + ssl_version=None, ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True): + return MockSSLSocket(sock) + + +def mockgetaddrinfo(host, port, unused, streamtype): + assert unused == 0 + assert streamtype == socket.SOCK_STREAM + if host.count('.') != 3: + host = '127.0.0.42' + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '', + (host, port))] + + +class HttpTestBase(object): + def setUp(self): + self.orig_socket = socket.socket + socket.socket = MockSocket + + self.orig_getaddrinfo = socket.getaddrinfo + socket.getaddrinfo = mockgetaddrinfo + + self.orig_select = http.select.select + http.select.select = mockselect + + self.orig_sslwrap = http.socketutil.wrap_socket + http.socketutil.wrap_socket = mocksslwrap + + def tearDown(self): + socket.socket = self.orig_socket + http.select.select = self.orig_select + http.socketutil.wrap_socket = self.orig_sslwrap + socket.getaddrinfo = self.orig_getaddrinfo + + def assertStringEqual(self, l, r): + try: + self.assertEqual(l, r, ('failed string equality check, ' + 'see stdout for details')) + except: + add_nl = lambda li: map(lambda x: x + '\n', li) + print 'failed expectation:' + print ''.join(difflib.unified_diff( + add_nl(l.splitlines()), add_nl(r.splitlines()), + fromfile='expected', tofile='got')) + raise + + def doPost(self, con, expect_body, body_to_send='This is some POST data'): + con.request('POST', '/', body=body_to_send, + expect_continue=True) + expected_req = ('POST / HTTP/1.1\r\n' + 'Host: 1.2.3.4\r\n' + 'content-length: %d\r\n' + 'Expect: 100-Continue\r\n' + 'accept-encoding: identity\r\n\r\n' % + len(body_to_send)) + if expect_body: + expected_req += body_to_send + return expected_req +# no-check-code |