diff options
Diffstat (limited to 'mercurial/httpclient/__init__.py')
| -rw-r--r-- | mercurial/httpclient/__init__.py | 214 |
1 files changed, 119 insertions, 95 deletions
diff --git a/mercurial/httpclient/__init__.py b/mercurial/httpclient/__init__.py index f5c3baf..227d60b 100644 --- a/mercurial/httpclient/__init__.py +++ b/mercurial/httpclient/__init__.py @@ -45,7 +45,6 @@ import rfc822 import select import socket -import _readers import socketutil logger = logging.getLogger(__name__) @@ -55,6 +54,8 @@ __all__ = ['HTTPConnection', 'HTTPResponse'] HTTP_VER_1_0 = 'HTTP/1.0' HTTP_VER_1_1 = 'HTTP/1.1' +_LEN_CLOSE_IS_END = -1 + OUTGOING_BUFFER_SIZE = 1 << 15 INCOMING_BUFFER_SIZE = 1 << 20 @@ -82,19 +83,23 @@ class HTTPResponse(object): The response will continue to load as available. If you need the complete response before continuing, check the .complete() method. """ - def __init__(self, sock, timeout, method): + def __init__(self, sock, timeout): self.sock = sock - self.method = method self.raw_response = '' + self._body = None self._headers_len = 0 + self._content_len = 0 self.headers = None self.will_close = False self.status_line = '' self.status = None - self.continued = False self.http_version = None self.reason = None - self._reader = None + self._chunked = False + self._chunked_done = False + self._chunked_until_next = 0 + self._chunked_skip_bytes = 0 + self._chunked_preloaded_block = None self._read_location = 0 self._eol = EOL @@ -112,12 +117,11 @@ class HTTPResponse(object): socket is closed, this will nearly always return False, even in cases where all the data has actually been loaded. """ - if self._reader: - return self._reader.done() - - def _close(self): - if self._reader is not None: - self._reader._close() + if self._chunked: + return self._chunked_done + if self._content_len == _LEN_CLOSE_IS_END: + return False + return self._body is not None and len(self._body) >= self._content_len def readline(self): """Read a single line from the response body. @@ -125,34 +129,30 @@ class HTTPResponse(object): This may block until either a line ending is found or the response is complete. """ - # TODO: move this into the reader interface where it can be - # smarter (and probably avoid copies) - bytes = [] - while not bytes: - try: - bytes = [self._reader.read(1)] - except _readers.ReadNotReady: - self._select() - while bytes[-1] != '\n' and not self.complete(): + eol = self._body.find('\n', self._read_location) + while eol == -1 and not self.complete(): self._select() - bytes.append(self._reader.read(1)) - if bytes[-1] != '\n': - next = self._reader.read(1) - while next and next != '\n': - bytes.append(next) - next = self._reader.read(1) - bytes.append(next) - return ''.join(bytes) + eol = self._body.find('\n', self._read_location) + if eol != -1: + eol += 1 + else: + eol = len(self._body) + data = self._body[self._read_location:eol] + self._read_location = eol + return data def read(self, length=None): # if length is None, unbounded read while (not self.complete() # never select on a finished read and (not length # unbounded, so we wait for complete() - or length > self._reader.available_data)): + or (self._read_location + length) > len(self._body))): self._select() if not length: - length = self._reader.available_data - r = self._reader.read(length) + length = len(self._body) - self._read_location + elif len(self._body) < (self._read_location + length): + length = len(self._body) - self._read_location + r = self._body[self._read_location:self._read_location + length] + self._read_location += len(r) if self.complete() and self.will_close: self.sock.close() return r @@ -160,11 +160,15 @@ class HTTPResponse(object): def _select(self): r, _, _ = select.select([self.sock], [], [], self._timeout) if not r: - # socket was not readable. If the response is not - # complete, raise a timeout. - if not self.complete(): + # socket was not readable. If the response is not complete + # and we're not a _LEN_CLOSE_IS_END response, raise a timeout. + # If we are a _LEN_CLOSE_IS_END response and we have no data, + # raise a timeout. + if not (self.complete() or + (self._content_len == _LEN_CLOSE_IS_END and self._body)): logger.info('timed out with timeout of %s', self._timeout) raise HTTPTimeoutException('timeout reading data') + logger.info('cl: %r body: %r', self._content_len, self._body) try: data = self.sock.recv(INCOMING_BUFFER_SIZE) except socket.sslerror, e: @@ -173,22 +177,68 @@ class HTTPResponse(object): logger.debug('SSL_WANT_READ in _select, should retry later') return True logger.debug('response read %d data during _select', len(data)) - # If the socket was readable and no data was read, that means - # the socket was closed. Inform the reader (if any) so it can - # raise an exception if this is an invalid situation. if not data: - if self._reader: - self._reader._close() + if self.headers and self._content_len == _LEN_CLOSE_IS_END: + self._content_len = len(self._body) return False else: self._load_response(data) return True + def _chunked_parsedata(self, data): + if self._chunked_preloaded_block: + data = self._chunked_preloaded_block + data + self._chunked_preloaded_block = None + while data: + logger.debug('looping with %d data remaining', len(data)) + # Slice out anything we should skip + if self._chunked_skip_bytes: + if len(data) <= self._chunked_skip_bytes: + self._chunked_skip_bytes -= len(data) + data = '' + break + else: + data = data[self._chunked_skip_bytes:] + self._chunked_skip_bytes = 0 + + # determine how much is until the next chunk + if self._chunked_until_next: + amt = self._chunked_until_next + logger.debug('reading remaining %d of existing chunk', amt) + self._chunked_until_next = 0 + body = data + else: + try: + amt, body = data.split(self._eol, 1) + except ValueError: + self._chunked_preloaded_block = data + logger.debug('saving %r as a preloaded block for chunked', + self._chunked_preloaded_block) + return + amt = int(amt, base=16) + logger.debug('reading chunk of length %d', amt) + if amt == 0: + self._chunked_done = True + + # read through end of what we have or the chunk + self._body += body[:amt] + if len(body) >= amt: + data = body[amt:] + self._chunked_skip_bytes = len(self._eol) + else: + self._chunked_until_next = amt - len(body) + self._chunked_skip_bytes = 0 + data = '' + def _load_response(self, data): - # Being here implies we're not at the end of the headers yet, - # since at the end of this method if headers were completely - # loaded we replace this method with the load() method of the - # reader we created. + if self._chunked: + self._chunked_parsedata(data) + return + elif self._body is not None: + self._body += data + return + + # We haven't seen end of headers yet self.raw_response += data # This is a bogus server with bad line endings if self._eol not in self.raw_response: @@ -212,7 +262,6 @@ class HTTPResponse(object): http_ver, status = hdrs.split(' ', 1) if status.startswith('100'): self.raw_response = body - self.continued = True logger.debug('continue seen, setting body to %r', body) return @@ -232,46 +281,23 @@ class HTTPResponse(object): if self._eol != EOL: hdrs = hdrs.replace(self._eol, '\r\n') headers = rfc822.Message(cStringIO.StringIO(hdrs)) - content_len = None if HDR_CONTENT_LENGTH in headers: - content_len = int(headers[HDR_CONTENT_LENGTH]) + self._content_len = int(headers[HDR_CONTENT_LENGTH]) if self.http_version == HTTP_VER_1_0: self.will_close = True elif HDR_CONNECTION_CTRL in headers: self.will_close = ( headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE) + if self._content_len == 0: + self._content_len = _LEN_CLOSE_IS_END if (HDR_XFER_ENCODING in headers and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED): - self._reader = _readers.ChunkedReader(self._eol) - logger.debug('using a chunked reader') - else: - # HEAD responses are forbidden from returning a body, and - # it's implausible for a CONNECT response to use - # close-is-end logic for an OK response. - if (self.method == 'HEAD' or - (self.method == 'CONNECT' and content_len is None)): - content_len = 0 - if content_len is not None: - logger.debug('using a content-length reader with length %d', - content_len) - self._reader = _readers.ContentLengthReader(content_len) - else: - # Response body had no length specified and is not - # chunked, so the end of the body will only be - # identifiable by the termination of the socket by the - # server. My interpretation of the spec means that we - # are correct in hitting this case if - # transfer-encoding, content-length, and - # connection-control were left unspecified. - self._reader = _readers.CloseIsEndReader() - logger.debug('using a close-is-end reader') - self.will_close = True - - if body: - self._reader._load(body) - logger.debug('headers complete') + self._body = '' + self._chunked_parsedata(body) + self._chunked = True + if self._body is None: + self._body = body self.headers = headers - self._load_response = self._reader._load class HTTPConnection(object): @@ -348,14 +374,13 @@ class HTTPConnection(object): {}, HTTP_VER_1_0) sock.send(data) sock.setblocking(0) - r = self.response_class(sock, self.timeout, 'CONNECT') + r = self.response_class(sock, self.timeout) timeout_exc = HTTPTimeoutException( 'Timed out waiting for CONNECT response from proxy') while not r.complete(): try: if not r._select(): - if not r.complete(): - raise timeout_exc + raise timeout_exc except HTTPTimeoutException: # This raise/except pattern looks goofy, but # _select can raise the timeout as well as the @@ -372,10 +397,6 @@ class HTTPConnection(object): else: sock = socketutil.create_connection((self.host, self.port)) if self.ssl: - # This is the default, but in the case of proxied SSL - # requests the proxy logic above will have cleared - # blocking mode, so reenable it just to be safe. - sock.setblocking(1) logger.debug('wrapping socket for ssl with options %r', self.ssl_opts) sock = socketutil.wrap_socket(sock, **self.ssl_opts) @@ -498,7 +519,7 @@ class HTTPConnection(object): out = outgoing_headers or body blocking_on_continue = False if expect_continue and not outgoing_headers and not ( - response and (response.headers or response.continued)): + response and response.headers): logger.info( 'waiting up to %s seconds for' ' continue response from server', @@ -521,6 +542,11 @@ class HTTPConnection(object): 'server, optimistically sending request body') else: raise HTTPTimeoutException('timeout sending data') + # TODO exceptional conditions with select? (what are those be?) + # TODO if the response is loading, must we finish sending at all? + # + # Certainly not if it's going to close the connection and/or + # the response is already done...I think. was_first = first # incoming data @@ -538,11 +564,11 @@ class HTTPConnection(object): logger.info('socket appears closed in read') self.sock = None self._current_response = None - if response is not None: - response._close() # This if/elif ladder is a bit subtle, # comments in each branch should help. - if response is not None and response.complete(): + if response is not None and ( + response.complete() or + response._content_len == _LEN_CLOSE_IS_END): # Server responded completely and then # closed the socket. We should just shut # things down and let the caller get their @@ -571,7 +597,7 @@ class HTTPConnection(object): 'response was missing or incomplete!') logger.debug('read %d bytes in request()', len(data)) if response is None: - response = self.response_class(r[0], self.timeout, method) + response = self.response_class(r[0], self.timeout) response._load_response(data) # Jump to the next select() call so we load more # data if the server is still sending us content. @@ -579,6 +605,10 @@ class HTTPConnection(object): except socket.error, e: if e[0] != errno.EPIPE and not was_first: raise + if (response._content_len + and response._content_len != _LEN_CLOSE_IS_END): + outgoing_headers = sent_data + outgoing_headers + reconnect('read') # outgoing data if w and out: @@ -623,7 +653,7 @@ class HTTPConnection(object): # close if the server response said to or responded before eating # the whole request if response is None: - response = self.response_class(self.sock, self.timeout, method) + response = self.response_class(self.sock, self.timeout) complete = response.complete() data_left = bool(outgoing_headers or body) if data_left: @@ -641,8 +671,7 @@ class HTTPConnection(object): raise httplib.ResponseNotReady() r = self._current_response while r.headers is None: - if not r._select() and not r.complete(): - raise _readers.HTTPRemoteClosedError() + r._select() if r.will_close: self.sock = None self._current_response = None @@ -664,11 +693,6 @@ class BadRequestData(httplib.HTTPException): class HTTPProxyConnectFailedException(httplib.HTTPException): """Connecting to the HTTP proxy failed.""" - class HTTPStateError(httplib.HTTPException): """Invalid internal state encountered.""" - -# Forward this exception type from _readers since it needs to be part -# of the public API. -HTTPRemoteClosedError = _readers.HTTPRemoteClosedError # no-check-code |