diff options
Diffstat (limited to 'pip/_vendor/requests/models.py')
| -rw-r--r-- | pip/_vendor/requests/models.py | 730 |
1 files changed, 730 insertions, 0 deletions
diff --git a/pip/_vendor/requests/models.py b/pip/_vendor/requests/models.py new file mode 100644 index 000000000..8fd973535 --- /dev/null +++ b/pip/_vendor/requests/models.py @@ -0,0 +1,730 @@ +# -*- coding: utf-8 -*- + +""" +requests.models +~~~~~~~~~~~~~~~ + +This module contains the primary objects that power Requests. +""" + +import collections +import logging +import datetime + +from io import BytesIO, UnsupportedOperation +from .hooks import default_hooks +from .structures import CaseInsensitiveDict + +from .auth import HTTPBasicAuth +from .cookies import cookiejar_from_dict, get_cookie_header +from .packages.urllib3.filepost import encode_multipart_formdata +from .packages.urllib3.util import parse_url +from .exceptions import ( + HTTPError, RequestException, MissingSchema, InvalidURL, + ChunkedEncodingError) +from .utils import ( + guess_filename, get_auth_from_url, requote_uri, + stream_decode_response_unicode, to_key_val_list, parse_header_links, + iter_slices, guess_json_utf, super_len, to_native_string) +from .compat import ( + cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, + is_py2, chardet, json, builtin_str, basestring, IncompleteRead) + +CONTENT_CHUNK_SIZE = 10 * 1024 +ITER_CHUNK_SIZE = 512 + +log = logging.getLogger(__name__) + + +class RequestEncodingMixin(object): + @property + def path_url(self): + """Build the path URL to use.""" + + url = [] + + p = urlsplit(self.url) + + path = p.path + if not path: + path = '/' + + url.append(path) + + query = p.query + if query: + url.append('?') + url.append(query) + + return ''.join(url) + + @staticmethod + def _encode_params(data): + """Encode parameters in a piece of data. + + Will successfully encode parameters when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary + if parameters are supplied as a dict. + """ + + if isinstance(data, (str, bytes)): + return data + elif hasattr(data, 'read'): + return data + elif hasattr(data, '__iter__'): + result = [] + for k, vs in to_key_val_list(data): + if isinstance(vs, basestring) or not hasattr(vs, '__iter__'): + vs = [vs] + for v in vs: + if v is not None: + result.append( + (k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v)) + return urlencode(result, doseq=True) + else: + return data + + @staticmethod + def _encode_files(files, data): + """Build the body for a multipart/form-data request. + + Will successfully encode files when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. + + """ + if (not files): + raise ValueError("Files must be provided.") + elif isinstance(data, basestring): + raise ValueError("Data must not be a string.") + + new_fields = [] + fields = to_key_val_list(data or {}) + files = to_key_val_list(files or {}) + + for field, val in fields: + if isinstance(val, basestring) or not hasattr(val, '__iter__'): + val = [val] + for v in val: + if v is not None: + # Don't call str() on bytestrings: in Py3 it all goes wrong. + if not isinstance(v, bytes): + v = str(v) + + new_fields.append( + (field.decode('utf-8') if isinstance(field, bytes) else field, + v.encode('utf-8') if isinstance(v, str) else v)) + + for (k, v) in files: + # support for explicit filename + ft = None + if isinstance(v, (tuple, list)): + if len(v) == 2: + fn, fp = v + else: + fn, fp, ft = v + else: + fn = guess_filename(v) or k + fp = v + if isinstance(fp, str): + fp = StringIO(fp) + if isinstance(fp, bytes): + fp = BytesIO(fp) + + if ft: + new_v = (fn, fp.read(), ft) + else: + new_v = (fn, fp.read()) + new_fields.append((k, new_v)) + + body, content_type = encode_multipart_formdata(new_fields) + + return body, content_type + + +class RequestHooksMixin(object): + def register_hook(self, event, hook): + """Properly register a hook.""" + + if event not in self.hooks: + raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + + if isinstance(hook, collections.Callable): + self.hooks[event].append(hook) + elif hasattr(hook, '__iter__'): + self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + + def deregister_hook(self, event, hook): + """Deregister a previously registered hook. + Returns True if the hook existed, False if not. + """ + + try: + self.hooks[event].remove(hook) + return True + except ValueError: + return False + + +class Request(RequestHooksMixin): + """A user-created :class:`Request <Request>` object. + + Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server. + + :param method: HTTP method to use. + :param url: URL to send. + :param headers: dictionary of headers to send. + :param files: dictionary of {filename: fileobject} files to multipart upload. + :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place. + :param params: dictionary of URL parameters to append to the URL. + :param auth: Auth handler or (user, pass) tuple. + :param cookies: dictionary or CookieJar of cookies to attach to this request. + :param hooks: dictionary of callback hooks, for internal usage. + + Usage:: + + >>> import requests + >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req.prepare() + <PreparedRequest [GET]> + + """ + def __init__(self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None): + + # Default empty dicts for dict params. + data = [] if data is None else data + files = [] if files is None else files + headers = {} if headers is None else headers + params = {} if params is None else params + hooks = {} if hooks is None else hooks + + self.hooks = default_hooks() + for (k, v) in list(hooks.items()): + self.register_hook(event=k, hook=v) + + self.method = method + self.url = url + self.headers = headers + self.files = files + self.data = data + self.params = params + self.auth = auth + self.cookies = cookies + + def __repr__(self): + return '<Request [%s]>' % (self.method) + + def prepare(self): + """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it.""" + p = PreparedRequest() + p.prepare( + method=self.method, + url=self.url, + headers=self.headers, + files=self.files, + data=self.data, + params=self.params, + auth=self.auth, + cookies=self.cookies, + hooks=self.hooks, + ) + return p + + +class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): + """The fully mutable :class:`PreparedRequest <PreparedRequest>` object, + containing the exact bytes that will be sent to the server. + + Generated from either a :class:`Request <Request>` object or manually. + + Usage:: + + >>> import requests + >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> r = req.prepare() + <PreparedRequest [GET]> + + >>> s = requests.Session() + >>> s.send(r) + <Response [200]> + + """ + + def __init__(self): + #: HTTP verb to send to the server. + self.method = None + #: HTTP URL to send the request to. + self.url = None + #: dictionary of HTTP headers. + self.headers = None + #: request body to send to the server. + self.body = None + #: dictionary of callback hooks, for internal usage. + self.hooks = default_hooks() + + def prepare(self, method=None, url=None, headers=None, files=None, + data=None, params=None, auth=None, cookies=None, hooks=None): + """Prepares the the entire request with the given parameters.""" + + self.prepare_method(method) + self.prepare_url(url, params) + self.prepare_headers(headers) + self.prepare_cookies(cookies) + self.prepare_body(data, files) + self.prepare_auth(auth, url) + # Note that prepare_auth must be last to enable authentication schemes + # such as OAuth to work on a fully prepared request. + + # This MUST go after prepare_auth. Authenticators could add a hook + self.prepare_hooks(hooks) + + def __repr__(self): + return '<PreparedRequest [%s]>' % (self.method) + + def copy(self): + p = PreparedRequest() + p.method = self.method + p.url = self.url + p.headers = self.headers + p.body = self.body + p.hooks = self.hooks + return p + + def prepare_method(self, method): + """Prepares the given HTTP method.""" + self.method = method + if self.method is not None: + self.method = self.method.upper() + + def prepare_url(self, url, params): + """Prepares the given HTTP URL.""" + #: Accept objects that have string representations. + try: + url = unicode(url) + except NameError: + # We're on Python 3. + url = str(url) + except UnicodeDecodeError: + pass + + # Support for unicode domain names and paths. + scheme, auth, host, port, path, query, fragment = parse_url(url) + + if not scheme: + raise MissingSchema("Invalid URL %r: No schema supplied" % url) + + if not host: + raise InvalidURL("Invalid URL %r: No host supplied" % url) + + # Only want to apply IDNA to the hostname + try: + host = host.encode('idna').decode('utf-8') + except UnicodeError: + raise InvalidURL('URL has an invalid label.') + + # Carefully reconstruct the network location + netloc = auth or '' + if netloc: + netloc += '@' + netloc += host + if port: + netloc += ':' + str(port) + + # Bare domains aren't valid URLs. + if not path: + path = '/' + + if is_py2: + if isinstance(scheme, str): + scheme = scheme.encode('utf-8') + if isinstance(netloc, str): + netloc = netloc.encode('utf-8') + if isinstance(path, str): + path = path.encode('utf-8') + if isinstance(query, str): + query = query.encode('utf-8') + if isinstance(fragment, str): + fragment = fragment.encode('utf-8') + + enc_params = self._encode_params(params) + if enc_params: + if query: + query = '%s&%s' % (query, enc_params) + else: + query = enc_params + + url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) + self.url = url + + def prepare_headers(self, headers): + """Prepares the given HTTP headers.""" + + if headers: + self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items()) + else: + self.headers = CaseInsensitiveDict() + + def prepare_body(self, data, files): + """Prepares the given HTTP body data.""" + + # Check if file, fo, generator, iterator. + # If not, run through normal process. + + # Nottin' on you. + body = None + content_type = None + length = None + + is_stream = all([ + hasattr(data, '__iter__'), + not isinstance(data, basestring), + not isinstance(data, list), + not isinstance(data, dict) + ]) + + try: + length = super_len(data) + except (TypeError, AttributeError, UnsupportedOperation): + length = None + + if is_stream: + body = data + + if files: + raise NotImplementedError('Streamed bodies and files are mutually exclusive.') + + if length is not None: + self.headers['Content-Length'] = str(length) + else: + self.headers['Transfer-Encoding'] = 'chunked' + else: + # Multi-part file uploads. + if files: + (body, content_type) = self._encode_files(files, data) + else: + if data: + body = self._encode_params(data) + if isinstance(data, str) or isinstance(data, builtin_str) or hasattr(data, 'read'): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' + + self.prepare_content_length(body) + + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type + + self.body = body + + def prepare_content_length(self, body): + if hasattr(body, 'seek') and hasattr(body, 'tell'): + body.seek(0, 2) + self.headers['Content-Length'] = str(body.tell()) + body.seek(0, 0) + elif body is not None: + l = super_len(body) + if l: + self.headers['Content-Length'] = str(l) + elif self.method not in ('GET', 'HEAD'): + self.headers['Content-Length'] = '0' + + def prepare_auth(self, auth, url=''): + """Prepares the given HTTP auth data.""" + + # If no Auth is explicitly provided, extract it from the URL first. + if auth is None: + url_auth = get_auth_from_url(self.url) + auth = url_auth if any(url_auth) else None + + if auth: + if isinstance(auth, tuple) and len(auth) == 2: + # special-case basic HTTP auth + auth = HTTPBasicAuth(*auth) + + # Allow auth to make its changes. + r = auth(self) + + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) + + # Recompute Content-Length + self.prepare_content_length(self.body) + + def prepare_cookies(self, cookies): + """Prepares the given HTTP cookie data.""" + + if isinstance(cookies, cookielib.CookieJar): + cookies = cookies + else: + cookies = cookiejar_from_dict(cookies) + + if 'cookie' not in self.headers: + cookie_header = get_cookie_header(cookies, self) + if cookie_header is not None: + self.headers['Cookie'] = cookie_header + + def prepare_hooks(self, hooks): + """Prepares the given hooks.""" + for event in hooks: + self.register_hook(event, hooks[event]) + + +class Response(object): + """The :class:`Response <Response>` object, which contains a + server's response to an HTTP request. + """ + + def __init__(self): + super(Response, self).__init__() + + self._content = False + self._content_consumed = False + + #: Integer Code of responded HTTP Status. + self.status_code = None + + #: Case-insensitive Dictionary of Response Headers. + #: For example, ``headers['content-encoding']`` will return the + #: value of a ``'Content-Encoding'`` response header. + self.headers = CaseInsensitiveDict() + + #: File-like object representation of response (for advanced usage). + #: Requires that ``stream=True` on the request. + # This requirement does not apply for use internally to Requests. + self.raw = None + + #: Final URL location of Response. + self.url = None + + #: Encoding to decode with when accessing r.text. + self.encoding = None + + #: A list of :class:`Response <Response>` objects from + #: the history of the Request. Any redirect responses will end + #: up here. The list is sorted from the oldest to the most recent request. + self.history = [] + + self.reason = None + + #: A CookieJar of Cookies the server sent back. + self.cookies = cookiejar_from_dict({}) + + #: The amount of time elapsed between sending the request + #: and the arrival of the response (as a timedelta) + self.elapsed = datetime.timedelta(0) + + def __repr__(self): + return '<Response [%s]>' % (self.status_code) + + def __bool__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + + def __nonzero__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + + def __iter__(self): + """Allows you to use a response as an iterator.""" + return self.iter_content(128) + + @property + def ok(self): + try: + self.raise_for_status() + except RequestException: + return False + return True + + @property + def apparent_encoding(self): + """The apparent encoding, provided by the lovely Charade library + (Thanks, Ian!).""" + return chardet.detect(self.content)['encoding'] + + def iter_content(self, chunk_size=1, decode_unicode=False): + """Iterates over the response data. When stream=True is set on the + request, this avoids reading the content at once into memory for + large responses. The chunk size is the number of bytes it should + read into memory. This is not necessarily the length of each item + returned as decoding can take place. + """ + if self._content_consumed: + # simulate reading small chunks of the content + return iter_slices(self._content, chunk_size) + + def generate(): + try: + # Special case for urllib3. + try: + for chunk in self.raw.stream(chunk_size, + decode_content=True): + yield chunk + except IncompleteRead as e: + raise ChunkedEncodingError(e) + except AttributeError: + # Standard file-like object. + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk + + self._content_consumed = True + + gen = generate() + + if decode_unicode: + gen = stream_decode_response_unicode(gen, self) + + return gen + + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None): + """Iterates over the response data, one line at a time. When + stream=True is set on the request, this avoids reading the + content at once into memory for large responses. + """ + + pending = None + + for chunk in self.iter_content(chunk_size=chunk_size, + decode_unicode=decode_unicode): + + if pending is not None: + chunk = pending + chunk + lines = chunk.splitlines() + + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + pending = lines.pop() + else: + pending = None + + for line in lines: + yield line + + if pending is not None: + yield pending + + @property + def content(self): + """Content of the response, in bytes.""" + + if self._content is False: + # Read the contents. + try: + if self._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed') + + if self.status_code == 0: + self._content = None + else: + self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() + + except AttributeError: + self._content = None + + self._content_consumed = True + # don't need to release the connection; that's been handled by urllib3 + # since we exhausted the data. + return self._content + + @property + def text(self): + """Content of the response, in unicode. + + if Response.encoding is None and chardet module is available, encoding + will be guessed. + """ + + # Try charset from content-type + content = None + encoding = self.encoding + + if not self.content: + return str('') + + # Fallback to auto-detected encoding. + if self.encoding is None: + encoding = self.apparent_encoding + + # Decode unicode from given encoding. + try: + content = str(self.content, encoding, errors='replace') + except (LookupError, TypeError): + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # A TypeError can be raised if encoding is None + # + # So we try blindly encoding. + content = str(self.content, errors='replace') + + return content + + def json(self, **kwargs): + """Returns the json-encoded content of a response, if any. + + :param \*\*kwargs: Optional arguments that ``json.loads`` takes. + """ + + if not self.encoding and len(self.content) > 3: + # No encoding set. JSON RFC 4627 section 3 states we should expect + # UTF-8, -16 or -32. Detect which one to use; If the detection or + # decoding fails, fall back to `self.text` (using chardet to make + # a best guess). + encoding = guess_json_utf(self.content) + if encoding is not None: + return json.loads(self.content.decode(encoding), **kwargs) + return json.loads(self.text or self.content, **kwargs) + + @property + def links(self): + """Returns the parsed header links of the response, if any.""" + + header = self.headers.get('link') + + # l = MultiDict() + l = {} + + if header: + links = parse_header_links(header) + + for link in links: + key = link.get('rel') or link.get('url') + l[key] = link + + return l + + def raise_for_status(self): + """Raises stored :class:`HTTPError`, if one occurred.""" + + http_error_msg = '' + + if 400 <= self.status_code < 500: + http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason) + + elif 500 <= self.status_code < 600: + http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason) + + if http_error_msg: + raise HTTPError(http_error_msg, response=self) + + def close(self): + """Closes the underlying file descriptor and releases the connection + back to the pool. + + *Note: Should not normally need to be called explicitly.* + """ + return self.raw.release_conn() |