diff options
| author | Cory Benfield <lukasaoz@gmail.com> | 2016-09-06 15:40:02 +0100 |
|---|---|---|
| committer | Cory Benfield <lukasaoz@gmail.com> | 2016-09-06 15:40:02 +0100 |
| commit | 15b2346de06d3eed2161a22e67a8a76b03231472 (patch) | |
| tree | ccebe32a66c46b73f869920b6c0729de3dfdd6ca | |
| parent | 52463752d3a79790b43e23182bcc750d6bb53554 (diff) | |
| parent | 7563abc132304b49747e364c7e29e7b734f78c89 (diff) | |
| download | urllib3-15b2346de06d3eed2161a22e67a8a76b03231472.tar.gz | |
Merging new release version: 1.17
62 files changed, 2199 insertions, 1536 deletions
diff --git a/.coveragerc b/.coveragerc index c21c72e1..c765f1b2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,7 @@ [run] +include = + urllib3/* + omit = urllib3/packages/* urllib3/contrib/appengine.py diff --git a/.travis.yml b/.travis.yml index 2a334142..24677ac1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,16 @@ language: python script: - - ./_travis/travis-run.sh + - tox before_install: - openssl version cache: directories: - ${HOME}/.cache install: -- ./_travis/travis-install.sh + - pip install tox==2.1.1 + - if [[ "${TOXENV}" == "gae" && ! -d ${GAE_PYTHONPATH} ]]; then + python _travis/fetch_gae_sdk.py ; + fi notifications: email: false env: @@ -20,11 +23,14 @@ env: - TOXENV=py27 - TOXENV=py33 - TOXENV=py34 - - TOXENV=pypy - TOXENV=gae # https://github.com/travis-ci/travis-ci/issues/4794 matrix: include: - python: 3.5 env: TOXENV=py35 + - python: pypy-5.4 + env: TOXENV=pypy + allow_failures: + - python: pypy-5.4 sudo: false diff --git a/CHANGES.rst b/CHANGES.rst index fdf6a6e8..8b03eb01 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,52 @@ Changes ======= +1.17 (2016-09-06) +----------------- + +* Accept ``SSLContext`` objects for use in SSL/TLS negotiation. (Issue #835) + +* ConnectionPool debug log now includes scheme, host, and port. (Issue #897) + +* Substantially refactored documentation. (Issue #887) + +* Used URLFetch default timeout on AppEngine, rather than hardcoding our own. + (Issue #858) + +* Normalize the scheme and host in the URL parser (Issue #833) + +* ``HTTPResponse`` contains the last ``Retry`` object, which now also + contains retries history. (Issue #848) + +* Timeout can no longer be set as boolean, and must be greater than zero. + (PR #924) + +* Removed pyasn1 and ndg-httpsclient from dependencies used for PyOpenSSL. We + now use cryptography and idna, both of which are already dependencies of + PyOpenSSL. (PR #930) + +* Fixed infinite loop in ``stream`` when amt=None. (Issue #928) + +* Try to use the operating system's certificates when we are using an + ``SSLContext``. (PR #941) + +* Updated cipher suite list to allow ChaCha20+Poly1305. AES-GCM is preferred to + ChaCha20, but ChaCha20 is then preferred to everything else. (PR #947) + +* Updated cipher suite list to remove 3DES-based cipher suites. (PR #958) + +* Removed the cipher suite fallback to allow HIGH ciphers. (PR #958) + +* Implemented ``length_remaining`` to determine remaining content + to be read. (PR #949) + +* Implemented ``enforce_content_length`` to enable exceptions when + incomplete data chunks are received. (PR #949) + +* Dropped connection start, dropped connection reset, redirect, forced retry, + and new HTTPS connection log levels to DEBUG, from INFO. (PR #967) + + 1.16 (2016-06-11) ----------------- diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 85df95ac..9ef5c69f 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -202,5 +202,15 @@ In chronological order: * Jeremy Cline <jeremy@jcline.org> * Added connection pool keys by scheme +* Aviv Palivoda <palaviv@gmail.com> + * History list to Retry object. + * HTTPResponse contains the last Retry object. + +* Nate Prewitt <nate.prewitt@gmail.com> + * Ensure timeouts are not booleans and greater than zero. + * Fixed infinite loop in ``stream`` when amt=None. + * Added length_remaining to determine remaining data to be read. + * Added enforce_content_length to raise exception when incorrect content-length received. + * [Your name or handle] <[email or website]> * [Brief summary of your changes] @@ -2,50 +2,61 @@ REQUIREMENTS_FILE=dev-requirements.txt REQUIREMENTS_OUT=dev-requirements.txt.log SETUP_OUT=*.egg-info - +.PHONY: all all: setup requirements +.PHONY: virtualenv virtualenv: ifndef VIRTUAL_ENV $(error Must be run inside of a virtualenv) endif +.PHONY: setup setup: virtualenv $(SETUP_OUT) +.PHONY: $(SETUP_OUT) $(SETUP_OUT): setup.py setup.cfg python setup.py develop touch $(SETUP_OUT) +.PHONY: requirements requirements: setup $(REQUIREMENTS_OUT) +.PHONY: piprot piprot: setup pip install piprot piprot -x $(REQUIREMENTS_FILE) +.PHONY: $(REQUIREMENTS_OUT) $(REQUIREMENTS_OUT): $(REQUIREMENTS_FILE) pip install -r $(REQUIREMENTS_FILE) | tee -a $(REQUIREMENTS_OUT) python setup.py develop +.PHONY: clean clean: find . -name "*.py[oc]" -delete find . -name "__pycache__" -delete rm -f $(REQUIREMENTS_OUT) rm -rf docs/_build build/ dist/ +.PHONY: test test: requirements nosetests +.PHONY: test-all test-all: requirements tox +.PHONY: test-gae test-gae: requirements tox -e gae +.PHONY: docs docs: - cd docs && pip install -r doc-requirements.txt && make html + tox -e docs +.PHONY: release release: ./release.sh -.PHONY: docs @@ -1,134 +1,71 @@ -======= urllib3 ======= .. image:: https://travis-ci.org/shazow/urllib3.png?branch=master + :alt: Build status on Travis :target: https://travis-ci.org/shazow/urllib3 -.. image:: https://www.bountysource.com/badge/tracker?tracker_id=192525 - :target: https://www.bountysource.com/trackers/192525-urllib3?utm_source=192525&utm_medium=shield&utm_campaign=TRACKER_BADGE - - -Highlights -========== - -- Re-use the same socket connection for multiple requests - (``HTTPConnectionPool`` and ``HTTPSConnectionPool``) - (with optional client-side certificate verification). -- File posting (``encode_multipart_formdata``). -- Built-in redirection and retries (optional). -- Supports gzip and deflate decoding. -- Proxy over HTTP or SOCKS. -- Thread-safe and sanity-safe. -- Works with AppEngine, gevent, and eventlib. -- Tested on Python 2.6+, Python 3.3+, and PyPy, with 100% unit test coverage. -- Small and easy to understand codebase perfect for extending and building upon. - For a more comprehensive solution, have a look at - `Requests <http://python-requests.org/>`_ which is also powered by ``urllib3``. - - -You might already be using urllib3! -=================================== - -``urllib3`` powers `many great Python libraries -<https://sourcegraph.com/search?q=package+urllib3>`_, including ``pip`` and -``requests``. - - -What's wrong with urllib and urllib2? -===================================== - -There are two critical features missing from the Python standard library: -Connection re-using/pooling and file posting. It's not terribly hard to -implement these yourself, but it's much easier to use a module that already -did the work for you. - -The Python standard libraries ``urllib`` and ``urllib2`` have little to do -with each other. They were designed to be independent and standalone, each -solving a different scope of problems, and ``urllib3`` follows in a similar -vein. - +.. image:: https://readthedocs.org/projects/urllib3/badge/?version=latest + :alt: Documentation Status + :target: https://urllib3.readthedocs.io/en/latest/ -Why do I want to reuse connections? -=================================== +.. image:: https://img.shields.io/pypi/v/urllib3.svg?maxAge=86400 + :alt: PyPI version + :target: https://pypi.python.org/pypi/urllib3 -Performance. When you normally do a urllib call, a separate socket -connection is created with each request. By reusing existing sockets -(supported since HTTP 1.1), the requests will take up less resources on the -server's end, and also provide a faster response time at the client's end. -With some simple benchmarks (see `test/benchmark.py -<https://github.com/shazow/urllib3/blob/master/test/benchmark.py>`_ -), downloading 15 URLs from google.com is about twice as fast when using -HTTPConnectionPool (which uses 1 connection) than using plain urllib (which -uses 15 connections). - -This library is perfect for: - -- Talking to an API -- Crawling a website -- Any situation where being able to post files, handle redirection, and - retrying is useful. It's relatively lightweight, so it can be used for - anything! - - -Examples -======== - -Go to `urllib3.readthedocs.org <https://urllib3.readthedocs.io>`_ -for more nice syntax-highlighted examples. - -But, long story short:: - - import urllib3 +.. image:: https://www.bountysource.com/badge/tracker?tracker_id=192525 + :alt: Bountysource + :target: https://www.bountysource.com/trackers/192525-urllib3?utm_source=192525&utm_medium=shield&utm_campaign=TRACKER_BADGE - http = urllib3.PoolManager() +urllib3 is a powerful, *sanity-friendly* HTTP client for Python. Much of the +Python ecosystem already uses urllib3 and you should too. +urllib3 brings many critical features that are missing from the Python +standard libraries: - r = http.request('GET', 'http://google.com/') +- Thread safety. +- Connection pooling. +- Client-side SSL/TLS verification. +- File uploads with multipart encoding. +- Helpers for retrying requests and dealing with HTTP redirects. +- Support for gzip and deflate encoding. +- Proxy support for HTTP and SOCKS. +- 100% test coverage. - print r.status, r.data +urllib3 is powerful and easy to use:: -The ``PoolManager`` will take care of reusing connections for you whenever -you request the same host. For more fine-grained control of your connection -pools, you should look at `ConnectionPool -<https://urllib3.readthedocs.io/#connectionpool>`_. + >>> import urllib3 + >>> http = urllib3.PoolManager() + >>> r = http.request('GET', 'http://httpbin.org/robots.txt') + >>> r.status + 200 + >>> r.data + 'User-agent: *\nDisallow: /deny\n' +Installing +---------- -Run the tests -============= +urllib3 can be installed with `pip <https://pip.pypa.io>`_:: -We use some external dependencies, multiple interpreters and code coverage -analysis while running test suite. Our ``Makefile`` handles much of this for -you as long as you're running it `inside of a virtualenv -<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_:: + $ pip install urllib3 - $ make test - [... magically installs dependencies and runs tests on your virtualenv] - Ran 182 tests in 1.633s +Alternatively, you can grab the latest source code from `GitHub <https://github.com/shazow/urllib3>`_:: - OK (SKIP=6) + $ git clone git://github.com/shazow/urllib3.git + $ python setup.py install -Note that code coverage less than 100% is regarded as a failing run. Some -platform-specific tests are skipped unless run in that platform. To make sure -the code works in all of urllib3's supported platforms, you can run our ``tox`` -suite:: - $ make test-all - [... tox creates a virtualenv for every platform and runs tests inside of each] - py26: commands succeeded - py27: commands succeeded - py32: commands succeeded - py33: commands succeeded - py34: commands succeeded +Documentation +------------- -Our test suite `runs continuously on Travis CI -<https://travis-ci.org/shazow/urllib3>`_ with every pull request. +urllib3 has usage and reference documentation at `urllib3.readthedocs.io <https://urllib3.readthedocs.io>`_. Contributing -============ +------------ -Thank you for giving back to urllib3. Please meet our jolly team -of code-sherpas: +urllib3 happily accepts contributions. Please see our +`contributing documentation <https://urllib3.readthedocs.io/en/latest/contributing.html>`_ +for some tips on getting started. Maintainers ----------- @@ -139,23 +76,8 @@ Maintainers 👋 -Getting Started ---------------- - -#. `Check for open issues <https://github.com/shazow/urllib3/issues>`_ or open - a fresh issue to start a discussion around a feature idea or a bug. There is - a *Contributor Friendly* tag for issues that should be ideal for people who - are not very familiar with the codebase yet. -#. Fork the `urllib3 repository on Github <https://github.com/shazow/urllib3>`_ - to start making your changes. -#. Write a test which shows that the bug was fixed or that the feature works - as expected. -#. Send a pull request and bug the maintainer until it gets merged and published. - :) Make sure to add yourself to ``CONTRIBUTORS.txt``. - - Sponsorship -=========== +----------- If your company benefits from this library, please consider `sponsoring its -development <https://urllib3.readthedocs.io/en/latest/#sponsorship>`_. +development <https://urllib3.readthedocs.io/en/latest/contributing.html#sponsorship>`_. diff --git a/_travis/fetch_gae_sdk.py b/_travis/fetch_gae_sdk.py index 9eb4e1f9..501e4d02 100644 --- a/_travis/fetch_gae_sdk.py +++ b/_travis/fetch_gae_sdk.py @@ -67,7 +67,13 @@ def main(argv): if len(argv) > 2: print('Usage: {0} [<destination_dir>]'.format(argv[0])) return 1 - dest_dir = argv[1] if len(argv) > 1 else '.' + if len(argv) > 1: + dest_dir = argv[1] + else: + try: + dest_dir = os.path.dirname(os.environ['GAE_PYTHONPATH']) + except IndexError: + dest_dir = '.' if not os.path.exists(dest_dir): os.makedirs(dest_dir) @@ -91,8 +97,8 @@ def main(argv): return 1 sdk_contents.seek(0) try: - zip_contents = zipfile.ZipFile(sdk_contents) - zip_contents.extractall(dest_dir) + with zipfile.ZipFile(sdk_contents) as zip_contents: + zip_contents.extractall(dest_dir) except: print('Error extracting SDK contents') return 1 diff --git a/_travis/travis-install.sh b/_travis/travis-install.sh deleted file mode 100755 index ba44c612..00000000 --- a/_travis/travis-install.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ev - -pip install tox==2.1.1 - -# Workaround Travis' old PyPy release. If Travis updates, we can remove this -# code. -if [[ "${TOXENV}" == pypy* ]]; then - git clone https://github.com/yyuu/pyenv.git ~/.pyenv - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - pyenv install pypy-4.0.1 - pyenv global pypy-4.0.1 - pyenv rehash -fi - -if [[ "${TOXENV}" == "gae" && ! -d ${GAE_PYTHONPATH} ]]; then - python _travis/fetch_gae_sdk.py `dirname ${GAE_PYTHONPATH}` -fi diff --git a/_travis/travis-run.sh b/_travis/travis-run.sh deleted file mode 100755 index c044a8d4..00000000 --- a/_travis/travis-run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -ev - -if [[ "${TOXENV}" == "pypy" ]]; then - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" -fi - -tox
\ No newline at end of file diff --git a/docs/README b/docs/README deleted file mode 100644 index 9126c735..00000000 --- a/docs/README +++ /dev/null @@ -1,14 +0,0 @@ -# Building the Docs - -First install Sphinx: - - pip install sphinx - -Install pyopenssl and certifi dependencies, to avoid some build errors. (Optional) - - # This step is optional - pip install ndg-httpsclient pyasn1 certifi - -Then build: - - cd docs && make html diff --git a/docs/_templates/fonts.html b/docs/_templates/fonts.html new file mode 100644 index 00000000..c52c8f20 --- /dev/null +++ b/docs/_templates/fonts.html @@ -0,0 +1 @@ +<link href='https://fonts.googleapis.com/css?family=Roboto|Roboto+Mono' rel='stylesheet' type='text/css'> diff --git a/docs/advanced-usage.rst b/docs/advanced-usage.rst new file mode 100644 index 00000000..f28c83c5 --- /dev/null +++ b/docs/advanced-usage.rst @@ -0,0 +1,265 @@ +Advanced Usage +============== + +.. currentmodule:: urllib3 + + +Customizing pool behavior +------------------------- + +The :class:`~poolmanager.PoolManager` class automatically handles creating +:class:`~connectionpool.ConnectionPool` instances for each host as needed. By +default, it will keep a maximum of 10 :class:`~connectionpool.ConnectionPool` +instances. If you're making requests to many different hosts it might improve +performance to increase this number:: + + >>> import urllib3 + >>> http = urllib3.PoolManager(num_pools=50) + +However, keep in mind that this does increase memory and socket consumption. + +Similarly, the :class:`~connectionpool.ConnectionPool` class keeps a pool +of individual :class:`~connection.HTTPConnection` instances. These connections +are used during an individual request and returned to the pool when the request +is complete. By default only one connection will be saved for re-use. If you +are making many requests to the same host simultaneously it might improve +performance to increase this number:: + + >>> import urllib3 + >>> http = urllib3.PoolManager(maxsize=10) + # Alternatively + >>> http = urllib3.HTTPConnectionPool('google.com', maxsize=10) + +The behavior of the pooling for :class:`~connectionpool.ConnectionPool` is +different from :class:`~poolmanager.PoolManager`. By default, if a new +request is made and there is no free connection in the pool then a new +connection will be created. However, this connection will not be saved if more +than ``maxsize`` connections exist. This means that ``maxsize`` does not +determine the maximum number of connections that can be open to a particular +host, just the maximum number of connections to keep in the pool. However, if you specify ``block=True`` then there can be at most ``maxsize`` connections +open to a particular host:: + + >>> http = urllib3.PoolManager(maxsize=10, block=True) + # Alternatively + >>> http = urllib3.HTTPConnectionPool('google.com', maxsize=10, block=True) + +Any new requests will block until a connection is available from the pool. +This is a great way to prevent flooding a host with too many connections in +multi-threaded applications. + +.. _stream: + +Streaming and IO +---------------- + +When dealing with large responses it's often better to stream the response +content:: + + >>> import urllib3 + >>> http = urllib3.PoolManager() + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/bytes/1024', + ... preload_content=False) + >>> for chunk in r.stream(32): + ... print(chunk) + b'...' + b'...' + ... + >>> r.release_conn() + +Setting ``preload_content`` to ``False`` means that urllib3 will stream the +response content. :meth:`~response.HTTPResponse.stream` lets you iterate over +chunks of the response content. + +.. note:: When using ``preload_content=False``, you should call + :meth:`~response.HTTPResponse.release_conn` to release the http connection + back to the connection pool so that it can be re-used. + +However, you can also treat the :class:`~response.HTTPResponse` instance as +a file-like object. This allows you to do buffering:: + + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/bytes/1024', + ... preload_content=False) + >>> r.read(4) + b'\x88\x1f\x8b\xe5' + +Calls to :meth:`~response.HTTPResponse.read()` will block until more response +data is available. + + >>> import io + >>> reader = io.BufferedReader(r, 8) + >>> reader.read(4) + >>> r.release_conn() + +You can use this file-like object to do things like decode the content using +:mod:`codecs`:: + + >>> import codecs + >>> reader = codecs.getreader('utf-8') + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/ip', + ... preload_content=False) + >>> json.load(reader(r)) + {'origin': '127.0.0.1'} + >>> r.release_conn() + +.. _proxies: + +Proxies +------- + +You can use :class:`~poolmanager.ProxyManager` to tunnel requests through an +HTTP proxy:: + + >>> import urllib3 + >>> proxy = urllib3.ProxyManager('http://localhost:3128/') + >>> proxy.request('GET', 'http://google.com/') + +The usage of :class:`~poolmanager.ProxyManager` is the same as +:class:`~poolmanager.PoolManager`. + +You can use :class:`~contrib.socks.SOCKSProxyManager` to connect to SOCKS4 or +SOCKS5 proxies. In order to use SOCKS proxies you will need to install +`PySocks <https://pypi.python.org/pypi/PySocks>`_ or install urllib3 with the +``socks`` extra:: + + pip install urllib3[socks] + +Once PySocks is installed, you can use +:class:`~contrib.socks.SOCKSProxyManager`:: + + >>> from urllib3.contrib.socks import SOCKSProxyManager + >>> proxy = SOCKSProxyManager('socks5://localhost:8889/') + >>> proxy.request('GET', 'http://google.com/') + + +.. _ssl_custom: + +Custom SSL certificates and client certificates +----------------------------------------------- + +Instead of using `certifi <https://certifi.io/en/latest>`_ you can provide your +own certificate authority bundle. This is useful for cases where you've +generated your own certificates or when you're using a private certificate +authority. Just provide the full path to the certificate bundle when creating a +:class:`~poolmanager.PoolManager`:: + + >>> import urllib3 + >>> http = urllib3.PoolManager( + ... cert_reqs='CERT_REQUIRED', + ... ca_certs='/path/to/your/certificate_bundle') + +When you specify your own certificate bundle only requests that can be +verified with that bundle will succeed. It's recommended to use a separate +:class:`~poolmanager.PoolManager` to make requests to URLs that do not need +the custom certificate. + +You can also specify a client certificate. This is useful when both the server +and the client need to verify each other's identity. Typically these +certificates are issued from the same authority. To use a client certificate, +provide the full path when creating a :class:`~poolmanager.PoolManager`:: + + >>> http = urllib3.PoolManager( + ... cert_file='/path/to/your/client_cert.pem', + ... cert_reqs='CERT_REQUIRED', + ... ca_certs='/path/to/your/certificate_bundle') + +.. _ssl_mac: + +Certificate validation and Mac OS X +----------------------------------- + +Apple-provided Python and OpenSSL libraries contain a patches that make them +automatically check the system keychain's certificates. This can be +surprising if you specify custom certificates and see requests unexpectedly +succeed. For example, if you are specifying your own certificate for validation +and the server presents a different certificate you would expect the connection +to fail. However, if that server presents a certificate that is in the system +keychain then the connection will succeed. + +`This article <https://hynek.me/articles/apple-openssl-verification-surprises/>`_ +has more in-depth analysis and explanation. + +If you have `homebrew <http://brew.sh>`_, you can configure homebrew Python to +use homebrew's OpenSSL instead of the system OpenSSL:: + + brew install openssl + brew install python --with-brewed-openssl + +.. _ssl_warnings: + +SSL Warnings +------------ + +urllib3 will issue several different warnings based on the level of certificate +verification support. These warning indicate particular situations and can +resolved in different ways. + +* :class:`~exceptions.InsecureRequestWarning` + This happens when an request is made to an HTTPS URL without certificate + verification enabled. Follow the :ref:`certificate verification <ssl>` + guide to resolve this warning. +* :class:`~exceptions.InsecurePlatformWarning` + This happens on Python 2 platforms that have an outdated :mod:`ssl` module. + These older :mod:`ssl` modules can cause some insecure requests to succeed + where they should fail and secure requests to fail where they should + succeed. Follow the :ref:`pyOpenSSL <ssl_py2>` guide to resolve this + warning. + +.. _sni_warning: + +* :class:`~exceptions.SNIMissingWarning` + This happens on Python 2 versions older than 2.7.9. These older versions + lack `SNI <https://en.wikipedia.org/wiki/Server_Name_Indication>`_ support. + This can cause servers to present a certificate that the client thinks is + invalid. Follow the :ref:`pyOpenSSL <ssl_py2>` guide to resolve this + warning. + +.. _disable_ssl_warnings: + +Making unverified HTTPS requests is **strongly** discouraged, however, if you +understand the risks and wish to disable these warnings, you can use :func:`~urllib3.disable_warnings`:: + + >>> import urllib3 + >>> urllib3.disable_warnings() + +Alternatively you can capture the warnings with the standard :mod:`logging` module:: + + >>> logging.captureWarnings(True) + +Finally, you can suppress the warnings at the interpreter level by setting the +``PYTHONWARNINGS`` environment variable or by using the +`-W flag <https://docs.python.org/2/using/cmdline.html#cmdoption-W>`_. + +Google App Engine +----------------- + +urllib3 supports `Google App Engine <https://cloud.google.com/appengine>`_ with +some caveats. + +If you're using the `Flexible environment +<https://cloud.google.com/appengine/docs/flexible/>`_, you do not have to do +any configuration- urllib3 will just work. However, if you're using the +`Standard environment <https://cloud.google.com/appengine/docs/python/>`_ then +you either have to use :mod:`urllib3.contrib.appengine`'s +:class:`~urllib3.contrib.appengine.AppEngineManager` or use the `Sockets API +<https://cloud.google.com/appengine/docs/python/sockets/>`_ + +To use :class:`~urllib3.contrib.appengine.AppEngineManager`:: + + >>> from urllib3.contrib.appengine import AppEngineManager + >>> http = AppEngineManager() + >>> http.request('GET', 'https://google.com/') + +To use the Sockets API, add the following to your app.yaml and use +:class:`~urllib3.poolmanager.PoolManager` as usual:: + + env_variables: + GAE_USE_SOCKETS_HTTPLIB : 'true' + +For more details on the limitations and gotchas, see +:mod:`urllib3.contrib.appengine`. diff --git a/docs/collections.rst b/docs/collections.rst deleted file mode 100644 index b3481402..00000000 --- a/docs/collections.rst +++ /dev/null @@ -1,13 +0,0 @@ -Collections -=========== - -These datastructures are used to implement the behaviour of various urllib3 -components in a decoupled and application-agnostic design. - -.. automodule:: urllib3._collections - - .. autoclass:: RecentlyUsedContainer - :members: - - .. autoclass:: HTTPHeaderDict - :members: diff --git a/docs/conf.py b/docs/conf.py index 7ac8393d..70a42819 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,6 +15,8 @@ from datetime import date import os import sys +import alabaster + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -22,6 +24,25 @@ import sys root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) sys.path.insert(0, root_path) +# Mock some expensive/platform-specific modules so build will work. +# (https://read-the-docs.readthedocs.io/en/latest/faq.html#\ +# i-get-import-errors-on-libraries-that-depend-on-c-modules) +import mock + + +class MockModule(mock.Mock): + @classmethod + def __getattr__(cls, name): + return MockModule() + + +MOCK_MODULES = ( + 'ntlm', +) + +sys.modules.update((mod_name, MockModule()) for mod_name in MOCK_MODULES) + + import urllib3 @@ -33,6 +54,7 @@ import urllib3 # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'alabaster', 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', @@ -105,15 +127,26 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'nature' +html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'description': 'Sanity-friendly HTTP client.', + 'github_user': 'shazow', + 'github_repo': 'urllib3', + 'github_button': False, + 'github_banner': True, + 'travis_button': True, + 'show_powered_by': False, + 'font_family': "'Roboto', Georgia, sans", + 'head_font_family': "'Roboto', Georgia, serif", + 'code_font_family': "'Roboto Mono', 'Consolas', monospace", +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +html_theme_path = [alabaster.get_path()] # The name for this set of Sphinx documents. If None, it defaults to # "<project> v<release> documentation". @@ -145,7 +178,15 @@ html_theme = 'nature' #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', + 'searchbox.html', + 'donate.html', + ] +} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -229,4 +270,5 @@ man_pages = [ [u'Andrey Petrov'], 1) ] -intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)} +intersphinx_mapping = { + 'python': ('https://docs.python.org/3.5', None),} diff --git a/docs/contrib.rst b/docs/contrib.rst deleted file mode 100644 index c85d3aaf..00000000 --- a/docs/contrib.rst +++ /dev/null @@ -1,102 +0,0 @@ -.. _contrib-modules: - -Contrib Modules -=============== - -These modules implement various extra features, that may not be ready for -prime time or that require optional third-party dependencies. - -.. _contrib-pyopenssl: - -SNI-support for Python 2 ------------------------- - -.. automodule:: urllib3.contrib.pyopenssl - - -.. _gae: - -Google App Engine ------------------ - -The :mod:`urllib3.contrib.appengine` module provides a pool manager that -uses Google App Engine's `URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. - -Example usage:: - - from urllib3 import PoolManager - from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox - - # This substitution will be done automagically once appengine code - # graduates from the contrib module. - if is_appengine_sandbox(): - # AppEngineManager uses AppEngine's URLFetch API behind the scenes - http = AppEngineManager() - else: - # PoolManager uses a socket-level API behind the scenes - http = PoolManager() - - # The client API should be consistent across managers, though some features are not available - # in URLFetch and you'll get warnings when you try to use them (like granular timeouts). - r = http.request('GET', 'https://google.com/') - - -There are `limitations <https://cloud.google.com/appengine/docs/python/urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be the best choice for your application. App Engine provides three options for urllib3 users: - -1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is cost-effective in many circumstances as long as your usage is within the limitations. -2. You can use a normal :class:`PoolManager` by enabling sockets. Sockets also have `limitations and restrictions <https://cloud.google.com/appengine/docs/python/sockets/#limitations-and-restrictions>`_ and have a lower free quota than URLFetch. To use sockets, be sure to specify the following in your ``app.yaml``:: - - env_variables: - GAE_USE_SOCKETS_HTTPLIB : 'true' - -3. If you are using `Managed VMs <https://cloud.google.com/appengine/docs/managed-vms/>`_, you can use the standard :class:`PoolManager` without any configuration or special environment variables. - - -.. _socks: - -SOCKS Proxies -------------- - -.. versionadded:: 1.14 - -The :mod:`urllib3.contrib.socks` module enables urllib3 to work with proxies -that use either the SOCKS4 or SOCKS5 protocols. These proxies are common in -environments that want to allow generic TCP/UDP traffic through their borders, -but don't want unrestricted traffic flows. - -To use it, either install ``PySocks`` or install urllib3 with the ``socks`` -extra, like so: - -.. code-block:: bash - - $ pip install -U urllib3[socks] - -The SOCKS module provides a -:class:`SOCKSProxyManager <urllib3.contrib.socks.SOCKSProxyManager>` that can -be used when SOCKS support is required. This class behaves very much like a -standard :class:`ProxyManager <urllib3.poolmanager.ProxyManager>`, but allows -the use of a SOCKS proxy instead. - -Using it is simple. For example, with a SOCKS5 proxy running on the local -machine, listening on port 8889: - -.. code-block:: python - - from urllib3.contrib.socks import SOCKSProxyManager - - http = SOCKSProxyManager('socks5://localhost:8889/') - r = http.request('GET', 'https://www.google.com/') - -The SOCKS implementation supports the full range of urllib3 features. It also -supports the following SOCKS features: - -- SOCKS4 -- SOCKS4a -- SOCKS5 -- Usernames and passwords for the SOCKS proxy - -The SOCKS module does have the following limitations: - -- No support for contacting a SOCKS proxy via IPv6. -- No support for reaching websites via a literal IPv6 address: domain names - must be used. diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 00000000..ad3dd2e9 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,83 @@ +Contributing +============ + +urllib3 is a community-maintained project and we happily accept contributions. + +If you wish to add a new feature or fix a bug: + +#. `Check for open issues <https://github.com/shazow/urllib3/issues>`_ or open + a fresh issue to start a discussion around a feature idea or a bug. There is + a *Contributor Friendly* tag for issues that should be ideal for people who + are not very familiar with the codebase yet. +#. Fork the `urllib3 repository on Github <https://github.com/shazow/urllib3>`_ + to start making your changes. +#. Write a test which shows that the bug was fixed or that the feature works + as expected. +#. Send a pull request and bug the maintainer until it gets merged and published. + :) Make sure to add yourself to ``CONTRIBUTORS.txt``. + + +Running the tests +----------------- + +We use some external dependencies, multiple interpreters and code coverage +analysis while running test suite. Our ``Makefile`` handles much of this for +you as long as you're running it `inside of a virtualenv +<http://docs.python-guide.org/en/latest/dev/virtualenvs/>`_:: + + $ make test + [... magically installs dependencies and runs tests on your virtualenv] + Ran 182 tests in 1.633s + + OK (SKIP=6) + +Note that code coverage less than 100% is regarded as a failing run. Some +platform-specific tests are skipped unless run in that platform. To make sure +the code works in all of urllib3's supported platforms, you can run our ``tox`` +suite:: + + $ make test-all + [... tox creates a virtualenv for every platform and runs tests inside of each] + py26: commands succeeded + py27: commands succeeded + py32: commands succeeded + py33: commands succeeded + py34: commands succeeded + +Our test suite `runs continuously on Travis CI +<https://travis-ci.org/shazow/urllib3>`_ with every pull request. + + +Sponsorship +----------- + +Please consider sponsoring urllib3 development, especially if your company +benefits from this library. + +We welcome your patronage on `Bountysource <https://www.bountysource.com/teams/urllib3>`_: + +* `Contribute a recurring amount to the team <https://salt.bountysource.com/checkout/amount?team=urllib3>`_ +* `Place a bounty on a specific feature <https://www.bountysource.com/teams/urllib3>`_ + +Your contribution will go towards adding new features to urllib3 and making +sure all functionality continues to meet our high quality standards. + + +Project Grant +------------- + +A grant for contiguous full-time development has the biggest impact for +progress. Periods of 3 to 10 days allow a contributor to tackle substantial +complex issues which are otherwise left to linger until somebody can't afford +to not fix them. + +Contact `@shazow <https://github.com/shazow>`_ to arrange a grant for a core +contributor. + +Huge thanks to all the companies and individuals who financially contributed to +the development of urllib3. Please send a PR if you've donated and would like +to be listed. + +* `Stripe <https://stripe.com/>`_ (June 23, 2014) + +.. * [Company] ([date]) diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt deleted file mode 100644 index b7b6d66d..00000000 --- a/docs/doc-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -ndg-httpsclient==0.3.2 -pyasn1==0.1.7 -Sphinx==1.2.2 -Jinja2==2.7.3 -MarkupSafe==0.23 -Pygments==1.6 -cryptography==0.4 -six==1.7.2 -cffi==0.8.2 -docutils==0.11 -pycparser==2.10 -certifi==14.05.14
\ No newline at end of file diff --git a/docs/exceptions.rst b/docs/exceptions.rst deleted file mode 100644 index cd451be5..00000000 --- a/docs/exceptions.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _exceptions: - -Exceptions -========== - -Custom exceptions defined by urllib3 - -.. automodule:: urllib3.exceptions - :members: diff --git a/docs/helpers.rst b/docs/helpers.rst deleted file mode 100644 index 6835e9aa..00000000 --- a/docs/helpers.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _helpers: - -Helpers -======= - -Useful methods for working with :mod:`httplib`, completely decoupled from -code specific to **urllib3**. - - -Timeouts --------- - -.. automodule:: urllib3.util.timeout - :members: - -Retries -------- - -.. automodule:: urllib3.util.retry - :members: - -URL Helpers ------------ - -.. automodule:: urllib3.util.url - :members: - -Filepost --------- - -.. automodule:: urllib3.filepost - :members: - -.. automodule:: urllib3.fields - :members: - -Request -------- - -.. automodule:: urllib3.request - :members: - -.. automodule:: urllib3.util.request - :members: - -Response --------- - -.. automodule:: urllib3.response - :members: - :undoc-members: - -SSL/TLS Helpers ---------------- - -.. automodule:: urllib3.util.ssl_ - :members: diff --git a/docs/index.rst b/docs/index.rst index c1882022..89b7baea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,408 +1,74 @@ -===================== -urllib3 Documentation -===================== +urllib3 +======= .. toctree:: :hidden: + :maxdepth: 2 - pools - managers - security - helpers - exceptions - collections - contrib - recipes + user-guide + advanced-usage + reference/index + contributing +urllib3 is a powerful, *sanity-friendly* HTTP client for Python. Much of the +Python ecosystem :ref:`already uses <who-uses>` urllib3 and you should too. +urllib3 brings many critical features that are missing from the Python +standard libraries: -Highlights -========== +- Thread safety. +- Connection pooling. +- Client-side SSL/TLS verification. +- File uploads with multipart encoding. +- Helpers for retrying requests and dealing with HTTP redirects. +- Support for gzip and deflate encoding. +- Proxy support for HTTP and SOCKS. +- 100% test coverage. -- Re-use the same socket connection for multiple requests, with optional - client-side certificate verification. See: - :class:`~urllib3.connectionpool.HTTPConnectionPool` and - :class:`~urllib3.connectionpool.HTTPSConnectionPool` - -- File posting with multipart encoding. See: - :func:`~urllib3.filepost.encode_multipart_formdata` - -- Built-in redirection and retries (optional). - -- Supports gzip and deflate decoding. See: - :func:`~urllib3.response.decode_gzip` and - :func:`~urllib3.response.decode_deflate` - -- Thread-safe and sanity-safe. - -- Proxy over :ref:`HTTP or SOCKS <proxymanager>`. - -- Tested on Python 2.6+ and Python 3.2+, 100% unit test coverage. - -- Works with AppEngine, gevent, eventlib, and the standard library :mod:`io` module. - -- Small and easy to understand codebase perfect for extending and building upon. - For a simplified abstraction, have a look at - `Requests <http://python-requests.org/>`_ which is also powered by urllib3. - - -Getting Started -=============== - -Installing ----------- - -``pip install urllib3`` or fetch the latest source from -`github.com/shazow/urllib3 <https://github.com/shazow/urllib3>`_. - -Usage ------ - -.. doctest :: +urllib3 is powerful and easy to use:: >>> import urllib3 >>> http = urllib3.PoolManager() - >>> r = http.request('GET', 'http://example.com/') + >>> r = http.request('GET', 'http://httpbin.org/robots.txt') >>> r.status 200 - >>> r.headers['server'] - 'ECS (iad/182A)' - >>> 'data: ' + r.data - 'data: ...' - - -**By default, urllib3 does not verify your HTTPS requests**. -You'll need to supply a root certificate bundle, or use `certifi -<https://certifi.io/>`_ - -.. doctest :: - - >>> import urllib3, certifi - >>> http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) - >>> r = http.request('GET', 'https://insecure.com/') - Traceback (most recent call last): - ... - SSLError: hostname 'insecure.com' doesn't match 'svn.nmap.org' - -For more on making secure SSL/TLS HTTPS requests, read the :ref:`Security -section <security>`. - - -urllib3's responses respect the :mod:`io` framework from Python's -standard library, allowing use of these standard objects for purposes -like buffering: - -.. doctest :: - - >>> http = urllib3.PoolManager() - >>> r = http.urlopen('GET','http://example.com/', preload_content=False) - >>> b = io.BufferedReader(r, 2048) - >>> firstpart = b.read(100) - >>> # ... your internet connection fails momentarily ... - >>> secondpart = b.read() - - -The response can be treated as a file-like object. -A file can be downloaded directly to a local file in a context without -being saved in memory. - -.. doctest :: - - >>> url = 'http://example.com/file' - >>> http = urllib3.PoolManager() - >>> with http.request('GET', url, preload_content=False) as r, open('filename', 'wb') as fp: - >>> .... shutil.copyfileobj(r, fp) - - -Upgrading & Versioning ----------------------- - -urllib3 uses a compatibility-based versioning scheme (let's call it -*compatver*). For the user, they indicate the required decision for upgrading. - -Given a version ``A.B.C``: - -``C.`` Strictly backwards-compatible, usually a bug-fix. **Always upgrade.** - -``B.`` Possibly partially incompatible, usually a new feature or a minor API -improvement. **Read the changelog and upgrade when ready.** - -``A.`` Major rewrite and possibly breaks everything. Not really an upgrade, -basically a new library under the same namespace, decide if you want to switch. - -For example, when going from urllib3 v1.2.3 to v1.2.4, you should always -upgrade without hesitation. When going from v1.2 to v1.3, you should read the -changes to make sure they're not going to affect you. - - -Components -========== - -:mod:`urllib3` tries to strike a fine balance between power, extendability, and -sanity. To achieve this, the codebase is a collection of small reusable -utilities and abstractions composed together in a few helpful layers. - - -PoolManager ------------ - -The highest level is the :doc:`PoolManager(...) <managers>`. - -The :class:`~urllib3.poolmanagers.PoolManager` will take care of reusing -connections for you whenever you request the same host. This should cover most -scenarios without significant loss of efficiency, but you can always drop down -to a lower level component for more granular control. - -.. doctest :: - - >>> import urllib3 - >>> http = urllib3.PoolManager(10) - >>> r1 = http.request('GET', 'http://example.com/') - >>> r2 = http.request('GET', 'http://httpbin.org/') - >>> r3 = http.request('GET', 'http://httpbin.org/get') - >>> len(http.pools) - 2 - -A :class:`~urllib3.poolmanagers.PoolManager` is a proxy for a collection of -:class:`ConnectionPool` objects. They both inherit from -:class:`~urllib3.request.RequestMethods` to make sure that their API is -similar, so that instances of either can be passed around interchangeably. - - -.. _proxymanager: - -ProxyManager ------------- - -HTTP Proxy -~~~~~~~~~~ - -The :class:`~urllib3.poolmanagers.ProxyManager` is an HTTP proxy-aware -subclass of :class:`~urllib3.poolmanagers.PoolManager`. It produces a single -:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP -connections and individual per-``server:port`` -:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled -HTTPS connections: - -:: - - >>> proxy = urllib3.ProxyManager('http://localhost:3128/') - >>> r1 = proxy.request('GET', 'http://google.com/') - >>> r2 = proxy.request('GET', 'http://httpbin.org/') - >>> len(proxy.pools) - 1 - >>> r3 = proxy.request('GET', 'https://httpbin.org/') - >>> r4 = proxy.request('GET', 'https://twitter.com/') - >>> len(proxy.pools) - 3 - + >>> r.data + 'User-agent: *\nDisallow: /deny\n' -SOCKS Proxy -~~~~~~~~~~~ - -The :ref:`contrib module <socks>` includes support for a :class:`SOCKSProxyManager <urllib3.contrib.socks.SOCKSProxyManager>`. - - -ConnectionPool --------------- - -The next layer is the :doc:`ConnectionPool(...) <pools>`. - -The :class:`~urllib3.connectionpool.HTTPConnectionPool` and -:class:`~urllib3.connectionpool.HTTPSConnectionPool` classes allow you to -define a pool of connections to a single host and make requests against this -pool with automatic **connection reusing** and **thread safety**. - -When the :mod:`ssl` module is available, then -:class:`~urllib3.connectionpool.HTTPSConnectionPool` objects can be configured -to check SSL certificates against specific provided certificate authorities. - -.. doctest :: - - >>> import urllib3 - >>> conn = urllib3.connection_from_url('http://httpbin.org/') - >>> r1 = conn.request('GET', 'http://httpbin.org/') - >>> r2 = conn.request('GET', '/user-agent') - >>> r3 = conn.request('GET', 'http://example.com') - Traceback (most recent call last): - ... - urllib3.exceptions.HostChangedError: HTTPConnectionPool(host='httpbin.org', port=None): Tried to open a foreign host with url: http://example.com - -Again, a ConnectionPool is a pool of connections to a specific host. Trying to -access a different host through the same pool will raise a ``HostChangedError`` -exception unless you specify ``assert_same_host=False``. Do this at your own -risk as the outcome is completely dependent on the behaviour of the host server. - -If you need to access multiple hosts and don't want to manage your own -collection of :class:`~urllib3.connectionpool.ConnectionPool` objects, then you -should use a :class:`~urllib3.poolmanager.PoolManager`. - -A :class:`~urllib3.connectionpool.ConnectionPool` is composed of a collection -of :class:`httplib.HTTPConnection` objects. - - -Timeout -------- - -A timeout can be set to abort socket operations on individual connections after -the specified duration. The timeout can be defined as a float or an instance of -:class:`~urllib3.util.timeout.Timeout` which gives more granular configuration -over how much time is allowed for different stages of the request. This can be -set for the entire pool or per-request. - -.. doctest :: - - >>> from urllib3 import PoolManager, Timeout - - >>> # Manager with 3 seconds combined timeout. - >>> http = PoolManager(timeout=3.0) - >>> r = http.request('GET', 'http://httpbin.org/delay/1') - - >>> # Manager with 2 second timeout for the read phase, no limit for the rest. - >>> http = PoolManager(timeout=Timeout(read=2.0)) - >>> r = http.request('GET', 'http://httpbin.org/delay/1') +Installing +---------- - >>> # Manager with no timeout but a request with a timeout of 1 seconds for - >>> # the connect phase and 2 seconds for the read phase. - >>> http = PoolManager() - >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(connect=1.0, read=2.0)) +urllib3 can be installed with `pip <https://pip.pypa.io>`_:: - >>> # Same Manager but request with a 5 second total timeout. - >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(total=5.0)) + $ pip install urllib3 -See the :class:`~urllib3.util.timeout.Timeout` definition for more details. +Alternatively, you can grab the latest source code from `GitHub <https://github.com/shazow/urllib3>`_:: + $ git clone git://github.com/shazow/urllib3.git + $ python setup.py install -Retry +Usage ----- -Retries can be configured by passing an instance of -:class:`~urllib3.util.retry.Retry`, or disabled by passing ``False``, to the -``retries`` parameter. - -Redirects are also considered to be a subset of retries but can be configured or -disabled individually. - -:: - - >>> from urllib3 import PoolManager, Retry - - >>> # Allow 3 retries total for all requests in this pool. These are the same: - >>> http = PoolManager(retries=3) - >>> http = PoolManager(retries=Retry(3)) - >>> http = PoolManager(retries=Retry(total=3)) - - >>> r = http.request('GET', 'http://httpbin.org/redirect/2') - >>> # r.status -> 200 - - >>> # Disable redirects for this request. - >>> r = http.request('GET', 'http://httpbin.org/redirect/2', retries=Retry(3, redirect=False)) - >>> # r.status -> 302 - - >>> # No total limit, but only do 5 connect retries, for this request. - >>> r = http.request('GET', 'http://httpbin.org/', retries=Retry(connect=5)) - +The :doc:`user-guide` is the place to go to learn how to use the library and +accomplish common tasks. The more in-depth :doc:`advanced-usage` guide is the place to go for lower-level tweaking. -See the :class:`~urllib3.util.retry.Retry` definition for more details. +The :doc:`reference/index` documentation provides API-level documentation. +.. _who-uses: -Stream ------- +Who uses urllib3? +----------------- -You may also stream your response and get data as they come (e.g. when using -``transfer-encoding: chunked``). In this case, method -:func:`~urllib3.response.HTTPResponse.stream` will return generator. +* `Requests <http://python-requests.org/>`_ +* `Pip <https://pip.pypa.io>`_ +* & more! -:: - - >>> import urllib3 - >>> http = urllib3.PoolManager() - - >>> r = http.request("GET", "http://httpbin.org/stream/3") - >>> r.getheader("transfer-encoding") - 'chunked' - - >>> for chunk in r.stream(): - ... print chunk - {"url": "http://httpbin.org/stream/3", ..., "id": 0, ...} - {"url": "http://httpbin.org/stream/3", ..., "id": 1, ...} - {"url": "http://httpbin.org/stream/3", ..., "id": 2, ...} - >>> r.closed - True - -Completely consuming the stream will auto-close the response and release -the connection back to the pool. If you're only partially consuming the -consuming a stream, make sure to manually call ``r.close()`` on the -response. - -Foundation ----------- - -At the very core, just like its predecessors, :mod:`urllib3` is built on top of -:mod:`httplib` -- the lowest level HTTP library included in the Python -standard library. - -To aid the limited functionality of the :mod:`httplib` module, :mod:`urllib3` -provides various helper methods which are used with the higher level components -but can also be used independently. - -* :ref:`helpers` -* :ref:`exceptions` - - -Contrib Modules ---------------- - -These modules implement various extra features, that may not be ready for -prime time or that require optional third-party dependencies. - -* :ref:`contrib-modules` +License +------- +urllib3 is made available under the MIT License. For more details, see `LICENSE.txt <https://github.com/shazow/urllib3/blob/master/LICENSE.txt>`_. Contributing -============ - -#. `Check for open issues <https://github.com/shazow/urllib3/issues>`_ or open - a fresh issue to start a discussion around a feature idea or a bug. There is - a *Contributor Friendly* tag for issues that should be ideal for people who - are not very familiar with the codebase yet. -#. Fork the `urllib3 repository on Github <https://github.com/shazow/urllib3>`_ - to start making your changes. -#. Write a test which shows that the bug was fixed or that the feature works - as expected. -#. Send a pull request and bug the maintainer until it gets merged and published. - :) Make sure to add yourself to ``CONTRIBUTORS.txt``. - - -Sponsorship -=========== - -Please consider sponsoring urllib3 development, especially if your company -benefits from this library. - -We welcome your patronage on `Bountysource <https://www.bountysource.com/teams/urllib3>`_: - -* `Contribute a recurring amount to the team <https://salt.bountysource.com/checkout/amount?team=urllib3>`_ -* `Place a bounty on a specific feature <https://www.bountysource.com/teams/urllib3>`_ - -Your contribution will go towards adding new features to urllib3 and making -sure all functionality continues to meet our high quality standards. - - -Project Grant -------------- - -A grant for contiguous full-time development has the biggest impact for -progress. Periods of 3 to 10 days allow a contributor to tackle substantial -complex issues which are otherwise left to linger until somebody can't afford -to not fix them. - -Contact `@shazow <https://github.com/shazow>`_ to arrange a grant for a core -contributor. - -Huge thanks to all the companies and individuals who financially contributed to -the development of urllib3. Please send a PR if you've donated and would like -to be listed. - -* `Stripe <https://stripe.com/>`_ (June 23, 2014) +------------ -.. * [Company] ([date]) +We happily welcome contributions, please see :doc:`contributing` for details. diff --git a/docs/managers.rst b/docs/managers.rst deleted file mode 100644 index e8e7e210..00000000 --- a/docs/managers.rst +++ /dev/null @@ -1,131 +0,0 @@ -PoolManager -=========== - -.. automodule:: urllib3.poolmanager - -A pool manager is an abstraction for a collection of -:doc:`ConnectionPools <pools>`. - -If you need to make requests to multiple hosts, then you can use a -:class:`.PoolManager`, which takes care of maintaining your pools -so you don't have to. - -.. doctest :: - - >>> from urllib3 import PoolManager - >>> manager = PoolManager(10) - >>> r = manager.request('GET', 'http://example.com') - >>> r.headers['server'] - 'ECS (iad/182A)' - >>> r = manager.request('GET', 'http://httpbin.org/') - >>> r.headers['server'] - 'gunicorn/18.0' - >>> r = manager.request('POST', 'http://httpbin.org/headers') - >>> r = manager.request('HEAD', 'http://httpbin.org/cookies') - >>> len(manager.pools) - 2 - >>> conn = manager.connection_from_host('httpbin.org') - >>> conn.num_requests - 3 - -A :class:`.PoolManager` will create a new :doc:`ConnectionPool <pools>` -when no :doc:`ConnectionPools <pools>` exist with a matching pool key. -The pool key is derived using the requested URL and the current values -of the ``connection_pool_kw`` instance variable on :class:`.PoolManager`. - -The keys in ``connection_pool_kw`` used when deriving the key are -configurable. For example, by default the ``my_field`` key is not -considered. - -.. doctest :: - - >>> from urllib3.poolmanager import PoolManager - >>> manager = PoolManager(10, my_field='wheat') - >>> manager.connection_from_url('http://example.com') - >>> manager.connection_pool_kw['my_field'] = 'barley' - >>> manager.connection_from_url('http://example.com') - >>> len(manager.pools) - 1 - -To make the pool manager create new pools when the value of -``my_field`` changes, you can define a custom pool key and alter -the ``key_fn_by_scheme`` instance variable on :class:`.PoolManager`. - -.. doctest :: - - >>> import functools - >>> from collections import namedtuple - >>> from urllib3.poolmanager import PoolManager, HTTPPoolKey - >>> from urllib3.poolmanager import default_key_normalizer as normalizer - >>> CustomKey = namedtuple('CustomKey', HTTPPoolKey._fields + ('my_field',)) - >>> manager = PoolManager(10, my_field='wheat') - >>> manager.key_fn_by_scheme['http'] = functools.partial(normalizer, CustomKey) - >>> manager.connection_from_url('http://example.com') - >>> manager.connection_pool_kw['my_field'] = 'barley' - >>> manager.connection_from_url('http://example.com') - >>> len(manager.pools) - 2 - -The API of a :class:`.PoolManager` object is similar to that of a -:doc:`ConnectionPool <pools>`, so they can be passed around interchangeably. - -The PoolManager uses a Least Recently Used (LRU) policy for discarding old -pools. That is, if you set the PoolManager ``num_pools`` to 10, then after -making requests to 11 or more different hosts, the least recently used pools -will be cleaned up eventually. - -Cleanup of stale pools does not happen immediately but can be forced when used -as a context manager. - -.. doctest :: - - >>> from urllib3 import PoolManager - >>> with PoolManager(10) as manager: - ... r = manager.request('GET', 'http://example.com') - ... r = manager.request('GET', 'http://httpbin.org/') - ... len(manager.pools) - ... - 2 - >>> len(manager.pools) - 0 - -You can read more about the implementation and the various adjustable variables -within :class:`~urllib3._collections.RecentlyUsedContainer`. - -API ---- - - .. autoclass:: PoolManager - :inherited-members: - .. autoclass:: BasePoolKey - :inherited-members: - .. autoclass:: HTTPPoolKey - :inherited-members: - .. autoclass:: HTTPSPoolKey - :inherited-members: - -ProxyManager -============ - -:class:`.ProxyManager` is an HTTP proxy-aware subclass of :class:`.PoolManager`. -It produces a single -:class:`~urllib3.connectionpool.HTTPConnectionPool` instance for all HTTP -connections and individual per-server:port -:class:`~urllib3.connectionpool.HTTPSConnectionPool` instances for tunnelled -HTTPS connections. - -Example using proxy authentication: - -:: - - >>> headers = urllib3.make_headers(proxy_basic_auth='myusername:mypassword') - >>> proxy = urllib3.ProxyManager('http://localhost:3128', proxy_headers=headers) - >>> r = proxy.request('GET', 'http://example.com/') - >>> r.status - 200 - - -API ---- - .. autoclass:: ProxyManager - diff --git a/docs/pools.rst b/docs/pools.rst deleted file mode 100644 index 9cc2be9a..00000000 --- a/docs/pools.rst +++ /dev/null @@ -1,86 +0,0 @@ -ConnectionPools -=============== - -.. automodule:: urllib3.connectionpool - -A connection pool is a container for a collection of connections to a specific -host. - -If you need to make requests to the same host repeatedly, then you should use a -:class:`.HTTPConnectionPool`. - -.. doctest :: - - >>> from urllib3 import HTTPConnectionPool - >>> pool = HTTPConnectionPool('ajax.googleapis.com', maxsize=1) - >>> r = pool.request('GET', '/ajax/services/search/web', - ... fields={'q': 'urllib3', 'v': '1.0'}) - >>> r.status - 200 - >>> r.headers['content-type'] - 'text/javascript; charset=utf-8' - >>> 'data: ' + r.data # Content of the response - 'data: ...' - >>> r = pool.request('GET', '/ajax/services/search/web', - ... fields={'q': 'python', 'v': '1.0'}) - >>> 'data: ' + r.data # Content of the response - 'data: ...' - >>> pool.num_connections - 1 - >>> pool.num_requests - 2 - -By default, the pool will cache just one connection. If you're planning on using -such a pool in a multithreaded environment, you should set the ``maxsize`` of -the pool to a higher number, such as the number of threads. You can also control -many other variables like timeout, blocking, and default headers. - -A ConnectionPool can be used as a context manager to automatically clear the -pool after usage. - -.. doctest :: - - >>> from urllib3 import HTTPConnectionPool - >>> with HTTPConnectionPool('ajax.googleapis.com', maxsize=1) as pool: - ... r = pool.request('GET', '/ajax/services/search/web', - ... fields={'q': 'urllib3', 'v': '1.0'}) - ... print(pool.pool) - ... - <queue.LifoQueue object at 0x7f67367dfcf8> - >>> print(pool.pool) - None - -Helpers -------- - -There are various helper functions provided for instantiating these -ConnectionPools more easily: - - .. autofunction:: connection_from_url - -API ---- - -:mod:`urllib3.connectionpool` comes with two connection pools: - - .. autoclass:: HTTPConnectionPool - :members: - :inherited-members: - - .. autoclass:: HTTPSConnectionPool - - -All of these pools inherit from a common base class: - - .. autoclass:: ConnectionPool - -.. module:: urllib3.connection - -Related Classes ---------------- - -urllib3 implements its own :class:`HTTPConnection` object to allow for more -flexibility than the standard library's implementation. - -.. autoclass:: HTTPConnection - :members: diff --git a/docs/recipes.rst b/docs/recipes.rst deleted file mode 100644 index fda6bef8..00000000 --- a/docs/recipes.rst +++ /dev/null @@ -1,41 +0,0 @@ -Recipes -======= - -This page includes a collection of recipes in the urlib3 cookbook. - -Decode HTTP Response Data in Concatenated Gzip Format ------------------------------------------------------ - -By default, urllib3 checks ``Content-Encoding`` header in HTTP response and decodes the data in ``gzip`` or ``deflate`` transparently. If ``Content-Encoding`` is not either of them, however, you will have to decode data in your application. - -This recipe shows how to decode data in the concatenated gzip format where multiple gzipped data chunks are concatenated in HTTP response. - -.. doctest :: - - import zlib - import urllib3 - - CHUNK_SIZE = 1024 - - def decode_gzip_raw_content(raw_data_fd): - obj = zlib.decompressobj(16 + zlib.MAX_WBITS) - output = [] - d = raw_data_fd.read(CHUNK_SIZE) - while d: - output.append(obj.decompress(d)) - while obj.unused_data != b'': - unused_data = obj.unused_data - obj = zlib.decompressobj(16 + zlib.MAX_WBITS) - output.append(obj.decompress(unused_data)) - d = raw_data_fd.read(CHUNK_SIZE) - return b''.join(output) - - - def test_urllib3_concatenated_gzip_in_http_response(): - # example for urllib3 - http = urllib3.PoolManager() - r = http.request('GET', 'http://example.com/abc.txt', - decode_content=False, preload_content=False) - content = decode_gzip_raw_content(r).decode('utf-8') - -``obj.unused_data`` includes the left over data in the previous ``obj.decompress`` method call. A new ``zlib.decompressobj`` is used to start decoding the next gzipped data chunk until no further data is given. diff --git a/docs/reference/index.rst b/docs/reference/index.rst new file mode 100644 index 00000000..bc1b7120 --- /dev/null +++ b/docs/reference/index.rst @@ -0,0 +1,90 @@ +Reference +========= + +.. contents:: + :local: + :backlinks: none + +Subpackages +----------- + +.. toctree:: + + urllib3.contrib + urllib3.util + +Submodules +---------- + +urllib3.connection module +------------------------- + +.. automodule:: urllib3.connection + :members: + :undoc-members: + :show-inheritance: + +urllib3.connectionpool module +----------------------------- + +.. automodule:: urllib3.connectionpool + :members: + :undoc-members: + :show-inheritance: + +urllib3.exceptions module +------------------------- + +.. automodule:: urllib3.exceptions + :members: + :undoc-members: + :show-inheritance: + +urllib3.fields module +--------------------- + +.. automodule:: urllib3.fields + :members: + :undoc-members: + :show-inheritance: + +urllib3.filepost module +----------------------- + +.. automodule:: urllib3.filepost + :members: + :undoc-members: + :show-inheritance: + +urllib3.poolmanager module +-------------------------- + +.. automodule:: urllib3.poolmanager + :members: + :undoc-members: + :show-inheritance: + +urllib3.request module +---------------------- + +.. automodule:: urllib3.request + :members: + :undoc-members: + :show-inheritance: + +urllib3.response module +----------------------- + +.. automodule:: urllib3.response + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: urllib3 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/urllib3.contrib.rst b/docs/reference/urllib3.contrib.rst new file mode 100644 index 00000000..af6f9461 --- /dev/null +++ b/docs/reference/urllib3.contrib.rst @@ -0,0 +1,37 @@ +urllib3.contrib package +======================= + +These modules implement various extra features, that may not be ready for +prime time or that require optional third-party dependencies. + +urllib3.contrib.appengine module +-------------------------------- + +.. automodule:: urllib3.contrib.appengine + :members: + :undoc-members: + :show-inheritance: + +urllib3.contrib.ntlmpool module +------------------------------- + +.. automodule:: urllib3.contrib.ntlmpool + :members: + :undoc-members: + :show-inheritance: + +urllib3.contrib.pyopenssl module +-------------------------------- + +.. automodule:: urllib3.contrib.pyopenssl + :members: + :undoc-members: + :show-inheritance: + +urllib3.contrib.socks module +---------------------------- + +.. automodule:: urllib3.contrib.socks + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/reference/urllib3.util.rst b/docs/reference/urllib3.util.rst new file mode 100644 index 00000000..f8b20a98 --- /dev/null +++ b/docs/reference/urllib3.util.rst @@ -0,0 +1,70 @@ +urllib3.util package +==================== + +Useful methods for working with :mod:`httplib`, completely decoupled from +code specific to **urllib3**. + +At the very core, just like its predecessors, :mod:`urllib3` is built on top of +:mod:`httplib` -- the lowest level HTTP library included in the Python +standard library. + +To aid the limited functionality of the :mod:`httplib` module, :mod:`urllib3` +provides various helper methods which are used with the higher level components +but can also be used independently. + +urllib3.util.connection module +------------------------------ + +.. automodule:: urllib3.util.connection + :members: + :undoc-members: + :show-inheritance: + +urllib3.util.request module +--------------------------- + +.. automodule:: urllib3.util.request + :members: + :undoc-members: + :show-inheritance: + +urllib3.util.response module +---------------------------- + +.. automodule:: urllib3.util.response + :members: + :undoc-members: + :show-inheritance: + +urllib3.util.retry module +------------------------- + +.. automodule:: urllib3.util.retry + :members: + :undoc-members: + :show-inheritance: + +urllib3.util.timeout module +--------------------------- + +.. automodule:: urllib3.util.timeout + :members: + :undoc-members: + :show-inheritance: + +urllib3.util.url module +----------------------- + +.. automodule:: urllib3.util.url + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: urllib3.util + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..05b104a8 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +-r ../dev-requirements.txt +ndg-httpsclient +sphinx +alabaster diff --git a/docs/security.rst b/docs/security.rst deleted file mode 100644 index 5117e329..00000000 --- a/docs/security.rst +++ /dev/null @@ -1,262 +0,0 @@ -.. _security: - -Security: Verified HTTPS with SSL/TLS -===================================== - -Very important fact: **By default, urllib3 does not verify HTTPS requests.** - -The historic reason for this is that we rely on ``httplib`` for some of the -HTTP protocol implementation, and ``httplib`` does not verify requests out of -the box. This is not a good reason, but here we are. - -Luckily, it's not too hard to enable verified HTTPS requests and there are a -few ways to do it. - - -Python with SSL enabled ------------------------ - -First we need to make sure your Python installation has SSL enabled. Easiest -way to check is to simply open a Python shell and type ``import ssl``:: - - >>> import ssl - Traceback (most recent call last): - ... - ImportError: No module named _ssl - -If you got an ``ImportError``, then your Python is not compiled with SSL support -and you'll need to re-install it. Read -`this StackOverflow thread <https://stackoverflow.com/questions/5128845/importerror-no-module-named-ssl>`_ -for details. - -Otherwise, if ``ssl`` imported cleanly, then we're ready to setup our certificates: -:ref:`certifi-with-urllib3`. - - -Enabling SSL on Google AppEngine -++++++++++++++++++++++++++++++++ - -If you're using Google App Engine, you'll need to add ``ssl`` as a library -dependency to your yaml file, like this:: - - libraries: - - name: ssl - version: latest - -If it's still not working, you may need to enable billing on your account -to `enable using sockets -<https://developers.google.com/appengine/docs/python/sockets/>`_. - - -.. _certifi-with-urllib3: - -Using Certifi with urllib3 --------------------------- - -`Certifi <http://certifi.io/>`_ is a package which ships with Mozilla's root -certificates for easy programmatic access. - -1. Install the Python ``certifi`` package:: - - $ pip install certifi - -2. Setup your pool to require a certificate and provide the certifi bundle:: - - import urllib3 - import certifi - - http = urllib3.PoolManager( - cert_reqs='CERT_REQUIRED', # Force certificate check. - ca_certs=certifi.where(), # Path to the Certifi bundle. - ) - - # You're ready to make verified HTTPS requests. - try: - r = http.request('GET', 'https://example.com/') - except urllib3.exceptions.SSLError as e: - # Handle incorrect certificate error. - ... - -Make sure to update your ``certifi`` package regularly to get the latest root -certificates. - - -Using your system's root certificates -------------------------------------- - -Your system's root certificates may be more up-to-date than maintaining your -own, but the trick is finding where they live. Different operating systems have -them in different places. - -For example, on most Linux distributions they're at -``/etc/ssl/certs/ca-certificates.crt``. On Windows and OS X? `It's not so simple -<https://stackoverflow.com/questions/10095676/openssl-reasonable-default-for-trusted-ca-certificates>`_. - -Once you find your root certificate file:: - - import urllib3 - - ca_certs = "/etc/ssl/certs/ca-certificates.crt" # Or wherever it lives. - - http = urllib3.PoolManager( - cert_reqs='CERT_REQUIRED', # Force certificate check. - ca_certs=ca_certs, # Path to your certificate bundle. - ) - - # You're ready to make verified HTTPS requests. - try: - r = http.request('GET', 'https://example.com/') - except urllib3.exceptions.SSLError as e: - # Handle incorrect certificate error. - ... - - -.. _pyopenssl: - -OpenSSL / PyOpenSSL -------------------- - -By default, we use the standard library's ``ssl`` module. Unfortunately, there -are several limitations which are addressed by PyOpenSSL: - -- (Python 2.x) SNI support. -- (Python 2.x-3.2) Disabling compression to mitigate `CRIME attack - <https://en.wikipedia.org/wiki/CRIME_(security_exploit)>`_. - -To use the Python OpenSSL bindings instead, you'll need to install the required -packages:: - - $ pip install pyopenssl ndg-httpsclient pyasn1 - -If ``cryptography`` fails to install as a dependency, make sure you have `libffi -<http://sourceware.org/libffi/>`_ available on your system and run -``pip install cryptography``. - -Once the packages are installed, you can tell urllib3 to switch the ssl backend -to PyOpenSSL with :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3`:: - - import urllib3.contrib.pyopenssl - urllib3.contrib.pyopenssl.inject_into_urllib3() - -Now you can continue using urllib3 as you normally would. - -For more details, check the :mod:`~urllib3.contrib.pyopenssl` module. - -Installing urllib3 with SNI support and certificates ----------------------------------------------------- - -By default, if you need to use SNI on Python 2.6 or Python 2.7.0-2.7.8, you -have to install PyOpenSSL, ndghttpsclient, and pyasn1 separately. Further, to -use certifi you have to install it separately. If you know that you want these -dependencies when you install urllib3, you can now do:: - - pip install urllib3[secure] - -This will install the SNI dependencies on Python 2.6 and 2.7 (we cannot yet -restrict the microversion for 2.7) and certifi on all versions of Python. - -.. note:: - - If you do this on linux, e.g., Ubuntu 14.04, you will need extra system - dependencies for PyOpenSSL. Specifically, PyOpenSSL requires cryptography - which will require you to install: - - - build-essential - - python-dev - - libffi-dev - - libssl-dev - - The package names may vary depending on the distribution of linux you are - using. - -.. _insecurerequestwarning: - -InsecureRequestWarning ----------------------- - -.. versionadded:: 1.9 - -Unverified HTTPS requests will trigger a warning via Python's ``warnings`` module:: - - urllib3/connectionpool.py:736: InsecureRequestWarning: Unverified HTTPS - request is being made. Adding certificate verification is strongly advised. - See: https://urllib3.readthedocs.io/en/latest/security.html - -This would be a great time to enable HTTPS verification: -:ref:`certifi-with-urllib3`. - -For info about disabling warnings, see `Disabling Warnings`_. - - -InsecurePlatformWarning ------------------------ - -.. versionadded:: 1.11 - -Certain Python platforms (specifically, versions of Python earlier than 2.7.9) -have restrictions in their ``ssl`` module that limit the configuration that -``urllib3`` can apply. In particular, this can cause HTTPS requests that would -succeed on more featureful platforms to fail, and can cause certain security -features to be unavailable. - -If you encounter this warning, it is strongly recommended you: - -- upgrade to a newer Python version -- upgrade ``ndg-httpsclient`` with ``pip install --upgrade ndg-httpsclient`` -- use pyOpenSSL as described in the :ref:`pyopenssl` section - -For info about disabling warnings, see `Disabling Warnings`_. - - -SNIMissingWarning ------------------ - -.. versionadded:: 1.13 - -Certain Python distributions (specifically, versions of Python earlier than -2.7.9) and older OpenSSLs have restrictions that prevent them from using the -SNI (Server Name Indication) extension. This can cause unexpected behaviour -when making some HTTPS requests, usually causing the server to present the a -TLS certificate that is not valid for the website you're trying to access. - -If you encounter this warning, it is strongly recommended that you upgrade -to a newer Python version, or that you use pyOpenSSL as described in the -:ref:`pyopenssl` section. - -For info about disabling warnings, see `Disabling Warnings`_. - - -Disabling Warnings ------------------- - -Making unverified HTTPS requests is strongly discouraged. ˙ ͜ʟ˙ - -But if you understand the ramifications and still want to do it... - -Within the code -+++++++++++++++ - -If you know what you're doing and would like to disable all ``urllib3`` warnings, -you can use :func:`~urllib3.disable_warnings`:: - - import urllib3 - urllib3.disable_warnings() - -Alternatively, if you are using Python's ``logging`` module, you can capture the -warnings to your own log:: - - logging.captureWarnings(True) - -Capturing the warnings to your own log is much preferred over simply disabling -the warnings. - -Without modifying code -++++++++++++++++++++++ - -If you are using a program that uses ``urllib3`` and don't want to change the -code, you can suppress warnings by setting the ``PYTHONWARNINGS`` environment -variable in Python 2.7+ or by using the ``-W`` flag with the Python -interpreter (see `docs -<https://docs.python.org/2/using/cmdline.html#cmdoption-W>`_), such as:: - - PYTHONWARNINGS="ignore:Unverified HTTPS request" ./do-insecure-request.py diff --git a/docs/user-guide.rst b/docs/user-guide.rst new file mode 100644 index 00000000..b37343aa --- /dev/null +++ b/docs/user-guide.rst @@ -0,0 +1,417 @@ +User Guide +========== + +.. currentmodule:: urllib3 + +Making requests +--------------- + +First things first, import the urllib3 module:: + + >>> import urllib3 + +You'll need a :class:`~poolmanager.PoolManager` instance to make requests. +This object handles all of the details of connection pooling and thread safety +so that you don't have to:: + + >>> http = urllib3.PoolManager() + +To make a request use :meth:`~poolmanager.PoolManager.request`:: + + >>> r = http.request('GET', 'http://httpbin.org/robots.txt') + >>> r.data + b'User-agent: *\nDisallow: /deny\n' + +``request()`` returns a :class:`~response.HTTPResponse` object, the +:ref:`response_content` section explains how to handle various responses. + +You can use :meth:`~poolmanager.PoolManager.request` to make requests using any +HTTP verb:: + + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... fields={'hello': 'world'}) + +The :ref:`request_data` section covers sending other kinds of requests data, +including JSON, files, and binary data. + +.. _response_content: + +Response content +---------------- + +The :class:`~response.HTTPResponse` object provides +:attr:`~response.HTTPResponse.status`, :attr:`~response.HTTPResponse.data`, and +:attr:`~response.HTTPResponse.header` attributes:: + + >>> r = http.request('GET', 'http://httpbin.org/ip') + >>> r.status + 200 + >>> r.data + b'{\n "origin": "104.232.115.37"\n}\n' + >>> r.headers + HTTPHeaderDict({'Content-Length': '33', ...}) + +JSON content +~~~~~~~~~~~~ + +JSON content can be loaded by decoding and deserializing the +:attr:`~response.HTTPResponse.data` attribute of the request:: + + >>> import json + >>> r = http.request('GET', 'http://httpbin.org/ip') + >>> json.loads(r.data.decode('utf-8')) + {'origin': '127.0.0.1'} + +Binary content +~~~~~~~~~~~~~~ + +The :attr:`~response.HTTPResponse.data` attribute of the response is always set +to a byte string representing the response content:: + + >>> r = http.request('GET', 'http://httpbin.org/bytes/8') + >>> r.data + b'\xaa\xa5H?\x95\xe9\x9b\x11' + +.. note:: For larger responses, it's sometimes better to :ref:`stream <stream>` + the response. + +.. _request_data: + +Request data +------------ + +Headers +~~~~~~~ + +You can specify headers as a dictionary in the ``headers`` argument in :meth:`~poolmanager.PoolManager.request`:: + + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/headers', + ... headers={ + ... 'X-Something': 'value' + ... }) + >>> json.loads(r.data.decode('utf-8'))['headers'] + {'X-Something': 'value', ...} + +Query parameters +~~~~~~~~~~~~~~~~ + +For ``GET``, ``HEAD``, and ``DELETE`` requests, you can simply pass the +arguments as a dictionary in the ``fields`` argument to +:meth:`~poolmanager.PoolManager.request`:: + + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/get', + ... fields={'arg': 'value'}) + >>> json.loads(r.data.decode('utf-8'))['args'] + {'arg': 'value'} + +For ``POST`` and ``PUT`` requests, you need to manually encode query parameters +in the URL:: + + >>> from urllib.parse import urlencode + >>> encoded_args = urlencode({'arg': 'value'}) + >>> url = 'http://httpbin.org/post?' + encoded_args + >>> r = http.request('POST', url) + >>> json.loads(r.data.decode('utf-8'))['args'] + {'arg': 'value'} + + +.. _form_data: + +Form data +~~~~~~~~~ + +For ``PUT`` and ``POST`` requests, urllib3 will automatically form-encode the +dictionary in the ``fields`` argument provided to +:meth:`~poolmanager.PoolManager.request`:: + + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... fields={'field': 'value'}) + >>> json.loads(r.data.decode('utf-8'))['form'] + {'field': 'value'} + +JSON +~~~~ + +You can sent JSON a request by specifying the encoded data as the ``body`` +argument and setting the ``Content-Type`` header when calling +:meth:`~poolmanager.PoolManager.request`:: + + >>> import json + >>> data = {'attribute': 'value'} + >>> encoded_data = json.dumps(data).encode('utf-8') + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... body=encoded_data, + ... headers={'Content-Type': 'application/json'}) + >>> json.loads(r.data.decode('utf-8'))['json'] + {'attribute': 'value'} + +Files & binary data +~~~~~~~~~~~~~~~~~~~ + +For uploading files using ``multipart/form-data`` encoding you can use the same +approach as :ref:`form_data` and specify the file field as a tuple of +``(file_name, file_data)``:: + + >>> with open('example.txt') as fp: + ... file_data = fp.read() + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... fields={ + ... 'filefield': ('example.txt', file_data), + ... }) + >>> json.loads(r.data.decode('utf-8'))['files'] + {'filefield': '...'} + +While specifying the filename is not strictly required, it's recommended in +order to match browser behavior. You can also pass a third item in the tuple +to specify the file's MIME type explicitly:: + + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... fields={ + ... 'filefield': ('example.txt', file_data, 'text/plain'), + ... }) + +For sending raw binary data simply specify the ``body`` argument. It's also +recommended to set the ``Content-Type`` header:: + + >>> with open('example.jpg', 'rb') as fp: + ... binary_data = fp.read() + >>> r = http.request( + ... 'POST', + ... 'http://httpbin.org/post', + ... body=binary_data, + ... headers={'Content-Type': 'image/jpeg'}) + >>> json.loads(r.data.decode('utf-8'))['data'] + b'...' + +.. _ssl: + +Certificate verification +------------------------ + +It is highly recommended to always use SSL certificate verification. +**By default, urllib3 does not verify HTTPS requests**. + +In order to enable verification you will need root certificate. The easiest +and most reliable method is to use the `certifi <https://certifi.io/en/latest>`_ package which provides Mozilla's root certificate bundle:: + + pip install certifi + +You can also install certifi along with urllib3 by using the ``secure`` +extra:: + + pip install urllib3[secure] + +.. warning:: If you're using Python 2 you may need additional packages. See the :ref:`section below <ssl_py2>` for more details. + +Once you have certificates, you can create a :class:`~poolmanager.PoolManager` +that verifies certificates when making requests:: + + >>> import certifi + >>> import urllib3 + >>> http = urllib3.PoolManager( + ... cert_reqs='CERT_REQUIRED', + ... ca_certs=certifi.where()) + +The :class:`~poolmanager.PoolManager` will automatically handle certificate +verification and will raise :class:`~exceptions.SSLError` if verification fails:: + + >>> http.request('GET', 'https://google.com') + (No exception) + >>> http.request('GET', 'https://expired.badssl.com') + urllib3.exceptions.SSLError ... + +.. note:: You can use OS-provided certificates if desired. Just specify the full + path to the certificate bundle as the ``ca_certs`` argument instead of + ``certifi.where()``. For example, most Linux systems store the certificates + at ``/etc/ssl/certs/ca-certificates.crt``. Other operating systems can + be `difficult <https://stackoverflow.com/questions/10095676/openssl-reasonable-default-for-trusted-ca-certificates>`_. + +.. _ssl_py2: + +Certificate verification in Python 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Older versions of Python 2 are built with an :mod:`ssl` module that lacks +:ref:`SNI support <sni_warning>` and can lag behind security updates. For these reasons it's recommended to use +`pyOpenSSL <https://pyopenssl.readthedocs.io/en/latest/>`_. + +If you install urllib3 with the ``secure`` extra, all required packages for +certificate verification on Python 2 will be installed:: + + pip install urllib3[secure] + +If you want to install the packages manually, you will need ``pyOpenSSL``, +``cryptography``, ``idna``, and ``certifi``. + +.. note:: If you are not using macOS or Windows, note that `cryptography + <https://cryptography.io/en/latest/>`_ requires additional system packages + to compile. See `building cryptography on Linux + <https://cryptography.io/en/latest/installation/#building-cryptography-on-linux>`_ + for the list of packages required. + +Once installed, you can tell urllib3 to use pyOpenSSL by using :mod:`urllib3.contrib.pyopenssl`:: + + >>> import urllib3.contrib.pyopenssl + >>> urllib3.contrib.pyopenssl.inject_into_urllib3() + +Finally, you can create a :class:`~poolmanager.PoolManager` that verifies +certificates when performing requests:: + + >>> import certifi + >>> import urllib3 + >>> http = urllib3.PoolManager( + ... cert_reqs='CERT_REQUIRED', + ... ca_certs=certifi.where()) + +If you do not wish to use pyOpenSSL, you can simply omit the call to +:func:`urllib3.contrib.pyopenssl.inject_into_urllib3`. urllib3 will fall back +to the standard-library :mod:`ssl` module. You may experience +:ref:`several warnings <ssl_warnings>` when doing this. + +.. warning:: If you do not use pyOpenSSL, Python must be compiled with ssl + support for certificate verification to work. It is uncommon, but it is + possible to compile Python without SSL support. See this + `Stackoverflow thread <https://stackoverflow.com/questions/5128845/importerror-no-module-named-ssl>`_ + for more details. + + If you are on Google App Engine, you must explicitly enable SSL + support in your ``app.yaml``:: + + libraries: + - name: ssl + version: latest + +Using timeouts +-------------- + +Timeouts allow you to control how long requests are allowed to run before +being aborted. In simple cases, you can specify a timeout as a ``float`` +to :meth:`~poolmanager.PoolManager.request`:: + + >>> http.request( + ... 'GET', 'http://httpbin.org/delay/3', timeout=4.0) + <urllib3.response.HTTPResponse> + >>> http.request( + ... 'GET', 'http://httpbin.org/delay/3', timeout=2.5) + MaxRetryError caused by ReadTimeoutError + +For more granular control you can use a :class:`~util.timeout.Timeout` +instance which lets you specify separate connect and read timeouts:: + + >>> http.request( + ... 'GET', + ... 'http://httpbin.org/delay/3', + ... timeout=urllib3.Timeout(connect=1.0)) + <urllib3.response.HTTPResponse> + >>> http.request( + ... 'GET', + ... 'http://httpbin.org/delay/3', + ... timeout=urllib3.Timeout(connect=1.0, read=2.0)) + MaxRetryError caused by ReadTimeoutError + + +If you want all requests to be subject to the same timeout, you can specify +the timeout at the :class:`~urllib3.poolmanager.PoolManager` level:: + + >>> http = urllib3.PoolManager(timeout=3.0) + >>> http = urllib3.PoolManager( + ... timeout=urllib3.Timeout(connect=1.0, read=2.0)) + +You still override this pool-level timeout by specifying ``timeout`` to +:meth:`~poolmanager.PoolManager.request`. + +Retrying requests +----------------- + +urllib3 can automatically retry idempotent requests. This same mechanism also +handles redirects. You can control the retries using the ``retries`` parameter +to :meth:`~poolmanager.PoolManager.request`. By default, urllib3 will retry +requests 3 times and follow up to 3 redirects. + +To change the number of retries just specify an integer:: + + >>> http.requests('GET', 'http://httpbin.org/ip', retries=10) + +To disable all retry and redirect logic specify ``retries=False``:: + + >>> http.request( + ... 'GET', 'http://nxdomain.example.com', retries=False) + NewConnectionError + >>> r = http.request( + ... 'GET', 'http://httpbin.org/redirect/1', retries=False) + >>> r.status + 302 + +To disable redirects but keep the retrying logic, specify ``redirect=False``:: + + >>> r = http.request( + ... 'GET', 'http://httpbin.org/redirect/1', redirect=False) + >>> r.status + 302 + +For more granular control you can use a :class:`~util.retry.Retry` instance. +This class allows you far greater control of how requests are retried. + +For example, to do a total of 3 retries, but limit to only 2 redirects:: + + >>> http.request( + ... 'GET', + ... 'http://httpbin.org/redirect/3', + ... retries=urllib3.Retries(3, redirect=2)) + MaxRetryError + +You can also disable exceptions for too many redirects and just return the +``302`` response:: + + >>> r = http.request( + ... 'GET', + ... 'http://httpbin.org/redirect/3', + ... retries=urllib3.Retries( + ... redirect=2, raise_on_redirect=False)) + >>> r.status + 302 + +If you want all requests to be subject to the same retry policy, you can +specify the retry at the :class:`~urllib3.poolmanager.PoolManager` level:: + + >>> http = urllib3.PoolManager(retries=False) + >>> http = urllib3.PoolManager( + ... retries=urllib3.Retry(5, redirect=2)) + +You still override this pool-level retry policy by specifying ``retries`` to +:meth:`~poolmanager.PoolManager.request`. + +Errors & Exceptions +------------------- + +urllib3 wraps lower-level exceptions, for example:: + + >>> try: + ... http.request('GET', 'nx.example.com', retries=False) + >>> except urllib3.exceptions.NewConnectionError: + ... print('Connection failed.') + +See :mod:`~urllib3.exceptions` for the full list of all exceptions. + +Logging +------- + +If you are using the standard library :mod:`logging` module urllib3 will +emit several logs. In some cases this can be undesirable. You can use the +standard logger interface to change the log level for urllib3's logger:: + + >>> logging.getLogger("urllib3").setLevel(logging.WARNING) diff --git a/dummyserver/certs/ca_path_test/98a2772e.0 b/dummyserver/certs/ca_path_test/98a2772e.0 index 1310cfcf..38d32dcd 120000..100644 --- a/dummyserver/certs/ca_path_test/98a2772e.0 +++ b/dummyserver/certs/ca_path_test/98a2772e.0 @@ -1 +1,23 @@ -cacert.pem
\ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIDzDCCAzWgAwIBAgIJALPrscov4b/jMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD +VQQGEwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQK +EwVkdW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJ +KoZIhvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsMB4XDTExMTIyMjA3NTYxNVoXDTIx +MTIxOTA3NTYxNVowgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwG +A1UEBxMFZHVtbXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8G +A1UEAxMIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWww +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMrPxr1fZJ82az1N9/I1oU78rjZ8 +CNQjV0AzUbxNWiPRrzVrLtbPhHtXXN+NcVP9ahFbThjrF6TRt9/Q62xb4CuKihTL +v6k9ietyGkBbSnuE+MfUMgFVpvTUIkyFDbh6v3ZDV0XhYG/jIqoRpXUhjPVy+q8I +ImABuxafUjwKdrWXAgMBAAGjggFIMIIBRDAdBgNVHQ4EFgQUGXd/I2JiQllF+3Wd +x3NyBLszCi0wgbYGA1UdIwSBrjCBq4AUGXd/I2JiQllF+3Wdx3NyBLszCi2hgYek +gYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt +bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8GA1UEAxMIU25h +a2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWyCCQCz67HKL+G/ +4zAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBBjAJBgNVHRIEAjAA +MCsGCWCGSAGG+EIBDQQeFhxUaW55Q0EgR2VuZXJhdGVkIENlcnRpZmljYXRlMA4G +A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQUFAAOBgQBnnwtO8onsyhGOvS6cS8af +IRZyAXgouuPeP3Zrf5W80iZcV23u94969sPEIsD8Ujv5u0hUSrToGl4ahOMEOFNL +R5ndQOkh3VsepJnoE+RklZzbHWxU8onWlVzsNBFbclxidzaU3UHmdgXJAJL5nVSd +Zpn44QSS0UXsaC0mBimVNw== +-----END CERTIFICATE----- diff --git a/dummyserver/certs/ca_path_test/b6b9ccf9.0 b/dummyserver/certs/ca_path_test/b6b9ccf9.0 index 1310cfcf..38d32dcd 120000..100644 --- a/dummyserver/certs/ca_path_test/b6b9ccf9.0 +++ b/dummyserver/certs/ca_path_test/b6b9ccf9.0 @@ -1 +1,23 @@ -cacert.pem
\ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIDzDCCAzWgAwIBAgIJALPrscov4b/jMA0GCSqGSIb3DQEBBQUAMIGBMQswCQYD +VQQGEwJGSTEOMAwGA1UECBMFZHVtbXkxDjAMBgNVBAcTBWR1bW15MQ4wDAYDVQQK +EwVkdW1teTEOMAwGA1UECxMFZHVtbXkxETAPBgNVBAMTCFNuYWtlT2lsMR8wHQYJ +KoZIhvcNAQkBFhBkdW1teUB0ZXN0LmxvY2FsMB4XDTExMTIyMjA3NTYxNVoXDTIx +MTIxOTA3NTYxNVowgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwG +A1UEBxMFZHVtbXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8G +A1UEAxMIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWww +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMrPxr1fZJ82az1N9/I1oU78rjZ8 +CNQjV0AzUbxNWiPRrzVrLtbPhHtXXN+NcVP9ahFbThjrF6TRt9/Q62xb4CuKihTL +v6k9ietyGkBbSnuE+MfUMgFVpvTUIkyFDbh6v3ZDV0XhYG/jIqoRpXUhjPVy+q8I +ImABuxafUjwKdrWXAgMBAAGjggFIMIIBRDAdBgNVHQ4EFgQUGXd/I2JiQllF+3Wd +x3NyBLszCi0wgbYGA1UdIwSBrjCBq4AUGXd/I2JiQllF+3Wdx3NyBLszCi2hgYek +gYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIEwVkdW1teTEOMAwGA1UEBxMFZHVt +bXkxDjAMBgNVBAoTBWR1bW15MQ4wDAYDVQQLEwVkdW1teTERMA8GA1UEAxMIU25h +a2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRlc3QubG9jYWyCCQCz67HKL+G/ +4zAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBBjAJBgNVHRIEAjAA +MCsGCWCGSAGG+EIBDQQeFhxUaW55Q0EgR2VuZXJhdGVkIENlcnRpZmljYXRlMA4G +A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQUFAAOBgQBnnwtO8onsyhGOvS6cS8af +IRZyAXgouuPeP3Zrf5W80iZcV23u94969sPEIsD8Ujv5u0hUSrToGl4ahOMEOFNL +R5ndQOkh3VsepJnoE+RklZzbHWxU8onWlVzsNBFbclxidzaU3UHmdgXJAJL5nVSd +Zpn44QSS0UXsaC0mBimVNw== +-----END CERTIFICATE----- diff --git a/dummyserver/certs/client.pem b/dummyserver/certs/client.pem index d23ad4d8..c8302bf2 120000..100644 --- a/dummyserver/certs/client.pem +++ b/dummyserver/certs/client.pem @@ -1 +1,21 @@ -server.crt
\ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIDczCCAtygAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgTELMAkGA1UEBhMCRkkx +DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQHDAVkdW1teTEOMAwGA1UECgwFZHVtbXkx +DjAMBgNVBAsMBWR1bW15MREwDwYDVQQDDAhTbmFrZU9pbDEfMB0GCSqGSIb3DQEJ +ARYQZHVtbXlAdGVzdC5sb2NhbDAeFw0xMTEyMjIwNzU4NDBaFw0yMTEyMTgwNzU4 +NDBaMGExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UEBwwFZHVt +bXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVkdW1teTESMBAGA1UEAwwJbG9j +YWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXe3FqmCWvP8XPxqtT ++0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB +0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN +3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo4IBGDCCARQwCQYDVR0TBAIwADAdBgNV +HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwgbYGA1UdIwSBrjCBq4AUGXd/I2Ji +QllF+3Wdx3NyBLszCi2hgYekgYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVk +dW1teTEOMAwGA1UEBwwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVk +dW1teTERMA8GA1UEAwwIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRl +c3QubG9jYWyCCQCz67HKL+G/4zAJBgNVHRIEAjAAMCQGA1UdEQQdMBuBDnJvb3RA +bG9jYWxob3N0gglsb2NhbGhvc3QwDQYJKoZIhvcNAQEFBQADgYEAgcW6X1ZUyufm +TFEqEAdpKXdL0rxDwcsM/qqqsXbkz17otH6ujPhBEagzdKtgeNKfy0aXz6rWZugk +lF0IqyC4mcI+vvfgGR5Iy4KdXMrIX98MbrvGJBfbdKhGW2b84wDV42DIDiD2ZGGe +6YZQQIo9LxjuOTf9jsvf+PIkbI4H0To= +-----END CERTIFICATE----- diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 75983296..2c47a814 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -1,6 +1,7 @@ from __future__ import print_function import collections +import contextlib import gzip import json import logging @@ -11,10 +12,8 @@ import zlib from io import BytesIO from tornado.web import RequestHandler -try: - from urllib.parse import urlsplit -except ImportError: - from urlparse import urlsplit +from urllib3.packages.six.moves.http_client import responses +from urllib3.packages.six.moves.urllib.parse import urlsplit log = logging.getLogger(__name__) @@ -159,6 +158,17 @@ class TestingApp(RequestHandler): headers = [('Location', target)] return Response(status='303 See Other', headers=headers) + def multi_redirect(self, request): + "Performs a redirect chain based on ``redirect_codes``" + codes = request.params.get('redirect_codes', '200').decode('utf-8') + head, tail = codes.split(',', 1) if "," in codes else (codes, None) + status = "{0} {1}".format(head, responses[int(head)]) + if not tail: + return Response("Done redirecting", status=status) + + headers = [('Location', '/multi_redirect?redirect_codes=%s' % tail)] + return Response(status=status, headers=headers) + def keepalive(self, request): if request.params.get('close', b'0') == b'1': headers = [('Connection', 'close')] @@ -190,9 +200,8 @@ class TestingApp(RequestHandler): if encoding == 'gzip': headers = [('Content-Encoding', 'gzip')] file_ = BytesIO() - zipfile = gzip.GzipFile('', mode='w', fileobj=file_) - zipfile.write(data) - zipfile.close() + with contextlib.closing(gzip.GzipFile('', mode='w', fileobj=file_)) as zipfile: + zipfile.write(data) data = file_.getvalue() elif encoding == 'deflate': headers = [('Content-Encoding', 'deflate')] @@ -18,8 +18,8 @@ provides-extra = secure socks requires-dist = - pyOpenSSL>=0.13; python_version<="2.7" and extra == 'secure' - ndg-httpsclient; python_version<="2.7" and extra == 'secure' - pyasn1; python_version<="2.7" and extra == 'secure' + pyOpenSSL>=0.14; python_version<="2.7" and extra == 'secure' + cryptography>=1.3.4; python_version<="2.7" and extra == 'secure' + idna>=2.0.0; python_version<="2.7" and extra == 'secure' certifi; extra == 'secure' - PySocks>=1.5.6,<2.0; extra == 'socks' + PySocks>=1.5.6,<2.0,!=1.5.7; extra == 'socks' @@ -9,13 +9,14 @@ import codecs base_path = os.path.dirname(__file__) # Get the version (borrowed from SQLAlchemy) -fp = open(os.path.join(base_path, 'urllib3', '__init__.py')) -VERSION = re.compile(r".*__version__ = '(.*?)'", - re.S).match(fp.read()).group(1) -fp.close() +with open(os.path.join(base_path, 'urllib3', '__init__.py')) as fp: + VERSION = re.compile(r".*__version__ = '(.*?)'", + re.S).match(fp.read()).group(1) -readme = codecs.open('README.rst', encoding='utf-8').read() -changes = codecs.open('CHANGES.rst', encoding='utf-8').read() +with codecs.open('README.rst', encoding='utf-8') as fp: + readme = fp.read() +with codecs.open('CHANGES.rst', encoding='utf-8') as fp: + changes = fp.read() version = VERSION setup(name='urllib3', @@ -54,13 +55,13 @@ setup(name='urllib3', test_suite='test', extras_require={ 'secure': [ - 'pyOpenSSL>=0.13', - 'ndg-httpsclient', - 'pyasn1', + 'pyOpenSSL>=0.14', + 'cryptography>=1.3.4', + 'idna>=2.0.0', 'certifi', ], 'socks': [ - 'PySocks>=1.5.6,<2.0', + 'PySocks>=1.5.6,<2.0,!=1.5.7', ] }, ) diff --git a/test/contrib/test_gae_manager.py b/test/contrib/test_gae_manager.py index aa909e91..e49dd74f 100644 --- a/test/contrib/test_gae_manager.py +++ b/test/contrib/test_gae_manager.py @@ -15,7 +15,7 @@ from urllib3.exceptions import ( ProtocolError, SSLError) from urllib3.util.url import Url -from urllib3.util.retry import Retry +from urllib3.util.retry import Retry, RequestHistory from test.with_dummyserver.test_connectionpool import ( TestConnectionPool, TestRetry) @@ -177,8 +177,24 @@ class TestGAERetry(TestRetry): retries=retry) self.assertEqual(resp.status, 200) + def test_retry_return_in_response(self): + headers = {'test-name': 'test_retry_return_in_response'} + retry = Retry(total=2, status_forcelist=[418]) + resp = self.pool.request('GET', '/successful_retry', + headers=headers, retries=retry) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.retries.total, 1) + # URLFetch use absolute urls. + self.assertEqual(resp.retries.history, + (RequestHistory('GET', + self.pool._absolute_url('/successful_retry'), + None, 418, None),)) + #test_max_retry = None #test_disabled_retry = None + # We don't need these tests because URLFetch resolves its own redirects. + test_retry_redirect_history = None + test_multi_redirect_history = None if __name__ == '__main__': diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 4c9e37a4..2fab0c6b 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -10,6 +10,8 @@ from urllib3.connectionpool import ( ) from urllib3.response import httplib, HTTPResponse from urllib3.util.timeout import Timeout +from urllib3.packages.six.moves.http_client import HTTPException +from urllib3.packages.six.moves.queue import Empty from urllib3.packages.ssl_match_hostname import CertificateError from urllib3.exceptions import ( ClosedPoolError, @@ -27,12 +29,7 @@ from .test_response import MockChunkedEncodingResponse, MockSock from socket import error as SocketError from ssl import SSLError as BaseSSLError -try: # Python 3 - from queue import Empty - from http.client import HTTPException -except ImportError: - from Queue import Empty - from httplib import HTTPException +from dummyserver.server import DEFAULT_CA class TestConnectionPool(unittest.TestCase): @@ -285,7 +282,7 @@ class TestConnectionPool(unittest.TestCase): c._absolute_url('path?query=foo')) def test_ca_certs_default_cert_required(self): - with connection_from_url('https://google.com:80', ca_certs='/etc/ssl/certs/custom.pem') as pool: + with connection_from_url('https://google.com:80', ca_certs=DEFAULT_CA) as pool: conn = pool._get_conn() self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED') diff --git a/test/test_response.py b/test/test_response.py index 47d05213..5c66f69d 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -2,12 +2,12 @@ import unittest from io import BytesIO, BufferedReader -try: - import http.client as httplib -except ImportError: - import httplib from urllib3.response import HTTPResponse -from urllib3.exceptions import DecodeError, ResponseNotChunked, ProtocolError +from urllib3.exceptions import ( + DecodeError, ResponseNotChunked, ProtocolError, InvalidHeader +) +from urllib3.packages.six.moves import http_client as httplib +from urllib3.util.retry import Retry from base64 import b64decode @@ -379,6 +379,73 @@ class TestResponse(unittest.TestCase): self.assertRaises(StopIteration, next, stream) + def test_length_no_header(self): + fp = BytesIO(b'12345') + resp = HTTPResponse(fp, preload_content=False) + self.assertEqual(resp.length_remaining, None) + + def test_length_w_valid_header(self): + headers = {"content-length": "5"} + fp = BytesIO(b'12345') + + resp = HTTPResponse(fp, headers=headers, preload_content=False) + self.assertEqual(resp.length_remaining, 5) + + def test_length_w_bad_header(self): + garbage = {'content-length': 'foo'} + fp = BytesIO(b'12345') + + resp = HTTPResponse(fp, headers=garbage, preload_content=False) + self.assertEqual(resp.length_remaining, None) + + garbage['content-length'] = "-10" + resp = HTTPResponse(fp, headers=garbage, preload_content=False) + self.assertEqual(resp.length_remaining, None) + + def test_length_when_chunked(self): + # This is expressly forbidden in RFC 7230 sec 3.3.2 + # We fall back to chunked in this case and try to + # handle response ignoring content length. + headers = {'content-length': '5', + 'transfer-encoding': 'chunked'} + fp = BytesIO(b'12345') + + resp = HTTPResponse(fp, headers=headers, preload_content=False) + self.assertEqual(resp.length_remaining, None) + + def test_length_with_multiple_content_lengths(self): + headers = {'content-length': '5, 5, 5'} + garbage = {'content-length': '5, 42'} + fp = BytesIO(b'abcde') + + resp = HTTPResponse(fp, headers=headers, preload_content=False) + self.assertEqual(resp.length_remaining, 5) + + self.assertRaises(InvalidHeader, HTTPResponse, fp, + headers=garbage, preload_content=False) + + def test_length_after_read(self): + headers = {"content-length": "5"} + + # Test no defined length + fp = BytesIO(b'12345') + resp = HTTPResponse(fp, preload_content=False) + resp.read() + self.assertEqual(resp.length_remaining, None) + + # Test our update from content-length + fp = BytesIO(b'12345') + resp = HTTPResponse(fp, headers=headers, preload_content=False) + resp.read() + self.assertEqual(resp.length_remaining, 0) + + # Test partial read + fp = BytesIO(b'12345') + resp = HTTPResponse(fp, headers=headers, preload_content=False) + data = resp.stream(2) + next(data) + self.assertEqual(resp.length_remaining, 3) + def test_mock_httpresponse_stream(self): # Mock out a HTTP Request that does enough to make it through urllib3's # read() and close() calls, and also exhausts and underlying file @@ -525,6 +592,14 @@ class TestResponse(unittest.TestCase): self.assertEqual(r.headers.get('host'), 'example.com') self.assertEqual(r.headers.get('Host'), 'example.com') + def test_retries(self): + fp = BytesIO(b'') + resp = HTTPResponse(fp) + self.assertEqual(resp.retries, None) + retry = Retry() + resp = HTTPResponse(fp, retries=retry) + self.assertEqual(resp.retries, retry) + class MockChunkedEncodingResponse(object): diff --git a/test/test_retry.py b/test/test_retry.py index 1e87585e..b6015c13 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -2,7 +2,7 @@ import unittest from urllib3.response import HTTPResponse from urllib3.packages.six.moves import xrange -from urllib3.util.retry import Retry +from urllib3.util.retry import Retry, RequestHistory from urllib3.exceptions import ( ConnectTimeoutError, MaxRetryError, @@ -131,6 +131,18 @@ class RetryTest(unittest.TestCase): retry = retry.increment() self.assertEqual(retry.get_backoff_time(), 0) + def test_backoff_reset_after_redirect(self): + retry = Retry(total=100, redirect=5, backoff_factor=0.2) + retry = retry.increment() + retry = retry.increment() + self.assertEqual(retry.get_backoff_time(), 0.4) + redirect_response = HTTPResponse(status=302, headers={'location': 'test'}) + retry = retry.increment(response=redirect_response) + self.assertEqual(retry.get_backoff_time(), 0) + retry = retry.increment() + retry = retry.increment() + self.assertEqual(retry.get_backoff_time(), 0.4) + def test_sleep(self): # sleep a very small amount of time so our code coverage is happy retry = Retry(backoff_factor=0.0001) @@ -211,3 +223,19 @@ class RetryTest(unittest.TestCase): except MaxRetryError as e: assert 'Caused by redirect' not in str(e) self.assertEqual(str(e.reason), 'conntimeout') + + def test_history(self): + retry = Retry(total=10) + self.assertEqual(retry.history, tuple()) + connection_error = ConnectTimeoutError('conntimeout') + retry = retry.increment('GET', '/test1', None, connection_error) + self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None),)) + read_error = ReadTimeoutError(None, "/test2", "read timed out") + retry = retry.increment('POST', '/test2', None, read_error) + self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None), + RequestHistory('POST', '/test2', read_error, None, None))) + response = HTTPResponse(status=500) + retry = retry.increment('GET', '/test3', response, None) + self.assertEqual(retry.history, (RequestHistory('GET', '/test1', connection_error, None, None), + RequestHistory('POST', '/test2', read_error, None, None), + RequestHistory('GET', '/test3', None, 500, None))) diff --git a/test/test_util.py b/test/test_util.py index 918b1fdc..72bd07ab 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -77,13 +77,13 @@ class TestUtil(unittest.TestCase): 'http://[2a00:1450:4001:c01::67]:80/test': ('http', '[2a00:1450:4001:c01::67]', 80), # More IPv6 from http://www.ietf.org/rfc/rfc2732.txt - 'http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8000/index.html': ('http', '[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]', 8000), - 'http://[1080:0:0:0:8:800:200C:417A]/index.html': ('http', '[1080:0:0:0:8:800:200C:417A]', None), + 'http://[fedc:ba98:7654:3210:fedc:ba98:7654:3210]:8000/index.html': ('http', '[fedc:ba98:7654:3210:fedc:ba98:7654:3210]', 8000), + 'http://[1080:0:0:0:8:800:200c:417a]/index.html': ('http', '[1080:0:0:0:8:800:200c:417a]', None), 'http://[3ffe:2a00:100:7031::1]': ('http', '[3ffe:2a00:100:7031::1]', None), - 'http://[1080::8:800:200C:417A]/foo': ('http', '[1080::8:800:200C:417A]', None), + 'http://[1080::8:800:200c:417a]/foo': ('http', '[1080::8:800:200c:417a]', None), 'http://[::192.9.5.5]/ipng': ('http', '[::192.9.5.5]', None), - 'http://[::FFFF:129.144.52.38]:42/index.html': ('http', '[::FFFF:129.144.52.38]', 42), - 'http://[2010:836B:4179::836B:4179]': ('http', '[2010:836B:4179::836B:4179]', None), + 'http://[::ffff:129.144.52.38]:42/index.html': ('http', '[::ffff:129.144.52.38]', 42), + 'http://[2010:836b:4179::836b:4179]': ('http', '[2010:836b:4179::836b:4179]', None), } for url, expected_host in url_host_map.items(): returned_host = get_host(url) @@ -100,6 +100,35 @@ class TestUtil(unittest.TestCase): for location in invalid_host: self.assertRaises(LocationParseError, get_host, location) + def test_host_normalization(self): + """Asserts the scheme and host is normalized to lower-case.""" + url_host_map = { + # Hosts + 'HTTP://GOOGLE.COM/mail/': ('http', 'google.com', None), + 'GOogle.COM/mail': ('http', 'google.com', None), + 'HTTP://GoOgLe.CoM:8000/mail/': ('http', 'google.com', 8000), + 'HTTP://user:password@EXAMPLE.COM:1234': ('http', 'example.com', 1234), + '173.194.35.7': ('http', '173.194.35.7', None), + 'HTTP://173.194.35.7': ('http', '173.194.35.7', None), + 'HTTP://[2a00:1450:4001:c01::67]:80/test': ('http', '[2a00:1450:4001:c01::67]', 80), + 'HTTP://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8000/index.html': ('http', '[fedc:ba98:7654:3210:fedc:ba98:7654:3210]', 8000), + 'HTTPS://[1080:0:0:0:8:800:200c:417A]/index.html': ('https', '[1080:0:0:0:8:800:200c:417a]', None), + } + for url, expected_host in url_host_map.items(): + returned_host = get_host(url) + self.assertEqual(returned_host, expected_host) + + def test_parse_url_normalization(self): + """Assert parse_url normalizes the scheme/host, and only the scheme/host""" + test_urls = [ + ('HTTP://GOOGLE.COM/MAIL/', 'http://google.com/MAIL/'), + ('HTTP://JeremyCline:Hunter2@Example.com:8080/', 'http://JeremyCline:Hunter2@example.com:8080/'), + ('HTTPS://Example.Com/?Key=Value', 'https://example.com/?Key=Value'), + ('Https://Example.Com/#Fragment', 'https://example.com/#Fragment'), + ] + for url, expected_normalized_url in test_urls: + actual_normalized_url = parse_url(url).url + self.assertEqual(actual_normalized_url, expected_normalized_url) parse_url_host_map = { 'http://google.com/mail': Url('http', host='google.com', path='/mail'), @@ -275,15 +304,29 @@ class TestUtil(unittest.TestCase): except ValueError as e: self.assertTrue('less than' in str(e)) - # Booleans are allowed also by socket.settimeout and converted to the - # equivalent float (1.0 for True, 0.0 for False) - Timeout(connect=False, read=True) + try: + Timeout(connect=False) + self.fail("boolean values should throw exception") + except ValueError as e: + self.assertTrue('cannot be a boolean' in str(e)) + + try: + Timeout(read=True) + self.fail("boolean values should throw exception") + except ValueError as e: + self.assertTrue('cannot be a boolean' in str(e)) + + try: + Timeout(connect=0) + self.fail("value <= 0 should throw exception") + except ValueError as e: + self.assertTrue('less than or equal' in str(e)) try: Timeout(read="foo") self.fail("string value should not be allowed") except ValueError as e: - self.assertTrue('int or float' in str(e)) + self.assertTrue('int, float or None' in str(e)) @patch('urllib3.util.timeout.current_time') @@ -350,7 +393,6 @@ class TestUtil(unittest.TestCase): def test_resolve_cert_reqs(self): self.assertEqual(resolve_cert_reqs(None), ssl.CERT_NONE) self.assertEqual(resolve_cert_reqs(ssl.CERT_NONE), ssl.CERT_NONE) - self.assertEqual(resolve_cert_reqs(ssl.CERT_REQUIRED), ssl.CERT_REQUIRED) self.assertEqual(resolve_cert_reqs('REQUIRED'), ssl.CERT_REQUIRED) self.assertEqual(resolve_cert_reqs('CERT_REQUIRED'), ssl.CERT_REQUIRED) @@ -394,6 +436,16 @@ class TestUtil(unittest.TestCase): mock_context.load_cert_chain.assert_called_once_with( '/path/to/certfile', None) + @patch('urllib3.util.ssl_.create_urllib3_context') + def test_ssl_wrap_socket_creates_new_context(self, + create_urllib3_context): + socket = object() + ssl_wrap_socket(sock=socket, cert_reqs='CERT_REQUIRED') + + create_urllib3_context.assert_called_once_with( + None, 'CERT_REQUIRED', ciphers=None + ) + def test_ssl_wrap_socket_loads_verify_locations(self): socket = object() mock_context = Mock() diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 0f31fa00..20327dbd 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -8,11 +8,6 @@ import warnings import mock -try: - from urllib.parse import urlencode -except: - from urllib import urlencode - from .. import ( requires_network, onlyPy3, onlyPy26OrOlder, TARPIT_HOST, VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES, @@ -32,7 +27,8 @@ from urllib3.exceptions import ( NewConnectionError, ) from urllib3.packages.six import b, u -from urllib3.util.retry import Retry +from urllib3.packages.six.moves.urllib.parse import urlencode +from urllib3.util.retry import Retry, RequestHistory from urllib3.util.timeout import Timeout from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase @@ -774,6 +770,37 @@ class TestRetry(HTTPDummyServerTestCase): headers=headers, retries=retry) self.assertEqual(resp.status, 200) + def test_retry_return_in_response(self): + headers = {'test-name': 'test_retry_return_in_response'} + retry = Retry(total=2, status_forcelist=[418]) + resp = self.pool.request('GET', '/successful_retry', + headers=headers, retries=retry) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.retries.total, 1) + self.assertEqual(resp.retries.history, (RequestHistory('GET', '/successful_retry', None, 418, None),)) + + def test_retry_redirect_history(self): + resp = self.pool.request('GET', '/redirect', fields={'target': '/'}) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.retries.history, (RequestHistory('GET', '/redirect?target=%2F', None, 303, '/'),)) + + def test_multi_redirect_history(self): + r = self.pool.request('GET', '/multi_redirect', fields={'redirect_codes': '303,302,200'}, redirect=False) + self.assertEqual(r.status, 303) + self.assertEqual(r.retries.history, tuple()) + + r = self.pool.request('GET', '/multi_redirect', retries=10, + fields={'redirect_codes': '303,302,301,307,302,200'}) + self.assertEqual(r.status, 200) + self.assertEqual(r.data, b'Done redirecting') + self.assertEqual([(request_history.status, request_history.redirect_location) for request_history in r.retries.history], [ + (303, '/multi_redirect?redirect_codes=302,301,307,302,200'), + (302, '/multi_redirect?redirect_codes=301,307,302,200'), + (301, '/multi_redirect?redirect_codes=307,302,200'), + (307, '/multi_redirect?redirect_codes=302,200'), + (302, '/multi_redirect?redirect_codes=200') + ]) + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 2c5f035e..28b6aa0e 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -77,6 +77,60 @@ class TestHTTPS(HTTPSDummyServerTestCase): r = https_pool.request('GET', '/') self.assertEqual(r.status, 200) + # Modern versions of Python, or systems using PyOpenSSL, don't + # emit warnings. + if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL: + self.assertFalse(warn.called, warn.call_args_list) + else: + self.assertTrue(warn.called) + if util.HAS_SNI: + call = warn.call_args_list[0] + else: + call = warn.call_args_list[1] + error = call[0][1] + self.assertEqual(error, InsecurePlatformWarning) + + def test_verified_with_context(self): + ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED) + ctx.load_verify_locations(cafile=DEFAULT_CA) + https_pool = HTTPSConnectionPool(self.host, self.port, + ssl_context=ctx) + + conn = https_pool._new_conn() + self.assertEqual(conn.__class__, VerifiedHTTPSConnection) + + with mock.patch('warnings.warn') as warn: + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + + # Modern versions of Python, or systems using PyOpenSSL, don't + # emit warnings. + if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL: + self.assertFalse(warn.called, warn.call_args_list) + else: + self.assertTrue(warn.called) + if util.HAS_SNI: + call = warn.call_args_list[0] + else: + call = warn.call_args_list[1] + error = call[0][1] + self.assertEqual(error, InsecurePlatformWarning) + + def test_context_combines_with_ca_certs(self): + ctx = util.ssl_.create_urllib3_context(cert_reqs=ssl.CERT_REQUIRED) + https_pool = HTTPSConnectionPool(self.host, self.port, + ca_certs=DEFAULT_CA, + ssl_context=ctx) + + conn = https_pool._new_conn() + self.assertEqual(conn.__class__, VerifiedHTTPSConnection) + + with mock.patch('warnings.warn') as warn: + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + + # Modern versions of Python, or systems using PyOpenSSL, don't + # emit warnings. if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL: self.assertFalse(warn.called, warn.call_args_list) else: @@ -166,8 +220,16 @@ class TestHTTPS(HTTPSDummyServerTestCase): self.assertEqual(r.status, 200) self.assertTrue(warn.called) - call, = warn.call_args_list - category = call[0][1] + # Modern versions of Python, or systems using PyOpenSSL, only emit + # the unverified warning. Older systems may also emit other + # warnings, which we want to ignore here. + calls = warn.call_args_list + if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL: + category = calls[0][0][1] + elif util.HAS_SNI: + category = calls[1][0][1] + else: + category = calls[2][0][1] self.assertEqual(category, InsecureRequestWarning) def test_ssl_unverified_with_ca_certs(self): @@ -180,6 +242,9 @@ class TestHTTPS(HTTPSDummyServerTestCase): self.assertEqual(r.status, 200) self.assertTrue(warn.called) + # Modern versions of Python, or systems using PyOpenSSL, only emit + # the unverified warning. Older systems may also emit other + # warnings, which we want to ignore here. calls = warn.call_args_list if sys.version_info >= (2, 7, 9) or util.IS_PYOPENSSL: category = calls[0][0][1] @@ -189,33 +254,6 @@ class TestHTTPS(HTTPSDummyServerTestCase): category = calls[2][0][1] self.assertEqual(category, InsecureRequestWarning) - @requires_network - def test_ssl_verified_with_platform_ca_certs(self): - """ - We should rely on the platform CA file to validate authenticity of SSL - certificates. Since this file is used by many components of the OS, - such as curl, apt-get, etc., we decided to not touch it, in order to - not compromise the security of the OS running the test suite (typically - urllib3 developer's OS). - - This test assumes that httpbin.org uses a certificate signed by a well - known Certificate Authority. - """ - try: - import urllib3.contrib.pyopenssl - except ImportError: - raise SkipTest('Test requires PyOpenSSL') - if (urllib3.connection.ssl_wrap_socket is - urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket): - # Not patched - raise SkipTest('Test should only be run after PyOpenSSL ' - 'monkey patching') - - https_pool = HTTPSConnectionPool('httpbin.org', 443, - cert_reqs=ssl.CERT_REQUIRED) - - https_pool.request('HEAD', '/') - def test_assert_hostname_false(self): https_pool = HTTPSConnectionPool('localhost', self.port, cert_reqs='CERT_REQUIRED', @@ -451,7 +489,7 @@ class TestHTTPS_TLSv1(HTTPSDummyServerTestCase): def test_set_cert_default_cert_required(self): conn = VerifiedHTTPSConnection(self.host, self.port) - conn.set_cert(ca_certs='/etc/ssl/certs/custom.pem') + conn.set_cert(ca_certs=DEFAULT_CA) self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED') diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index b37d8bbc..05169919 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -300,6 +300,16 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase): except MaxRetryError as e: self.assertEqual(type(e.reason), ConnectTimeoutError) + def test_scheme_host_case_insensitive(self): + """Assert that upper-case schemes and hosts are normalized.""" + http = proxy_from_url(self.proxy_url.upper()) + + r = http.request('GET', '%s/' % self.http_url.upper()) + self.assertEqual(r.status, 200) + + r = http.request('GET', '%s/' % self.https_url.upper()) + self.assertEqual(r.status, 200) + class TestIPv6HTTPProxyManager(IPv6HTTPDummyProxyTestCase): diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 3048a348..3df468a1 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1045,3 +1045,99 @@ class TestHEAD(SocketDummyServerTestCase): # stream will use the read method here. self.assertEqual([], list(r.stream())) + + +class TestStream(SocketDummyServerTestCase): + def test_stream_none_unchunked_response_does_not_hang(self): + done_event = Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf += sock.recv(65536) + + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 12\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + b'hello, world' + ) + done_event.wait(5) + sock.close() + + self._start_server(socket_handler) + pool = HTTPConnectionPool(self.host, self.port, retries=False) + r = pool.request('GET', '/', timeout=1, preload_content=False) + + # Stream should read to the end. + self.assertEqual([b'hello, world'], list(r.stream(None))) + + done_event.set() + +class TestBadContentLength(SocketDummyServerTestCase): + def test_enforce_content_length_get(self): + done_event = Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf += sock.recv(65536) + + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 22\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + b'hello, world' + ) + done_event.wait(1) + sock.close() + + self._start_server(socket_handler) + conn = HTTPConnectionPool(self.host, self.port, maxsize=1) + + # Test stream read when content length less than headers claim + get_response = conn.request('GET', url='/', preload_content=False, + enforce_content_length=True) + data = get_response.stream(100) + # Read "good" data before we try to read again. + # This won't trigger till generator is exhausted. + next(data) + self.assertRaises(ProtocolError, next, data) + + done_event.set() + + def test_enforce_content_length_no_body(self): + done_event = Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf += sock.recv(65536) + + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 22\r\n' + b'Content-type: text/plain\r\n' + b'\r\n' + ) + done_event.wait(1) + sock.close() + + self._start_server(socket_handler) + conn = HTTPConnectionPool(self.host, self.port, maxsize=1) + + #Test stream on 0 length body + head_response = conn.request('HEAD', url='/', preload_content=False, + enforce_content_length=True) + data = [chunk for chunk in head_response.stream(1)] + self.assertEqual(len(data), 0) + + done_event.set() @@ -35,4 +35,15 @@ deps= flake8 commands= flake8 --version - flake8 + flake8 setup.py docs dummyserver urllib3 + +[testenv:docs] +basepython = python2.7 +deps= + -r{toxinidir}/docs/requirements.txt +commands= + rm -r {toxinidir}/docs/_build + make -C {toxinidir}/docs html +whitelist_externals= + make + rm diff --git a/urllib3/__init__.py b/urllib3/__init__.py index c3536742..6c78fafd 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.16' +__version__ = '1.17' __all__ = ( 'HTTPConnectionPool', diff --git a/urllib3/connection.py b/urllib3/connection.py index 5ce00804..054e7fdd 100644 --- a/urllib3/connection.py +++ b/urllib3/connection.py @@ -7,13 +7,8 @@ import socket from socket import error as SocketError, timeout as SocketTimeout import warnings from .packages import six - -try: # Python 3 - from http.client import HTTPConnection as _HTTPConnection - from http.client import HTTPException # noqa: unused in this module -except ImportError: - from httplib import HTTPConnection as _HTTPConnection - from httplib import HTTPException # noqa: unused in this module +from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection +from .packages.six.moves.http_client import HTTPException # noqa: F401 try: # Compiled with SSL? import ssl @@ -44,8 +39,9 @@ from .packages.ssl_match_hostname import match_hostname, CertificateError from .util.ssl_ import ( resolve_cert_reqs, resolve_ssl_version, - ssl_wrap_socket, assert_fingerprint, + create_urllib3_context, + ssl_wrap_socket ) @@ -203,14 +199,18 @@ class HTTPConnection(_HTTPConnection, object): class HTTPSConnection(HTTPConnection): default_port = port_by_scheme['https'] + ssl_version = None + def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw): + strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) self.key_file = key_file self.cert_file = cert_file + self.ssl_context = ssl_context # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -219,7 +219,19 @@ class HTTPSConnection(HTTPConnection): def connect(self): conn = self._new_conn() self._prepare_conn(conn) - self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file) + + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(None), + cert_reqs=resolve_cert_reqs(None), + ) + + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ssl_context=self.ssl_context, + ) class VerifiedHTTPSConnection(HTTPSConnection): @@ -237,9 +249,18 @@ class VerifiedHTTPSConnection(HTTPSConnection): cert_reqs=None, ca_certs=None, assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None): - - if (ca_certs or ca_cert_dir) and cert_reqs is None: - cert_reqs = 'CERT_REQUIRED' + """ + This method should only be called once, before the connection is used. + """ + # If cert_reqs is not provided, we can try to guess. If the user gave + # us a cert database, we assume they want to use it: otherwise, if + # they gave us an SSL Context object we should use whatever is set for + # it. + if cert_reqs is None: + if ca_certs or ca_cert_dir: + cert_reqs = 'CERT_REQUIRED' + elif self.ssl_context is not None: + cert_reqs = self.ssl_context.verify_mode self.key_file = key_file self.cert_file = cert_file @@ -253,9 +274,6 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Add certificate verification conn = self._new_conn() - resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) - resolved_ssl_version = resolve_ssl_version(self.ssl_version) - hostname = self.host if getattr(self, '_tunnel_host', None): # _tunnel_host was added in Python 2.6.3 @@ -281,17 +299,27 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Wrap socket using verification with the root certs in # trusted_root_certs - self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, - cert_reqs=resolved_cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, - ssl_version=resolved_ssl_version) + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(self.ssl_version), + cert_reqs=resolve_cert_reqs(self.cert_reqs), + ) + + context = self.ssl_context + context.verify_mode = resolve_cert_reqs(self.cert_reqs) + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + server_hostname=hostname, + ssl_context=context) if self.assert_fingerprint: assert_fingerprint(self.sock.getpeercert(binary_form=True), self.assert_fingerprint) - elif resolved_cert_reqs != ssl.CERT_NONE \ + elif context.verify_mode != ssl.CERT_NONE \ and self.assert_hostname is not False: cert = self.sock.getpeercert() if not cert.get('subjectAltName', ()): @@ -304,8 +332,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): ) _match_hostname(cert, self.assert_hostname or hostname) - self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or - self.assert_fingerprint is not None) + self.is_verified = ( + context.verify_mode == ssl.CERT_REQUIRED or + self.assert_fingerprint is not None + ) def _match_hostname(cert, asserted_hostname): diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index ab634cb4..13563ca1 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -7,13 +7,6 @@ import warnings from socket import error as SocketError, timeout as SocketTimeout import socket -try: # Python 3 - from queue import LifoQueue, Empty, Full -except ImportError: - from Queue import LifoQueue, Empty, Full - # Queue is imported for side effects on MS Windows - import Queue as _unused_module_Queue # noqa: unused - from .exceptions import ( ClosedPoolError, @@ -32,6 +25,7 @@ from .exceptions import ( ) from .packages.ssl_match_hostname import CertificateError from .packages import six +from .packages.six.moves.queue import LifoQueue, Empty, Full from .connection import ( port_by_scheme, DummyConnection, @@ -48,6 +42,10 @@ from .util.timeout import Timeout from .util.url import get_host, Url +if six.PY2: + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: F401 + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -210,8 +208,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -246,7 +244,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s", self.host) + log.debug("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -397,8 +395,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s", method, url, http_version, - httplib_response.status, httplib_response.length) + log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, + method, url, http_version, httplib_response.status, + httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -600,10 +599,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # mess. response_conn = conn if not release_conn else None + # Pass method to Response for length checking + response_kw['request_method'] = method + # Import httplib's response into our own wrapper object response = self.ResponseCls.from_httplib(httplib_response, pool=self, connection=response_conn, + retries=retries, **response_kw) # Everything went great! @@ -683,7 +686,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s", url, redirect_location) + log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -703,7 +706,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response retries.sleep() - log.info("Forced retry: %s", url) + log.debug("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -775,7 +778,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version - return conn def _prepare_proxy(self, conn): @@ -801,8 +803,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -834,7 +836,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.io/en/latest/security.html'), + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings'), InsecureRequestWarning) diff --git a/urllib3/contrib/appengine.py b/urllib3/contrib/appengine.py index 1579476c..863b0b46 100644 --- a/urllib3/contrib/appengine.py +++ b/urllib3/contrib/appengine.py @@ -1,3 +1,43 @@ +""" +This module provides a pool manager that uses Google App Engine's +`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. + +Example usage:: + + from urllib3 import PoolManager + from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox + + if is_appengine_sandbox(): + # AppEngineManager uses AppEngine's URLFetch API behind the scenes + http = AppEngineManager() + else: + # PoolManager uses a socket-level API behind the scenes + http = PoolManager() + + r = http.request('GET', 'https://google.com/') + +There are `limitations <https://cloud.google.com/appengine/docs/python/\ +urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be +the best choice for your application. There are three options for using +urllib3 on Google App Engine: + +1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is + cost-effective in many circumstances as long as your usage is within the + limitations. +2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. + Sockets also have `limitations and restrictions + <https://cloud.google.com/appengine/docs/python/sockets/\ + #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. + To use sockets, be sure to specify the following in your ``app.yaml``:: + + env_variables: + GAE_USE_SOCKETS_HTTPLIB : 'true' + +3. If you are using `App Engine Flexible +<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard +:class:`PoolManager` without any configuration or special environment variables. +""" + from __future__ import absolute_import import logging import os @@ -41,13 +81,12 @@ class AppEngineManager(RequestMethods): This manager uses the URLFetch service directly instead of using the emulated httplib, and is subject to URLFetch limitations as described in - the App Engine documentation here: - - https://cloud.google.com/appengine/docs/python/urlfetch + the App Engine documentation `here + <https://cloud.google.com/appengine/docs/python/urlfetch>`_. - Notably it will raise an AppEnginePlatformError if: + Notably it will raise an :class:`AppEnginePlatformError` if: * URLFetch is not available. - * If you attempt to use this on GAEv2 (Managed VMs), as full socket + * If you attempt to use this on App Engine Flexible, as full socket support is available. * If a request size is more than 10 megabytes. * If a response size is more than 32 megabtyes. @@ -133,7 +172,7 @@ class AppEngineManager(RequestMethods): "URLFetch does not support method: %s" % method, e) http_response = self._urlfetch_response_to_http_response( - response, **response_kw) + response, retries=retries, **response_kw) # Check for redirect response if (http_response.get_redirect_location() and @@ -183,12 +222,13 @@ class AppEngineManager(RequestMethods): def _get_absolute_timeout(self, timeout): if timeout is Timeout.DEFAULT_TIMEOUT: - return 5 # 5s is the default timeout for URLFetch. + return None # Defer to URLFetch's default. if isinstance(timeout, Timeout): - if timeout._read is not timeout._connect: + if timeout._read is not None or timeout._connect is not None: warnings.warn( "URLFetch does not support granular timeout settings, " - "reverting to total timeout.", AppEnginePlatformWarning) + "reverting to total or default URLFetch timeout.", + AppEnginePlatformWarning) return timeout.total return timeout diff --git a/urllib3/contrib/ntlmpool.py b/urllib3/contrib/ntlmpool.py index 11d0b5c3..642e99ed 100644 --- a/urllib3/contrib/ntlmpool.py +++ b/urllib3/contrib/ntlmpool.py @@ -5,14 +5,11 @@ Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 """ from __future__ import absolute_import -try: - from http.client import HTTPSConnection -except ImportError: - from httplib import HTTPSConnection from logging import getLogger from ntlm import ntlm -from urllib3 import HTTPSConnectionPool +from .. import HTTPSConnectionPool +from ..packages.six.moves.http_client import HTTPSConnection log = getLogger(__name__) diff --git a/urllib3/contrib/pyopenssl.py b/urllib3/contrib/pyopenssl.py index ed3b9cc3..520b306f 100644 --- a/urllib3/contrib/pyopenssl.py +++ b/urllib3/contrib/pyopenssl.py @@ -1,17 +1,21 @@ -'''SSL with SNI_-support for Python 2. Follow these instructions if you would +""" +SSL with SNI_-support for Python 2. Follow these instructions if you would like to verify SSL certificates in Python 2. Note, the default libraries do *not* do certificate checking; you need to do additional work to validate certificates yourself. This needs the following packages installed: -* pyOpenSSL (tested with 0.13) -* ndg-httpsclient (tested with 0.3.2) -* pyasn1 (tested with 0.1.6) +* pyOpenSSL (tested with 16.0.0) +* cryptography (minimum 1.3.4, from pyopenssl) +* idna (minimum 2.0, from cryptography) + +However, pyopenssl depends on cryptography, which depends on idna, so while we +use all three directly here we end up having relatively few packages required. You can install them with the following command: - pip install pyopenssl ndg-httpsclient pyasn1 + pip install pyopenssl cryptography idna To activate certificate checking, call :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code @@ -34,45 +38,38 @@ compression in Python 2 (see `CRIME attack`_). If you want to configure the default list of supported cipher suites, you can set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. -Module Variables ----------------- - -:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites. - .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) - -''' +""" from __future__ import absolute_import -try: - from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT - from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName -except SyntaxError as e: - raise ImportError(e) - +import idna import OpenSSL.SSL -from pyasn1.codec.der import decoder as der_decoder -from pyasn1.type import univ, constraint +from cryptography import x509 +from cryptography.hazmat.backends.openssl import backend as openssl_backend +from cryptography.hazmat.backends.openssl.x509 import _Certificate + from socket import timeout, error as SocketError +from io import BytesIO try: # Platform-specific: Python 2 from socket import _fileobject except ImportError: # Platform-specific: Python 3 _fileobject = None - from urllib3.packages.backports.makefile import backport_makefile + from ..packages.backports.makefile import backport_makefile +import logging import ssl import select import six +import sys -from .. import connection from .. import util __all__ = ['inject_into_urllib3', 'extract_from_urllib3'] -# SNI only *really* works if we can read the subjectAltName of certificates. -HAS_SNI = SUBJ_ALT_NAME_SUPPORT +# SNI always works. +HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { @@ -98,71 +95,101 @@ _openssl_verify = { OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } +#: The list of supported SSL/TLS cipher suites. DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 orig_util_HAS_SNI = util.HAS_SNI -orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket +orig_util_SSLContext = util.ssl_.SSLContext + + +log = logging.getLogger(__name__) def inject_into_urllib3(): 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' - connection.ssl_wrap_socket = ssl_wrap_socket + util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI + util.ssl_.HAS_SNI = HAS_SNI util.IS_PYOPENSSL = True + util.ssl_.IS_PYOPENSSL = True def extract_from_urllib3(): 'Undo monkey-patching by :func:`inject_into_urllib3`.' - connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket + util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI + util.ssl_.HAS_SNI = orig_util_HAS_SNI util.IS_PYOPENSSL = False + util.ssl_.IS_PYOPENSSL = False -# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. -class SubjectAltName(BaseSubjectAltName): - '''ASN.1 implementation for subjectAltNames support''' +def _dnsname_to_stdlib(name): + """ + Converts a dNSName SubjectAlternativeName field to the form used by the + standard library on the given Python version. - # There is no limit to how many SAN certificates a certificate may have, - # however this needs to have some limit so we'll set an arbitrarily high - # limit. - sizeSpec = univ.SequenceOf.sizeSpec + \ - constraint.ValueSizeConstraint(1, 1024) + Cryptography produces a dNSName as a unicode string that was idna-decoded + from ASCII bytes. We need to idna-encode that string to get it back, and + then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib + uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + """ + name = idna.encode(name) + if sys.version_info >= (3, 0): + name = name.decode('utf-8') + return name -# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. def get_subj_alt_name(peer_cert): - # Search through extensions - dns_name = [] - if not SUBJ_ALT_NAME_SUPPORT: - return dns_name - - general_names = SubjectAltName() - for i in range(peer_cert.get_extension_count()): - ext = peer_cert.get_extension(i) - ext_name = ext.get_short_name() - if ext_name != b'subjectAltName': - continue - - # PyOpenSSL returns extension data in ASN.1 encoded form - ext_dat = ext.get_data() - decoded_dat = der_decoder.decode(ext_dat, - asn1Spec=general_names) - - for name in decoded_dat: - if not isinstance(name, SubjectAltName): - continue - for entry in range(len(name)): - component = name.getComponentByPosition(entry) - if component.getName() != 'dNSName': - continue - dns_name.append(str(component.getComponent())) - - return dns_name + """ + Given an PyOpenSSL certificate, provides all the subject alternative names. + """ + # Pass the cert to cryptography, which has much better APIs for this. + # This is technically using private APIs, but should work across all + # relevant versions until PyOpenSSL gets something proper for this. + cert = _Certificate(openssl_backend, peer_cert._x509) + + # We want to find the SAN extension. Ask Cryptography to locate it (it's + # faster than looping in Python) + try: + ext = cert.extensions.get_extension_for_class( + x509.SubjectAlternativeName + ).value + except x509.ExtensionNotFound: + # No such extension, return the empty list. + return [] + except (x509.DuplicateExtension, x509.UnsupportedExtension, + x509.UnsupportedGeneralNameType, UnicodeError) as e: + # A problem has been found with the quality of the certificate. Assume + # no SAN field is present. + log.warning( + "A problem was encountered with the certificate that prevented " + "urllib3 from finding the SubjectAlternativeName field. This can " + "affect certificate validation. The error was %s", + e, + ) + return [] + + # We want to return dNSName and iPAddress fields. We need to cast the IPs + # back to strings because the match_hostname function wants them as + # strings. + # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 + # decoded. This is pretty frustrating, but that's what the standard library + # does with certificates, and so we need to attempt to do the same. + names = [ + ('DNS', _dnsname_to_stdlib(name)) + for name in ext.get_values_for_type(x509.DNSName) + ] + names.extend( + ('IP Address', str(name)) + for name in ext.get_values_for_type(x509.IPAddress) + ) + + return names class WrappedSocket(object): @@ -282,10 +309,7 @@ class WrappedSocket(object): 'subject': ( (('commonName', x509.get_subject().CN),), ), - 'subjectAltName': [ - ('DNS', value) - for value in get_subj_alt_name(x509) - ] + 'subjectAltName': get_subj_alt_name(x509) } def _reuse(self): @@ -308,6 +332,86 @@ else: # Platform-specific: Python 3 WrappedSocket.makefile = makefile +class PyOpenSSLContext(object): + """ + I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible + for translating the interface of the standard library ``SSLContext`` object + to calls into PyOpenSSL. + """ + def __init__(self, protocol): + self.protocol = _openssl_versions[protocol] + self._ctx = OpenSSL.SSL.Context(self.protocol) + self._options = 0 + self.check_hostname = False + + @property + def options(self): + return self._options + + @options.setter + def options(self, value): + self._options = value + self._ctx.set_options(value) + + @property + def verify_mode(self): + return self._ctx.get_verify_mode() + + @verify_mode.setter + def verify_mode(self, value): + self._ctx.set_verify(value, _verify_callback) + + def set_default_verify_paths(self): + self._ctx.set_default_verify_paths() + + def set_ciphers(self, ciphers): + if isinstance(ciphers, six.text_type): + ciphers = ciphers.encode('utf-8') + self._ctx.set_cipher_list(ciphers) + + def load_verify_locations(self, cafile=None, capath=None, cadata=None): + if cafile is not None: + cafile = cafile.encode('utf-8') + if capath is not None: + capath = capath.encode('utf-8') + self._ctx.load_verify_locations(cafile, capath) + if cadata is not None: + self._ctx.load_verify_locations(BytesIO(cadata)) + + def load_cert_chain(self, certfile, keyfile=None, password=None): + self._ctx.use_certificate_file(certfile) + if password is not None: + self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) + self._ctx.use_privatekey_file(keyfile or certfile) + + def wrap_socket(self, sock, server_side=False, + do_handshake_on_connect=True, suppress_ragged_eofs=True, + server_hostname=None): + cnx = OpenSSL.SSL.Connection(self._ctx, sock) + + if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') + + if server_hostname is not None: + cnx.set_tlsext_host_name(server_hostname) + + cnx.set_connect_state() + + while True: + try: + cnx.do_handshake() + except OpenSSL.SSL.WantReadError: + rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + if not rd: + raise timeout('select timed out') + continue + except OpenSSL.SSL.Error as e: + raise ssl.SSLError('bad handshake: %r' % e) + break + + return WrappedSocket(cnx, sock) + + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 @@ -315,6 +419,8 @@ def _verify_callback(cnx, x509, err_no, err_depth, return_code): def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, ssl_version=None, ca_cert_dir=None): + # This function is no longer used by urllib3. We should strongly consider + # removing it. ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: keyfile = keyfile or certfile # Match behaviour of the normal python ssl library diff --git a/urllib3/contrib/socks.py b/urllib3/contrib/socks.py index 81970fa6..c8fa8409 100644 --- a/urllib3/contrib/socks.py +++ b/urllib3/contrib/socks.py @@ -1,17 +1,23 @@ # -*- coding: utf-8 -*- """ -SOCKS support for urllib3 -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This contrib module contains provisional support for SOCKS proxies from within +This module contains provisional support for SOCKS proxies from within urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and SOCKS5. To enable its functionality, either install PySocks or install this module with the ``socks`` extra. +The SOCKS implementation supports the full range of urllib3 features. It also +supports the following SOCKS features: + +- SOCKS4 +- SOCKS4a +- SOCKS5 +- Usernames and passwords for the SOCKS proxy + Known Limitations: - Currently PySocks does not support contacting remote websites via literal - IPv6 addresses. Any such connection attempt will fail. + IPv6 addresses. Any such connection attempt will fail. You must use a domain + name. - Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any such connection attempt will fail. """ diff --git a/urllib3/exceptions.py b/urllib3/exceptions.py index f2e65917..cdc2bc24 100644 --- a/urllib3/exceptions.py +++ b/urllib3/exceptions.py @@ -1,4 +1,7 @@ from __future__ import absolute_import +from .packages.six.moves.http_client import ( + IncompleteRead as httplib_IncompleteRead +) # Base Exceptions @@ -193,6 +196,25 @@ class ResponseNotChunked(ProtocolError, ValueError): pass +class IncompleteRead(HTTPError, httplib_IncompleteRead): + """ + Response length doesn't match expected Content-Length + + Subclass of http_client.IncompleteRead to allow int value + for `partial` to avoid creating large objects on streamed + reads. + """ + def __init__(self, partial, expected): + message = ('IncompleteRead(%i bytes read, ' + '%i more expected)' % (partial, expected)) + httplib_IncompleteRead.__init__(self, message) + + +class InvalidHeader(HTTPError): + "The header provided was somehow invalid." + pass + + class ProxySchemeUnknown(AssertionError, ValueError): "ProxyManager does not support the supplied scheme" # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. diff --git a/urllib3/filepost.py b/urllib3/filepost.py index 97a2843c..cd11cee4 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -13,7 +13,7 @@ writer = codecs.lookup('utf-8')[3] def choose_boundary(): """ - Our embarassingly-simple replacement for mimetools.choose_boundary. + Our embarrassingly-simple replacement for mimetools.choose_boundary. """ return uuid4().hex diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py index 7ed00b1c..276b54dd 100644 --- a/urllib3/poolmanager.py +++ b/urllib3/poolmanager.py @@ -3,15 +3,11 @@ import collections import functools import logging -try: # Python 3 - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry @@ -23,7 +19,7 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir') + 'ssl_version', 'ca_cert_dir', 'ssl_context') # The base fields to use when determining what pool to get a connection from; # these do not rely on the ``connection_pool_kw`` and can be determined by the diff --git a/urllib3/request.py b/urllib3/request.py index d5aa62d8..c0fddff0 100644 --- a/urllib3/request.py +++ b/urllib3/request.py @@ -1,10 +1,7 @@ from __future__ import absolute_import -try: - from urllib.parse import urlencode -except ImportError: - from urllib import urlencode from .filepost import encode_multipart_formdata +from .packages.six.moves.urllib.parse import urlencode __all__ = ['RequestMethods'] diff --git a/urllib3/response.py b/urllib3/response.py index 55679032..be2accda 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -2,18 +2,22 @@ from __future__ import absolute_import from contextlib import contextmanager import zlib import io +import logging from socket import timeout as SocketTimeout from socket import error as SocketError from ._collections import HTTPHeaderDict from .exceptions import ( - ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked + ProtocolError, DecodeError, ReadTimeoutError, + ResponseNotChunked, IncompleteRead, InvalidHeader ) from .packages.six import string_types as basestring, binary_type, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head +log = logging.getLogger(__name__) + class DeflateDecoder(object): @@ -89,6 +93,14 @@ class HTTPResponse(io.IOBase): When this HTTPResponse wrapper is generated from an httplib.HTTPResponse object, it's convenient to include the original for debug purposes. It's otherwise unused. + + :param retries: + The retries contains the last :class:`~urllib3.util.retry.Retry` that + was used during the request. + + :param enforce_content_length: + Enforce content length checking. Body returned by server must match + value of Content-Length header, if present. Otherwise, raise error. """ CONTENT_DECODERS = ['gzip', 'deflate'] @@ -96,7 +108,8 @@ class HTTPResponse(io.IOBase): def __init__(self, body='', headers=None, status=0, version=0, reason=None, strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None): + original_response=None, pool=None, connection=None, + retries=None, enforce_content_length=False, request_method=None): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -107,6 +120,8 @@ class HTTPResponse(io.IOBase): self.reason = reason self.strict = strict self.decode_content = decode_content + self.retries = retries + self.enforce_content_length = enforce_content_length self._decoder = None self._body = None @@ -132,6 +147,9 @@ class HTTPResponse(io.IOBase): if "chunked" in encodings: self.chunked = True + # Determine length of response + self.length_remaining = self._init_length(request_method) + # If requested, preload the body. if preload_content and not self._body: self._body = self.read(decode_content=decode_content) @@ -177,9 +195,57 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_length(self, request_method): + """ + Set initial length value for Response content if available. + """ + length = self.headers.get('content-length') + + if length is not None and self.chunked: + # This Response will fail with an IncompleteRead if it can't be + # received as chunked. This method falls back to attempt reading + # the response before raising an exception. + log.warning("Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked.") + return None + + elif length is not None: + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = set([int(val) for val in length.split(',')]) + if len(lengths) > 1: + raise InvalidHeader("Content-Length contained multiple " + "unmatching values (%s)" % length) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + # Convert status to int for comparison + # In some cases, httplib returns a status of "_UNKNOWN" + try: + status = int(self.status) + except ValueError: + status = 0 + + # Check for responses that shouldn't include a body + if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': + length = 0 + + return length + def _init_decoder(self): """ - Set-up the _decoder attribute if necessar. + Set-up the _decoder attribute if necessary. """ # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 @@ -322,9 +388,18 @@ class HTTPResponse(io.IOBase): # no harm in redundantly calling close. self._fp.close() flush_decoder = True + if self.enforce_content_length and self.length_remaining not in (0, None): + # This is an edge case that httplib failed to cover due + # to concerns of backward compatibility. We're + # addressing it here to make sure IncompleteRead is + # raised during streaming, so all calls with incorrect + # Content-Length are caught. + raise IncompleteRead(self._fp_bytes_read, self.length_remaining) if data: self._fp_bytes_read += len(data) + if self.length_remaining is not None: + self.length_remaining -= len(data) data = self._decode(data, decode_content, flush_decoder) diff --git a/urllib3/util/response.py b/urllib3/util/response.py index 0b5c75c1..67cf730a 100644 --- a/urllib3/util/response.py +++ b/urllib3/util/response.py @@ -13,6 +13,13 @@ def is_fp_closed(obj): """ try: + # Check `isclosed()` first, in case Python3 doesn't set `closed`. + # GH Issue #928 + return obj.isclosed() + except AttributeError: + pass + + try: # Check via the official file-like-object way. return obj.closed except AttributeError: diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py index d379833c..f8f21810 100644 --- a/urllib3/util/retry.py +++ b/urllib3/util/retry.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import time import logging +from collections import namedtuple +from itertools import takewhile from ..exceptions import ( ConnectTimeoutError, @@ -14,6 +16,10 @@ from ..packages import six log = logging.getLogger(__name__) +# Data structure for representing the metadata of requests that result in a retry. +RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", + "status", "redirect_location"]) + class Retry(object): """ Retry configuration. @@ -113,6 +119,10 @@ class Retry(object): whether we should raise an exception, or return a response, if status falls in ``status_forcelist`` range and retries have been exhausted. + + :param tuple history: The history of the request encountered during + each call to :meth:`~Retry.increment`. The list is in the order + the requests occurred. Each list item is of class :class:`RequestHistory`. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -124,7 +134,7 @@ class Retry(object): def __init__(self, total=10, connect=None, read=None, redirect=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - _observed_errors=0): + history=None): self.total = total self.connect = connect @@ -140,7 +150,7 @@ class Retry(object): self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect self.raise_on_status = raise_on_status - self._observed_errors = _observed_errors # TODO: use .history instead? + self.history = history or tuple() def new(self, **kw): params = dict( @@ -151,7 +161,7 @@ class Retry(object): backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, - _observed_errors=self._observed_errors, + history=self.history, ) params.update(kw) return type(self)(**params) @@ -175,10 +185,13 @@ class Retry(object): :rtype: float """ - if self._observed_errors <= 1: + # We want to consider only the last consecutive errors sequence (Ignore redirects). + consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, + reversed(self.history)))) + if consecutive_errors_len <= 1: return 0 - backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1)) + backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) return min(self.BACKOFF_MAX, backoff_value) def sleep(self): @@ -241,11 +254,12 @@ class Retry(object): if total is not None: total -= 1 - _observed_errors = self._observed_errors connect = self.connect read = self.read redirect = self.redirect cause = 'unknown' + status = None + redirect_location = None if error and self._is_connection_error(error): # Connect retry? @@ -253,7 +267,6 @@ class Retry(object): raise six.reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 - _observed_errors += 1 elif error and self._is_read_error(error): # Read retry? @@ -261,27 +274,30 @@ class Retry(object): raise six.reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 - _observed_errors += 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = 'too many redirects' + redirect_location = response.get_redirect_location() + status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and a the given method is in the whitelist - _observed_errors += 1 cause = ResponseError.GENERIC_ERROR if response and response.status: cause = ResponseError.SPECIFIC_ERROR.format( status_code=response.status) + status = response.status + + history = self.history + (RequestHistory(method, url, error, status, redirect_location),) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, - _observed_errors=_observed_errors) + history=history) if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) diff --git a/urllib3/util/ssl_.py b/urllib3/util/ssl_.py index 4a64d7ef..c4c55df6 100644 --- a/urllib3/util/ssl_.py +++ b/urllib3/util/ssl_.py @@ -11,7 +11,6 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None HAS_SNI = False -create_default_context = None IS_PYOPENSSL = False # Maps the length of a digest to a possible hash function producing this digest @@ -63,14 +62,25 @@ except ImportError: # The general intent is: # - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), # - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM over any AES-CBC for better performance and security, -# - use 3DES as fallback which is secure but slow, +# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and +# security, +# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, # - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5' -) +DEFAULT_CIPHERS = ':'.join([ + 'ECDH+AESGCM', + 'ECDH+CHACHA20', + 'DH+AESGCM', + 'DH+CHACHA20', + 'ECDH+AES256', + 'DH+AES256', + 'ECDH+AES128', + 'DH+AES', + 'RSA+AESGCM', + 'RSA+AES', + '!aNULL', + '!eNULL', + '!MD5', +]) try: from ssl import SSLContext # Modern SSL? @@ -117,8 +127,8 @@ except ImportError: 'urllib3 from configuring SSL appropriately and may cause ' 'certain SSL connections to fail. You can upgrade to a newer ' 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/security.html' - '#insecureplatformwarning.', + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', InsecurePlatformWarning ) kwargs = { @@ -287,6 +297,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, """ context = ssl_context if context is None: + # Note: This branch of code and all the variables in it are no longer + # used by urllib3 itself. We should consider deprecating and removing + # this code. context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) @@ -301,6 +314,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise + elif getattr(context, 'load_default_certs', None) is not None: + # try to load OS default certs; works well on Windows (require Python3.4+) + context.load_default_certs() if certfile: context.load_cert_chain(certfile, keyfile) @@ -313,8 +329,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 'This may cause the server to present an incorrect TLS ' 'certificate, which can cause validation failures. You can upgrade to ' 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/security.html' - '#snimissingwarning.', + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', SNIMissingWarning ) return context.wrap_socket(sock) diff --git a/urllib3/util/timeout.py b/urllib3/util/timeout.py index ff62f476..1eaa6a35 100644 --- a/urllib3/util/timeout.py +++ b/urllib3/util/timeout.py @@ -111,8 +111,8 @@ class Timeout(object): :param name: The name of the timeout attribute to validate. This is used to specify in error messages. :return: The validated and casted version of the given value. - :raises ValueError: If the type is not an integer or a float, or if it - is a numeric value less than zero. + :raises ValueError: If it is a numeric value less than or equal to + zero, or the type is not an integer, float, or None. """ if value is _Default: return cls.DEFAULT_TIMEOUT @@ -120,20 +120,23 @@ class Timeout(object): if value is None or value is cls.DEFAULT_TIMEOUT: return value + if isinstance(value, bool): + raise ValueError("Timeout cannot be a boolean value. It must " + "be an int, float or None.") try: float(value) except (TypeError, ValueError): raise ValueError("Timeout value %s was %s, but it must be an " - "int or float." % (name, value)) + "int, float or None." % (name, value)) try: - if value < 0: + if value <= 0: raise ValueError("Attempted to set %s timeout to %s, but the " "timeout cannot be set to a value less " - "than 0." % (name, value)) + "than or equal to 0." % (name, value)) except TypeError: # Python 3 raise ValueError("Timeout value %s was %s, but it must be an " - "int or float." % (name, value)) + "int, float or None." % (name, value)) return value diff --git a/urllib3/util/url.py b/urllib3/util/url.py index e996204a..f685e7f2 100644 --- a/urllib3/util/url.py +++ b/urllib3/util/url.py @@ -10,14 +10,19 @@ url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] class Url(namedtuple('Url', url_attrs)): """ Datastructure for representing an HTTP URL. Used as a return value for - :func:`parse_url`. + :func:`parse_url`. Both the scheme and host are normalized as they are + both case-insensitive according to RFC 3986. """ - slots = () + __slots__ = () def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): if path and not path.startswith('/'): path = '/' + path + if scheme: + scheme = scheme.lower() + if host: + host = host.lower() return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) @@ -211,7 +216,7 @@ def parse_url(url): def get_host(url): """ - Deprecated. Use :func:`.parse_url` instead. + Deprecated. Use :func:`parse_url` instead. """ p = parse_url(url) return p.scheme or 'http', p.hostname, p.port |