Pass urllib3.SKIP_HEADER when headers should be unsetbug/5671

urllib3 introduced some default headers and a way to skip them if
desired. Let's use that sentinel value to pass along information about
Requests' users desire to skip those headers as well.

Closes gh-5671

5 files changed, 64 insertions, 18 deletions

diff --git a/requests/compat.py b/requests/compat.py
index 5de0769f..c5220389 100644
--- a/requests/compat.py
+++ b/requests/compat.py
@@ -30,6 +30,16 @@ try:
 except ImportError:
     import json
 
+
+import urllib3
+
+try:
+    SKIP_HEADER = urllib3.util.SKIP_HEADER
+    SKIPPABLE_HEADERS = urllib3.util.SKIPPABLE_HEADERS
+except AttributeError:
+    SKIP_HEADER = None
+    SKIPPABLE_HEADERS = frozenset([])
+
 # ---------
 # Specifics
 # ---------
diff --git a/requests/models.py b/requests/models.py
index ec2edc20..a5c0c8b7 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -15,6 +15,7 @@ import sys
 # such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
 import encodings.idna
 
+import urllib3
 from urllib3.fields import RequestField
 from urllib3.filepost import encode_multipart_formdata
 from urllib3.util import parse_url
@@ -36,9 +37,21 @@ from .utils import (
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
     iter_slices, guess_json_utf, super_len, check_header_validity)
 from .compat import (
-    Callable, Mapping,
-    cookielib, urlunparse, urlsplit, urlencode, str, bytes,
-    is_py2, chardet, builtin_str, basestring)
+    SKIP_HEADER,
+    SKIPPABLE_HEADERS,
+    Callable,
+    Mapping,
+    cookielib,
+    urlunparse,
+    urlsplit,
+    urlencode,
+    str,
+    bytes,
+    is_py2,
+    chardet,
+    builtin_str,
+    basestring,
+)
 from .compat import json as complexjson
 from .status_codes import codes
 
@@ -447,9 +460,14 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.headers = CaseInsensitiveDict()
         if headers:
             for header in headers.items():
+                name, value = header
+                if value is None:
+                    if name.lower() in SKIPPABLE_HEADERS:
+                        value = SKIP_HEADER
+                    else:
+                        continue
                 # Raise exception on invalid header value.
                 check_header_validity(header)
-                name, value = header
                 self.headers[to_native_string(name)] = value
 
     def prepare_body(self, data, files, json=None):
diff --git a/requests/sessions.py b/requests/sessions.py
index fdf7e9fe..108be7bd 100644
--- a/requests/sessions.py
+++ b/requests/sessions.py
@@ -47,7 +47,9 @@ else:
     preferred_clock = time.time
 
 
-def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
+def merge_setting(
+    request_setting, session_setting, dict_class=OrderedDict, delete_none=True
+):
     """Determines appropriate setting for a given request, taking into account
     the explicit setting on that request, and the setting in the session. If a
     setting is a dictionary, they will be merged together using `dict_class`
@@ -69,11 +71,12 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
     merged_setting = dict_class(to_key_val_list(session_setting))
     merged_setting.update(to_key_val_list(request_setting))
 
-    # Remove keys that are set to None. Extract keys first to avoid altering
-    # the dictionary during iteration.
-    none_keys = [k for (k, v) in merged_setting.items() if v is None]
-    for key in none_keys:
-        del merged_setting[key]
+    if delete_none:
+        # Remove keys that are set to None. Extract keys first to avoid altering
+        # the dictionary during iteration.
+        none_keys = [k for (k, v) in merged_setting.items() if v is None]
+        for key in none_keys:
+            del merged_setting[key]
 
     return merged_setting
 
@@ -459,7 +462,12 @@ class Session(SessionRedirectMixin):
             files=request.files,
             data=request.data,
             json=request.json,
-            headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
+            headers=merge_setting(
+                request.headers,
+                self.headers,
+                dict_class=CaseInsensitiveDict,
+                delete_none=False,
+            ),
             params=merge_setting(request.params, self.params),
             auth=merge_setting(auth, self.auth),
             cookies=merged_cookies,
diff --git a/requests/utils.py b/requests/utils.py
index db67938e..905c9330 100644
--- a/requests/utils.py
+++ b/requests/utils.py
@@ -947,6 +947,8 @@ def check_header_validity(header):
     :param header: tuple, in the format (name, value).
     """
     name, value = header
+    if value is None:
+        return
 
     if isinstance(value, bytes):
         pat = _CLEAN_HEADER_REGEX_BYTE
diff --git a/tests/test_requests.py b/tests/test_requests.py
index 5b6a7f58..6a0727b8 100644
--- a/tests/test_requests.py
+++ b/tests/test_requests.py
@@ -17,10 +17,15 @@ import pytest
 from requests.adapters import HTTPAdapter
 from requests.auth import HTTPDigestAuth, _basic_auth_str
 from requests.compat import (
-    Morsel, cookielib, getproxies, str, urlparse,
-    builtin_str)
-from requests.cookies import (
-    cookiejar_from_dict, morsel_to_cookie)
+    Morsel,
+    cookielib,
+    getproxies,
+    str,
+    urlparse,
+    builtin_str,
+    SKIP_HEADER,
+)
+from requests.cookies import cookiejar_from_dict, morsel_to_cookie
 from requests.exceptions import (
     ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL,
     MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects,
@@ -438,10 +443,13 @@ class TestRequests:
     def test_headers_on_session_with_None_are_not_sent(self, httpbin):
         """Do not send headers in Session.headers with None values."""
         ses = requests.Session()
-        ses.headers['Accept-Encoding'] = None
-        req = requests.Request('GET', httpbin('get'))
+        ses.headers["Accept-Encoding"] = None
+        req = requests.Request("GET", httpbin("get"))
         prep = ses.prepare_request(req)
-        assert 'Accept-Encoding' not in prep.headers
+        if not SKIP_HEADER:
+            assert "Accept-Encoding" not in prep.headers
+        else:
+            assert SKIP_HEADER == prep.headers["Accept-Encoding"]
 
     def test_headers_preserve_order(self, httpbin):
         """Preserve order when headers provided as OrderedDict."""