Fixed keep-alive support in manage.py runserver.

Ticket #25619 changed the default protocol to HTTP/1.1 but did not
properly implement keep-alive. As a "fix" keep-alive was disabled in
ticket #28440 to prevent clients from hanging (they expect the server to
send more data if the connection is not closed and there is no content
length set).

The combination of those two fixes resulted in yet another problem:
HTTP/1.1 by default allows a client to assume that keep-alive is
supported unless the server disables it via 'Connection: close' -- see
RFC2616 8.1.2.1 for details on persistent connection negotiation. Now if
the client receives a response from Django without 'Connection: close'
and immediately sends a new request (on the same tcp connection) before
our server closes the tcp connection, it will error out at some point
because the connection does get closed a few milli seconds later.

This patch fixes the mentioned issues by always sending 'Connection:
close' if we cannot determine a content length. The code is inefficient
in the sense that it does not allow for persistent connections when
chunked responses are used, but that should not really cause any
problems (Django does not generate those) and it only affects the
development server anyways.

Refs #25619, #28440.

3 files changed, 54 insertions, 19 deletions

diff --git a/tests/servers/tests.py b/tests/servers/tests.py
index ce08eb4a3f..e38cb5eb07 100644
--- a/tests/servers/tests.py
+++ b/tests/servers/tests.py
@@ -4,8 +4,7 @@ Tests for django.core.servers.
 import errno
 import os
 import socket
-import sys
-from http.client import HTTPConnection, RemoteDisconnected
+from http.client import HTTPConnection
 from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import urlopen
@@ -57,29 +56,60 @@ class LiveServerViews(LiveServerBase):
         with self.urlopen('/example_view/') as f:
             self.assertEqual(f.version, 11)
 
-    @override_settings(MIDDLEWARE=[])
     def test_closes_connection_without_content_length(self):
         """
-        The server doesn't support keep-alive because Python's http.server
-        module that it uses hangs if a Content-Length header isn't set (for
-        example, if CommonMiddleware isn't enabled or if the response is a
-        StreamingHttpResponse) (#28440 / https://bugs.python.org/issue31076).
+        A HTTP 1.1 server is supposed to support keep-alive. Since our
+        development server is rather simple we support it only in cases where
+        we can detect a content length from the response. This should be doable
+        for all simple views and streaming responses where an iterable with
+        length of one is passed. The latter follows as result of `set_content_length`
+        from https://github.com/python/cpython/blob/master/Lib/wsgiref/handlers.py.
+
+        If we cannot detect a content length we explicitly set the `Connection`
+        header to `close` to notify the client that we do not actually support
+        it.
         """
         conn = HTTPConnection(LiveServerViews.server_thread.host, LiveServerViews.server_thread.port, timeout=1)
         try:
-            conn.request('GET', '/example_view/', headers={'Connection': 'keep-alive'})
-            response = conn.getresponse().read()
-            conn.request('GET', '/example_view/', headers={'Connection': 'close'})
-            # macOS may give ConnectionResetError.
-            with self.assertRaises((RemoteDisconnected, ConnectionResetError)):
-                try:
-                    conn.getresponse()
-                except ConnectionAbortedError:
-                    if sys.platform == 'win32':
-                        self.skipTest('Ignore nondeterministic failure on Windows.')
+            conn.request('GET', '/streaming_example_view/', headers={'Connection': 'keep-alive'})
+            response = conn.getresponse()
+            self.assertTrue(response.will_close)
+            self.assertEqual(response.read(), b'Iamastream')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.getheader('Connection'), 'close')
+
+            conn.request('GET', '/streaming_example_view/', headers={'Connection': 'close'})
+            response = conn.getresponse()
+            self.assertTrue(response.will_close)
+            self.assertEqual(response.read(), b'Iamastream')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.getheader('Connection'), 'close')
+        finally:
+            conn.close()
+
+    def test_keep_alive_on_connection_with_content_length(self):
+        """
+        See `test_closes_connection_without_content_length` for details. This
+        is a follow up test, which ensure that we do not close the connection
+        if not needed, hence allowing us to take advantage of keep-alive.
+        """
+        conn = HTTPConnection(LiveServerViews.server_thread.host, LiveServerViews.server_thread.port)
+        try:
+            conn.request('GET', '/example_view/', headers={"Connection": "keep-alive"})
+            response = conn.getresponse()
+            self.assertFalse(response.will_close)
+            self.assertEqual(response.read(), b'example view')
+            self.assertEqual(response.status, 200)
+            self.assertIsNone(response.getheader('Connection'))
+
+            conn.request('GET', '/example_view/', headers={"Connection": "close"})
+            response = conn.getresponse()
+            self.assertFalse(response.will_close)
+            self.assertEqual(response.read(), b'example view')
+            self.assertEqual(response.status, 200)
+            self.assertIsNone(response.getheader('Connection'))
         finally:
             conn.close()
-        self.assertEqual(response, b'example view')
 
     def test_404(self):
         with self.assertRaises(HTTPError) as err:
diff --git a/tests/servers/urls.py b/tests/servers/urls.py
index 4963bde357..9017161808 100644
--- a/tests/servers/urls.py
+++ b/tests/servers/urls.py
@@ -4,6 +4,7 @@ from . import views
 
 urlpatterns = [
     url(r'^example_view/$', views.example_view),
+    url(r'^streaming_example_view/$', views.streaming_example_view),
     url(r'^model_view/$', views.model_view),
     url(r'^create_model_instance/$', views.create_model_instance),
     url(r'^environ_view/$', views.environ_view),
diff --git a/tests/servers/views.py b/tests/servers/views.py
index 3bae0834ab..078be67f46 100644
--- a/tests/servers/views.py
+++ b/tests/servers/views.py
@@ -1,6 +1,6 @@
 from urllib.request import urlopen
 
-from django.http import HttpResponse
+from django.http import HttpResponse, StreamingHttpResponse
 
 from .models import Person
 
@@ -9,6 +9,10 @@ def example_view(request):
     return HttpResponse('example view')
 
 
+def streaming_example_view(request):
+    return StreamingHttpResponse((b'I', b'am', b'a', b'stream'))
+
+
 def model_view(request):
     people = Person.objects.all()
     return HttpResponse('\n'.join(person.name for person in people))