summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--django/contrib/csrf/middleware.py7
-rw-r--r--django/core/handlers/base.py6
-rw-r--r--django/http/__init__.py41
-rw-r--r--django/middleware/common.py16
-rw-r--r--django/middleware/gzip.py20
-rw-r--r--django/middleware/http.py5
-rw-r--r--django/utils/text.py18
-rw-r--r--docs/ref/request-response.txt29
-rw-r--r--tests/regressiontests/response_streaming/__init__.py0
-rw-r--r--tests/regressiontests/response_streaming/models.py0
-rw-r--r--tests/regressiontests/response_streaming/tests.py32
-rw-r--r--tests/regressiontests/response_streaming/urls.py7
-rw-r--r--tests/regressiontests/response_streaming/views.py13
-rw-r--r--tests/urls.py3
14 files changed, 183 insertions, 14 deletions
diff --git a/django/contrib/csrf/middleware.py b/django/contrib/csrf/middleware.py
index 0d0a8eca9e..7edaefedc6 100644
--- a/django/contrib/csrf/middleware.py
+++ b/django/contrib/csrf/middleware.py
@@ -64,6 +64,8 @@ class CsrfResponseMiddleware(object):
csrfmiddlewaretoken if the response/request have an active
session.
"""
+ streaming_safe = True
+
def process_response(self, request, response):
if getattr(response, 'csrf_exempt', False):
return response
@@ -102,6 +104,11 @@ class CsrfResponseMiddleware(object):
# Modify any POST forms
response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
+ # Handle streaming responses
+ if getattr(response, "content_generator", False):
+ response.content = (_POST_FORM_RE.sub(add_csrf_field, chunk) for chunk in response.content_generator)
+ else:
+ response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
return response
class CsrfMiddleware(CsrfViewMiddleware, CsrfResponseMiddleware):
diff --git a/django/core/handlers/base.py b/django/core/handlers/base.py
index ff534764aa..d3ba538b8d 100644
--- a/django/core/handlers/base.py
+++ b/django/core/handlers/base.py
@@ -74,9 +74,13 @@ class BaseHandler(object):
response = self.get_response(request)
# Apply response middleware
+ streaming = getattr(response, "content_generator", False)
+ streaming_safe = lambda x: getattr(x.im_self, "streaming_safe", False)
if not isinstance(response, http.HttpResponseSendFile):
for middleware_method in self._response_middleware:
- response = middleware_method(request, response)
+ if not streaming or streaming_safe(middleware_method):
+ print middleware_method
+ response = middleware_method(request, response)
response = self.apply_response_fixes(request, response)
finally:
signals.request_finished.send(sender=self.__class__)
diff --git a/django/http/__init__.py b/django/http/__init__.py
index eaa8cb3232..51d6ac8b35 100644
--- a/django/http/__init__.py
+++ b/django/http/__init__.py
@@ -436,6 +436,47 @@ class HttpResponse(object):
def tell(self):
return sum([len(chunk) for chunk in self._container])
+class HttpResponseStreaming(HttpResponse):
+ """
+ This class behaves the same as HttpResponse, except that the content
+ attribute is an unconsumed generator or iterator.
+ """
+ def __init__(self, content='', mimetype=None, status=None,
+ content_type=None, request=None):
+ super(HttpResponseStreaming, self).__init__('', mimetype,
+ status, content_type, request)
+
+ self._container = content
+ self._is_string = False
+
+ def _consume_content(self):
+ if not self._is_string:
+ content = self._container
+ self._container = [''.join(content)]
+ if hasattr(content, 'close'):
+ content.close()
+ self._is_string = True
+
+ def _get_content(self):
+ self._consume_content()
+ return super(HttpResponseStreaming, self)._get_content()
+
+ def _set_content(self, value):
+ if not isinstance(value, basestring) and hasattr(value, "__iter__"):
+ self._container = value
+ self._is_string = False
+ else:
+ self._container = [value]
+ self._is_string = True
+
+ content = property(_get_content, _set_content)
+
+ def _get_content_generator(self):
+ if not self._is_string:
+ return self._container
+
+ content_generator = property(_get_content_generator)
+
class HttpResponseSendFile(HttpResponse):
sendfile_fh = None
diff --git a/django/middleware/common.py b/django/middleware/common.py
index b2c97c6740..302375c794 100644
--- a/django/middleware/common.py
+++ b/django/middleware/common.py
@@ -27,6 +27,7 @@ class CommonMiddleware(object):
the entire page content and Not Modified responses will be returned
appropriately.
"""
+ streaming_safe = True
def process_request(self, request):
"""
@@ -100,14 +101,15 @@ class CommonMiddleware(object):
if settings.USE_ETAGS:
if response.has_header('ETag'):
etag = response['ETag']
- else:
+ # Do not consume the content of HttpResponseStreaming
+ elif not getattr(response, "content_generator", False):
etag = '"%s"' % md5_constructor(response.content).hexdigest()
- if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
- cookies = response.cookies
- response = http.HttpResponseNotModified()
- response.cookies = cookies
- else:
- response['ETag'] = etag
+ if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
+ cookies = response.cookies
+ response = http.HttpResponseNotModified()
+ response.cookies = cookies
+ else:
+ response['ETag'] = etag
return response
diff --git a/django/middleware/gzip.py b/django/middleware/gzip.py
index 47f75aa416..37c12487b9 100644
--- a/django/middleware/gzip.py
+++ b/django/middleware/gzip.py
@@ -1,6 +1,6 @@
import re
-from django.utils.text import compress_string
+from django.utils.text import compress_sequence, compress_string
from django.utils.cache import patch_vary_headers
re_accepts_gzip = re.compile(r'\bgzip\b')
@@ -11,9 +11,15 @@ class GZipMiddleware(object):
It sets the Vary header accordingly, so that caches will base their storage
on the Accept-Encoding header.
"""
- def process_response(self, request, response):
+ streaming_safe = True
+
+ def process_response(self, request, response):
+ # Do not consume the content of HttpResponseStreaming responses just to
+ # check content length
+ streaming = getattr(response, "content_generator", False)
+
# It's not worth compressing non-OK or really short responses.
- if response.status_code != 200 or len(response.content) < 200:
+ if response.status_code != 200 or (not streaming and len(response.content) < 200):
return response
patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,7 +38,11 @@ class GZipMiddleware(object):
if not re_accepts_gzip.search(ae):
return response
- response.content = compress_string(response.content)
+ if streaming:
+ response.content = compress_sequence(response.content_generator)
+ del response['Content-Length']
+ else:
+ response.content = compress_string(response.content)
+ response['Content-Length'] = str(len(response.content))
response['Content-Encoding'] = 'gzip'
- response['Content-Length'] = str(len(response.content))
return response
diff --git a/django/middleware/http.py b/django/middleware/http.py
index 53b65c1034..f699c27f1b 100644
--- a/django/middleware/http.py
+++ b/django/middleware/http.py
@@ -8,9 +8,12 @@ class ConditionalGetMiddleware(object):
Also sets the Date and Content-Length response-headers.
"""
+ streaming_safe = True
+
def process_response(self, request, response):
response['Date'] = http_date()
- if not response.has_header('Content-Length'):
+ streaming = getattr(response, "content_generator", False)
+ if not response.has_header('Content-Length') and not streaming:
response['Content-Length'] = str(len(response.content))
if response.has_header('ETag'):
diff --git a/django/utils/text.py b/django/utils/text.py
index fe46e26b52..63d1ccec24 100644
--- a/django/utils/text.py
+++ b/django/utils/text.py
@@ -176,6 +176,24 @@ def compress_string(s):
zfile.close()
return zbuf.getvalue()
+# WARNING - be aware that compress_sequence does not achieve the same
+# level of compression as compress_string
+def compress_sequence(sequence):
+ import cStringIO, gzip
+ zbuf = cStringIO.StringIO()
+ zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
+ yield zbuf.getvalue()
+ for item in sequence:
+ position = zbuf.tell()
+ zfile.write(item)
+ zfile.flush()
+ zbuf.seek(position)
+ yield zbuf.read()
+ position = zbuf.tell()
+ zfile.close()
+ zbuf.seek(position)
+ yield zbuf.read()
+
ustring_re = re.compile(u"([\u0080-\uffff])")
def javascript_quote(s, quote_double_quotes=False):
diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt
index ae23c3a507..ebb20505af 100644
--- a/docs/ref/request-response.txt
+++ b/docs/ref/request-response.txt
@@ -597,6 +597,35 @@ live in :mod:`django.http`.
**Note:** Response middleware is bypassed by HttpResponseSendFile.
+.. class:: HttpResponseStreaming
+
+ .. versionadded:: 1.1
+
+ A special response class that does not consume generators before returning
+ the response. To do this, it bypasses middleware that is not useful for
+ chunked responses, and is treated specially by middleware that is useful.
+
+ It is primarily useful for sending large responses that would cause
+ timeouts if sent with a normal HttpResponse.
+
+ **Note:** Of the built-in response middleware, this class works correctly with:
+
+ * :class:`django.middleware.common.CommonMiddleware`
+
+ * :class:`django.middleware.gzip.GZipMiddleware`
+
+ * :class:`django.middleware.http.ConditionalGetMiddleware`
+
+ * :class:`django.contrib.csrf.middleware.CsrfMiddleware`
+
+ Developers of third-party middleware who wish to make it work with this class
+ should note that any time they access :class:`HttpResponseStreaming.content`, it will
+ break the functionality of this class. Instead, replace :attr:`HttpResponseStreaming.content`
+ by wrapping the value of :attr:`HttpResponseStreaming.content_generator`. :class:`django.middleware.gzip.GZipMiddleware`
+ is a good example to follow. To inform the handler to send :class:`HttpResponseStreaming`
+ responses through your middleware, add the class attribute ``streaming_safe = True``
+ to your middleware class.
+
.. class:: HttpResponseRedirect
The constructor takes a single argument -- the path to redirect to. This
diff --git a/tests/regressiontests/response_streaming/__init__.py b/tests/regressiontests/response_streaming/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/regressiontests/response_streaming/__init__.py
diff --git a/tests/regressiontests/response_streaming/models.py b/tests/regressiontests/response_streaming/models.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tests/regressiontests/response_streaming/models.py
diff --git a/tests/regressiontests/response_streaming/tests.py b/tests/regressiontests/response_streaming/tests.py
new file mode 100644
index 0000000000..79eac2ec8e
--- /dev/null
+++ b/tests/regressiontests/response_streaming/tests.py
@@ -0,0 +1,32 @@
+import urllib, os
+
+from django.test import TestCase
+from django.conf import settings
+from django.core.files import temp as tempfile
+
+def x():
+ for i in range(0, 10):
+ yield unicode(i) + u'\n'
+
+class ResponseStreamingTests(TestCase):
+ def test_streaming(self):
+ response = self.client.get('/streaming/stream_file/')
+
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response['Content-Disposition'],
+ 'attachment; filename=test.csv')
+ self.assertEqual(response['Content-Type'], 'text/csv')
+ self.assertTrue(not response._is_string)
+ self.assertEqual("".join(iter(response)), "".join(x()))
+ self.assertTrue(not response._is_string)
+
+ def test_bad_streaming(self):
+ response = self.client.get('/streaming/stream_file/')
+
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response['Content-Disposition'],
+ 'attachment; filename=test.csv')
+ self.assertEqual(response['Content-Type'], 'text/csv')
+ self.assertTrue(not response._is_string)
+ self.assertEqual(response.content, "".join(x()))
+ self.assertTrue(response._is_string)
diff --git a/tests/regressiontests/response_streaming/urls.py b/tests/regressiontests/response_streaming/urls.py
new file mode 100644
index 0000000000..52d9ab43a1
--- /dev/null
+++ b/tests/regressiontests/response_streaming/urls.py
@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import patterns
+
+import views
+
+urlpatterns = patterns('',
+ (r'^stream_file/$', views.test_streaming),
+)
diff --git a/tests/regressiontests/response_streaming/views.py b/tests/regressiontests/response_streaming/views.py
new file mode 100644
index 0000000000..2ed11ab75f
--- /dev/null
+++ b/tests/regressiontests/response_streaming/views.py
@@ -0,0 +1,13 @@
+import urllib
+
+from django.http import HttpResponseStreaming
+from time import sleep
+
+def x():
+ for i in range(0, 10):
+ yield unicode(i) + u'\n'
+
+def test_streaming(request):
+ response = HttpResponseStreaming(content=x(), mimetype='text/csv')
+ response['Content-Disposition'] = 'attachment; filename=test.csv'
+ return response
diff --git a/tests/urls.py b/tests/urls.py
index 5c0a04da2b..f66bd124de 100644
--- a/tests/urls.py
+++ b/tests/urls.py
@@ -36,6 +36,9 @@ urlpatterns = patterns('',
# HttpResponseSendfile tests
(r'^sendfile/', include('regressiontests.sendfile.urls')),
+ # HttpResponseStreaming tests
+ (r'^streaming/', include('regressiontests.response_streaming.urls')),
+
# conditional get views
(r'condition/', include('regressiontests.conditional_processing.urls')),
)
View Lorry Controller Status