wsgi: writelines() doesn't handle partial writeswsgi-writelines-295

Instead, use `socket.sendall()` for any wsgi network writing operations.

https://github.com/eventlet/eventlet/issues/295

Upstream CPython bug:

http://bugs.python.org/issue26292

4 files changed, 93 insertions, 53 deletions

diff --git a/eventlet/greenio/base.py b/eventlet/greenio/base.py
index 5f0108c..d7831bf 100644
--- a/eventlet/greenio/base.py
+++ b/eventlet/greenio/base.py
@@ -430,7 +430,7 @@ greenpipe_doc = """
     GreenPipe is a cooperative replacement for file class.
     It will cooperate on pipes. It will block on regular file.
     Differneces from file class:
-    - mode is r/w property. Should re r/o
+    - mode is r/w property. Should be r/o
     - encoding property not implemented
     - write/writelines will not raise TypeError exception when non-string data is written
       it will write str(data) instead
diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py
index 88e7752..4f78882 100644
--- a/eventlet/wsgi.py
+++ b/eventlet/wsgi.py
@@ -75,9 +75,8 @@ class Input(object):
                  rfile,
                  content_length,
                  sock,
-                 wfile=None,
-                 wfile_line=None,
-                 chunked_input=False):
+                 chunked_input=False,
+                 env=None):
 
         self.rfile = rfile
         self._sock = sock
@@ -85,43 +84,37 @@ class Input(object):
             content_length = int(content_length)
         self.content_length = content_length
 
-        self.wfile = wfile
-        self.wfile_line = wfile_line
-
         self.position = 0
         self.chunked_input = chunked_input
         self.chunk_length = -1
 
+        # None - do not send, True - yet to send, False - already sent
+        self.state_hundred_continue = None
+        if env and (env.get('HTTP_EXPECT', '').lower() == '100-continue'):
+            state_hundred_continue = True
         # (optional) headers to send with a "100 Continue" response. Set by
         # calling set_hundred_continue_respose_headers() on env['wsgi.input']
-        self.hundred_continue_headers = None
-        self.is_hundred_continue_response_sent = False
+        self.hundred_continue_headers = ()
 
     def send_hundred_continue_response(self):
-        towrite = []
-
-        # 100 Continue status line
-        towrite.append(self.wfile_line)
-
-        # Optional headers
-        if self.hundred_continue_headers is not None:
-            # 100 Continue headers
-            for header in self.hundred_continue_headers:
-                towrite.append(six.b('%s: %s\r\n' % header))
+        if not self.state_hundred_continue:
+            return
 
-        # Blank line
+        towrite = [b'HTTP/1.1 100 Continue\r\n']
+        towrite.extend(
+            six.b('%s: %s\r\n' % header)
+            for header in self.hundred_continue_headers
+        )
         towrite.append(b'\r\n')
-
-        self.wfile.writelines(towrite)
+        self._sock.sendall(b''.join(towrite))
 
         # Reinitialize chunk_length (expect more data)
         self.chunk_length = -1
 
+        self.state_hundred_continue = False
+
     def _do_read(self, reader, length=None):
-        if self.wfile is not None and not self.is_hundred_continue_response_sent:
-            # 100 Continue response
-            self.send_hundred_continue_response()
-            self.is_hundred_continue_response_sent = True
+        self.send_hundred_continue_response()
         if (self.content_length is not None) and (
                 length is None or length > self.content_length - self.position):
             length = self.content_length - self.position
@@ -135,10 +128,7 @@ class Input(object):
         return read
 
     def _chunked_read(self, rfile, length=None, use_readline=False):
-        if self.wfile is not None and not self.is_hundred_continue_response_sent:
-            # 100 Continue response
-            self.send_hundred_continue_response()
-            self.is_hundred_continue_response_sent = True
+        self.send_hundred_continue_response()
         try:
             if length == 0:
                 return ""
@@ -290,14 +280,14 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
     def setup(self):
         # overriding SocketServer.setup to correctly handle SSL.Connection objects
         conn = self.connection = self.request
+        # wfile is only for flush/close by base class
+        self.wfile = six.StringIO()
         try:
             self.rfile = conn.makefile('rb', self.rbufsize)
-            self.wfile = conn.makefile('wb', self.wbufsize)
         except (AttributeError, NotImplementedError):
             if hasattr(conn, 'send') and hasattr(conn, 'recv'):
                 # it's an SSL.Connection
                 self.rfile = socket._fileobject(conn, "rb", self.rbufsize)
-                self.wfile = socket._fileobject(conn, "wb", self.wbufsize)
             else:
                 # it's a SSLObject, or a martian
                 raise NotImplementedError("wsgi.py doesn't support sockets "
@@ -314,7 +304,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
         try:
             self.raw_requestline = self.rfile.readline(self.server.url_length_limit)
             if len(self.raw_requestline) == self.server.url_length_limit:
-                self.wfile.write(
+                self.connection.sendall(
                     b"HTTP/1.0 414 Request URI Too Long\r\n"
                     b"Connection: close\r\nContent-length: 0\r\n\r\n")
                 self.close_connection = 1
@@ -336,13 +326,13 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
             if not self.parse_request():
                 return
         except HeaderLineTooLong:
-            self.wfile.write(
+            self.connection.sendall(
                 b"HTTP/1.0 400 Header Line Too Long\r\n"
                 b"Connection: close\r\nContent-length: 0\r\n\r\n")
             self.close_connection = 1
             return
         except HeadersTooLarge:
-            self.wfile.write(
+            self.connection.sendall(
                 b"HTTP/1.0 400 Headers Too Large\r\n"
                 b"Connection: close\r\nContent-length: 0\r\n\r\n")
             self.close_connection = 1
@@ -355,7 +345,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
             try:
                 int(content_length)
             except ValueError:
-                self.wfile.write(
+                self.connection.sendall(
                     b"HTTP/1.0 400 Bad Request\r\n"
                     b"Connection: close\r\nContent-length: 0\r\n\r\n")
                 self.close_connection = 1
@@ -379,13 +369,12 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
         headers_set = []
         headers_sent = []
 
-        wfile = self.wfile
         result = None
         use_chunked = [False]
         length = [0]
         status_code = [200]
 
-        def write(data, _writelines=wfile.writelines):
+        def write(data):
             towrite = []
             if not headers_set:
                 raise AssertionError("write() before start_response()")
@@ -406,9 +395,9 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
                 if self.close_connection == 0 and \
                    self.server.keepalive and (client_conn == 'keep-alive' or
                                               (self.request_version == 'HTTP/1.1' and
-                                               not client_conn == 'close')):
-                        # only send keep-alives back to clients that sent them,
-                        # it's redundant for 1.1 connections
+                                               client_conn != 'close')):
+                    # only send keep-alives back to clients that sent them,
+                    # it's redundant for 1.1 connections
                     send_keep_alive = (client_conn == 'keep-alive')
                     self.close_connection = 0
                 else:
@@ -418,7 +407,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
                     if self.request_version == 'HTTP/1.1':
                         use_chunked[0] = True
                         towrite.append(b'Transfer-Encoding: chunked\r\n')
-                    elif 'content-length' not in header_list:
+                    else:
                         # client is 1.0 and therefore must read to EOF
                         self.close_connection = 1
 
@@ -434,8 +423,8 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
                 towrite.append(six.b("%x" % (len(data),)) + b"\r\n" + data + b"\r\n")
             else:
                 towrite.append(data)
-            _writelines(towrite)
             length[0] = length[0] + sum(map(len, towrite))
+            self.connection.sendall(b''.join(towrite))
 
         def start_response(status, response_headers, exc_info=None):
             status_code[0] = status.split()[0]
@@ -513,7 +502,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
             if (request_input.chunked_input or
                     request_input.position < (request_input.content_length or 0)):
                 # Read and discard body if there was no pending 100-continue
-                if not request_input.wfile and self.close_connection == 0:
+                if request_input.state_hundred_continue is None and self.close_connection == 0:
                     try:
                         request_input.discard()
                     except ChunkReadError as e:
@@ -596,16 +585,14 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler):
             else:
                 env[envk] = v
 
-        if env.get('HTTP_EXPECT') == '100-continue':
-            wfile = self.wfile
-            wfile_line = b'HTTP/1.1 100 Continue\r\n'
-        else:
-            wfile = None
-            wfile_line = None
         chunked = env.get('HTTP_TRANSFER_ENCODING', '').lower() == 'chunked'
-        env['wsgi.input'] = env['eventlet.input'] = Input(
-            self.rfile, length, self.connection, wfile=wfile, wfile_line=wfile_line,
-            chunked_input=chunked)
+        env['wsgi.input'] = env['eventlet.input'] = request_input = Input(
+            rfile=self.rfile,
+            content_length=length,
+            sock=self.connection,
+            chunked_input=chunked,
+            env=env,
+        )
         env['eventlet.posthooks'] = []
 
         return env
diff --git a/tests/isolated/wsgi_partial_write.py b/tests/isolated/wsgi_partial_write.py
new file mode 100644
index 0000000..3ccb36d
--- /dev/null
+++ b/tests/isolated/wsgi_partial_write.py
@@ -0,0 +1,49 @@
+from __future__ import print_function
+# no standard tests in this file, ignore
+__test__ = False
+
+
+class PartialFile(object):
+    def __init__(self, sock):
+        self.sock = sock
+
+    def writelines(self, lines):
+        print('partial writelines', lines)
+        self.sock.send(lines[0][:-1])
+
+
+def main():
+    import eventlet
+    from eventlet import wsgi
+    from eventlet.support import six
+    from tests import wsgi_test
+
+    original_makefile = eventlet.greenio.base.GreenSocket.makefile
+
+    def test_makefile(sock, mode, *a, **kw):
+        if 'r' in mode:
+            return original_makefile(sock, mode, *a, **kw)
+        return PartialFile(sock)
+
+    eventlet.greenio.base.GreenSocket.makefile = test_makefile
+
+    server_sock = eventlet.listen(('localhost', 0))
+    server_port = server_sock.getsockname()[1]
+    eventlet.spawn(
+        wsgi.server,
+        sock=server_sock,
+        site=wsgi_test.hello_world,
+        log=six.StringIO(),
+    )
+
+    sock = eventlet.connect(('localhost', server_port))
+    sock.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
+    result = wsgi_test.read_http(sock)
+    sock.close()
+    assert result.status == 'HTTP/1.1 200 OK'
+    assert result.body == b'hello world'
+
+    print('pass')
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/wsgi_test.py b/tests/wsgi_test.py
index 8e2b50f..0ab18b1 100644
--- a/tests/wsgi_test.py
+++ b/tests/wsgi_test.py
@@ -1575,6 +1575,10 @@ class TestHttpd(_TestBase):
         finally:
             shutil.rmtree(tempdir)
 
+    def test_partial_write(self):
+        # https://github.com/eventlet/eventlet/issues/295
+        tests.run_isolated('wsgi_partial_write.py')
+
 
 def read_headers(sock):
     fd = sock.makefile('rb')