helper coroutines for http protocol

5 files changed, 292 insertions, 41 deletions

diff --git a/srv.py b/srv.py
index 545ba4d..0a7c87a 100644
--- a/srv.py
+++ b/srv.py
@@ -1,9 +1,9 @@
 """Simple server written using an event loop."""
 
+import http.client
 import email.message
 import email.parser
 import os
-import re
 
 import tulip
 import tulip.http
@@ -19,36 +19,31 @@ class HttpServer(tulip.Protocol):
 
     @tulip.task
     def handle_request(self):
-        line = yield from self.reader.readline()
-        print('request line', line)
-        match = re.match(rb'([A-Z]+) (\S+) HTTP/(1.\d)\r?\n\Z', line)
-        if not match:
+        try:
+            method, path, version = yield from self.reader.read_request_line()
+        except http.client.BadStatusLine:
             self.transport.close()
             return
-        bmethod, bpath, bversion = match.groups()
+
         print('method = {!r}; path = {!r}; version = {!r}'.format(
-            bmethod, bpath, bversion))
-        try:
-            path = bpath.decode('ascii')
-        except UnicodeError as exc:
-            print('not ascii', repr(bpath), exc)
+            method, path, version))
+
+        if (not (path.isprintable() and path.startswith('/')) or '/.' in path):
+            print('bad path', repr(path))
             path = None
         else:
-            if (not (path.isprintable() and path.startswith('/')) or
-                '/.' in path):
-                print('bad path', repr(path))
+            path = '.' + path
+            if not os.path.exists(path):
+                print('no file', repr(path))
                 path = None
             else:
-                path = '.' + path
-                if not os.path.exists(path):
-                    print('no file', repr(path))
-                    path = None
-                else:
-                    isdir = os.path.isdir(path)
+                isdir = os.path.isdir(path)
+
         if not path:
             self.transport.write(b'HTTP/1.0 404 Not found\r\n\r\n')
             self.transport.close()
             return
+
         lines = []
         while True:
             line = yield from self.reader.readline()
@@ -58,10 +53,13 @@ class HttpServer(tulip.Protocol):
             lines.append(line)
             if line == b'\r\n':
                 break
+
         parser = email.parser.BytesHeaderParser()
-        headers = parser.parsebytes(b''.join(lines))
+        parser.parsebytes(b''.join(lines))
+
         write = self.transport.write
         if isdir and not path.endswith('/'):
+            bpath = path.encode('ascii')
             write(b'HTTP/1.0 302 Redirected\r\n'
                   b'URI: ' + bpath + b'/\r\n'
                   b'Location: ' + bpath + b'/\r\n'
@@ -79,7 +77,7 @@ class HttpServer(tulip.Protocol):
                 if name.isprintable() and not name.startswith('.'):
                     try:
                         bname = name.encode('ascii')
-                    except UnicodeError as exc:
+                    except UnicodeError:
                         pass
                     else:
                         if os.path.isdir(os.path.join(path, name)):
@@ -93,14 +91,14 @@ class HttpServer(tulip.Protocol):
             try:
                 with open(path, 'rb') as f:
                     write(f.read())
-            except OSError as exc:
+            except OSError:
                 write(b'Cannot open\r\n')
         self.transport.close()
 
     def connection_made(self, transport):
         self.transport = transport
         print('connection made', transport, transport.get_extra_info('socket'))
-        self.reader = tulip.http.StreamReader()
+        self.reader = tulip.http.HttpStreamReader()
         self.handler = self.handle_request()
 
     def data_received(self, data):
diff --git a/tests/http_protocol_test.py b/tests/http_protocol_test.py
new file mode 100644
index 0000000..61d7ebc
--- /dev/null
+++ b/tests/http_protocol_test.py
@@ -0,0 +1,142 @@
+"""Tests for http/protocol.py"""
+
+import http.client
+import unittest
+import unittest.mock
+
+import tulip
+from tulip.http import protocol
+from tulip.test_utils import LogTrackingTestCase
+
+
+class HttpStreamReaderTests(LogTrackingTestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.suppress_log_errors()
+
+        self.loop = tulip.new_event_loop()
+        tulip.set_event_loop(self.loop)
+
+        self.transport = unittest.mock.Mock()
+        self.stream = protocol.HttpStreamReader()
+
+    def tearDown(self):
+        self.loop.close()
+        super().tearDown()
+
+    def test_request_line(self):
+        self.stream.feed_data(b'get /path HTTP/1.1\r\n')
+        self.assertEqual(
+            ('GET', '/path', (1, 1)),
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_request_line())))
+
+    def test_request_line_two_slashes(self):
+        self.stream.feed_data(b'get //path HTTP/1.1\r\n')
+        self.assertEqual(
+            ('GET', '//path', (1, 1)),
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_request_line())))
+
+    def test_request_line_non_ascii(self):
+        self.stream.feed_data(b'get /path\xd0\xb0 HTTP/1.1\r\n')
+
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_request_line()))
+
+        self.assertEqual(
+            b'get /path\xd0\xb0 HTTP/1.1\r\n', cm.exception.args[0])
+
+    def test_request_line_bad_status_line(self):
+        self.stream.feed_data(b'\r\n')
+        self.assertRaises(
+            http.client.BadStatusLine,
+            self.loop.run_until_complete,
+            tulip.Task(self.stream.read_request_line()))
+
+    def test_request_line_bad_method(self):
+        self.stream.feed_data(b'!12%()+=~$ /get HTTP/1.1\r\n')
+        self.assertRaises(
+            http.client.BadStatusLine,
+            self.loop.run_until_complete,
+            tulip.Task(self.stream.read_request_line()))
+
+    def test_request_line_bad_version(self):
+        self.stream.feed_data(b'GET //get HT/11\r\n')
+        self.assertRaises(
+            http.client.BadStatusLine,
+            self.loop.run_until_complete,
+            tulip.Task(self.stream.read_request_line()))
+
+    def test_response_status_bad_status_line(self):
+        self.stream.feed_data(b'\r\n')
+        self.assertRaises(
+            http.client.BadStatusLine,
+            self.loop.run_until_complete,
+            tulip.Task(self.stream.read_response_status()))
+
+    def test_response_status_bad_status_line_eof(self):
+        self.stream.feed_eof()
+        self.assertRaises(
+            http.client.BadStatusLine,
+            self.loop.run_until_complete,
+            tulip.Task(self.stream.read_response_status()))
+
+    def test_response_status_bad_status_non_ascii(self):
+        self.stream.feed_data(b'HTTP/1.1 200 \xd0\xb0\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertEqual(b'HTTP/1.1 200 \xd0\xb0\r\n', cm.exception.args[0])
+
+    def test_response_status_bad_version(self):
+        self.stream.feed_data(b'HT/11 200 Ok\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertEqual('HT/11 200 Ok', cm.exception.args[0])
+
+    def test_response_status_no_reason(self):
+        self.stream.feed_data(b'HTTP/1.1 200\r\n')
+
+        v, s, r = self.loop.run_until_complete(
+            tulip.Task(self.stream.read_response_status()))
+        self.assertEqual(v, (1, 1))
+        self.assertEqual(s, 200)
+        self.assertEqual(r, '')
+
+    def test_response_status_bad(self):
+        self.stream.feed_data(b'HTT/1\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertIn('HTT/1', str(cm.exception))
+
+    def test_response_status_bad_code_under_100(self):
+        self.stream.feed_data(b'HTTP/1.1 99 test\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertIn('HTTP/1.1 99 test', str(cm.exception))
+
+    def test_response_status_bad_code_above_999(self):
+        self.stream.feed_data(b'HTTP/1.1 9999 test\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertIn('HTTP/1.1 9999 test', str(cm.exception))
+
+    def test_response_status_bad_code_not_int(self):
+        self.stream.feed_data(b'HTTP/1.1 ttt test\r\n')
+        with self.assertRaises(http.client.BadStatusLine) as cm:
+            self.loop.run_until_complete(
+                tulip.Task(self.stream.read_response_status()))
+
+        self.assertIn('HTTP/1.1 ttt test', str(cm.exception))
diff --git a/tulip/http/__init__.py b/tulip/http/__init__.py
index cb07d9f..16f4383 100644
--- a/tulip/http/__init__.py
+++ b/tulip/http/__init__.py
@@ -1,5 +1,8 @@
 # This relies on each of the submodules having an __all__ variable.
+
+from .protocol import *
 from .http_client import *
 
 
-__all__ = http_client.__all__
+__all__ = (protocol.__all__ +
+           http_client.__all__)
diff --git a/tulip/http/http_client.py b/tulip/http/http_client.py
index 71f97c5..520d162 100644
--- a/tulip/http/http_client.py
+++ b/tulip/http/http_client.py
@@ -26,21 +26,21 @@ TODO: How do we do connection keep alive?  Pooling?
 __all__ = ['HttpClientProtocol']
 
 
-import collections
 import email.message
 import email.parser
-import re
 
 import tulip
 
+from . import protocol
+
 
 class HttpClientProtocol:
     """This Protocol class is also used to initiate the connection.
 
     Usage:
       p = HttpClientProtocol(url, ...)
-      f = p.connect()  # Returns a Future
-      ...now what?...
+      sts, headers, stream = yield from p.connect()
+
     """
 
     def __init__(self, host, port=None, *,
@@ -107,19 +107,14 @@ class HttpClientProtocol:
 
     @tulip.coroutine
     def connect(self):
-        yield from self.event_loop.create_connection(lambda: self,
-                                                     self.host,
-                                                     self.port,
-                                                     ssl=self.ssl)
+        yield from self.event_loop.create_connection(
+            lambda: self, self.host, self.port, ssl=self.ssl)
+
         # TODO: A better mechanism to return all info from the
         # status line, all headers, and the buffer, without having
         # an N-tuple return value.
-        status_line = yield from self.stream.readline()
-        m = re.match(rb'HTTP/(\d\.\d)\s+(\d\d\d)\s+([^\r\n]+)\r?\n\Z',
-                     status_line)
-        if not m:
-            raise 'Invalid HTTP status line ({!r})'.format(status_line)
-        version, status, message = m.groups()
+        version, status, message = yield from self.stream.read_response_status()
+
         raw_headers = []
         while True:
             header = yield from self.stream.readline()
@@ -137,7 +132,7 @@ class HttpClientProtocol:
             # TODO: A wrapping stream that limits how much it can read
             # without reading it all into memory at once.
             body = yield from self.stream.readexactly(content_length)
-            stream = StreamReader()
+            stream = protocol.HttpStreamReader()
             stream.feed_data(body)
             stream.feed_eof()
         sts = '{} {}'.format(self.decode(status), self.decode(message))
@@ -176,7 +171,7 @@ class HttpClientProtocol:
         for key, value in self.headers.items():
             self.write_str('{}: {}\r\n'.format(key, value))
         self.transport.write(b'\r\n')
-        self.stream = StreamReader()
+        self.stream = protocol.HttpStreamReader()
         if self.make_body is not None:
             if self.chunked:
                 self.make_body(self.write_chunked, self.write_chunked_eof)
diff --git a/tulip/http/protocol.py b/tulip/http/protocol.py
new file mode 100644
index 0000000..c6f5a60
--- /dev/null
+++ b/tulip/http/protocol.py
@@ -0,0 +1,113 @@
+"""Http related helper utils."""
+
+__all__ = ['HttpStreamReader', 'RequestLine', 'ResponseStatus']
+
+import collections
+import http.client
+import re
+
+import tulip
+
+METHRE = re.compile('[A-Z0-9$-_.]+')
+VERSRE = re.compile('HTTP/(\d+).(\d+)')
+
+
+RequestLine = collections.namedtuple(
+    'RequestLine', ['method', 'uri', 'version'])
+
+
+ResponseStatus = collections.namedtuple(
+    'ResponseStatus', ['version', 'code', 'reason'])
+
+
+class HttpStreamReader(tulip.StreamReader):
+
+    @tulip.coroutine
+    def read_request_line(self):
+        """Read request status line. Exception http.client.BadStatusLine
+        could be raised in case of any errors in status line.
+        Returns three values (method, uri, version)
+
+        Example:
+
+            GET /path HTTP/1.1
+
+            >> yield from reader.read_request_line()
+            ('GET', '/path', (1, 1))
+
+        """
+        bline = yield from self.readline()
+        try:
+            line = bline.decode('ascii').rstrip()
+        except UnicodeDecodeError:
+            raise http.client.BadStatusLine(bline) from None
+
+        try:
+            method, uri, version = line.split(None, 2)
+        except ValueError:
+            raise http.client.BadStatusLine(line) from None
+
+        # method
+        method = method.upper()
+        if not METHRE.match(method):
+            raise http.client.BadStatusLine(method)
+
+        # version
+        match = VERSRE.match(version)
+        if match is None:
+            raise http.client.BadStatusLine(version)
+        version = (int(match.group(1)), int(match.group(2)))
+
+        return RequestLine(method, uri, version)
+
+    @tulip.coroutine
+    def read_response_status(self):
+        """Read response status line. Exception http.client.BadStatusLine
+        could be raised in case of any errors in status line.
+        Returns three values (version, status_code, reason)
+
+        Example:
+
+            HTTP/1.1 200 Ok
+
+            >> yield from reader.read_response_status()
+            ((1, 1), 200, 'Ok')
+
+        """
+        bline = yield from self.readline()
+        if not bline:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise http.client.BadStatusLine(bline)
+
+        try:
+            line = bline.decode('ascii').rstrip()
+        except UnicodeDecodeError:
+            raise http.client.BadStatusLine(bline) from None
+
+        try:
+            version, status = line.split(None, 1)
+        except ValueError:
+            raise http.client.BadStatusLine(line) from None
+        else:
+            try:
+                status, reason = status.split(None, 1)
+            except ValueError:
+                reason = ''
+
+        # version
+        match = VERSRE.match(version)
+        if match is None:
+            raise http.client.BadStatusLine(line)
+        version = (int(match.group(1)), int(match.group(2)))
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+        except ValueError:
+            raise http.client.BadStatusLine(line) from None
+
+        if status < 100 or status > 999:
+            raise http.client.BadStatusLine(line)
+
+        return ResponseStatus(version, status, reason.strip())