Implement speedups in cython (#311)

* Use __slots__ in GenericContent

* Remove unnecessary keyword parameters

* Added intial cython support

* Fixed unittests

* Revert delivery_info change

* Fix installing library

* Fix flake8

* Fixed failing unittests

* Added optional installation of speedups

* Comment that FrameSyntaxError is not raised and fixed unittests

* Fix flake8

* Rename PY_AMQP_ENABLE_CYTHON -> CELERY_ENABLE_CYTHON

* Add cython headers for serializers, removed optional parameters from _write_item, update unittests

* Add test_speedups stage to CI

* Rename CELERY_ENABLE_CYTHON -> CELERY_ENABLE_SPEEDUPS

* Remove unnecesary changes

6 files changed, 96 insertions, 32 deletions

diff --git a/.travis.yml b/.travis.yml
index 9ae7446..bd31c88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ env:
     PYTHONUNBUFFERED=yes
   matrix:
     - MATRIX_TOXENV=unit
+    - MATRIX_TOXENV=unit CELERY_ENABLE_SPEEDUPS=1
 
 stages:
   - test
@@ -24,6 +25,11 @@ stages:
 
 matrix:
   fast_finish: true
+  exclude:
+    - python: pypy2.7-6.0
+      env: MATRIX_TOXENV=unit CELERY_ENABLE_SPEEDUPS=1
+    - python: pypy3.5-6.0
+      env: MATRIX_TOXENV=unit CELERY_ENABLE_SPEEDUPS=1
   include:
     - python: 2.7
       env: TOXENV=flake8
@@ -98,6 +104,7 @@ before_install:
 install:
   - pip --disable-pip-version-check install -U pip setuptools wheel | cat
   - pip --disable-pip-version-check install -U --upgrade-strategy eager tox tox-docker | cat
+  - python setup.py develop
 script: tox -v -- -v
 after_success:
   - .tox/$TRAVIS_PYTHON_VERSION/bin/coverage xml
diff --git a/amqp/serialization.pxd b/amqp/serialization.pxd
new file mode 100644
index 0000000..5a9b878
--- /dev/null
+++ b/amqp/serialization.pxd
@@ -0,0 +1,25 @@
+import cython
+
+cdef int _flushbits(list bits, write)
+
+# Does not raise FrameSyntaxError due performance reasons
+@cython.locals(blen=cython.int, limit=cython.int, keylen=cython.int, tlen=cython.int, alen=cython.int, blen=cython.int, slen=cython.int, d=cython.int)
+cpdef tuple _read_item(buf, int offset)
+
+# Does not raise FrameSyntaxError due performance reasons
+@cython.locals(bitcount=cython.int, bits=cython.int, tlen=cython.int, limit=cython.int, slen=cython.int, keylen=cython.int)
+cpdef tuple loads(format, buf, int offset)
+
+@cython.locals(bitcount=cython.int, shift=cython.int)
+cpdef dumps(format, values)
+
+# Does not raise FrameSyntaxError due performance reasons
+cpdef int _write_table(d, write, bits) except -1
+
+# Does not raise FrameSyntaxError due performance reasons
+cdef int _write_array(l, write, bits) except -1
+
+@cython.locals(slen=cython.int, flags=cython.ushort)
+cdef tuple decode_properties_basic(buf, int offset)
+
+cdef int _write_item(v, write, bits) except -1
diff --git a/amqp/serialization.py b/amqp/serialization.py
index 758a3c0..21c589c 100644
--- a/amqp/serialization.py
+++ b/amqp/serialization.py
@@ -34,7 +34,7 @@ ILLEGAL_TABLE_TYPE_WITH_VALUE = """\
 """
 
 
-def _read_item(buf, offset=0, unpack_from=unpack_from, ftype_t=ftype_t):
+def _read_item(buf, offset):
     ftype = ftype_t(buf[offset]) if ftype_t else buf[offset]
     offset += 1
 
@@ -144,9 +144,7 @@ def _read_item(buf, offset=0, unpack_from=unpack_from, ftype_t=ftype_t):
     return val, offset
 
 
-def loads(format, buf, offset=0,
-          ord=ord, unpack_from=unpack_from,
-          _read_item=_read_item, pstr_t=pstr_t):
+def loads(format, buf, offset):
     """Deserialize amqp format.
 
     bit = b
@@ -245,7 +243,7 @@ def loads(format, buf, offset=0,
     return values, offset
 
 
-def _flushbits(bits, write, pack=pack):
+def _flushbits(bits, write):
     if bits:
         write(pack('B' * len(bits), *bits))
         bits[:] = []
@@ -325,7 +323,7 @@ def dumps(format, values):
     return out.getvalue()
 
 
-def _write_table(d, write, bits, pack=pack):
+def _write_table(d, write, bits):
     out = BytesIO()
     twrite = out.write
     for k, v in items(d):
@@ -343,7 +341,7 @@ def _write_table(d, write, bits, pack=pack):
     write(table_data)
 
 
-def _write_array(l, write, bits, pack=pack):
+def _write_array(l, write, bits):
     out = BytesIO()
     awrite = out.write
     for v in l:
@@ -357,11 +355,7 @@ def _write_array(l, write, bits, pack=pack):
     write(array_data)
 
 
-def _write_item(v, write, bits, pack=pack,
-                string_t=string_t, bytes=bytes, string=string, bool=bool,
-                float=float, int_types=int_types, Decimal=Decimal,
-                datetime=datetime, dict=dict, list=list, tuple=tuple,
-                None_t=None):
+def _write_item(v, write, bits):
     if isinstance(v, (string_t, bytes)):
         if isinstance(v, string):
             v = v.encode('utf-8', 'surrogatepass')
@@ -393,14 +387,13 @@ def _write_item(v, write, bits, pack=pack,
     elif isinstance(v, (list, tuple)):
         write(b'A')
         _write_array(v, write, bits)
-    elif v is None_t:
+    elif v is None:
         write(b'V')
     else:
         raise ValueError()
 
 
-def decode_properties_basic(buf, offset=0,
-                            unpack_from=unpack_from, pstr_t=pstr_t):
+def decode_properties_basic(buf, offset):
     """Decode basic properties."""
     properties = {}
 
@@ -486,6 +479,11 @@ class GenericContent(object):
     CLASS_ID = None
     PROPERTIES = [('dummy', 's')]
 
+    __slots__ = (
+        'frame_method', 'frame_args', '_pending_chunks', 'body',
+        'body_received', 'body_size', 'ready', 'properties'
+    )
+
     def __init__(self, frame_method=None, frame_args=None, **props):
         self.frame_method = frame_method
         self.frame_args = frame_args
@@ -507,8 +505,7 @@ class GenericContent(object):
             return self.properties[name]
         raise AttributeError(name)
 
-    def _load_properties(self, class_id, buf, offset=0,
-                         classes=PROPERTY_CLASSES, unpack_from=unpack_from):
+    def _load_properties(self, class_id, buf, offset):
         """Load AMQP properties.
 
         Given the raw bytes containing the property-flags and property-list
@@ -516,7 +513,7 @@ class GenericContent(object):
         stored in this object as an attribute named 'properties'.
         """
         # Read 16-bit shorts until we get one with a low bit set to zero
-        props, offset = classes[class_id](buf, offset)
+        props, offset = PROPERTY_CLASSES[class_id](buf, offset)
         self.properties = props
         return offset
 
diff --git a/setup.py b/setup.py
index 4d306af..08b25d3 100644
--- a/setup.py
+++ b/setup.py
@@ -103,6 +103,27 @@ class pytest(setuptools.command.test.test):
         sys.exit(pytest.main(pytest_args))
 
 
+if os.environ.get("CELERY_ENABLE_SPEEDUPS"):
+    setup_requires=['Cython']
+    ext_modules = [
+        setuptools.Extension(
+            'amqp.serialization',
+            ["amqp/serialization.py"],
+        ),
+        setuptools.Extension(
+            'amqp.basic_message',
+            ["amqp/basic_message.py"],
+        ),
+        setuptools.Extension(
+            'amqp.method_framing',
+            ["amqp/method_framing.py"],
+        ),
+    ]
+else:
+    setup_requires = []
+    ext_modules = []
+
+
 setuptools.setup(
     name=NAME,
     packages=setuptools.find_packages(exclude=['ez_setup', 't', 't.*']),
@@ -119,7 +140,9 @@ setuptools.setup(
     classifiers=classifiers,
     python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
     install_requires=reqs('default.txt'),
+    setup_requires=setup_requires,
     tests_require=reqs('test.txt'),
     cmdclass={'test': pytest},
     zip_safe=False,
+    ext_modules = ext_modules,
 )
diff --git a/t/integration/test_integration.py b/t/integration/test_integration.py
index cf7c55b..e21350b 100644
--- a/t/integration/test_integration.py
+++ b/t/integration/test_integration.py
@@ -123,7 +123,7 @@ class DataComparator(object):
         self.items = items
 
     def __eq__(self, other):
-        values, offset = loads(self.argsig, other)
+        values, offset = loads(self.argsig, other, 0)
         return tuple(values) == tuple(self.items)
 
 
diff --git a/t/unit/test_serialization.py b/t/unit/test_serialization.py
index 9a3a0bf..17c0eee 100644
--- a/t/unit/test_serialization.py
+++ b/t/unit/test_serialization.py
@@ -36,12 +36,12 @@ class test_serialization:
         ('f', b'f' + pack('>f', 33.3), 34.0, ceil),
     ])
     def test_read_item(self, descr, frame, expected, cast):
-        actual = _read_item(frame)[0]
+        actual = _read_item(frame, 0)[0]
         actual = cast(actual) if cast else actual
         assert actual == expected
 
     def test_read_item_V(self):
-        assert _read_item(b'V')[0] is None
+        assert _read_item(b'V', 0)[0] is None
 
     def test_roundtrip(self):
         format = b'bobBlLbsbSTx'
@@ -51,7 +51,7 @@ class test_serialization:
             datetime(2015, 3, 13, 10, 23),
             b'thequick\xff'
         ])
-        y = loads(format, x)
+        y = loads(format, x, 0)
         assert [
             True, 32, False, 3415, 4513134, 13241923419,
             True, 'thequickbrownfox', False, 'jumpsoverthelazydog',
@@ -63,18 +63,18 @@ class test_serialization:
         x = dumps(format, [
             {'a': -2147483649, 'b': 2147483648},  # celery/celery#3121
         ])
-        y = loads(format, x)
+        y = loads(format, x, 0)
         assert y[0] == [{
             'a': -2147483649, 'b': 2147483648,  # celery/celery#3121
         }]
 
     def test_loads_unknown_type(self):
         with pytest.raises(FrameSyntaxError):
-            loads('y', 'asdsad')
+            loads('y', 'asdsad', 0)
 
     def test_float(self):
-        assert (int(loads(b'fb', dumps(b'fb', [32.31, False]))[0][0] * 100) ==
-                3231)
+        data = int(loads(b'fb', dumps(b'fb', [32.31, False]), 0)[0][0] * 100)
+        assert(data == 3231)
 
     def test_table(self):
         table = {
@@ -85,7 +85,19 @@ class test_serialization:
                 1, True, 'bar'
             ]
         }
-        assert loads(b'F', dumps(b'F', [table]))[0][0] == table
+        assert loads(b'F', dumps(b'F', [table]), 0)[0][0] == table
+
+    def test_table__unknown_type(self):
+        table = {
+            'foo': object(),
+            'bar': 'baz',
+            'nil': None,
+            'array': [
+                1, True, 'bar'
+            ]
+        }
+        with pytest.raises(FrameSyntaxError):
+            dumps(b'F', [table])
 
     def test_array(self):
         array = [
@@ -99,7 +111,7 @@ class test_serialization:
         expected = list(array)
         expected[6] = _ANY()
 
-        assert expected == loads('A', dumps('A', [array]))[0][0]
+        assert expected == loads('A', dumps('A', [array]), 0)[0][0]
 
     def test_array_unknown_type(self):
         with pytest.raises(FrameSyntaxError):
@@ -109,14 +121,14 @@ class test_serialization:
         expected = [50, "quick", "fox", True,
                     False, False, True, True, {"prop1": True}]
         buf = dumps('BssbbbbbF', expected)
-        actual, _ = loads('BssbbbbbF', buf)
+        actual, _ = loads('BssbbbbbF', buf, 0)
         assert actual == expected
 
     def test_sixteen_bitflags(self):
         expected = [True, False] * 8
         format = 'b' * len(expected)
         buf = dumps(format, expected)
-        actual, _ = loads(format, buf)
+        actual, _ = loads(format, buf, 0)
         assert actual == expected
 
 
@@ -157,7 +169,7 @@ class test_GenericContent:
         }
         s = m._serialize_properties()
         m2 = Message()
-        m2._load_properties(m2.CLASS_ID, s)
+        m2._load_properties(m2.CLASS_ID, s, 0)
         assert m2.properties == m.properties
 
     def test_load_properties__some_missing(self):
@@ -176,7 +188,7 @@ class test_GenericContent:
         }
         s = m._serialize_properties()
         m2 = Message()
-        m2._load_properties(m2.CLASS_ID, s)
+        m2._load_properties(m2.CLASS_ID, s, 0)
 
     def test_inbound_header(self):
         m = Message()