summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbillyevans <pervushinai@gmail.com>2018-07-20 14:11:41 -0700
committerJeff Widman <jeff@jeffwidman.com>2018-10-29 12:57:42 -0700
commit3689da3d5c02e362d872cf1fb2d65201419c4b93 (patch)
treea6952876d9086a5b975eef0ab155a182e24876b7
parent4d13713c515796afa535e980b15fa0c2c86ba0eb (diff)
downloadkafka-python-3689da3d5c02e362d872cf1fb2d65201419c4b93.tar.gz
Pre-compile pack/unpack function calls
I noticed that pack/unpack functions from https://github.com/dpkp/kafka-python/blob/master/kafka/protocol/types.py might be slightly improved. I made pre-compilation for them. It gives about 10% better performance compared to the current implementation. Consumption of 100msg: ``` 239884 0.187 0.000 0.287 0.000 types.py:18(_unpack) # new version 239884 0.192 0.000 0.323 0.000 types.py:17(_unpack) ``` I also made some profiling for producers/consumers. It gives about 1-1.5% time savings.
-rw-r--r--kafka/protocol/types.py42
1 files changed, 29 insertions, 13 deletions
diff --git a/kafka/protocol/types.py b/kafka/protocol/types.py
index 5ccb83e..d508b26 100644
--- a/kafka/protocol/types.py
+++ b/kafka/protocol/types.py
@@ -1,13 +1,14 @@
from __future__ import absolute_import
-from struct import pack, unpack, error
+import struct
+from struct import error
from kafka.protocol.abstract import AbstractType
def _pack(f, value):
try:
- return pack(f, value)
+ return f(value)
except error as e:
raise ValueError("Error encountered when attempting to convert value: "
"{!r} to struct format: '{}', hit error: {}"
@@ -16,7 +17,7 @@ def _pack(f, value):
def _unpack(f, data):
try:
- (value,) = unpack(f, data)
+ (value,) = f(data)
return value
except error as e:
raise ValueError("Error encountered when attempting to convert value: "
@@ -25,43 +26,55 @@ def _unpack(f, data):
class Int8(AbstractType):
+ _pack = struct.Struct('>b').pack
+ _unpack = struct.Struct('>b').unpack
+
@classmethod
def encode(cls, value):
- return _pack('>b', value)
+ return _pack(cls._pack, value)
@classmethod
def decode(cls, data):
- return _unpack('>b', data.read(1))
+ return _unpack(cls._unpack, data.read(1))
class Int16(AbstractType):
+ _pack = struct.Struct('>h').pack
+ _unpack = struct.Struct('>h').unpack
+
@classmethod
def encode(cls, value):
- return _pack('>h', value)
+ return _pack(cls._pack, value)
@classmethod
def decode(cls, data):
- return _unpack('>h', data.read(2))
+ return _unpack(cls._unpack, data.read(2))
class Int32(AbstractType):
+ _pack = struct.Struct('>i').pack
+ _unpack = struct.Struct('>i').unpack
+
@classmethod
def encode(cls, value):
- return _pack('>i', value)
+ return _pack(cls._pack, value)
@classmethod
def decode(cls, data):
- return _unpack('>i', data.read(4))
+ return _unpack(cls._unpack, data.read(4))
class Int64(AbstractType):
+ _pack = struct.Struct('>q').pack
+ _unpack = struct.Struct('>q').unpack
+
@classmethod
def encode(cls, value):
- return _pack('>q', value)
+ return _pack(cls._pack, value)
@classmethod
def decode(cls, data):
- return _unpack('>q', data.read(8))
+ return _unpack(cls._unpack, data.read(8))
class String(AbstractType):
@@ -108,13 +121,16 @@ class Bytes(AbstractType):
class Boolean(AbstractType):
+ _pack = struct.Struct('>?').pack
+ _unpack = struct.Struct('>?').unpack
+
@classmethod
def encode(cls, value):
- return _pack('>?', value)
+ return _pack(cls._pack, value)
@classmethod
def decode(cls, data):
- return _unpack('>?', data.read(1))
+ return _unpack(cls._unpack, data.read(1))
class Schema(AbstractType):