diff options
| -rw-r--r-- | .travis.yml | 14 | ||||
| -rw-r--r-- | ChangeLog.rst | 3 | ||||
| -rw-r--r-- | README.rst | 39 | ||||
| -rw-r--r-- | msgpack/_msgpack.pyx | 95 | ||||
| -rw-r--r-- | msgpack/unpack.h | 5 | ||||
| -rw-r--r-- | msgpack/unpack_template.h | 4 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rw-r--r-- | test/test_obj.py | 21 | ||||
| -rw-r--r-- | test/test_unpack_raw.py | 34 |
9 files changed, 176 insertions, 42 deletions
diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..11835cb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: python +python: + - 2.6 + - 2.7 + - 3.2 + +install: + - sudo apt-get update -qq + - sudo apt-get install -q cython + - cython --cplus msgpack/_msgpack.pyx + - pip install six --use-mirrors + - python setup.py install + +script: "nosetests -w test" diff --git a/ChangeLog.rst b/ChangeLog.rst index 6a4d27b..43ee723 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,7 +5,8 @@ Changes ------- * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) - +* Add capturing feature. You can pass the writable object to + ``Unpacker.unpack()`` as a second parameter. 0.2.3 ======= @@ -6,6 +6,9 @@ MessagePack Python Binding :version: 0.2.0 :date: 2012-06-27 +.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png + :target: https://travis-ci.org/#!/msgpack/msgpack-python + HOW TO USE ----------- @@ -54,13 +57,45 @@ stream. unpacker = msgpack.Unpacker() while True: - data = buf.read(4) + data = buf.read(16) if not data: break - unpacker.seed(buf.read(16)) + unpacker.feed(data) + for unpacked in unpacker: print unpacked +packing/unpacking of custom data type +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Also possible to pack/unpack user's data types. Here is an example for +``datetime.datetime``. + +:: + + import datetime + + import msgpack + + useful_dict = { + "id": 1, + "created": datetime.datetime.now(), + } + + def decode_datetime(obj): + if b'__datetime__' in obj: + obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") + return obj + + def encode_datetime(obj): + if isinstance(obj, datetime.datetime): + return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} + return obj + + + packed_dict = msgpack.packb(useful_dict, default=encode_datetime) + this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) + INSTALL --------- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 728e4f3..38940f0 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,8 +1,6 @@ # coding: utf-8 #cython: embedsignature=True -import warnings - from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -233,7 +231,9 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, + object object_hook, object object_pairs_hook, object list_hook, + bint use_list, char* encoding, char* unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = <PyObject*>NULL @@ -259,20 +259,8 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object object_pa raise TypeError("list_hook must be a callable.") ctx.user.list_hook = <PyObject*>list_hook - if encoding is None: - ctx.user.encoding = NULL - ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - _bencoding = encoding.encode('ascii') - else: - _bencoding = encoding - ctx.user.encoding = PyBytes_AsString(_bencoding) - if isinstance(unicode_errors, unicode): - _berrors = unicode_errors.encode('ascii') - else: - _berrors = unicode_errors - ctx.user.unicode_errors = PyBytes_AsString(_berrors) + ctx.user.encoding = encoding + ctx.user.unicode_errors = unicode_errors def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", @@ -288,10 +276,22 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + cdef char* cenc = NULL + cdef char* cerr = NULL PyObject_AsReadBuffer(packed, <const_void_ptr*>&buf, &buf_len) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) ret = template_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = template_data(&ctx) @@ -370,10 +370,7 @@ cdef class Unpacker(object): cdef object file_like_read cdef Py_ssize_t read_size cdef object object_hook - cdef object _bencoding - cdef object _berrors - cdef char *encoding - cdef char *unicode_errors + cdef object encoding, unicode_errors cdef size_t max_buffer_size def __cinit__(self): @@ -387,6 +384,8 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): + cdef char *cenc=NULL, *cerr=NULL + self.file_like = file_like if file_like: self.file_like_read = file_like.read @@ -406,7 +405,20 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + self.encoding = encoding + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) def feed(self, object next_bytes): cdef char* buf @@ -467,11 +479,16 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, execute_fn execute): + cdef object _unpack(self, execute_fn execute, object write_bytes): cdef int ret cdef object obj + cdef size_t prev_head while 1: + prev_head = self.buf_head ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if write_bytes is not None: + write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) + if ret == 1: obj = template_data(&self.ctx) template_init(&self.ctx) @@ -494,27 +511,35 @@ cdef class Unpacker(object): ret += self.file_like.read(nbytes - len(ret)) return ret - def unpack(self): - """unpack one object""" - return self._unpack(template_construct) + def unpack(self, object write_bytes=None): + """ + unpack one object + + If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + """ + return self._unpack(template_construct, write_bytes) + + def skip(self, object write_bytes=None): + """ + read and ignore one object, returning None - def skip(self): - """read and ignore one object, returning None""" - return self._unpack(template_skip) + If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + """ + return self._unpack(template_skip, write_bytes) - def read_array_header(self): + def read_array_header(self, object write_bytes=None): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents.""" - return self._unpack(read_array_header) + return self._unpack(read_array_header, write_bytes) - def read_map_header(self): + def read_map_header(self, object write_bytes=None): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs.""" - return self._unpack(read_map_header) + return self._unpack(read_map_header, write_bytes) def __iter__(self): return self def __next__(self): - return self._unpack(template_construct) + return self._unpack(template_construct, None) # for debug. #def _buf(self): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 5ec7dbc..3dc88e5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -163,6 +163,8 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj { if (u->list_hook) { PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); + if (!new_c) + return -1; Py_DECREF(*c); *c = new_c; } @@ -207,6 +209,9 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec { if (u->object_hook) { PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); + if (!new_c) + return -1; + Py_DECREF(*c); *c = new_c; } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 7d07601..8a57f0d 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -347,7 +347,7 @@ _push: if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_array_end)(user, &obj); + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -361,7 +361,7 @@ _push: if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_map_end)(user, &obj); + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -34,8 +34,7 @@ Install Cython >= 0.16 or install msgpack from PyPI. os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython): cythonize(pyx) - else: - return src + return src class BuildExt(build_ext): diff --git a/test/test_obj.py b/test/test_obj.py index 881e627..1d9024b 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -49,5 +49,26 @@ def test_array_hook(): unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') + +class DecodeError(Exception): + pass + +def bad_complex_decoder(o): + raise DecodeError("Ooops!") + + +@raises(DecodeError) +def test_an_exception_in_objecthook1(): + packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) + unpackb(packed, object_hook=bad_complex_decoder) + + +@raises(DecodeError) +def test_an_exception_in_objecthook2(): + packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]}) + unpackb(packed, list_hook=bad_complex_decoder, use_list=1) + + + if __name__ == '__main__': main() diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py new file mode 100644 index 0000000..15d9c93 --- /dev/null +++ b/test/test_unpack_raw.py @@ -0,0 +1,34 @@ +"""Tests for cases where the user seeks to obtain packed msgpack objects""" + +from nose import main +from nose.tools import * +import six +from msgpack import Unpacker, packb + + +def test_write_bytes(): + unpacker = Unpacker() + unpacker.feed(b'abc') + f = six.BytesIO() + assert_equal(unpacker.unpack(f.write), ord('a')) + assert_equal(f.getvalue(), b'a') + f = six.BytesIO() + assert unpacker.skip(f.write) is None + assert_equal(f.getvalue(), b'b') + f = six.BytesIO() + assert unpacker.skip() is None + assert_equal(f.getvalue(), b'') + + +def test_write_bytes_multi_buffer(): + long_val = (5) * 100 + expected = packb(long_val) + unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3) + + f = six.BytesIO() + unpacked = unpacker.unpack(f.write) + assert_equal(unpacked, long_val) + assert_equal(f.getvalue(), expected) + +if __name__ == '__main__': + main() |