From 28058fb53d21097947d190bcc47e3609a6794e7a Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sat, 22 Sep 2012 22:57:00 +1000 Subject: A first implementation of Unpacker.skip() --- msgpack/_msgpack.pyx | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'msgpack/_msgpack.pyx') diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 80d34ab..c1e3e75 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -200,7 +200,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) except -1 + size_t len, size_t* off, bool construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -246,7 +246,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off) + ret = template_execute(&ctx, buf, buf_len, &off, True) if ret == 1: return template_data(&ctx) else: @@ -440,15 +440,12 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef unpack(self): - """unpack one object""" + cpdef _unpack(self, bool construct): cdef int ret while 1: - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - o = template_data(&self.ctx) - template_init(&self.ctx) - return o + return elif ret == 0: if self.file_like is not None: self.fill_buffer() @@ -457,6 +454,18 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) + cpdef unpack(self): + """unpack one object""" + self._unpack(True) + o = template_data(&self.ctx) + template_init(&self.ctx) + + + cpdef skip(self): + """read and ignore one object, returning None""" + self._unpack(False) + template_init(&self.ctx) + def __iter__(self): return self -- cgit v1.2.1 From eaf9891b4255f3b1ca5cf2ea5b631091523b913d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:00:18 +0900 Subject: clean some cython code. --- msgpack/_msgpack.pyx | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'msgpack/_msgpack.pyx') diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9061d42..c8ee7bb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -209,7 +209,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bool construct) except -1 + size_t len, size_t* off, bint construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -255,7 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off, True) + ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) if off < buf_len: @@ -451,12 +451,18 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef _unpack(self, bool construct): + cdef _unpack(self, bint construct): cdef int ret + cdef object obj while 1: ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - return + if construct: + obj = template_data(&self.ctx) + else: + obj = None + template_init(&self.ctx) + return obj elif ret == 0: if self.file_like is not None: self.read_from_file() @@ -465,23 +471,19 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) - cpdef unpack(self): + def unpack(self): """unpack one object""" - self._unpack(True) - o = template_data(&self.ctx) - template_init(&self.ctx) - + return self._unpack(1) - cpdef skip(self): + def skip(self): """read and ignore one object, returning None""" - self._unpack(False) - template_init(&self.ctx) + return self._unpack(0) def __iter__(self): return self def __next__(self): - return self.unpack() + return self._unpack(1) # for debug. #def _buf(self): -- cgit v1.2.1 From 48d693c1b9613fd976a3bf668f692ec22ad4a520 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:09:51 +0900 Subject: Add test for `.skip()` --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'msgpack/_msgpack.pyx') diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c8ee7bb..8d37aaa 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -451,7 +451,7 @@ cdef class Unpacker(object): else: self.file_like = None - cdef _unpack(self, bint construct): + cdef object _unpack(self, bint construct): cdef int ret cdef object obj while 1: -- cgit v1.2.1 From b06ed8eb75563111ef88a119f9f7a45e67f61736 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 18:11:49 +1000 Subject: Factor context initialisation from unpackb and Unpacker --- msgpack/_msgpack.pyx | 82 ++++++++++++++++++---------------------------------- 1 file changed, 28 insertions(+), 54 deletions(-) (limited to 'msgpack/_msgpack.pyx') diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..823ed62 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -213,6 +213,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) +cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): + template_init(ctx) + ctx.user.use_list = use_list + ctx.user.object_hook = ctx.user.list_hook = NULL + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + ctx.user.object_hook = object_hook + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("list_hook must be a callable.") + ctx.user.list_hook = list_hook + if encoding is None: + ctx.user.encoding = NULL + ctx.user.unicode_errors = NULL + else: + if isinstance(encoding, unicode): + _bencoding = encoding.encode('ascii') + else: + _bencoding = encoding + ctx.user.encoding = PyBytes_AsString(_bencoding) + if isinstance(unicode_errors, unicode): + _berrors = unicode_errors.encode('ascii') + else: + _berrors = unicode_errors + ctx.user.unicode_errors = PyBytes_AsString(_berrors) def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", @@ -229,34 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - if encoding is None: - enc = NULL - err = NULL - else: - if isinstance(encoding, unicode): - bencoding = encoding.encode('ascii') - else: - bencoding = encoding - if isinstance(unicode_errors, unicode): - berrors = unicode_errors.encode('ascii') - else: - berrors = unicode_errors - enc = PyBytes_AsString(bencoding) - err = PyBytes_AsString(berrors) - - template_init(&ctx) - ctx.user.use_list = use_list - ctx.user.object_hook = ctx.user.list_hook = NULL - ctx.user.encoding = enc - ctx.user.unicode_errors = err - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - ctx.user.list_hook = list_hook + init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -348,7 +347,6 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, - object object_pairs_hook=None, ): self.use_list = use_list self.file_like = file_like @@ -370,31 +368,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - template_init(&self.ctx) - self.ctx.user.use_list = use_list - self.ctx.user.object_hook = self.ctx.user.list_hook = NULL - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - self.ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - self.ctx.user.list_hook = list_hook - if encoding is None: - self.ctx.user.encoding = NULL - self.ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - self._bencoding = encoding.encode('ascii') - else: - self._bencoding = encoding - self.ctx.user.encoding = PyBytes_AsString(self._bencoding) - if isinstance(unicode_errors, unicode): - self._berrors = unicode_errors.encode('ascii') - else: - self._berrors = unicode_errors - self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors) + init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf -- cgit v1.2.1 From 77942514db0c5a80e9f3f9bcb1e1939ecc8705e6 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 19:37:28 +1000 Subject: Implement object_pairs_hook --- msgpack/_msgpack.pyx | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'msgpack/_msgpack.pyx') diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 823ed62..b6d8e8b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -197,6 +197,7 @@ cdef extern from "unpack.h": ctypedef struct msgpack_user: int use_list PyObject* object_hook + bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook char *encoding char *unicode_errors @@ -213,18 +214,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL + + if object_hook is not None and object_pairs_hook is not None: + raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") + if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") ctx.user.object_hook = object_hook + + if object_pairs_hook is None: + ctx.user.has_pairs_hook = False + else: + if not PyCallable_Check(object_pairs_hook): + raise TypeError("object_pairs_hook must be a callable.") + ctx.user.object_hook = object_pairs_hook + ctx.user.has_pairs_hook = True + if list_hook is not None: if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook + if encoding is None: ctx.user.encoding = NULL ctx.user.unicode_errors = NULL @@ -240,7 +255,7 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook _berrors = unicode_errors ctx.user.unicode_errors = PyBytes_AsString(_berrors) -def unpackb(object packed, object object_hook=None, object list_hook=None, +def unpackb(object packed, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -255,7 +270,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -266,7 +281,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, return None -def unpack(object stream, object object_hook=None, object list_hook=None, +def unpack(object stream, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. @@ -274,7 +289,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, Raises `ValueError` when `stream` has extra bytes. """ return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, list_hook=list_hook, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, ) @@ -294,7 +309,10 @@ cdef class Unpacker(object): Otherwise, it is deserialized to Python tuple. (default: False) `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it when deserializing key-value. + and Unpacker calls it with a dict argument after deserializing a map. + + `object_pairs_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it with a list of key-value pairs after deserializing a map. `encoding` is encoding used for decoding msgpack bytes. If it is None (default), msgpack bytes is deserialized to Python bytes. @@ -345,7 +363,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, - object object_hook=None, object list_hook=None, + object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): self.use_list = use_list @@ -368,7 +386,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf -- cgit v1.2.1