diff options
Diffstat (limited to 'msgpack')
| -rw-r--r-- | msgpack/_packer.pyx | 22 | ||||
| -rw-r--r-- | msgpack/_unpacker.pyx | 42 | ||||
| -rw-r--r-- | msgpack/fallback.py | 4 | ||||
| -rw-r--r-- | msgpack/pack.h | 34 | ||||
| -rw-r--r-- | msgpack/unpack.h | 6 | ||||
| -rw-r--r-- | msgpack/unpack_template.h | 14 |
6 files changed, 57 insertions, 65 deletions
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index fcd20a7..0a6513c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -135,13 +135,21 @@ cdef class Packer(object): ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): # PyInt_Check(long) is True for Python 3. - # Sow we should test long before int. - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) + # So we should test long before int. + try: + if o > 0: + ullval = o + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = o + ret = msgpack_pack_long_long(&self.pk, llval) + except OverflowError as oe: + if not default_used and self._default is not None: + o = self._default(o) + default_used = True + continue + else: + raise elif PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b98957e..1aefc64 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -37,16 +37,17 @@ cdef extern from "unpack.h": ctypedef struct unpack_context: msgpack_user user PyObject* obj - size_t count + Py_ssize_t count ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, - size_t len, size_t* off) except? -1 + Py_ssize_t len, Py_ssize_t* off) except? -1 execute_fn unpack_construct execute_fn unpack_skip execute_fn read_array_header execute_fn read_map_header void unpack_init(unpack_context* ctx) object unpack_data(unpack_context* ctx) + void unpack_clear(unpack_context* ctx) cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, @@ -112,7 +113,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, See :class:`Unpacker` for options. """ cdef unpack_context ctx - cdef size_t off = 0 + cdef Py_ssize_t off = 0 cdef int ret cdef char* buf @@ -141,14 +142,18 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj - else: - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + unpack_clear(&ctx) + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, ext_hook=ExtType - ): + object_pairs_hook=None, ext_hook=ExtType, + Py_ssize_t max_str_len=2147483647, # 2**32-1 + Py_ssize_t max_bin_len=2147483647, + Py_ssize_t max_array_len=2147483647, + Py_ssize_t max_map_len=2147483647, + Py_ssize_t max_ext_len=2147483647): """ Unpack an object from `stream`. @@ -158,7 +163,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None, """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook + encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len, ) @@ -233,14 +243,14 @@ cdef class Unpacker(object): """ cdef unpack_context ctx cdef char* buf - cdef size_t buf_size, buf_head, buf_tail + cdef Py_ssize_t buf_size, buf_head, buf_tail cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size # To maintain refcnt. cdef object object_hook, object_pairs_hook, list_hook, ext_hook cdef object encoding, unicode_errors - cdef size_t max_buffer_size + cdef Py_ssize_t max_buffer_size def __cinit__(self): self.buf = NULL @@ -325,10 +335,10 @@ cdef class Unpacker(object): cdef: char* buf = self.buf char* new_buf - size_t head = self.buf_head - size_t tail = self.buf_tail - size_t buf_size = self.buf_size - size_t new_size + Py_ssize_t head = self.buf_head + Py_ssize_t tail = self.buf_tail + Py_ssize_t buf_size = self.buf_size + Py_ssize_t new_size if tail + _buf_len > buf_size: if ((tail - head) + _buf_len) <= buf_size: @@ -374,7 +384,7 @@ cdef class Unpacker(object): cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): cdef int ret cdef object obj - cdef size_t prev_head + cdef Py_ssize_t prev_head if self.buf_head >= self.buf_tail and self.file_like is not None: self.read_from_file() @@ -408,7 +418,7 @@ cdef class Unpacker(object): def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" - cdef size_t nread + cdef Py_ssize_t nread nread = min(self.buf_tail - self.buf_head, nbytes) ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) self.buf_head += nread diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5b5085e..f682611 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -655,6 +655,10 @@ class Packer(object): return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) + if not default_used and self._default is not None: + obj = self._default(obj) + default_used = True + continue raise PackValueError("Integer value out of range") if self._use_bin_type and isinstance(obj, bytes): n = len(obj) diff --git a/msgpack/pack.h b/msgpack/pack.h index 971065c..a75bdb0 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -39,40 +39,6 @@ typedef struct msgpack_packer { typedef struct Packer Packer; -static inline int msgpack_pack_int(msgpack_packer* pk, int d); -static inline int msgpack_pack_long(msgpack_packer* pk, long d); -static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); - -static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); - -static inline int msgpack_pack_float(msgpack_packer* pk, float d); -static inline int msgpack_pack_double(msgpack_packer* pk, double d); - -static inline int msgpack_pack_nil(msgpack_packer* pk); -static inline int msgpack_pack_true(msgpack_packer* pk); -static inline int msgpack_pack_false(msgpack_packer* pk); - -static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); - -static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); - -static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l); -static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -static inline int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l); - static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { char* buf = pk->buf; diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 5deb7cd..92f4f11 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -33,7 +33,7 @@ typedef struct unpack_user { typedef PyObject* msgpack_unpack_object; struct unpack_context; typedef struct unpack_context unpack_context; -typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off); +typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off); static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) { @@ -69,7 +69,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromSize_t((size_t)d); + p = PyInt_FromLong((long)d); } if (!p) return -1; @@ -100,7 +100,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac { PyObject *p; if (d > LONG_MAX || d < LONG_MIN) { - p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d); + p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { p = PyInt_FromLong((long)d); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index d34eced..e1e08fe 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -24,8 +24,8 @@ typedef struct unpack_stack { PyObject* obj; - size_t size; - size_t count; + Py_ssize_t size; + Py_ssize_t count; unsigned int ct; PyObject* map_key; } unpack_stack; @@ -70,9 +70,13 @@ static inline PyObject* unpack_data(unpack_context* ctx) return (ctx)->stack[0].obj; } +static inline void unpack_clear(unpack_context *ctx) +{ + Py_CLEAR(ctx->stack[0].obj); +} template <bool construct> -static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -89,7 +93,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l */ unpack_user* user = &ctx->user; - PyObject* obj; + PyObject* obj = NULL; unpack_stack* c = NULL; int ret; @@ -409,7 +413,7 @@ _end: #undef start_container template <unsigned int fixed_offset, unsigned int var_offset> -static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); uint32_t size; |