diff options
author | Antonio Cuni <anto.cuni@gmail.com> | 2013-10-19 17:27:16 +0200 |
---|---|---|
committer | Antonio Cuni <anto.cuni@gmail.com> | 2013-10-19 17:27:16 +0200 |
commit | 56dd1650a42a454027ba335b494100a9f211758e (patch) | |
tree | bb40490af6a940e75d91d0e2e19a9975c78c8a15 | |
parent | 985d4c1496d8c9186079ebc4e42aee319e67c385 (diff) | |
download | msgpack-python-56dd1650a42a454027ba335b494100a9f211758e.tar.gz |
implement unpacking for all the fixtext formats
-rw-r--r-- | msgpack/_unpacker.pyx | 21 | ||||
-rw-r--r-- | msgpack/unpack.h | 18 | ||||
-rw-r--r-- | msgpack/unpack_define.h | 14 | ||||
-rw-r--r-- | msgpack/unpack_template.h | 22 | ||||
-rw-r--r-- | setup.py | 1 | ||||
-rw-r--r-- | test/test_extension.py | 1 |
6 files changed, 62 insertions, 15 deletions
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e05b9ed..6500ef7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -25,6 +25,7 @@ cdef extern from "unpack.h": PyObject* object_hook bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook + PyObject* ext_type_hook char *encoding char *unicode_errors @@ -46,6 +47,7 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, + object ext_type_hook, bint use_list, char* encoding, char* unicode_errors): unpack_init(ctx) ctx.user.use_list = use_list @@ -72,9 +74,17 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("list_hook must be a callable.") ctx.user.list_hook = <PyObject*>list_hook + if ext_type_hook is not None: + if not PyCallable_Check(ext_type_hook): + raise TypeError("ext_type_hook must be a callable.") + ctx.user.ext_type_hook = <PyObject*>ext_type_hook + ctx.user.encoding = encoding ctx.user.unicode_errors = unicode_errors +def default_read_extended_type(typecode, data): + raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) + def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, @@ -107,7 +117,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, unicode_errors = unicode_errors.encode('ascii') cerr = PyBytes_AsString(unicode_errors) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type, + use_list, cenc, cerr) ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = unpack_data(&ctx) @@ -249,7 +260,10 @@ cdef class Unpacker(object): self.unicode_errors = unicode_errors cerr = PyBytes_AsString(self.unicode_errors) - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + ext_type_hook = self.read_extended_type + Py_INCREF(ext_type_hook) + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, + ext_type_hook, use_list, cenc, cerr) def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" @@ -404,6 +418,9 @@ cdef class Unpacker(object): """ return self._unpack(read_map_header, write_bytes) + def read_extended_type(self, typecode, data): + return default_read_extended_type(typecode, data) + def __iter__(self): return self diff --git a/msgpack/unpack.h b/msgpack/unpack.h index baeed1f..97ebd3f 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -24,6 +24,7 @@ typedef struct unpack_user { PyObject *object_hook; bool has_pairs_hook; PyObject *list_hook; + PyObject *ext_type_hook; const char *encoding; const char *unicode_errors; } unpack_user; @@ -226,4 +227,21 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* return 0; } +static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, + unsigned int lenght, msgpack_unpack_object* o) +{ + PyObject *py; + int8_t typecode = (int8_t)*pos++; + if (!u->ext_type_hook) { + PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL"); + return -1; + } + // lenght also includes the typecode, so the actual data is lenght-1 + py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1); + if (!py) + return -1; + *o = py; + return 0; +} + #include "unpack_template.h" diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index c81b990..986fa91 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -59,12 +59,12 @@ typedef enum { CS_INT_32 = 0x12, CS_INT_64 = 0x13, - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, + CS_FIXEXT1 = 0x14, + CS_FIXEXT2 = 0x15, + CS_FIXEXT4 = 0x16, + CS_FIXEXT8 = 0x17, + CS_FIXEXT16 = 0x18, + CS_RAW_16 = 0x1a, CS_RAW_32 = 0x1b, CS_ARRAY_16 = 0x1c, @@ -75,6 +75,8 @@ typedef enum { //ACS_BIG_INT_VALUE, //ACS_BIG_FLOAT_VALUE, ACS_RAW_VALUE, + ACS_EXT_VALUE, + } msgpack_unpack_state; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 29ac935..b051075 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -202,12 +202,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 + case 0xd4: // fixext 1 + case 0xd5: // fixext 2 + case 0xd6: // fixext 4 + case 0xd7: // fixext 8 + again_fixed_trail_if_zero(ACS_EXT_VALUE, + (1 << (((unsigned int)*p) & 0x03))+1, + _ext_zero); + case 0xd8: // fixext 16 + again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero); + //case 0xd9: case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -298,6 +302,10 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l _raw_zero: push_variable_value(_raw, data, n, trail); + case ACS_EXT_VALUE: + _ext_zero: + push_variable_value(_ext, data, n, trail); + case CS_ARRAY_16: start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: @@ -309,7 +317,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case CS_MAP_32: /* FIXME security guard */ start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - + default: goto _failed; } @@ -92,6 +92,7 @@ if not hasattr(sys, 'pypy_version_info'): libraries=libraries, include_dirs=['.'], define_macros=macros, + extra_compile_args=['-O0'], )) del libraries, macros diff --git a/test/test_extension.py b/test/test_extension.py index 96944a3..94117e1 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,3 +1,4 @@ +import py import array import struct import msgpack |