From 3a9f74e79c3d1912d8c0c1ad4d1478c611caba0a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: Make msgpack package instead of module. --- python/msgpack/unpack.h | 122 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 python/msgpack/unpack.h (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h new file mode 100644 index 0000000..e51557f --- /dev/null +++ b/python/msgpack/unpack.h @@ -0,0 +1,122 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack_define.h" + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + static inline ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ return NULL; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_None); *o = Py_None; return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_True); *o = Py_True; return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ Py_INCREF(Py_False); *o = Py_False; return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" -- cgit v1.2.1 From 87f5df1503e70efd2bb9e9565259f1ed37a45933 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: Use std::stack. --- python/msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } -- cgit v1.2.1 From 46d7c656214073cde4c458c7cf1eb03e2d33dbd9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: Fix compile error. --- python/msgpack/unpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index daeb54c..b753493 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { -- cgit v1.2.1 From dd53b141ef6d2495d0cc14c8949180a74ae3caf6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: Remove unneccessary value. --- python/msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index b753493..12702d8 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; -- cgit v1.2.1 From 3fd28d07928c51cb563f35d9dddb0c7867ac9875 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: Remove duplicated values. --- python/msgpack/unpack.h | 53 ++++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 38 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} - -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; +{ *o = PyList_New(n); return 0; } - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" -- cgit v1.2.1 From f61b282886bb9d7f7932fbcbe2affee06d150a5f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: Reduce memory footprint. --- python/msgpack/unpack.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; -- cgit v1.2.1 From 5d4189306ae0b6ea23e9410d34434b41aee384b3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: Check return value of c-api. --- python/msgpack/unpack.h | 118 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 26 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + return template_callback_uint16(u, d, o); +} -static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + return template_callback_int32(u, d, o); +} -static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" -- cgit v1.2.1 From 479263989b02693f0e449ee591ca58ed84fd904b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: Stop unnecessary caching. --- python/msgpack/unpack.h | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" -- cgit v1.2.1 From 3e396ef146940ada28fc8b154189ae9bb206babc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: Don't use C++. --- python/msgpack/unpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/msgpack/unpack.h') diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index dd23130..40058d0 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ -- cgit v1.2.1