Add bin type support.

6 files changed, 81 insertions, 26 deletions

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6289192..b9ae505 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -26,6 +26,7 @@ cdef extern from "pack.h":
     int msgpack_pack_array(msgpack_packer* pk, size_t l)
     int msgpack_pack_map(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
@@ -56,6 +57,9 @@ cdef class Packer(object):
     :param bool autoreset:
         Reset buffer after each pack and return it's content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -64,6 +68,7 @@ cdef class Packer(object):
     cdef char *encoding
     cdef char *unicode_errors
     cdef bool use_float
+    cdef bool use_bin_type
     cdef bint autoreset
 
     def __cinit__(self):
@@ -74,11 +79,13 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                 use_single_float=False, bint autoreset=1, bint use_bin_type=0):
         """
         """
         self.use_float = use_single_float
         self.autoreset = autoreset
+        self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
@@ -110,6 +117,7 @@ cdef class Packer(object):
         cdef char* rawval
         cdef int ret
         cdef dict d
+        cdef size_t L
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -140,9 +148,10 @@ cdef class Packer(object):
                ret = msgpack_pack_double(&self.pk, dval)
         elif PyBytes_Check(o):
             rawval = o
-            ret = msgpack_pack_raw(&self.pk, len(o))
+            L = len(o)
+            ret = msgpack_pack_bin(&self.pk, L)
             if ret == 0:
-                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
         elif PyUnicode_Check(o):
             if not self.encoding:
                 raise TypeError("Can't encode unicode string: no encoding is specified")
@@ -247,21 +256,26 @@ cdef class Packer(object):
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
 
 
-def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
+def pack(object o, object stream,
+         default=None, str encoding='utf-8', str unicode_errors='strict',
+         bint use_single_float=False, bint use_bin_type=False):
     """
     pack an object `o` and write it to stream)
 
     See :class:`Packer` for options.
     """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
+    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
+                    use_single_float=use_single_float, use_bin_type=use_bin_type)
     stream.write(packer.pack(o))
 
-def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
+def packb(object o,
+          default=None, str encoding='utf-8', str unicode_errors='strict',
+          bint use_single_float=False, bint use_bin_type=False):
     """
     pack o and return packed bytes
 
     See :class:`Packer` for options.
     """
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
-                    use_single_float=use_single_float)
+                    use_single_float=use_single_float, use_bin_type=use_bin_type)
     return packer.pack(o)
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 1539991..001a0c1 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -34,11 +34,11 @@ typedef struct msgpack_packer {
     char *buf;
     size_t length;
     size_t buf_size;
+    bool use_bin_type;
 } msgpack_packer;
 
 typedef struct Packer Packer;
 
-static inline int msgpack_pack_short(msgpack_packer* pk, short d);
 static inline int msgpack_pack_int(msgpack_packer* pk, int d);
 static inline int msgpack_pack_long(msgpack_packer* pk, long d);
 static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
@@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
 static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
 
 static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 9e00d7e..ff1cbaa 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -667,7 +667,10 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
     if(l < 32) {
         unsigned char d = 0xa0 | (uint8_t)l;
         msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
-    } else if(l < 65536) {
+    } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.
+        unsigned char buf[2] = {0xd9, (uint8_t)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
         unsigned char buf[3];
         buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
         msgpack_pack_append_buffer(x, buf, 3);
@@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
     }
 }
 
+/*
+ * bin
+ */
+static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
+{
+    if (!x->use_bin_type) {
+        return msgpack_pack_raw(x, l)
+    }
+    if (l < 256) {
+        unsigned char buf[2] = {0xc4, (unsigned char)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3] = {0xc5};
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5] = {0xc6};
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
 static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
     msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index baeed1f..03c735e 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py = PyBytes_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
 #include "unpack_template.h"
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index c81b990..0b14f52 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -34,6 +34,7 @@ extern "C" {
 #endif
 
 
+// CS is first byte & 0x1f
 typedef enum {
     CS_HEADER            = 0x00,  // nil
 
@@ -41,9 +42,9 @@ typedef enum {
     //CS_                = 0x02,  // false
     //CS_                = 0x03,  // true
 
-    //CS_                = 0x04,
-    //CS_                = 0x05,
-    //CS_                = 0x06,
+    CS_BIN_8             = 0x04,
+    CS_BIN_16            = 0x05,
+    CS_BIN_32            = 0x06,
     //CS_                = 0x07,
 
     //CS_                = 0x08,
@@ -59,12 +60,7 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    //CS_                = 0x14,
-    //CS_                = 0x15,
-    //CS_BIG_INT_16        = 0x16,
-    //CS_BIG_INT_32        = 0x17,
-    //CS_BIG_FLOAT_16      = 0x18,
-    //CS_BIG_FLOAT_32      = 0x19,
+    CS_RAW_8             = 0x19,
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
     CS_ARRAY_16          = 0x1c,
@@ -72,9 +68,8 @@ typedef enum {
     CS_MAP_16            = 0x1e,
     CS_MAP_32            = 0x1f,
 
-    //ACS_BIG_INT_VALUE,
-    //ACS_BIG_FLOAT_VALUE,
     ACS_RAW_VALUE,
+    ACS_BIN_VALUE,
 } msgpack_unpack_state;
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 29ac935..25229ac 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
     */ \
     goto _header_again
 
-#define NEXT_CS(p) \
-    ((unsigned int)*p & 0x1f)
+#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
 
 #ifdef USE_CASE_RANGE
 #define SWITCH_RANGE_BEGIN     switch(*p) {
@@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                     push_simple_value(_false);
                 case 0xc3:  // true
                     push_simple_value(_true);
-                //case 0xc4:
-                //case 0xc5:
-                //case 0xc6:
                 //case 0xc7:
                 //case 0xc8:
                 //case 0xc9:
@@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 case 0xd2:  // signed int 32
                 case 0xd3:  // signed int 64
                     again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                case 0xc4:  // bin 8
+                case 0xc5:  // bin 16
+                case 0xc6:  // bin 32
                 //case 0xd4:
                 //case 0xd5:
                 //case 0xd6:  // big integer 16
                 //case 0xd7:  // big integer 32
                 //case 0xd8:  // big float 16
-                //case 0xd9:  // big float 32
+                case 0xd9:  // raw 8
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
@@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             //    // FIXME
             //    push_variable_value(_big_float, data, n, trail);
 
+            case CS_BIN_8:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
+            case CS_BIN_16:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
+            case CS_BIN_32:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
+            case ACS_BIN_VALUE:
+            _bin_zero:
+                push_variable_value(_bin, data, n, trail);
+
+            case CS_RAW_8:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
             case CS_RAW_16:
                 again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
             case CS_RAW_32: