From 59603b902adf2abe0d274f41520569fde387a841 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 15 Jun 2010 17:51:24 +0900 Subject: Python: add "load(s)/dump(s)" alias for compatibility to simplejson/marshal/pickle. --- python/msgpack/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py index 797b29c..26bd2dd 100644 --- a/python/msgpack/__init__.py +++ b/python/msgpack/__init__.py @@ -1,3 +1,10 @@ # coding: utf-8 from _msgpack import * +# alias for compatibility to simplejson/marshal/pickle. +load = unpack +loads = unpackb + +dump = pack +dumps = packb + -- cgit v1.2.1 From f222f5ed9b49e9b9f4d31693969c4260ad97cef0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 15 Jun 2010 18:06:58 +0900 Subject: Python: 0.1.4 --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 8d8a6f4..64e71ed 100755 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.1.3' +version = '0.1.4' # take care of extension modules. if have_cython: -- cgit v1.2.1 From fd8069342052142449d4703f8a12014cf283fedf Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 22 Jun 2010 11:15:18 +0900 Subject: erlang: tests improved and code refined. --- erlang/OMakefile | 2 +- erlang/msgpack.erl | 114 +++++++++++++++++++++++-------------------- erlang/testcase_generator.rb | 10 ++-- 3 files changed, 67 insertions(+), 59 deletions(-) diff --git a/erlang/OMakefile b/erlang/OMakefile index ee72f78..34c590f 100644 --- a/erlang/OMakefile +++ b/erlang/OMakefile @@ -36,7 +36,7 @@ msgpack.beam: msgpack.erl erlc $< test: msgpack.beam - erl -s msgpack test -s init stop + erl -noshell -s msgpack test -s init stop clean: -rm *.beam diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index ca9769e..5a468a1 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -31,9 +31,51 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). +-type reason() :: enomem. +% ===== external APIs ===== % +pack(O) when is_integer(O) andalso O < 0 -> + pack_int_(O); +pack(O) when is_integer(O) -> + pack_uint_(O); +pack(O) when is_float(O)-> + pack_double(O); +pack(nil) -> + pack_nil(); +pack(Bool) when is_atom(Bool) -> + pack_bool(Bool); +pack(Bin) when is_binary(Bin)-> + pack_raw(Bin); +pack(List) when is_list(List)-> + pack_array(List); +pack({dict, Map})-> + pack_map({dict, Map}); +pack(_) -> + undefined. --type reason() :: enomem. +% unpacking. +% if failed in decoding and not end, get more data +% and feed more Bin into this function. +% TODO: error case for imcomplete format when short for any type formats. +-spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +unpack(Bin) when not is_binary(Bin)-> + {error, badard}; +unpack(Bin) when bit_size(Bin) >= 8 -> + << Flag:8/unsigned-integer, Payload/binary >> = Bin, + unpack_(Flag, Payload); +unpack(_)-> % when bit_size(Bin) < 8 -> + {more, 8}. + +unpack_all(Data)-> + case unpack(Data) of + { Term, Binary } when bit_size(Binary) =:= 0 -> + [Term]; + { Term, Binary } when is_binary(Binary) -> + [Term|unpack_all(Binary)] + end. + + +% ===== internal APIs ===== % % positive fixnum pack_uint_(N) when is_integer( N ) , N < 128 -> @@ -113,7 +155,7 @@ pack_array(L) when is_list(L)-> end. pack_array_([])-> <<>>; pack_array_([Head|Tail])-> - << (pack_object(Head))/binary, (pack_array_(Tail))/binary >>. + << (pack(Head))/binary, (pack_array_(Tail))/binary >>. unpack_array_(<<>>, 0)-> []; unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain]; @@ -134,7 +176,7 @@ pack_map({dict,M})-> pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> - << (pack_object(Key)),(pack_object(Value)),(pack_map_(Tail)) >>. + << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. unpack_map_(<<>>, 0)-> []; unpack_map_(Bin, 0) when is_binary(Bin)-> [Bin]; @@ -143,38 +185,7 @@ unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) -> { Value, Rest2 } = unpack(Rest), [{Key,Value}|unpack_map_(Rest2,Len-1)]. -pack_object(O) when is_integer(O) andalso O < 0 -> - pack_int_(O); -pack_object(O) when is_integer(O) -> - pack_uint_(O); -pack_object(O) when is_float(O)-> - pack_double(O); -pack_object(nil) -> - pack_nil(); -pack_object(Bool) when is_atom(Bool) -> - pack_bool(Bool); -pack_object(Bin) when is_binary(Bin)-> - pack_raw(Bin); -pack_object(List) when is_list(List)-> - pack_array(List); -pack_object({dict, Map})-> - pack_map({dict, Map}); -pack_object(_) -> - undefined. - -pack(Obj)-> - pack_object(Obj). - - -% unpacking. -% if failed in decoding and not end, get more data -% and feed more Bin into this function. -% TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. -unpack(Bin) when not is_binary(Bin)-> - {error, badard}; -unpack(Bin) when bit_size(Bin) >= 8 -> - << Flag:8/unsigned-integer, Payload/binary >> = Bin, +unpack_(Flag, Payload)-> case Flag of 16#C0 -> {nil, Payload}; @@ -305,20 +316,17 @@ unpack(Bin) when bit_size(Bin) >= 8 -> _Other -> erlang:display(_Other), {error, no_code_matches} - end; -unpack(_)-> % when bit_size(Bin) < 8 -> - {more, 8}. - -unpack_all(Data)-> - case unpack(Data) of - { Term, Binary } when bit_size(Binary) =:= 0 -> - [Term]; - { Term, Binary } when is_binary(Binary) -> - [Term|unpack_all(Binary)] end. -ifdef(EUNIT). +compare_all([], [])-> ok; +compare_all([], R)-> {toomuchrhs, R}; +compare_all(L, [])-> {toomuchlhs, L}; +compare_all([LH|LTL], [RH|RTL]) -> + LH=RH, + compare_all(LTL, RTL). + test_data()-> [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, @@ -339,7 +347,6 @@ port_test()-> {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])), Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]), true = port_command(Port, msgpack:pack(Tests) ), - %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner receive {Port, {data, Data}}-> {Tests, <<>>}=msgpack:unpack(Data) after 1024-> ?assert(false) end, @@ -348,24 +355,23 @@ port_test()-> unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, -% "hogehoge", "243546rf7g68h798j", - 123.123 %-234.4355, 1.0e-34, 1.0e64, -% [23, 234, 0.23] -% [0,42,"sum", [1,2]], [1,42, nil, [3]] + <<"hogehoge">>, <<"243546rf7g68h798j">>, +% 123.123, %FIXME +% -234.4355, 1.0e-34, 1.0e64, % FIXME + [23, 234, 0.23], + [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], + 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), - %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner receive {Port, {data, Data}}-> - Tests=msgpack:unpack_all(Data) -% io:format("~p~n", [Tests]) + compare_all(Tests, msgpack:unpack_all(Data)) after 1024-> ?assert(false) end, port_close(Port). test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), -% io:format("testing: ~p => ~p~n", [S, Pack]), {S, <<>>} = msgpack:unpack( Pack ), 1+test_(Rest). diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index a173790..a7c76c5 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -39,10 +39,12 @@ end objs = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, -# "hogehoge", "243546rf7g68h798j", - 123.123, #-234.4355, 1.0e-34, 1.0e64, -# [23, 234, 0.23] -# [0,42,"sum", [1,2]], [1,42, nil, [3]] + "hogehoge", "243546rf7g68h798j", +# 123.123 , #FIXME +# -234.4355, 1.0e-34, 1.0e64, + [23, 234, 0.23], + [0,42,"sum", [1,2]], [1,42, nil, [3]], + 42 ] begin objs.each do |obj| -- cgit v1.2.1 From b1e66256ce35ea4a953d255a2a98e9c8ddbe6401 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 22 Jun 2010 11:28:36 +0900 Subject: erlang: external APIs' type/specs. --- erlang/msgpack.erl | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 5a468a1..f2380fc 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -18,37 +18,38 @@ -module(msgpack). -author('kuenishi+msgpack@gmail.com'). -%% tuples, atoms are not supported. lists, integers, double, and so on. +%% tuples, atoms are not supported. lists, integers, double, and so on. %% see http://msgpack.sourceforge.jp/spec for %% supported formats. APIs are almost compatible %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). --export([pack/1, unpack/1, unpack_all/1, test/0]). - --include_lib("eunit/include/eunit.hrl"). +-export([pack/1, unpack/1, unpack_all/1]). % compile: % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem. +-type reason() :: enomem | badarg. +-type map() :: any(). % there's no 'dict' type... +-type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. % ===== external APIs ===== % +-spec pack(Term::msgpack_term()) -> binary(). pack(O) when is_integer(O) andalso O < 0 -> pack_int_(O); pack(O) when is_integer(O) -> pack_uint_(O); -pack(O) when is_float(O)-> +pack(O) when is_float(O) -> pack_double(O); pack(nil) -> pack_nil(); pack(Bool) when is_atom(Bool) -> pack_bool(Bool); -pack(Bin) when is_binary(Bin)-> +pack(Bin) when is_binary(Bin) -> pack_raw(Bin); -pack(List) when is_list(List)-> +pack(List) when is_list(List) -> pack_array(List); -pack({dict, Map})-> +pack({dict, Map}) -> pack_map({dict, Map}); pack(_) -> undefined. @@ -57,15 +58,16 @@ pack(_) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +-spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> - {error, badard}; + {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> << Flag:8/unsigned-integer, Payload/binary >> = Bin, unpack_(Flag, Payload); unpack(_)-> % when bit_size(Bin) < 8 -> {more, 8}. +-spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> case unpack(Data) of { Term, Binary } when bit_size(Binary) =:= 0 -> @@ -318,6 +320,8 @@ unpack_(Flag, Payload)-> {error, no_code_matches} end. +% ===== test codes ===== % +-include_lib("eunit/include/eunit.hrl"). -ifdef(EUNIT). compare_all([], [])-> ok; -- cgit v1.2.1 From 230ee3a03b5148d11c614fc18c7c7603c9a8af14 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 23 Jun 2010 01:26:10 +0900 Subject: erlang: too short binary to decode causes error {more, Int}. --- erlang/msgpack.erl | 211 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 125 insertions(+), 86 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index f2380fc..d2f7068 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -29,7 +29,7 @@ % erl> c(msgpack). % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). --type reason() :: enomem | badarg. +-type reason() :: enomem | badarg | no_code_matches. -type map() :: any(). % there's no 'dict' type... -type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. @@ -58,14 +58,15 @@ pack(_) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. +-spec unpack( binary() )-> + {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> {error, badarg}; unpack(Bin) when bit_size(Bin) >= 8 -> << Flag:8/unsigned-integer, Payload/binary >> = Bin, unpack_(Flag, Payload); -unpack(_)-> % when bit_size(Bin) < 8 -> - {more, 8}. +unpack(<<>>)-> % when bit_size(Bin) < 8 -> + {more, 1}. -spec unpack_all( binary() ) -> [msgpack_term()]. unpack_all(Data)-> @@ -129,12 +130,9 @@ pack_bool(false)-> << 16#C2:8 >>. pack_double(F) when is_float(F)-> << 16#CB:8, F:64/big-float-unit:1 >>. -power(N,0) when is_integer(N) -> 1; -power(N,D) when is_integer(N) and is_integer(D) -> N * power(N, D-1). - % raw bytes pack_raw(Bin) when is_binary(Bin)-> - MaxLen = power(2,16), + MaxLen = 16#10000, % 65536 case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; @@ -146,7 +144,7 @@ pack_raw(Bin) when is_binary(Bin)-> % list / tuple pack_array(L) when is_list(L)-> - MaxLen = power(2,16), + MaxLen = 16#10000, %65536 case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; @@ -159,14 +157,19 @@ pack_array_([])-> <<>>; pack_array_([Head|Tail])-> << (pack(Head))/binary, (pack_array_(Tail))/binary >>. -unpack_array_(<<>>, 0)-> []; -unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain]; -unpack_array_(Bin, RestLen) when is_binary(Bin)-> - {Term, Rest} = unpack(Bin), - [Term|unpack_array_(Rest, RestLen-1)]. +% FIXME! this should be tail-recursive and without lists:reverse/1 +unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; +unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain}; +unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; +unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> + case unpack(Bin) of + {more, Len} -> {more, Len+RestLen-1}; + {Term, Rest}-> + unpack_array_(Rest, RestLen-1, [Term|RetList]) + end. pack_map({dict,M})-> - MaxLen = power(2,16), + MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; @@ -180,14 +183,25 @@ pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. -unpack_map_(<<>>, 0)-> []; -unpack_map_(Bin, 0) when is_binary(Bin)-> [Bin]; -unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) -> - { Key, Rest } = unpack(Bin), - { Value, Rest2 } = unpack(Rest), - [{Key,Value}|unpack_map_(Rest2,Len-1)]. +-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> + {more, non_neg_integer()} | { any(), binary()}. +unpack_map_(Bin, 0, TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin}; +unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) -> + case unpack(Bin) of + { more, MoreLen } -> { more, MoreLen+Len-1 }; + { Key, Rest } -> + case unpack(Rest) of + {more, MoreLen} -> { more, MoreLen+Len-1 }; + { Value, Rest2 }-> + unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap]) + end + end. +% {more, +-spec unpack_(Flag::integer(), Payload::binary())-> + {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. unpack_(Flag, Payload)-> + PayloadLen = byte_size(Payload), case Flag of 16#C0 -> {nil, Payload}; @@ -196,86 +210,101 @@ unpack_(Flag, Payload)-> 16#C3 -> {true, Payload}; - 16#CA -> % 32bit float + 16#CA when PayloadLen >= 4 -> % 32bit float << Return:32/float-unit:1, Rest/binary >> = Payload, {Return, Rest}; - 16#CB -> % 64bit float + 16#CA -> + {more, 4-PayloadLen}; % at least more + + 16#CB when PayloadLen >= 8 -> % 64bit float << Return:64/float-unit:1, Rest/binary >> = Payload, {Return, Rest}; + 16#CB -> + {more, 8-PayloadLen}; - 16#CC -> % uint 8 + 16#CC when PayloadLen >= 1 -> % uint 8 << Int:8/unsigned-integer, Rest/binary >> = Payload, {Int, Rest}; - 16#CD -> % uint 16 + 16#CC -> + {more, 1}; + + 16#CD when PayloadLen >= 2 -> % uint 16 << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#CE -> + 16#CD -> + {more, 2-PayloadLen}; + + 16#CE when PayloadLen >= 4 -> << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#CF -> + 16#CE -> + {more, 4-PayloadLen}; % at least more + + 16#CF when PayloadLen >= 8 -> << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; + 16#CF -> + {more, 8-PayloadLen}; - 16#D0 -> % int 8 + 16#D0 when PayloadLen >= 1 -> % int 8 << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D1 -> % int 16 + 16#D0 -> + {more, 1}; + + 16#D1 when PayloadLen >= 2 -> % int 16 << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D2 -> % int 32 + 16#D1 -> + {more, 2-PayloadLen}; + + 16#D2 when PayloadLen >= 4 -> % int 32 << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#D3 -> % int 64 + 16#D2 -> + {more, 4-PayloadLen}; + + 16#D3 when PayloadLen >= 8 -> % int 64 << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload, {Int, Rest}; - 16#DA -> % raw 16 + 16#D3 -> + {more, 8-PayloadLen}; + + 16#DA when PayloadLen >= 2 -> % raw 16 << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload, << Return:Len/binary, Remain/binary >> = Rest, {Return, Remain}; - 16#DB -> % raw 32 + 16#DA -> + {more, 16-PayloadLen}; + + 16#DB when PayloadLen >= 4 -> % raw 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, << Return:Len/binary, Remain/binary >> = Rest, {Return, Remain}; - 16#DC -> % array 16 + 16#DB -> + {more, 4-PayloadLen}; + + 16#DC when PayloadLen >= 2 -> % array 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_array_(Rest, Len), - case length(Array) of - Len -> {Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; - 16#DD -> % array 32 + unpack_array_(Rest, Len, []); + 16#DC -> + {more, 2-PayloadLen}; + + 16#DD when PayloadLen >= 4 -> % array 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_array_(Rest, Len), - case length(Array) of - Len -> {Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; - 16#DE -> % map 16 + unpack_array_(Rest, Len, []); + 16#DD -> + {more, 4-PayloadLen}; + + 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_map_(Rest, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; - 16#DF -> % map 32 + unpack_map_(Rest, Len, []); + 16#DE -> + {more, 2-PayloadLen}; + + 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - Array=unpack_map_(Rest, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; + unpack_map_(Rest, Len, []); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -294,29 +323,15 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10010000 , Code < 2#10100000 -> % 1001XXXX for FixArray Len = Code rem 2#10010000, - Array=unpack_array_(Payload, Len), - case length(Array) of - Len -> { Array, <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {Return, Remain} - end; + unpack_array_(Payload, Len, []); Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - Array=unpack_map_(Payload, Len), - case length(Array) of - Len -> { dict:from_list(Array), <<>>}; - _ -> - {Return, RemainRest} = lists:split(Len, Array), - [Remain] = RemainRest, - {dict:from_list(Return), Remain} - end; + unpack_map_(Payload, Len, []); _Other -> - erlang:display(_Other), +% erlang:display(_Other), {error, no_code_matches} end. @@ -356,6 +371,26 @@ port_test()-> after 1024-> ?assert(false) end, port_close(Port). +test_p(Len,Term,OrigBin,Len) -> + {Term, <<>>}=msgpack:unpack(OrigBin); +test_p(I,_,OrigBin,Len) -> + <> = OrigBin, + {more, N}=msgpack:unpack(Bin), + ?assert(0 < N), + ?assert(N < Len). + +partial_test()-> % error handling test. + Term = lists:seq(0, 45), + Bin=msgpack:pack(Term), + BinLen = byte_size(Bin), + [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. + +long_test()-> + Longer = lists:seq(0, 655), %55), +%% Longest = lists:seq(0,12345), + {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)). +%% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, @@ -377,6 +412,10 @@ test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), {S, <<>>} = msgpack:unpack( Pack ), +% ?debugVal( hoge ), 1+test_(Rest). +other_test()-> + {more,1}=msgpack:unpack(<<>>). + -endif. -- cgit v1.2.1 From bc0c5f0cdca4d5244b3237b63305cfd01d1e9a9d Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 23 Jun 2010 09:02:53 +0900 Subject: erlang: (un)pack_map improved, incremental unpacking --- erlang/msgpack.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index d2f7068..764de11 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -167,7 +167,8 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. - + +% FIXME: write test for pack_map/1 pack_map({dict,M})-> MaxLen = 16#10000, %65536 case dict:size(M) of @@ -183,17 +184,18 @@ pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. +% FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin}; -unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) -> +unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> { {dict, Dict}, Bin}; +unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap]) + unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict)) end end. @@ -298,13 +300,13 @@ unpack_(Flag, Payload)-> 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); + unpack_map_(Rest, Len, dict:new()); 16#DE -> {more, 2-PayloadLen}; 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, []); + unpack_map_(Rest, Len, dict:new()); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -328,7 +330,7 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - unpack_map_(Payload, Len, []); + unpack_map_(Payload, Len, dict:new()); _Other -> % erlang:display(_Other), @@ -373,7 +375,7 @@ port_test()-> test_p(Len,Term,OrigBin,Len) -> {Term, <<>>}=msgpack:unpack(OrigBin); -test_p(I,_,OrigBin,Len) -> +test_p(I,_,OrigBin,Len) when I < Len-> <> = OrigBin, {more, N}=msgpack:unpack(Bin), ?assert(0 < N), -- cgit v1.2.1 From 2cdfbd8970756742af7b378612756c59d184d86b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 24 Jun 2010 07:26:34 +0900 Subject: erlang: testing pack_map/unpack_map with a silly bug --- erlang/msgpack.erl | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 764de11..dca5888 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -49,10 +49,10 @@ pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> pack_array(List); -pack({dict, Map}) -> - pack_map({dict, Map}); -pack(_) -> - undefined. +pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> + pack_map(Map); +pack(_O) -> + {error, undefined}. % unpacking. % if failed in decoding and not end, get more data @@ -169,25 +169,25 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> end. % FIXME: write test for pack_map/1 -pack_map({dict,M})-> +pack_map(M)-> MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; Len when Len < MaxLen -> - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> end. pack_map_([])-> <<>>; pack_map_([{Key,Value}|Tail]) -> - << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>. + << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>. % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> { {dict, Dict}, Bin}; +unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> {Dict, Bin}; unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; @@ -333,7 +333,6 @@ unpack_(Flag, Payload)-> unpack_map_(Payload, Len, dict:new()); _Other -> -% erlang:display(_Other), {error, no_code_matches} end. @@ -388,10 +387,25 @@ partial_test()-> % error handling test. [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. long_test()-> - Longer = lists:seq(0, 655), %55), -%% Longest = lists:seq(0,12345), - {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)). + Longer = lists:seq(0, 65), %55), +% Longest = lists:seq(0,12345), + {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)), +% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ok. + +map_test()-> + Ints = lists:seq(0, 65), %55), + Map = dict:from_list([ {X, X*2} || X <- Ints ]), + S=msgpack:pack(Map), +% ?debugVal(msgpack:unpack(S)), + {Map2, <<>>} = msgpack:unpack(S), + ?assertEqual(dict:size(Map), dict:size(Map2)), +% ?debugVal(dict:to_list(Map2)), + OrdMap = orddict:from_list( dict:to_list(Map) ), + OrdMap2 = orddict:from_list( dict:to_list(Map2) ), +% ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug. %% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ok. unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -- cgit v1.2.1 From 92d192277e14be4f32135a85cbca233e7dca3182 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 00:22:53 +0900 Subject: erlang: unpack_map's silly bug fixed. use dict:store/3.... --- erlang/msgpack.erl | 22 +++++++++------------- erlang/testcase_generator.rb | 5 +++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index dca5888..0d4151b 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -195,7 +195,7 @@ unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict)) + unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict)) end end. @@ -352,7 +352,7 @@ test_data()-> -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], - "hogehoge", "243546rf7g68h798j", + <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, [0,42,"sum", [1,2]], [1,42, nil, [3]] ]. @@ -387,34 +387,31 @@ partial_test()-> % error handling test. [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)]. long_test()-> - Longer = lists:seq(0, 65), %55), + Longer = lists:seq(0, 655), % Longest = lists:seq(0,12345), {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)), % {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). ok. map_test()-> - Ints = lists:seq(0, 65), %55), + Ints = lists:seq(0, 65), Map = dict:from_list([ {X, X*2} || X <- Ints ]), - S=msgpack:pack(Map), -% ?debugVal(msgpack:unpack(S)), - {Map2, <<>>} = msgpack:unpack(S), + {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), ?assertEqual(dict:size(Map), dict:size(Map2)), -% ?debugVal(dict:to_list(Map2)), OrdMap = orddict:from_list( dict:to_list(Map) ), OrdMap2 = orddict:from_list( dict:to_list(Map2) ), -% ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug. -%% {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)). + ?assertEqual(OrdMap, OrdMap2), ok. unknown_test()-> Tests = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, <<"hogehoge">>, <<"243546rf7g68h798j">>, -% 123.123, %FIXME -% -234.4355, 1.0e-34, 1.0e64, % FIXME + 123.123, + -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], + dict:from_list([{1,2},{<<"hoge">>,nil}]), 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), @@ -428,7 +425,6 @@ test_([]) -> 0; test_([S|Rest])-> Pack = msgpack:pack(S), {S, <<>>} = msgpack:unpack( Pack ), -% ?debugVal( hoge ), 1+test_(Rest). other_test()-> diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index a7c76c5..cfc36f9 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -40,10 +40,11 @@ end objs = [0, 1, 2, 123, 512, 1230, 678908, -1, -23, -512, -1230, -567898, "hogehoge", "243546rf7g68h798j", -# 123.123 , #FIXME -# -234.4355, 1.0e-34, 1.0e64, + 123.123, + -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,"sum", [1,2]], [1,42, nil, [3]], + { 1 => 2, "hoge" => nil }, 42 ] begin -- cgit v1.2.1 From 57f0598373ce542f7499218792a15fa0eee161f7 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 00:44:14 +0900 Subject: erlang: code refined and tests added --- erlang/msgpack.erl | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 0d4151b..789ed53 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -116,8 +116,7 @@ pack_int_( N ) when is_integer( N )-> << 16#D3:8, N:64/big-signed-integer-unit:1 >>. % nil -pack_nil()-> - << 16#C0:8 >>. +pack_nil()-> << 16#C0:8 >>. % pack_true / pack_false pack_bool(true)-> << 16#C3:8 >>; pack_bool(false)-> << 16#C2:8 >>. @@ -132,11 +131,10 @@ pack_double(F) when is_float(F)-> % raw bytes pack_raw(Bin) when is_binary(Bin)-> - MaxLen = 16#10000, % 65536 case byte_size(Bin) of Len when Len < 6-> << 2#101:3, Len:5, Bin/binary >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DA:8, Len:16/big-unsigned-integer-unit:1, Bin/binary >>; Len -> << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >> @@ -144,11 +142,10 @@ pack_raw(Bin) when is_binary(Bin)-> % list / tuple pack_array(L) when is_list(L)-> - MaxLen = 16#10000, %65536 case length(L) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>; Len -> << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >> @@ -164,17 +161,15 @@ unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 -> {more, RestLen}; unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> case unpack(Bin) of {more, Len} -> {more, Len+RestLen-1}; - {Term, Rest}-> - unpack_array_(Rest, RestLen-1, [Term|RetList]) + {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. % FIXME: write test for pack_map/1 pack_map(M)-> - MaxLen = 16#10000, %65536 case dict:size(M) of Len when Len < 16 -> << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; - Len when Len < MaxLen -> + Len when Len < 16#10000 -> % 65536 << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> @@ -348,13 +343,15 @@ compare_all([LH|LTL], [RH|RTL]) -> compare_all(LTL, RTL). test_data()-> - [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, + [true, false, nil, + 0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, - [0,42,"sum", [1,2]], [1,42, nil, [3]] + [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]], + 42 ]. basic_test()-> @@ -395,7 +392,7 @@ long_test()-> map_test()-> Ints = lists:seq(0, 65), - Map = dict:from_list([ {X, X*2} || X <- Ints ]), + Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]), {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), ?assertEqual(dict:size(Map), dict:size(Map2)), OrdMap = orddict:from_list( dict:to_list(Map) ), -- cgit v1.2.1 From ad052cb510e3be659df0322cbf33e4840c010b9b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 25 Jun 2010 01:26:57 +0900 Subject: updated readme --- erlang/README.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/erlang/README.md b/erlang/README.md index 50d446d..8616d5e 100644 --- a/erlang/README.md +++ b/erlang/README.md @@ -2,18 +2,8 @@ MessagePack for Erlang ====================== Binary-based efficient object serialization library. -## Status +see wiki ( http://redmine.msgpack.org/projects/msgpack/wiki/QuickStartErlang ) for details -still in development. +# Status -TODOs: - - decide string specification. - -## Installation - -## Example - -## License - - - - +0.1.0 released. -- cgit v1.2.1 From 0cca90c21deb2517de95c83a837bf90efc6b9fa0 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Fri, 25 Jun 2010 17:32:11 +0200 Subject: Fix encoding of fixmap type. The tag value was wrong, and a missing /binary flag caused an error. --- erlang/msgpack.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 789ed53..bbf9e64 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -168,7 +168,7 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> pack_map(M)-> case dict:size(M) of Len when Len < 16 -> - << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len when Len < 16#10000 -> % 65536 << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; Len -> -- cgit v1.2.1 From a1b2b41cdcb4bbc98e21bcd334b729ec6e7b90d5 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Sat, 26 Jun 2010 08:40:36 +0900 Subject: erlang: bugfix(serialization of -234 goes <<208,22>> while it should go int16 <<0xD1, ...>>) --- erlang/msgpack.erl | 20 ++++++++++---------- erlang/testcase_generator.rb | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index 789ed53..63d648c 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -86,15 +86,12 @@ pack_uint_(N) when is_integer( N ) , N < 128 -> % uint 8 pack_uint_( N ) when is_integer( N ) andalso N < 256 -> << 16#CC:8, N:8 >>; - % uint 16 pack_uint_( N ) when is_integer( N ) andalso N < 65536 -> << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>; - % uint 32 pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF-> << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>; - % uint 64 pack_uint_( N ) when is_integer( N )-> << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>. @@ -103,13 +100,13 @@ pack_uint_( N ) when is_integer( N )-> pack_int_( N ) when is_integer( N ) , N >= -32-> << 2#111:3, N:5 >>; % int 8 -pack_int_( N ) when is_integer( N ) , N >= -256 -> - << 16#D0:8, N:8 >>; +pack_int_( N ) when is_integer( N ) , N > -128 -> + << 16#D0:8, N:8/big-signed-integer-unit:1 >>; % int 16 -pack_int_( N ) when is_integer( N ), N >= -65536 -> +pack_int_( N ) when is_integer( N ), N > -32768 -> << 16#D1:8, N:16/big-signed-integer-unit:1 >>; % int 32 -pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF -> +pack_int_( N ) when is_integer( N ), N > -16#FFFFFFFF -> << 16#D2:8, N:32/big-signed-integer-unit:1 >>; % int 64 pack_int_( N ) when is_integer( N )-> @@ -351,6 +348,7 @@ test_data()-> <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>, <<"hoasfdafdas][">>, [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]], + -234, -40000, -16#10000000, -16#100000000, 42 ]. @@ -409,6 +407,7 @@ unknown_test()-> [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], dict:from_list([{1,2},{<<"hoge">>,nil}]), + -234, -50000, 42 ], Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]), @@ -419,9 +418,10 @@ unknown_test()-> port_close(Port). test_([]) -> 0; -test_([S|Rest])-> - Pack = msgpack:pack(S), - {S, <<>>} = msgpack:unpack( Pack ), +test_([Before|Rest])-> + Pack = msgpack:pack(Before), + {After, <<>>} = msgpack:unpack( Pack ), + ?assertEqual(Before, After), 1+test_(Rest). other_test()-> diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb index cfc36f9..8445bdd 100644 --- a/erlang/testcase_generator.rb +++ b/erlang/testcase_generator.rb @@ -45,6 +45,7 @@ objs = [0, 1, 2, 123, 512, 1230, 678908, [23, 234, 0.23], [0,42,"sum", [1,2]], [1,42, nil, [3]], { 1 => 2, "hoge" => nil }, + -234, -50000, 42 ] begin -- cgit v1.2.1 From 279121f87f322f3c1a2430751eb3a1032030853d Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 11:56:12 +0200 Subject: erlang: Use a simple proplist instead of a dict. A dict is overkill (code, cpu, memory) in most cases, and proplist<->dict conversion can easily be done by the libray user if desired. This is in line with other erlang libraries I've seen for various encoding schemes. The map encoder had a bug until I looked at it (see previous commit), so I guess it wasn't used much yet and a change is ok at this stage. The chosen representation for maps is a tuple containing the proplist as the only element. --- erlang/msgpack.erl | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index bbf9e64..bc63677 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -30,8 +30,7 @@ % erl> S = . % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ). -type reason() :: enomem | badarg | no_code_matches. --type map() :: any(). % there's no 'dict' type... --type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}. +-type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float(). % ===== external APIs ===== % -spec pack(Term::msgpack_term()) -> binary(). @@ -49,8 +48,10 @@ pack(Bin) when is_binary(Bin) -> pack_raw(Bin); pack(List) when is_list(List) -> pack_array(List); -pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> +pack({Map}) when is_list(Map) -> pack_map(Map); +pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> + pack_map(dict:from_list(Map)); pack(_O) -> {error, undefined}. @@ -166,13 +167,13 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> % FIXME: write test for pack_map/1 pack_map(M)-> - case dict:size(M) of + case length(M) of Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >> end. pack_map_([])-> <<>>; @@ -182,15 +183,15 @@ pack_map_([{Key,Value}|Tail]) -> % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> {more, non_neg_integer()} | { any(), binary()}. -unpack_map_(Bin, 0, Dict) when is_binary(Bin) -> {Dict, Bin}; -unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) -> +unpack_map_(Bin, 0, Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin}; +unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> case unpack(Bin) of { more, MoreLen } -> { more, MoreLen+Len-1 }; { Key, Rest } -> case unpack(Rest) of {more, MoreLen} -> { more, MoreLen+Len-1 }; - { Value, Rest2 }-> - unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict)) + { Value, Rest2 } -> + unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]) end end. @@ -295,13 +296,13 @@ unpack_(Flag, Payload)-> 16#DE when PayloadLen >= 2 -> % map 16 << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, dict:new()); + unpack_map_(Rest, Len, []); 16#DE -> {more, 2-PayloadLen}; 16#DF when PayloadLen >= 4 -> % map 32 << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload, - unpack_map_(Rest, Len, dict:new()); + unpack_map_(Rest, Len, []); % positive fixnum Code when Code >= 2#00000000, Code < 2#10000000-> @@ -325,7 +326,7 @@ unpack_(Flag, Payload)-> Code when Code >= 2#10000000 , Code < 2#10010000 -> % 1000XXXX for FixMap Len = Code rem 2#10000000, - unpack_map_(Payload, Len, dict:new()); + unpack_map_(Payload, Len, []); _Other -> {error, no_code_matches} @@ -392,12 +393,9 @@ long_test()-> map_test()-> Ints = lists:seq(0, 65), - Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]), + Map = {[ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]}, {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)), - ?assertEqual(dict:size(Map), dict:size(Map2)), - OrdMap = orddict:from_list( dict:to_list(Map) ), - OrdMap2 = orddict:from_list( dict:to_list(Map2) ), - ?assertEqual(OrdMap, OrdMap2), + ?assertEqual(Map, Map2), ok. unknown_test()-> -- cgit v1.2.1 From 537322e3b59978d26fcf096433fc718c16cc8b84 Mon Sep 17 00:00:00 2001 From: Vincent de Phily Date: Mon, 28 Jun 2010 14:17:44 +0200 Subject: Big speedup (around 40%) of maps and arrays encoding by using proper tail recursion. --- erlang/msgpack.erl | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index bc63677..255542b 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -59,7 +59,7 @@ pack(_O) -> % if failed in decoding and not end, get more data % and feed more Bin into this function. % TODO: error case for imcomplete format when short for any type formats. --spec unpack( binary() )-> +-spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}. unpack(Bin) when not is_binary(Bin)-> {error, badarg}; @@ -82,7 +82,7 @@ unpack_all(Data)-> % ===== internal APIs ===== % % positive fixnum -pack_uint_(N) when is_integer( N ) , N < 128 -> +pack_uint_(N) when is_integer( N ) , N < 128 -> << 2#0:1, N:7 >>; % uint 8 pack_uint_( N ) when is_integer( N ) andalso N < 256 -> @@ -145,15 +145,15 @@ pack_raw(Bin) when is_binary(Bin)-> pack_array(L) when is_list(L)-> case length(L) of Len when Len < 16 -> - << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>; + << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>; + << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>; Len -> - << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >> + << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >> end. -pack_array_([])-> <<>>; -pack_array_([Head|Tail])-> - << (pack(Head))/binary, (pack_array_(Tail))/binary >>. +pack_array_([], Acc) -> Acc; +pack_array_([Head|Tail], Acc) -> + pack_array_(Tail, <>). % FIXME! this should be tail-recursive and without lists:reverse/1 unpack_array_(<<>>, 0, RetList) -> {lists:reverse(RetList), <<>>}; @@ -169,16 +169,16 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> pack_map(M)-> case length(M) of Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>; + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>; + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> end. -pack_map_([])-> <<>>; -pack_map_([{Key,Value}|Tail]) -> - << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>. +pack_map_([], Acc) -> Acc; +pack_map_([{Key,Value}|Tail], Acc) -> + pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). % FIXME: write test for unpack_map/1 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])-> @@ -200,7 +200,7 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}. unpack_(Flag, Payload)-> PayloadLen = byte_size(Payload), - case Flag of + case Flag of 16#C0 -> {nil, Payload}; 16#C2 -> @@ -313,7 +313,7 @@ unpack_(Flag, Payload)-> {(Code - 16#100), Payload}; Code when Code >= 2#10100000 , Code < 2#11000000 -> -% 101XXXXX for FixRaw +% 101XXXXX for FixRaw Len = Code rem 2#10100000, << Return:Len/binary, Remain/binary >> = Payload, {Return, Remain}; @@ -344,7 +344,7 @@ compare_all([LH|LTL], [RH|RTL]) -> compare_all(LTL, RTL). test_data()-> - [true, false, nil, + [true, false, nil, 0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF, -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF, 123.123, -234.4355, 1.0e-34, 1.0e64, -- cgit v1.2.1 From b471e52e281d189396a78cef3eb7ecacbab51d21 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 29 Jun 2010 00:21:47 +0900 Subject: erlang: explicit API for serializing proplists, so as not to make wrong call of pack({proplists()}). --- erlang/msgpack.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index c807065..29e1728 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -24,6 +24,7 @@ %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). +-export([pack_map/1]) % compile: % erl> c(msgpack). @@ -51,7 +52,7 @@ pack(List) when is_list(List) -> pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:from_list(Map)); + pack_map(dict:to_list(Map)); pack(_O) -> {error, undefined}. @@ -78,6 +79,15 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. +pack_map(M)-> + case length(M) of + Len when Len < 16 -> + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len when Len < 16#10000 -> % 65536 + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len -> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> + end. % ===== internal APIs ===== % @@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. -% FIXME: write test for pack_map/1 -pack_map(M)-> - case length(M) of - Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> - end. - pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). @@ -404,7 +403,7 @@ unknown_test()-> -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], - dict:from_list([{1,2},{<<"hoge">>,nil}]), + {[{1,2},{<<"hoge">>,nil}]}, -234, -50000, 42 ], -- cgit v1.2.1 From 90e305d789d231258e26a2dc546cfdb6f8c09ce8 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 29 Jun 2010 00:21:47 +0900 Subject: erlang: explicit API for serializing proplists, so as not to make wrong call of pack({proplists()}). --- erlang/msgpack.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl index c807065..a168b55 100644 --- a/erlang/msgpack.erl +++ b/erlang/msgpack.erl @@ -24,6 +24,7 @@ %% for C API (http://msgpack.sourceforge.jp/c:doc) %% except buffering functions (both copying and zero-copying). -export([pack/1, unpack/1, unpack_all/1]). +-export([pack_map/1]). % compile: % erl> c(msgpack). @@ -51,7 +52,7 @@ pack(List) when is_list(List) -> pack({Map}) when is_list(Map) -> pack_map(Map); pack(Map) when is_tuple(Map), element(1,Map)=:=dict -> - pack_map(dict:from_list(Map)); + pack_map(dict:to_list(Map)); pack(_O) -> {error, undefined}. @@ -78,6 +79,15 @@ unpack_all(Data)-> [Term|unpack_all(Binary)] end. +pack_map(M)-> + case length(M) of + Len when Len < 16 -> + << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len when Len < 16#10000 -> % 65536 + << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; + Len -> + << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> + end. % ===== internal APIs ===== % @@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)-> {Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList]) end. -% FIXME: write test for pack_map/1 -pack_map(M)-> - case length(M) of - Len when Len < 16 -> - << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len when Len < 16#10000 -> % 65536 - << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>; - Len -> - << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >> - end. - pack_map_([], Acc) -> Acc; pack_map_([{Key,Value}|Tail], Acc) -> pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>). @@ -404,7 +403,7 @@ unknown_test()-> -234.4355, 1.0e-34, 1.0e64, [23, 234, 0.23], [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]], - dict:from_list([{1,2},{<<"hoge">>,nil}]), + {[{1,2},{<<"hoge">>,nil}]}, -234, -50000, 42 ], -- cgit v1.2.1