From 766295ae1ff8e59b61c93b345624eda3f146e9f4 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 26 Aug 2010 18:46:36 +0100 Subject: implementing topic routing with tries; adding better test for topic routing --- src/rabbit_exchange_type_topic.erl | 253 +++++++++++++++++++++++++++++++------ src/rabbit_mnesia.erl | 17 +++ src/rabbit_router.erl | 6 + src/rabbit_tests.erl | 114 +++++++++++++---- 4 files changed, 328 insertions(+), 62 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index e796acf3..35f25ccb 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -30,6 +30,7 @@ %% -module(rabbit_exchange_type_topic). +-include_lib("stdlib/include/qlc.hrl"). -include("rabbit.hrl"). -behaviour(rabbit_exchange_type). @@ -46,59 +47,231 @@ {requires, rabbit_exchange_type_registry}, {enables, kernel_ready}]}). --export([topic_matches/2]). +-export([which_matches/2]). -ifdef(use_specs). --spec(topic_matches/2 :: (binary(), binary()) -> boolean()). +-spec(which_matches/2 :: + (rabbit_exchange:name(), rabbit_router:routing_key()) -> + [rabbit_amqqueue:name()]). -endif. +%%---------------------------------------------------------------------------- + description() -> [{name, <<"topic">>}, {description, <<"AMQP topic exchange, as per the AMQP specification">>}]. -publish(#exchange{name = Name}, Delivery = - #delivery{message = #basic_message{routing_key = RoutingKey}}) -> - rabbit_router:deliver(rabbit_router:match_bindings( - Name, fun (#binding{key = BindingKey}) -> - topic_matches(BindingKey, RoutingKey) - end), - Delivery). - -split_topic_key(Key) -> - string:tokens(binary_to_list(Key), "."). - -topic_matches(PatternKey, RoutingKey) -> - P = split_topic_key(PatternKey), - R = split_topic_key(RoutingKey), - topic_matches1(P, R). - -topic_matches1(["#"], _R) -> - true; -topic_matches1(["#" | PTail], R) -> - last_topic_match(PTail, [], lists:reverse(R)); -topic_matches1([], []) -> - true; -topic_matches1(["*" | PatRest], [_ | ValRest]) -> - topic_matches1(PatRest, ValRest); -topic_matches1([PatElement | PatRest], [ValElement | ValRest]) - when PatElement == ValElement -> - topic_matches1(PatRest, ValRest); -topic_matches1(_, _) -> - false. - -last_topic_match(P, R, []) -> - topic_matches1(P, R); -last_topic_match(P, R, [BacktrackNext | BacktrackList]) -> - topic_matches1(P, R) or - last_topic_match(P, [BacktrackNext | R], BacktrackList). +publish(#exchange{name = X}, Delivery = + #delivery{message = #basic_message{routing_key = Key}}) -> + rabbit_router:deliver_by_queue_names(which_matches(X, Key), Delivery). validate(_X) -> ok. create(_X) -> ok. recover(_X, _Bs) -> ok. -delete(_X, _Bs) -> ok. -add_binding(_X, _B) -> ok. -remove_bindings(_X, _Bs) -> ok. + +delete(#exchange{name = X}, _Bs) -> + rabbit_misc:execute_mnesia_transaction(fun() -> trie_remove_all_edges(X), + trie_remove_all_bindings(X) + end), + ok. + +add_binding(_Exchange, #binding{exchange_name = X, key = K, queue_name = Q}) -> + rabbit_misc:execute_mnesia_transaction( + fun() -> FinalNode = follow_down_create(X, split_topic_key(K)), + trie_add_binding(X, FinalNode, Q) + end), + ok. + +remove_bindings(_X, Bs) -> + rabbit_misc:execute_mnesia_transaction( + fun() -> lists:foreach(fun remove_binding/1, Bs) end), + ok. + +remove_binding(#binding{exchange_name = X, key = K, queue_name = Q}) -> + Path = follow_down_get_path(X, split_topic_key(K)), + {FinalNode, _} = hd(Path), + trie_remove_binding(X, FinalNode, Q), + remove_path_if_empty(X, Path), + ok. + assert_args_equivalence(X, Args) -> rabbit_exchange:assert_args_equivalence(X, Args). + +%% NB: This function may return duplicate results in some situations (that's ok) +which_matches(X, Key) -> + Words = split_topic_key(Key), + mnesia:async_dirty(fun trie_match/2, [X, Words]). + +%%---------------------------------------------------------------------------- + +trie_match(X, Words) -> + trie_match(X, root, Words). +trie_match(X, Node, []) -> + FinalRes = trie_bindings(X, Node), + HashRes = case trie_child(X, Node, "#") of + {ok, HashNode} -> trie_match(X, HashNode, []); + error -> [] + end, + FinalRes ++ HashRes; +trie_match(X, Node, [W | RestW] = Words) -> + ExactRes = case trie_child(X, Node, W) of + {ok, NextNode} -> trie_match(X, NextNode, RestW); + error -> [] + end, + StarRes = case trie_child(X, Node, "*") of + {ok, StarNode} -> trie_match(X, StarNode, RestW); + error -> [] + end, + HashRes = case trie_child(X, Node, "#") of + {ok, HashNode} -> trie_match_skip_any(X, HashNode, Words); + error -> [] + end, + ExactRes ++ StarRes ++ HashRes. + +trie_match_skip_any(X, Node, []) -> + trie_match(X, Node, []); +trie_match_skip_any(X, Node, [_ | RestW] = Words) -> + trie_match(X, Node, Words) ++ + trie_match_skip_any(X, Node, RestW). + +follow_down(X, Words) -> + follow_down(X, root, Words). +follow_down(_X, CurNode, []) -> + {ok, CurNode}; +follow_down(X, CurNode, [W | RestW]) -> + case trie_child(X, CurNode, W) of + {ok, NextNode} -> follow_down(X, NextNode, RestW); + error -> {error, CurNode, [W | RestW]} + end. + +follow_down_create(X, Words) -> + case follow_down(X, Words) of + {ok, FinalNode} -> + FinalNode; + {error, Node, RestW} -> + lists:foldl( + fun(W, CurNode) -> + NewNode = new_node(), + trie_add_edge(X, CurNode, NewNode, W), + NewNode + end, Node, RestW) + end. + +follow_down_get_path(X, Words) -> + follow_down_get_path(X, root, Words, [{root, none}]). +follow_down_get_path(_, _, [], PathAcc) -> + PathAcc; +follow_down_get_path(X, CurNode, [W | RestW], PathAcc) -> + {ok, NextNode} = trie_child(X, CurNode, W), + follow_down_get_path(X, NextNode, RestW, [{NextNode, W} | PathAcc]). + +remove_path_if_empty(_, [{root, none}]) -> + ok; +remove_path_if_empty(X, [{Node, W} | [{Parent, _} | _] = RestPath]) -> + case trie_has_any_bindings(X, Node) orelse + trie_has_any_children(X, Node) of + true -> ok; + false -> trie_remove_edge(X, Parent, Node, W), + remove_path_if_empty(X, RestPath) + end. + +trie_child(X, Node, Word) -> + Query = qlc:q([NextNode || + #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X1, + node_id = Node1, + word = Word1}, + node_id = NextNode} + <- mnesia:table(rabbit_topic_trie_edge), + X1 == X, + Node1 == Node, + Word1 == Word]), + case qlc:e(Query) of + [NextNode] -> {ok, NextNode}; + [] -> error + end. + +trie_bindings(X, Node) -> + MatchHead = #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X, + node_id = Node, + queue_name = '$1'}}, + mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$1']}]). + +trie_add_edge(X, FromNode, ToNode, W) -> + trie_edge_op(X, FromNode, ToNode, W, fun mnesia:write/3). +trie_remove_edge(X, FromNode, ToNode, W) -> + trie_edge_op(X, FromNode, ToNode, W, fun mnesia:delete_object/3). +trie_edge_op(X, FromNode, ToNode, W, Op) -> + ok = Op(rabbit_topic_trie_edge, + #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + node_id = FromNode, + word = W}, + node_id = ToNode}, + write). + +trie_add_binding(X, Node, Q) -> + trie_binding_op(X, Node, Q, fun mnesia:write/3). +trie_remove_binding(X, Node, Q) -> + trie_binding_op(X, Node, Q, fun mnesia:delete_object/3). +trie_binding_op(X, Node, Q, Op) -> + ok = Op(rabbit_topic_trie_binding, + #topic_trie_binding{trie_binding = #trie_binding{exchange_name = X, + node_id = Node, + queue_name = Q}}, + write). + +trie_has_any_children(X, Node) -> + MatchHead = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + node_id = Node, + word = '$1'}, + _='_'}, + Select = mnesia:select(rabbit_topic_trie_edge, + [{MatchHead, [], ['$1']}], 1, read), + select_while_no_result(Select) /= '$end_of_table'. + +trie_has_any_bindings(X, Node) -> + MatchHead = #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X, + node_id = Node, + queue_name = '$1'}, + _='_'}, + Select = mnesia:select(rabbit_topic_trie_binding, + [{MatchHead, [], ['$1']}], 1, read), + select_while_no_result(Select) /= '$end_of_table'. + +select_while_no_result({[], Cont}) -> + select_while_no_result(mnesia:select(Cont)); +select_while_no_result(Other) -> + Other. + +trie_remove_all_edges(X) -> + Query = qlc:q([Entry || + Entry = #topic_trie_edge{ + trie_edge = #trie_edge{exchange_name = X1, + _='_'}, + _='_'} + <- mnesia:table(rabbit_topic_trie_edge), + X1 == X]), + lists:foreach( + fun(O) -> mnesia:delete_object(rabbit_topic_trie_edge, O, write) end, + qlc:e(Query)). + +trie_remove_all_bindings(X) -> + Query = qlc:q([Entry || + Entry = #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X1, + _='_'}, + _='_'} + <- mnesia:table(rabbit_topic_trie_binding), + X1 == X]), + lists:foreach( + fun(O) -> mnesia:delete_object(rabbit_topic_trie_binding, O, write) end, + qlc:e(Query)). + +new_node() -> + now(). % UUID + +split_topic_key(Key) -> + string:tokens(binary_to_list(Key), "."). diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 4a5adfae..37708b22 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -194,6 +194,17 @@ table_definitions() -> {type, ordered_set}, {match, #reverse_route{reverse_binding = reverse_binding_match(), _='_'}}]}, + {rabbit_topic_trie_edge, + [{record_name, topic_trie_edge}, + {attributes, record_info(fields, topic_trie_edge)}, + {type, ordered_set}, + {match, #topic_trie_edge{trie_edge = trie_edge_match(), _='_'}}]}, + {rabbit_topic_trie_binding, + [{record_name, topic_trie_binding}, + {attributes, record_info(fields, topic_trie_binding)}, + {type, ordered_set}, + {match, #topic_trie_binding{trie_binding = trie_binding_match(), + _='_'}}]}, %% Consider the implications to nodes_of_type/1 before altering %% the next entry. {rabbit_durable_exchange, @@ -223,6 +234,12 @@ reverse_binding_match() -> #reverse_binding{queue_name = queue_name_match(), exchange_name = exchange_name_match(), _='_'}. +trie_edge_match() -> + #trie_edge{exchange_name = exchange_name_match(), + _='_'}. +trie_binding_match() -> + #trie_edge{exchange_name = exchange_name_match(), + _='_'}. exchange_name_match() -> resource_match(exchange). queue_name_match() -> diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index ec049a1a..d7d6d0ad 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -34,6 +34,7 @@ -include("rabbit.hrl"). -export([deliver/2, + deliver_by_queue_names/2, match_bindings/2, match_routing_key/2]). @@ -48,6 +49,8 @@ -spec(deliver/2 :: ([pid()], rabbit_types:delivery()) -> {routing_result(), [pid()]}). +-spec(deliver_by_queue_names/2 :: + ([binary()], rabbit_types:delivery()) -> {routing_result(), [pid()]}). -endif. @@ -77,6 +80,9 @@ deliver(QPids, Delivery) -> check_delivery(Delivery#delivery.mandatory, Delivery#delivery.immediate, {Routed, Handled}). +deliver_by_queue_names(Qs, Delivery) -> + deliver(lookup_qpids(Qs), Delivery). + %% TODO: Maybe this should be handled by a cursor instead. %% TODO: This causes a full scan for each entry with the same exchange match_bindings(Name, Match) -> diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 082e7877..1e7f533a 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -584,30 +584,100 @@ sequence_with_content(Sequence) -> rabbit_framing_amqp_0_9_1), Sequence). -test_topic_match(P, R) -> - test_topic_match(P, R, true). - -test_topic_match(P, R, Expected) -> - case rabbit_exchange_type_topic:topic_matches(list_to_binary(P), - list_to_binary(R)) of - Expected -> - passed; - _ -> - {topic_match_failure, P, R} - end. +test_topic_expect_match(#exchange{name = XName}, List) -> + lists:foreach( + fun({Key, Expected}) -> + Res = rabbit_exchange_type_topic:which_matches( + XName, list_to_binary(Key)), + ExpectedRes = lists:map( + fun(Q) -> #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)} + end, Expected), + true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) + end, List). test_topic_matching() -> - passed = test_topic_match("#", "test.test"), - passed = test_topic_match("#", ""), - passed = test_topic_match("#.T.R", "T.T.R"), - passed = test_topic_match("#.T.R", "T.R.T.R"), - passed = test_topic_match("#.Y.Z", "X.Y.Z.X.Y.Z"), - passed = test_topic_match("#.test", "test"), - passed = test_topic_match("#.test", "test.test"), - passed = test_topic_match("#.test", "ignored.test"), - passed = test_topic_match("#.test", "more.ignored.test"), - passed = test_topic_match("#.test", "notmatched", false), - passed = test_topic_match("#.z", "one.two.three.four", false), + XName = #resource{virtual_host = <<"/">>, + kind = exchange, + name = <<"test_exchange">>}, + X = #exchange{name = XName, type = topic, durable = false, + auto_delete = false, arguments = []}, + %% create + rabbit_exchange_type_topic:validate(X), + rabbit_exchange_type_topic:create(X), + + %% add some bindings + Bindings = lists:map( + fun({Key, Q}) -> + #binding{exchange_name = XName, + key = list_to_binary(Key), + queue_name = #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)}} + end, [{"a.b.c", "t1"}, + {"a.*.c", "t2"}, + {"a.#.b", "t3"}, + {"a.b.b.c", "t4"}, + {"#", "t5"}, + {"#.#", "t6"}, + {"#.b", "t7"}, + {"*.*", "t8"}, + {"a.*", "t9"}, + {"*.b.c", "t10"}, + {"a.#", "t11"}, + {"a.#.#", "t12"}, + {"b.b.c", "t13"}, + {"a.b.b", "t14"}, + {"a.b", "t15"}, + {"b.c", "t16"}, + {"", "t17"}, + {"*.*.*", "t18"}, + {"vodka.martini", "t19"}, + {"a.b.c", "t20"}]), + lists:foreach(fun(B) -> rabbit_exchange_type_topic:add_binding(X, B) end, + Bindings), + + %% test some matches + test_topic_expect_match(X, + [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", "t18", "t20"]}, + {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", "t12", "t15"]}, + {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", "t18"]}, + {"", ["t5", "t6", "t17"]}, + {"b.c.c", ["t5", "t6", "t18"]}, + {"a.a.a.a.a", ["t5", "t6", "t11", "t12"]}, + {"vodka.gin", ["t5", "t6", "t8"]}, + {"vodka.martini", ["t5", "t6", "t8", "t19"]}, + {"b.b.c", ["t5", "t6", "t10", "t13", "t18"]}, + {"nothing.here.at.all", ["t5", "t6"]}, + {"un_der_sc.ore", ["t5", "t6", "t8"]}]), + + %% remove some bindings + RemovedBindings = [lists:nth(1, Bindings), lists:nth(5, Bindings), + lists:nth(11, Bindings)], + rabbit_exchange_type_topic:remove_bindings(X, RemovedBindings), + RemainingBindings = ordsets:to_list( + ordsets:subtract(ordsets:from_list(Bindings), + ordsets:from_list(RemovedBindings))), + + %% test some matches + test_topic_expect_match(X, + [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20"]}, + {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15"]}, + {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18"]}, + {"", ["t6", "t17"]}, + {"b.c.c", ["t6", "t18"]}, + {"a.a.a.a.a", ["t6", "t12"]}, + {"vodka.gin", ["t6", "t8"]}, + {"vodka.martini", ["t6", "t8", "t19"]}, + {"b.b.c", ["t6", "t10", "t13", "t18"]}, + {"nothing.here.at.all", ["t6"]}, + {"un_der_sc.ore", ["t6", "t8"]}]), + + %% remove the entire exchange + rabbit_exchange_type_topic:delete(X, RemainingBindings), + %% none should match now + test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), passed. test_app_management() -> -- cgit v1.2.1 From 2ae1ddd84c5896bd8888d807533fcfcbd5ad303f Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 26 Aug 2010 19:12:36 +0100 Subject: minor cosmetic --- src/rabbit_exchange_type_topic.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 35f25ccb..e2114b5d 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -262,7 +262,7 @@ trie_remove_all_bindings(X) -> Query = qlc:q([Entry || Entry = #topic_trie_binding{ trie_binding = #trie_binding{exchange_name = X1, - _='_'}, + _='_'}, _='_'} <- mnesia:table(rabbit_topic_trie_binding), X1 == X]), -- cgit v1.2.1 From 21a74891c40dee991f0cd0084f7cd7e49e331b60 Mon Sep 17 00:00:00 2001 From: Vlad Ionescu Date: Mon, 27 Sep 2010 19:44:44 +0100 Subject: using rabbit_guid; cosmetic --- src/rabbit_exchange_type_topic.erl | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index e2114b5d..8e6918d0 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -133,8 +133,7 @@ trie_match(X, Node, [W | RestW] = Words) -> trie_match_skip_any(X, Node, []) -> trie_match(X, Node, []); trie_match_skip_any(X, Node, [_ | RestW] = Words) -> - trie_match(X, Node, Words) ++ - trie_match_skip_any(X, Node, RestW). + trie_match(X, Node, Words) ++ trie_match_skip_any(X, Node, RestW). follow_down(X, Words) -> follow_down(X, root, Words). @@ -148,15 +147,13 @@ follow_down(X, CurNode, [W | RestW]) -> follow_down_create(X, Words) -> case follow_down(X, Words) of - {ok, FinalNode} -> - FinalNode; - {error, Node, RestW} -> - lists:foldl( - fun(W, CurNode) -> - NewNode = new_node(), - trie_add_edge(X, CurNode, NewNode, W), - NewNode - end, Node, RestW) + {ok, FinalNode} -> FinalNode; + {error, Node, RestW} -> lists:foldl( + fun(W, CurNode) -> + NewNode = new_node(), + trie_add_edge(X, CurNode, NewNode, W), + NewNode + end, Node, RestW) end. follow_down_get_path(X, Words) -> @@ -271,7 +268,7 @@ trie_remove_all_bindings(X) -> qlc:e(Query)). new_node() -> - now(). % UUID + rabbit_guid:guid(). split_topic_key(Key) -> string:tokens(binary_to_list(Key), "."). -- cgit v1.2.1 From 75206c6124f4be2d6e578186eacc5fbc7f331d99 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Tue, 28 Sep 2010 17:33:00 +0100 Subject: using recursive split; more tests --- src/rabbit_exchange_type_topic.erl | 18 +++++++- src/rabbit_tests.erl | 89 ++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 38 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 8e6918d0..bbd5d357 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -108,6 +108,7 @@ which_matches(X, Key) -> trie_match(X, Words) -> trie_match(X, root, Words). + trie_match(X, Node, []) -> FinalRes = trie_bindings(X, Node), HashRes = case trie_child(X, Node, "#") of @@ -137,6 +138,7 @@ trie_match_skip_any(X, Node, [_ | RestW] = Words) -> follow_down(X, Words) -> follow_down(X, root, Words). + follow_down(_X, CurNode, []) -> {ok, CurNode}; follow_down(X, CurNode, [W | RestW]) -> @@ -158,6 +160,7 @@ follow_down_create(X, Words) -> follow_down_get_path(X, Words) -> follow_down_get_path(X, root, Words, [{root, none}]). + follow_down_get_path(_, _, [], PathAcc) -> PathAcc; follow_down_get_path(X, CurNode, [W | RestW], PathAcc) -> @@ -198,8 +201,10 @@ trie_bindings(X, Node) -> trie_add_edge(X, FromNode, ToNode, W) -> trie_edge_op(X, FromNode, ToNode, W, fun mnesia:write/3). + trie_remove_edge(X, FromNode, ToNode, W) -> trie_edge_op(X, FromNode, ToNode, W, fun mnesia:delete_object/3). + trie_edge_op(X, FromNode, ToNode, W, Op) -> ok = Op(rabbit_topic_trie_edge, #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, @@ -210,8 +215,10 @@ trie_edge_op(X, FromNode, ToNode, W, Op) -> trie_add_binding(X, Node, Q) -> trie_binding_op(X, Node, Q, fun mnesia:write/3). + trie_remove_binding(X, Node, Q) -> trie_binding_op(X, Node, Q, fun mnesia:delete_object/3). + trie_binding_op(X, Node, Q, Op) -> ok = Op(rabbit_topic_trie_binding, #topic_trie_binding{trie_binding = #trie_binding{exchange_name = X, @@ -271,4 +278,13 @@ new_node() -> rabbit_guid:guid(). split_topic_key(Key) -> - string:tokens(binary_to_list(Key), "."). + split_topic_key(Key, [], []). + +split_topic_key(<<>>, [], []) -> + []; +split_topic_key(<<>>, RevWordAcc, RevResAcc) -> + lists:reverse([lists:reverse(RevWordAcc) | RevResAcc]); +split_topic_key(<<$., Rest/binary>>, RevWordAcc, RevResAcc) -> + split_topic_key(Rest, [], [lists:reverse(RevWordAcc) | RevResAcc]); +split_topic_key(<>, RevWordAcc, RevResAcc) -> + split_topic_key(Rest, [C | RevWordAcc], RevResAcc). diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index fb0faebb..0308f539 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -581,19 +581,6 @@ sequence_with_content(Sequence) -> rabbit_framing_amqp_0_9_1), Sequence). -test_topic_expect_match(#exchange{name = XName}, List) -> - lists:foreach( - fun({Key, Expected}) -> - Res = rabbit_exchange_type_topic:which_matches( - XName, list_to_binary(Key)), - ExpectedRes = lists:map( - fun(Q) -> #resource{virtual_host = <<"/">>, - kind = queue, - name = list_to_binary(Q)} - end, Expected), - true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) - end, List). - test_topic_matching() -> XName = #resource{virtual_host = <<"/">>, kind = exchange, @@ -631,27 +618,37 @@ test_topic_matching() -> {"", "t17"}, {"*.*.*", "t18"}, {"vodka.martini", "t19"}, - {"a.b.c", "t20"}]), + {"a.b.c", "t20"}, + {"*.#", "t21"}, + {"#.*.#", "t22"}, + {"*.#.#", "t23"}, + {"#.#.#", "t24"}, + {"*", "t25"}]), lists:foreach(fun(B) -> rabbit_exchange_type_topic:add_binding(X, B) end, Bindings), %% test some matches test_topic_expect_match(X, - [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", "t18", "t20"]}, - {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", "t12", "t15"]}, - {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", "t18"]}, - {"", ["t5", "t6", "t17"]}, - {"b.c.c", ["t5", "t6", "t18"]}, - {"a.a.a.a.a", ["t5", "t6", "t11", "t12"]}, - {"vodka.gin", ["t5", "t6", "t8"]}, - {"vodka.martini", ["t5", "t6", "t8", "t19"]}, - {"b.b.c", ["t5", "t6", "t10", "t13", "t18"]}, - {"nothing.here.at.all", ["t5", "t6"]}, - {"un_der_sc.ore", ["t5", "t6", "t8"]}]), + [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", "t18", "t20", + "t21", "t22", "t23", "t24"]}, + {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", "t12", "t15", + "t21", "t22", "t23", "t24"]}, + {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", "t18", "t21", + "t22", "t23", "t24"]}, + {"", ["t5", "t6", "t17", "t24"]}, + {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", "t24"]}, + {"a.a.a.a.a", ["t5", "t6", "t11", "t12", "t21", "t22", "t23", "t24"]}, + {"vodka.gin", ["t5", "t6", "t8", "t21", "t22", "t23", "t24"]}, + {"vodka.martini", ["t5", "t6", "t8", "t19", "t21", "t22", "t23", + "t24"]}, + {"b.b.c", ["t5", "t6", "t10", "t13", "t18", "t21", "t22", "t23", + "t24"]}, + {"nothing.here.at.all", ["t5", "t6", "t21", "t22", "t23", "t24"]}, + {"oneword", ["t5", "t6", "t21", "t22", "t23", "t24", "t25"]}]), %% remove some bindings RemovedBindings = [lists:nth(1, Bindings), lists:nth(5, Bindings), - lists:nth(11, Bindings)], + lists:nth(11, Bindings), lists:nth(21, Bindings)], rabbit_exchange_type_topic:remove_bindings(X, RemovedBindings), RemainingBindings = ordsets:to_list( ordsets:subtract(ordsets:from_list(Bindings), @@ -659,17 +656,20 @@ test_topic_matching() -> %% test some matches test_topic_expect_match(X, - [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20"]}, - {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15"]}, - {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18"]}, - {"", ["t6", "t17"]}, - {"b.c.c", ["t6", "t18"]}, - {"a.a.a.a.a", ["t6", "t12"]}, - {"vodka.gin", ["t6", "t8"]}, - {"vodka.martini", ["t6", "t8", "t19"]}, - {"b.b.c", ["t6", "t10", "t13", "t18"]}, - {"nothing.here.at.all", ["t6"]}, - {"un_der_sc.ore", ["t6", "t8"]}]), + [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20", "t22", "t23", + "t24"]}, + {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15", "t22", "t23", + "t24"]}, + {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18", "t22", "t23", + "t24"]}, + {"", ["t6", "t17", "t24"]}, + {"b.c.c", ["t6", "t18", "t22", "t23", "t24"]}, + {"a.a.a.a.a", ["t6", "t12", "t22", "t23", "t24"]}, + {"vodka.gin", ["t6", "t8", "t22", "t23", "t24"]}, + {"vodka.martini", ["t6", "t8", "t19", "t22", "t23", "t24"]}, + {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", "t24"]}, + {"nothing.here.at.all", ["t6", "t22", "t23", "t24"]}, + {"oneword", ["t6", "t22", "t23", "t24", "t25"]}]), %% remove the entire exchange rabbit_exchange_type_topic:delete(X, RemainingBindings), @@ -677,6 +677,21 @@ test_topic_matching() -> test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), passed. +test_topic_expect_match(#exchange{name = XName}, List) -> + lists:foreach( + fun({Key, Expected}) -> + io:format("~p ~p~n", [Key, Expected]), + Res = rabbit_exchange_type_topic:which_matches( + XName, list_to_binary(Key)), + io:format("Res: ~p~n", [Res]), + ExpectedRes = lists:map( + fun(Q) -> #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)} + end, Expected), + true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) + end, List). + test_app_management() -> %% starting, stopping, status ok = control_action(stop_app, []), -- cgit v1.2.1 From 6db65737ad2ad3d67d50a1b5adc0cfa28b999029 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Wed, 29 Sep 2010 19:36:24 +0100 Subject: using mnesia:read rather than qlc --- src/rabbit_exchange_type_topic.erl | 17 +++++------------ src/rabbit_tests.erl | 4 +--- 2 files changed, 6 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index bbd5d357..9091d385 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -178,18 +178,11 @@ remove_path_if_empty(X, [{Node, W} | [{Parent, _} | _] = RestPath]) -> end. trie_child(X, Node, Word) -> - Query = qlc:q([NextNode || - #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X1, - node_id = Node1, - word = Word1}, - node_id = NextNode} - <- mnesia:table(rabbit_topic_trie_edge), - X1 == X, - Node1 == Node, - Word1 == Word]), - case qlc:e(Query) of - [NextNode] -> {ok, NextNode}; - [] -> error + case mnesia:read(rabbit_topic_trie_edge, #trie_edge{exchange_name = X, + node_id = Node, + word = Word}) of + [#topic_trie_edge{node_id = NextNode}] -> {ok, NextNode}; + [] -> error end. trie_bindings(X, Node) -> diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 53aeea7f..cee728ea 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -673,16 +673,14 @@ test_topic_matching() -> %% remove the entire exchange rabbit_exchange_type_topic:delete(X, RemainingBindings), %% none should match now - test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), + test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), passed. test_topic_expect_match(#exchange{name = XName}, List) -> lists:foreach( fun({Key, Expected}) -> - io:format("~p ~p~n", [Key, Expected]), Res = rabbit_exchange_type_topic:which_matches( XName, list_to_binary(Key)), - io:format("Res: ~p~n", [Res]), ExpectedRes = lists:map( fun(Q) -> #resource{virtual_host = <<"/">>, kind = queue, -- cgit v1.2.1 From acc77d592b4b13c25a8d147f5889d7703c3fd401 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 30 Sep 2010 15:20:58 +0100 Subject: minor --- src/rabbit_exchange_type_topic.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 9091d385..078bacb6 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -222,20 +222,20 @@ trie_binding_op(X, Node, Q, Op) -> trie_has_any_children(X, Node) -> MatchHead = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, node_id = Node, - word = '$1'}, + _='_'}, _='_'}, Select = mnesia:select(rabbit_topic_trie_edge, - [{MatchHead, [], ['$1']}], 1, read), + [{MatchHead, [], ['$_']}], 1, read), select_while_no_result(Select) /= '$end_of_table'. trie_has_any_bindings(X, Node) -> MatchHead = #topic_trie_binding{ trie_binding = #trie_binding{exchange_name = X, node_id = Node, - queue_name = '$1'}, + _='_'}, _='_'}, Select = mnesia:select(rabbit_topic_trie_binding, - [{MatchHead, [], ['$1']}], 1, read), + [{MatchHead, [], ['$_']}], 1, read), select_while_no_result(Select) /= '$end_of_table'. select_while_no_result({[], Cont}) -> -- cgit v1.2.1 From aee4d8dac6bc4c1b1caafd87b41453c61f6b225f Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 30 Sep 2010 16:06:20 +0100 Subject: avoid using qlc --- src/rabbit_exchange_type_topic.erl | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 078bacb6..15ce487d 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -30,7 +30,7 @@ %% -module(rabbit_exchange_type_topic). --include_lib("stdlib/include/qlc.hrl"). + -include("rabbit.hrl"). -behaviour(rabbit_exchange_type). @@ -152,7 +152,7 @@ follow_down_create(X, Words) -> {ok, FinalNode} -> FinalNode; {error, Node, RestW} -> lists:foldl( fun(W, CurNode) -> - NewNode = new_node(), + NewNode = new_node_id(), trie_add_edge(X, CurNode, NewNode, W), NewNode end, Node, RestW) @@ -244,30 +244,23 @@ select_while_no_result(Other) -> Other. trie_remove_all_edges(X) -> - Query = qlc:q([Entry || - Entry = #topic_trie_edge{ - trie_edge = #trie_edge{exchange_name = X1, - _='_'}, - _='_'} - <- mnesia:table(rabbit_topic_trie_edge), - X1 == X]), + MatchHead = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + _='_'}, + _='_'}, lists:foreach( - fun(O) -> mnesia:delete_object(rabbit_topic_trie_edge, O, write) end, - qlc:e(Query)). - + fun(R) -> mnesia:delete_object(rabbit_topic_trie_edge, R, write) end, + mnesia:select(rabbit_topic_trie_edge, [{MatchHead, [], ['$_']}])). + trie_remove_all_bindings(X) -> - Query = qlc:q([Entry || - Entry = #topic_trie_binding{ - trie_binding = #trie_binding{exchange_name = X1, - _='_'}, - _='_'} - <- mnesia:table(rabbit_topic_trie_binding), - X1 == X]), + MatchHead = #topic_trie_binding{trie_binding = + #trie_binding{exchange_name = X, + _='_'}, + _='_'}, lists:foreach( - fun(O) -> mnesia:delete_object(rabbit_topic_trie_binding, O, write) end, - qlc:e(Query)). + fun(R) -> mnesia:delete_object(rabbit_topic_trie_binding, R, write) end, + mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$_']}])). -new_node() -> +new_node_id() -> rabbit_guid:guid(). split_topic_key(Key) -> -- cgit v1.2.1 From cac0907c331a99082bbe0c5f0bc4eba990c3b98b Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 30 Sep 2010 20:09:28 +0100 Subject: using mnesia:match_object when selecting all records with a given pattern --- src/rabbit_exchange_type_topic.erl | 17 ++++++++--------- src/rabbit_tests.erl | 28 +++++++++++++++------------- 2 files changed, 23 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 15ce487d..2c2e589a 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -244,21 +244,20 @@ select_while_no_result(Other) -> Other. trie_remove_all_edges(X) -> - MatchHead = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - _='_'}, - _='_'}, + Pattern = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + _='_'}, + _='_'}, lists:foreach( fun(R) -> mnesia:delete_object(rabbit_topic_trie_edge, R, write) end, - mnesia:select(rabbit_topic_trie_edge, [{MatchHead, [], ['$_']}])). + mnesia:match_object(rabbit_topic_trie_edge, Pattern, write)). trie_remove_all_bindings(X) -> - MatchHead = #topic_trie_binding{trie_binding = - #trie_binding{exchange_name = X, - _='_'}, - _='_'}, + Pattern = #topic_trie_binding{trie_binding = #trie_binding{exchange_name =X, + _='_'}, + _='_'}, lists:foreach( fun(R) -> mnesia:delete_object(rabbit_topic_trie_binding, R, write) end, - mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$_']}])). + mnesia:match_object(rabbit_topic_trie_binding, Pattern, write)). new_node_id() -> rabbit_guid:guid(). diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index cee728ea..32c31bbf 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -622,32 +622,34 @@ test_topic_matching() -> {"#.*.#", "t22"}, {"*.#.#", "t23"}, {"#.#.#", "t24"}, - {"*", "t25"}]), + {"*", "t25"}, + {"#.b.#", "t26"}]), lists:foreach(fun(B) -> rabbit_exchange_type_topic:add_binding(X, B) end, Bindings), %% test some matches test_topic_expect_match(X, [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", "t18", "t20", - "t21", "t22", "t23", "t24"]}, + "t21", "t22", "t23", "t24", "t26"]}, {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", "t12", "t15", - "t21", "t22", "t23", "t24"]}, + "t21", "t22", "t23", "t24", "t26"]}, {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", "t18", "t21", - "t22", "t23", "t24"]}, + "t22", "t23", "t24", "t26"]}, {"", ["t5", "t6", "t17", "t24"]}, - {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", "t24"]}, + {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", "t24", "t26"]}, {"a.a.a.a.a", ["t5", "t6", "t11", "t12", "t21", "t22", "t23", "t24"]}, {"vodka.gin", ["t5", "t6", "t8", "t21", "t22", "t23", "t24"]}, {"vodka.martini", ["t5", "t6", "t8", "t19", "t21", "t22", "t23", "t24"]}, {"b.b.c", ["t5", "t6", "t10", "t13", "t18", "t21", "t22", "t23", - "t24"]}, + "t24", "t26"]}, {"nothing.here.at.all", ["t5", "t6", "t21", "t22", "t23", "t24"]}, {"oneword", ["t5", "t6", "t21", "t22", "t23", "t24", "t25"]}]), %% remove some bindings RemovedBindings = [lists:nth(1, Bindings), lists:nth(5, Bindings), - lists:nth(11, Bindings), lists:nth(21, Bindings)], + lists:nth(11, Bindings), lists:nth(19, Bindings), + lists:nth(21, Bindings)], rabbit_exchange_type_topic:remove_bindings(X, RemovedBindings), RemainingBindings = ordsets:to_list( ordsets:subtract(ordsets:from_list(Bindings), @@ -656,17 +658,17 @@ test_topic_matching() -> %% test some matches test_topic_expect_match(X, [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20", "t22", "t23", - "t24"]}, + "t24", "t26"]}, {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15", "t22", "t23", - "t24"]}, + "t24", "t26"]}, {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18", "t22", "t23", - "t24"]}, + "t24", "t26"]}, {"", ["t6", "t17", "t24"]}, - {"b.c.c", ["t6", "t18", "t22", "t23", "t24"]}, + {"b.c.c", ["t6", "t18", "t22", "t23", "t24", "t26"]}, {"a.a.a.a.a", ["t6", "t12", "t22", "t23", "t24"]}, {"vodka.gin", ["t6", "t8", "t22", "t23", "t24"]}, - {"vodka.martini", ["t6", "t8", "t19", "t22", "t23", "t24"]}, - {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", "t24"]}, + {"vodka.martini", ["t6", "t8", "t22", "t23", "t24"]}, + {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", "t24", "t26"]}, {"nothing.here.at.all", ["t6", "t22", "t23", "t24"]}, {"oneword", ["t6", "t22", "t23", "t24", "t25"]}]), -- cgit v1.2.1 From eba47400be1446e878ddfb6ec6799b2bd2d712b2 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Tue, 5 Oct 2010 11:41:39 +0100 Subject: cosmetics --- src/rabbit_exchange_type_topic.erl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 2c2e589a..486c4f80 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -72,21 +72,21 @@ create(_X) -> ok. recover(_X, _Bs) -> ok. delete(#exchange{name = X}, _Bs) -> - rabbit_misc:execute_mnesia_transaction(fun() -> trie_remove_all_edges(X), - trie_remove_all_bindings(X) + rabbit_misc:execute_mnesia_transaction(fun () -> trie_remove_all_edges(X), + trie_remove_all_bindings(X) end), ok. add_binding(_Exchange, #binding{exchange_name = X, key = K, queue_name = Q}) -> rabbit_misc:execute_mnesia_transaction( - fun() -> FinalNode = follow_down_create(X, split_topic_key(K)), - trie_add_binding(X, FinalNode, Q) + fun () -> FinalNode = follow_down_create(X, split_topic_key(K)), + trie_add_binding(X, FinalNode, Q) end), ok. remove_bindings(_X, Bs) -> rabbit_misc:execute_mnesia_transaction( - fun() -> lists:foreach(fun remove_binding/1, Bs) end), + fun () -> lists:foreach(fun remove_binding/1, Bs) end), ok. remove_binding(#binding{exchange_name = X, key = K, queue_name = Q}) -> @@ -151,10 +151,10 @@ follow_down_create(X, Words) -> case follow_down(X, Words) of {ok, FinalNode} -> FinalNode; {error, Node, RestW} -> lists:foldl( - fun(W, CurNode) -> - NewNode = new_node_id(), - trie_add_edge(X, CurNode, NewNode, W), - NewNode + fun (W, CurNode) -> + NewNode = new_node_id(), + trie_add_edge(X, CurNode, NewNode, W), + NewNode end, Node, RestW) end. @@ -170,8 +170,7 @@ follow_down_get_path(X, CurNode, [W | RestW], PathAcc) -> remove_path_if_empty(_, [{root, none}]) -> ok; remove_path_if_empty(X, [{Node, W} | [{Parent, _} | _] = RestPath]) -> - case trie_has_any_bindings(X, Node) orelse - trie_has_any_children(X, Node) of + case trie_has_any_bindings(X, Node) orelse trie_has_any_children(X, Node) of true -> ok; false -> trie_remove_edge(X, Parent, Node, W), remove_path_if_empty(X, RestPath) @@ -248,7 +247,7 @@ trie_remove_all_edges(X) -> _='_'}, _='_'}, lists:foreach( - fun(R) -> mnesia:delete_object(rabbit_topic_trie_edge, R, write) end, + fun (R) -> mnesia:delete_object(rabbit_topic_trie_edge, R, write) end, mnesia:match_object(rabbit_topic_trie_edge, Pattern, write)). trie_remove_all_bindings(X) -> @@ -256,7 +255,7 @@ trie_remove_all_bindings(X) -> _='_'}, _='_'}, lists:foreach( - fun(R) -> mnesia:delete_object(rabbit_topic_trie_binding, R, write) end, + fun (R) -> mnesia:delete_object(rabbit_topic_trie_binding, R, write) end, mnesia:match_object(rabbit_topic_trie_binding, Pattern, write)). new_node_id() -> -- cgit v1.2.1 From 45a97a0535d43f2556cb8caa39af0ff00538ea5c Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 14 Oct 2010 20:26:19 +0100 Subject: fixing merge conflicts --- src/rabbit_exchange_type_topic.erl | 45 ++++++++++++++------------------------ src/rabbit_router.erl | 3 --- src/rabbit_tests.erl | 36 ++++++++++++++++-------------- 3 files changed, 35 insertions(+), 49 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 89598958..3b0f1505 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -47,25 +47,17 @@ {requires, rabbit_exchange_type_registry}, {enables, kernel_ready}]}). --export([which_matches/2]). - --ifdef(use_specs). - --spec(which_matches/2 :: - (rabbit_exchange:name(), rabbit_router:routing_key()) -> - [rabbit_amqqueue:name()]). - --endif. - %%---------------------------------------------------------------------------- description() -> [{name, <<"topic">>}, {description, <<"AMQP topic exchange, as per the AMQP specification">>}]. -route(#exchange{name = X}, Delivery = - #delivery{message = #basic_message{routing_key = Key}}) -> - which_matches(X, Key). +%% NB: This may return duplicate results in some situations (that's ok) +route(#exchange{name = X}, + #delivery{message = #basic_message{routing_key = Key}}) -> + Words = split_topic_key(Key), + mnesia:async_dirty(fun trie_match/2, [X, Words]). validate(_X) -> ok. create(_X) -> ok. @@ -77,10 +69,10 @@ delete(#exchange{name = X}, _Bs) -> end), ok. -add_binding(_Exchange, #binding{exchange_name = X, key = K, queue_name = Q}) -> +add_binding(_Exchange, #binding{source = X, key = K, destination = D}) -> rabbit_misc:execute_mnesia_transaction( fun () -> FinalNode = follow_down_create(X, split_topic_key(K)), - trie_add_binding(X, FinalNode, Q) + trie_add_binding(X, FinalNode, D) end), ok. @@ -89,21 +81,16 @@ remove_bindings(_X, Bs) -> fun () -> lists:foreach(fun remove_binding/1, Bs) end), ok. -remove_binding(#binding{exchange_name = X, key = K, queue_name = Q}) -> +remove_binding(#binding{source = X, key = K, destination = D}) -> Path = follow_down_get_path(X, split_topic_key(K)), {FinalNode, _} = hd(Path), - trie_remove_binding(X, FinalNode, Q), + trie_remove_binding(X, FinalNode, D), remove_path_if_empty(X, Path), ok. assert_args_equivalence(X, Args) -> rabbit_exchange:assert_args_equivalence(X, Args). -%% NB: This function may return duplicate results in some situations (that's ok) -which_matches(X, Key) -> - Words = split_topic_key(Key), - mnesia:async_dirty(fun trie_match/2, [X, Words]). - %%---------------------------------------------------------------------------- trie_match(X, Words) -> @@ -188,7 +175,7 @@ trie_bindings(X, Node) -> MatchHead = #topic_trie_binding{ trie_binding = #trie_binding{exchange_name = X, node_id = Node, - queue_name = '$1'}}, + destination = '$1'}}, mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$1']}]). trie_add_edge(X, FromNode, ToNode, W) -> @@ -205,17 +192,17 @@ trie_edge_op(X, FromNode, ToNode, W, Op) -> node_id = ToNode}, write). -trie_add_binding(X, Node, Q) -> - trie_binding_op(X, Node, Q, fun mnesia:write/3). +trie_add_binding(X, Node, D) -> + trie_binding_op(X, Node, D, fun mnesia:write/3). -trie_remove_binding(X, Node, Q) -> - trie_binding_op(X, Node, Q, fun mnesia:delete_object/3). +trie_remove_binding(X, Node, D) -> + trie_binding_op(X, Node, D, fun mnesia:delete_object/3). -trie_binding_op(X, Node, Q, Op) -> +trie_binding_op(X, Node, D, Op) -> ok = Op(rabbit_topic_trie_binding, #topic_trie_binding{trie_binding = #trie_binding{exchange_name = X, node_id = Node, - queue_name = Q}}, + destination = D}}, write). trie_has_any_children(X, Node) -> diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index 05bda8b0..00df1ce1 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -84,9 +84,6 @@ deliver(QNames, Delivery) -> check_delivery(Delivery#delivery.mandatory, Delivery#delivery.immediate, {Routed, Handled}). -deliver_by_queue_names(Qs, Delivery) -> - deliver(lookup_qpids(Qs), Delivery). - %% TODO: Maybe this should be handled by a cursor instead. %% TODO: This causes a full scan for each entry with the same source match_bindings(SrcName, Match) -> diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 641fb6fb..4b3059be 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -592,12 +592,12 @@ test_topic_matching() -> %% add some bindings Bindings = lists:map( - fun({Key, Q}) -> - #binding{exchange_name = XName, - key = list_to_binary(Key), - queue_name = #resource{virtual_host = <<"/">>, - kind = queue, - name = list_to_binary(Q)}} + fun ({Key, Q}) -> + #binding{source = XName, + key = list_to_binary(Key), + destination = #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)}} end, [{"a.b.c", "t1"}, {"a.*.c", "t2"}, {"a.#.b", "t3"}, @@ -624,7 +624,7 @@ test_topic_matching() -> {"#.#.#", "t24"}, {"*", "t25"}, {"#.b.#", "t26"}]), - lists:foreach(fun(B) -> rabbit_exchange_type_topic:add_binding(X, B) end, + lists:foreach(fun (B) -> rabbit_exchange_type_topic:add_binding(X, B) end, Bindings), %% test some matches @@ -678,17 +678,19 @@ test_topic_matching() -> test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), passed. -test_topic_expect_match(#exchange{name = XName}, List) -> +test_topic_expect_match(X, List) -> lists:foreach( - fun({Key, Expected}) -> - Res = rabbit_exchange_type_topic:which_matches( - XName, list_to_binary(Key)), - ExpectedRes = lists:map( - fun(Q) -> #resource{virtual_host = <<"/">>, - kind = queue, - name = list_to_binary(Q)} - end, Expected), - true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) + fun ({Key, Expected}) -> + BinKey = list_to_binary(Key), + Res = rabbit_exchange_type_topic:route( + X, #delivery{message = #basic_message{routing_key = + BinKey}}), + ExpectedRes = lists:map( + fun (Q) -> #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)} + end, Expected), + true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) end, List). test_app_management() -> -- cgit v1.2.1 From 930102bb5d272318c64d28860b3ff0d7435aa79b Mon Sep 17 00:00:00 2001 From: Marek Majkowski Date: Wed, 8 Dec 2010 10:01:04 +0000 Subject: Add 'return' stats --- src/rabbit_channel.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 0c8ad00a..ada63ca2 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1219,10 +1219,16 @@ is_message_persistent(Content) -> IsPersistent end. -process_routing_result(unroutable, _, MsgSeqNo, Message, State) -> +process_routing_result(unroutable, _, MsgSeqNo, + Message = #basic_message{exchange_name = ExchangeName}, + State) -> + maybe_incr_stats([{ExchangeName, 1}], return, State), ok = basic_return(Message, State#ch.writer_pid, no_route), send_or_enqueue_ack(MsgSeqNo, undefined, State); -process_routing_result(not_delivered, _, MsgSeqNo, Message, State) -> +process_routing_result(not_delivered, _, MsgSeqNo, + Message = #basic_message{exchange_name = ExchangeName}, + State) -> + maybe_incr_stats([{ExchangeName, 1}], return, State), ok = basic_return(Message, State#ch.writer_pid, no_consumers), send_or_enqueue_ack(MsgSeqNo, undefined, State); process_routing_result(routed, [], MsgSeqNo, _, State) -> -- cgit v1.2.1 From 98c743c563055ded2d8dc34182b05795c295c17c Mon Sep 17 00:00:00 2001 From: Marek Majkowski Date: Wed, 8 Dec 2010 13:09:26 +0000 Subject: moved maybe_incr_stats to basic_return --- src/rabbit_channel.erl | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index ada63ca2..50677fc6 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1070,11 +1070,12 @@ binding_action(Fun, ExchangeNameBin, DestinationType, DestinationNameBin, basic_return(#basic_message{exchange_name = ExchangeName, routing_key = RoutingKey, content = Content}, - WriterPid, Reason) -> + State, Reason) -> + maybe_incr_stats([{ExchangeName, 1}], return, State), {_Close, ReplyCode, ReplyText} = rabbit_framing_amqp_0_9_1:lookup_amqp_exception(Reason), ok = rabbit_writer:send_command( - WriterPid, + State#ch.writer_pid, #'basic.return'{reply_code = ReplyCode, reply_text = ReplyText, exchange = ExchangeName#resource.name, @@ -1219,17 +1220,11 @@ is_message_persistent(Content) -> IsPersistent end. -process_routing_result(unroutable, _, MsgSeqNo, - Message = #basic_message{exchange_name = ExchangeName}, - State) -> - maybe_incr_stats([{ExchangeName, 1}], return, State), - ok = basic_return(Message, State#ch.writer_pid, no_route), +process_routing_result(unroutable, _, MsgSeqNo, Message, State) -> + ok = basic_return(Message, State, no_route), send_or_enqueue_ack(MsgSeqNo, undefined, State); -process_routing_result(not_delivered, _, MsgSeqNo, - Message = #basic_message{exchange_name = ExchangeName}, - State) -> - maybe_incr_stats([{ExchangeName, 1}], return, State), - ok = basic_return(Message, State#ch.writer_pid, no_consumers), +process_routing_result(not_delivered, _, MsgSeqNo, Message, State) -> + ok = basic_return(Message, State, no_consumers), send_or_enqueue_ack(MsgSeqNo, undefined, State); process_routing_result(routed, [], MsgSeqNo, _, State) -> send_or_enqueue_ack(MsgSeqNo, undefined, State); -- cgit v1.2.1 From d39b09caeb77f61ead9d1621bf808b6d5272d9bb Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Thu, 6 Jan 2011 19:04:33 +0000 Subject: Sender-specified distribution First attempt for direct exchanges only --- src/rabbit_exchange_type_direct.erl | 16 +++++++++++++--- src/rabbit_misc.erl | 11 ++++++++++- src/rabbit_router.erl | 21 +++++++++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index d49d0199..ab688853 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -31,6 +31,7 @@ -module(rabbit_exchange_type_direct). -include("rabbit.hrl"). +-include("rabbit_framing.hrl"). -behaviour(rabbit_exchange_type). @@ -50,9 +51,18 @@ description() -> [{name, <<"direct">>}, {description, <<"AMQP direct exchange, as per the AMQP specification">>}]. -route(#exchange{name = Name}, - #delivery{message = #basic_message{routing_key = RoutingKey}}) -> - rabbit_router:match_routing_key(Name, RoutingKey). +route(#exchange{name = #resource{virtual_host = VHost} = Name}, + #delivery{message = #basic_message{routing_key = RoutingKey, + content = Content}}) -> + BindingRoutes = rabbit_router:match_routing_key(Name, RoutingKey), + HeaderRKeys = + case (Content#content.properties)#'P_basic'.headers of + undefined -> []; + Headers -> rabbit_misc:table_lookup(Headers, <<"CC">>, <<0>>) ++ + rabbit_misc:table_lookup(Headers, <<"BCC">>, <<0>>) + end, + HeaderRoutes = [rabbit_misc:r(VHost, queue, RKey) || RKey <- HeaderRKeys], + lists:usort(BindingRoutes ++ HeaderRoutes). validate(_X) -> ok. create(_X) -> ok. diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index 15ba787a..604346ed 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -40,7 +40,7 @@ protocol_error/3, protocol_error/4, protocol_error/1]). -export([not_found/1, assert_args_equivalence/4]). -export([dirty_read/1]). --export([table_lookup/2]). +-export([table_lookup/3, table_lookup/2]). -export([r/3, r/2, r_arg/4, rs/1]). -export([enable_cover/0, report_cover/0]). -export([enable_cover/1, report_cover/1]). @@ -112,6 +112,8 @@ 'ok' | rabbit_types:connection_exit()). -spec(dirty_read/1 :: ({atom(), any()}) -> rabbit_types:ok_or_error2(any(), 'not_found')). +-spec(table_lookup/3 :: + (rabbit_framing:amqp_table(), binary(), binary()) -> [binary()]). -spec(table_lookup/2 :: (rabbit_framing:amqp_table(), binary()) -> 'undefined' | {rabbit_framing:amqp_field_type(), any()}). @@ -253,6 +255,13 @@ dirty_read(ReadSpec) -> [] -> {error, not_found} end. +table_lookup(Table, Key, Separator) -> + case table_lookup(Table, Key) of + undefined -> []; + {longstr, BinVal} -> binary:split(BinVal, Separator, [global]); + _ -> [] + end. + table_lookup(Table, Key) -> case lists:keysearch(Key, 1, Table) of {value, {_, TypeBin, ValueBin}} -> {TypeBin, ValueBin}; diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index d49c072c..2f556df7 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -32,6 +32,7 @@ -module(rabbit_router). -include_lib("stdlib/include/qlc.hrl"). -include("rabbit.hrl"). +-include("rabbit_framing.hrl"). -export([deliver/2, match_bindings/2, match_routing_key/2]). @@ -68,22 +69,38 @@ deliver(QNames, Delivery = #delivery{mandatory = false, %% is preserved. This scales much better than the non-immediate %% case below. QPids = lookup_qpids(QNames), + ModifiedDelivery = strip_header(Delivery, <<"BCC">>), delegate:invoke_no_result( - QPids, fun (Pid) -> rabbit_amqqueue:deliver(Pid, Delivery) end), + QPids, fun (Pid) -> rabbit_amqqueue:deliver(Pid, ModifiedDelivery) end), {routed, QPids}; deliver(QNames, Delivery = #delivery{mandatory = Mandatory, immediate = Immediate}) -> QPids = lookup_qpids(QNames), + ModifiedDelivery = strip_header(Delivery, <<"BCC">>), {Success, _} = delegate:invoke(QPids, fun (Pid) -> - rabbit_amqqueue:deliver(Pid, Delivery) + rabbit_amqqueue:deliver(Pid, ModifiedDelivery) end), {Routed, Handled} = lists:foldl(fun fold_deliveries/2, {false, []}, Success), check_delivery(Mandatory, Immediate, {Routed, Handled}). +strip_header(Delivery = #delivery{message = Message = #basic_message{ + content = Content = #content{ + properties = Props = #'P_basic'{headers = Headers}}}}, + Key) when Headers =/= undefined -> + case lists:keyfind(Key, 1, Headers) of + false -> Delivery; + Tuple -> Headers0 = lists:delete(Tuple, Headers), + Delivery#delivery{message = Message#basic_message{ + content = Content#content{ + properties_bin = none, + properties = Props#'P_basic'{headers = Headers0}}}} + end; +strip_header(Delivery, _Key) -> + Delivery. %% TODO: Maybe this should be handled by a cursor instead. %% TODO: This causes a full scan for each entry with the same source -- cgit v1.2.1 From 02c2dd6844e132b64abef90ecfdf01c9e9124d8d Mon Sep 17 00:00:00 2001 From: Alexandru Scvortov Date: Wed, 12 Jan 2011 11:59:09 +0000 Subject: swap union and intersection --- src/rabbit_variable_queue.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index c678236f..07297f63 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -1423,8 +1423,8 @@ msgs_written_to_disk(QPid, GuidSet, written) -> msgs_confirmed(gb_sets:intersection(GuidSet, MIOD), State #vqstate { msgs_on_disk = - gb_sets:intersection( - gb_sets:union(MOD, GuidSet), UC) }) + gb_sets:union( + MOD, gb_sets:intersection(UC, GuidSet)) }) end). msg_indices_written_to_disk(QPid, GuidSet) -> @@ -1435,8 +1435,8 @@ msg_indices_written_to_disk(QPid, GuidSet) -> msgs_confirmed(gb_sets:intersection(GuidSet, MOD), State #vqstate { msg_indices_on_disk = - gb_sets:intersection( - gb_sets:union(MIOD, GuidSet), UC) }) + gb_sets:union( + MIOD, gb_sets:intersection(UC, GuidSet)) }) end). %%---------------------------------------------------------------------------- -- cgit v1.2.1 From 4a7803245612862f0eaa34597affb4ed1bcbbc77 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Tue, 18 Jan 2011 14:18:36 +0000 Subject: Er, and this too, of course :) --- src/rabbit_multi.erl | 362 --------------------------------------------------- 1 file changed, 362 deletions(-) delete mode 100644 src/rabbit_multi.erl (limited to 'src') diff --git a/src/rabbit_multi.erl b/src/rabbit_multi.erl deleted file mode 100644 index 0030216e..00000000 --- a/src/rabbit_multi.erl +++ /dev/null @@ -1,362 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developers of the Original Code are LShift Ltd, -%% Cohesive Financial Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created before 22-Nov-2008 00:00:00 GMT by LShift Ltd, -%% Cohesive Financial Technologies LLC, or Rabbit Technologies Ltd -%% are Copyright (C) 2007-2008 LShift Ltd, Cohesive Financial -%% Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created by LShift Ltd are Copyright (C) 2007-2010 LShift -%% Ltd. Portions created by Cohesive Financial Technologies LLC are -%% Copyright (C) 2007-2010 Cohesive Financial Technologies -%% LLC. Portions created by Rabbit Technologies Ltd are Copyright -%% (C) 2007-2010 Rabbit Technologies Ltd. -%% -%% All Rights Reserved. -%% -%% Contributor(s): ______________________________________. -%% - --module(rabbit_multi). --include("rabbit.hrl"). - --export([start/0, stop/0]). - --define(RPC_SLEEP, 500). - -%%---------------------------------------------------------------------------- - --ifdef(use_specs). - --spec(start/0 :: () -> no_return()). --spec(stop/0 :: () -> 'ok'). --spec(usage/0 :: () -> no_return()). - --endif. - -%%---------------------------------------------------------------------------- - -start() -> - RpcTimeout = - case init:get_argument(maxwait) of - {ok,[[N1]]} -> 1000 * list_to_integer(N1); - _ -> ?MAX_WAIT - end, - case init:get_plain_arguments() of - [] -> - usage(); - FullCommand -> - {Command, Args} = parse_args(FullCommand), - case catch action(Command, Args, RpcTimeout) of - ok -> - io:format("done.~n"), - halt(); - {'EXIT', {function_clause, [{?MODULE, action, _} | _]}} -> - print_error("invalid command '~s'", - [string:join(FullCommand, " ")]), - usage(); - timeout -> - print_error("timeout starting some nodes.", []), - halt(1); - Other -> - print_error("~p", [Other]), - halt(2) - end - end. - -print_error(Format, Args) -> - rabbit_misc:format_stderr("Error: " ++ Format ++ "~n", Args). - -parse_args([Command | Args]) -> - {list_to_atom(Command), Args}. - -stop() -> - ok. - -usage() -> - io:format("~s", [rabbit_multi_usage:usage()]), - halt(1). - -action(start_all, [NodeCount], RpcTimeout) -> - io:format("Starting all nodes...~n", []), - application:load(rabbit), - {_NodeNamePrefix, NodeHost} = NodeName = rabbit_misc:nodeparts( - getenv("RABBITMQ_NODENAME")), - case net_adm:names(NodeHost) of - {error, EpmdReason} -> - throw({cannot_connect_to_epmd, NodeHost, EpmdReason}); - {ok, _} -> - ok - end, - {NodePids, Running} = - case list_to_integer(NodeCount) of - 1 -> {NodePid, Started} = start_node(rabbit_misc:makenode(NodeName), - RpcTimeout), - {[NodePid], Started}; - N -> start_nodes(N, N, [], true, NodeName, - get_node_tcp_listener(), RpcTimeout) - end, - write_pids_file(NodePids), - case Running of - true -> ok; - false -> timeout - end; - -action(status, [], RpcTimeout) -> - io:format("Status of all running nodes...~n", []), - call_all_nodes( - fun ({Node, Pid}) -> - RabbitRunning = - case is_rabbit_running(Node, RpcTimeout) of - false -> not_running; - true -> running - end, - io:format("Node '~p' with Pid ~p: ~p~n", - [Node, Pid, RabbitRunning]) - end); - -action(stop_all, [], RpcTimeout) -> - io:format("Stopping all nodes...~n", []), - call_all_nodes(fun ({Node, Pid}) -> - io:format("Stopping node ~p~n", [Node]), - rpc:call(Node, rabbit, stop_and_halt, []), - case kill_wait(Pid, RpcTimeout, false) of - false -> kill_wait(Pid, RpcTimeout, true); - true -> ok - end, - io:format("OK~n", []) - end), - delete_pids_file(); - -action(rotate_logs, [], RpcTimeout) -> - action(rotate_logs, [""], RpcTimeout); - -action(rotate_logs, [Suffix], RpcTimeout) -> - io:format("Rotating logs for all nodes...~n", []), - BinarySuffix = list_to_binary(Suffix), - call_all_nodes( - fun ({Node, _}) -> - io:format("Rotating logs for node ~p", [Node]), - case rpc:call(Node, rabbit, rotate_logs, - [BinarySuffix], RpcTimeout) of - {badrpc, Error} -> io:format(": ~p.~n", [Error]); - ok -> io:format(": ok.~n", []) - end - end). - -%% PNodePid is the list of PIDs -%% Running is a boolean exhibiting success at some moment -start_nodes(0, _, PNodePid, Running, _, _, _) -> {PNodePid, Running}; - -start_nodes(N, Total, PNodePid, Running, NodeNameBase, Listener, RpcTimeout) -> - {NodePre, NodeSuff} = NodeNameBase, - NodeNumber = Total - N, - NodePre1 = case NodeNumber of - %% For compatibility with running a single node - 0 -> NodePre; - _ -> NodePre ++ "_" ++ integer_to_list(NodeNumber) - end, - Node = rabbit_misc:makenode({NodePre1, NodeSuff}), - os:putenv("RABBITMQ_NODENAME", atom_to_list(Node)), - case Listener of - {NodeIpAddress, NodePortBase} -> - NodePort = NodePortBase + NodeNumber, - os:putenv("RABBITMQ_NODE_PORT", integer_to_list(NodePort)), - os:putenv("RABBITMQ_NODE_IP_ADDRESS", NodeIpAddress); - undefined -> - ok - end, - {NodePid, Started} = start_node(Node, RpcTimeout), - start_nodes(N - 1, Total, [NodePid | PNodePid], - Started and Running, NodeNameBase, Listener, RpcTimeout). - -start_node(Node, RpcTimeout) -> - io:format("Starting node ~s...~n", [Node]), - case rpc:call(Node, os, getpid, []) of - {badrpc, _} -> - Port = run_rabbitmq_server(), - Started = wait_for_rabbit_to_start(Node, RpcTimeout, Port), - Pid = case rpc:call(Node, os, getpid, []) of - {badrpc, _} -> throw(cannot_get_pid); - PidS -> list_to_integer(PidS) - end, - io:format("~s~n", [case Started of - true -> "OK"; - false -> "timeout" - end]), - {{Node, Pid}, Started}; - PidS -> - Pid = list_to_integer(PidS), - throw({node_already_running, Node, Pid}) - end. - -wait_for_rabbit_to_start(_ , RpcTimeout, _) when RpcTimeout < 0 -> - false; -wait_for_rabbit_to_start(Node, RpcTimeout, Port) -> - case is_rabbit_running(Node, RpcTimeout) of - true -> true; - false -> receive - {'EXIT', Port, PosixCode} -> - throw({node_start_failed, PosixCode}) - after ?RPC_SLEEP -> - wait_for_rabbit_to_start( - Node, RpcTimeout - ?RPC_SLEEP, Port) - end - end. - -run_rabbitmq_server() -> - with_os([{unix, fun run_rabbitmq_server_unix/0}, - {win32, fun run_rabbitmq_server_win32/0}]). - -run_rabbitmq_server_unix() -> - CmdLine = getenv("RABBITMQ_SCRIPT_HOME") ++ "/rabbitmq-server -noinput", - erlang:open_port({spawn, CmdLine}, [nouse_stdio]). - -run_rabbitmq_server_win32() -> - Cmd = filename:nativename(os:find_executable("cmd")), - CmdLine = "\"" ++ getenv("RABBITMQ_SCRIPT_HOME") ++ - "\\rabbitmq-server.bat\" -noinput -detached", - erlang:open_port({spawn_executable, Cmd}, - [{arg0, Cmd}, {args, ["/q", "/s", "/c", CmdLine]}, - nouse_stdio]). - -is_rabbit_running(Node, RpcTimeout) -> - case rpc:call(Node, rabbit, status, [], RpcTimeout) of - {badrpc, _} -> false; - Status -> case proplists:get_value(running_applications, Status) of - undefined -> false; - Apps -> lists:keymember(rabbit, 1, Apps) - end - end. - -with_os(Handlers) -> - {OsFamily, _} = os:type(), - case proplists:get_value(OsFamily, Handlers) of - undefined -> throw({unsupported_os, OsFamily}); - Handler -> Handler() - end. - -pids_file() -> getenv("RABBITMQ_PIDS_FILE"). - -write_pids_file(Pids) -> - FileName = pids_file(), - Handle = case file:open(FileName, [write]) of - {ok, Device} -> - Device; - {error, Reason} -> - throw({cannot_create_pids_file, FileName, Reason}) - end, - try - ok = io:write(Handle, Pids), - ok = io:put_chars(Handle, [$.]) - after - case file:close(Handle) of - ok -> ok; - {error, Reason1} -> - throw({cannot_create_pids_file, FileName, Reason1}) - end - end, - ok. - -delete_pids_file() -> - FileName = pids_file(), - case file:delete(FileName) of - ok -> ok; - {error, enoent} -> ok; - {error, Reason} -> throw({cannot_delete_pids_file, FileName, Reason}) - end. - -read_pids_file() -> - FileName = pids_file(), - case file:consult(FileName) of - {ok, [Pids]} -> Pids; - {error, enoent} -> []; - {error, Reason} -> throw({cannot_read_pids_file, FileName, Reason}) - end. - -kill_wait(Pid, TimeLeft, Forceful) when TimeLeft < 0 -> - Cmd = with_os([{unix, fun () -> if Forceful -> "kill -9"; - true -> "kill" - end - end}, - %% Kill forcefully always on Windows, since erl.exe - %% seems to completely ignore non-forceful killing - %% even when everything is working - {win32, fun () -> "taskkill /f /pid" end}]), - os:cmd(Cmd ++ " " ++ integer_to_list(Pid)), - false; % Don't assume what we did just worked! - -% Returns true if the process is dead, false otherwise. -kill_wait(Pid, TimeLeft, Forceful) -> - timer:sleep(?RPC_SLEEP), - io:format(".", []), - is_dead(Pid) orelse kill_wait(Pid, TimeLeft - ?RPC_SLEEP, Forceful). - -% Test using some OS clunkiness since we shouldn't trust -% rpc:call(os, getpid, []) at this point -is_dead(Pid) -> - PidS = integer_to_list(Pid), - with_os([{unix, fun () -> - system("kill -0 " ++ PidS - ++ " >/dev/null 2>&1") /= 0 - end}, - {win32, fun () -> - Res = os:cmd("tasklist /nh /fi \"pid eq " ++ - PidS ++ "\" 2>&1"), - case re:run(Res, "erl\\.exe", [{capture, none}]) of - match -> false; - _ -> true - end - end}]). - -% Like system(3) -system(Cmd) -> - ShCmd = "sh -c '" ++ escape_quotes(Cmd) ++ "'", - Port = erlang:open_port({spawn, ShCmd}, [exit_status,nouse_stdio]), - receive {Port, {exit_status, Status}} -> Status end. - -% Escape the quotes in a shell command so that it can be used in "sh -c 'cmd'" -escape_quotes(Cmd) -> - lists:flatten(lists:map(fun ($') -> "'\\''"; (Ch) -> Ch end, Cmd)). - -call_all_nodes(Func) -> - case read_pids_file() of - [] -> throw(no_nodes_running); - NodePids -> lists:foreach(Func, NodePids) - end. - -getenv(Var) -> - case os:getenv(Var) of - false -> throw({missing_env_var, Var}); - Value -> Value - end. - -get_node_tcp_listener() -> - try - {getenv("RABBITMQ_NODE_IP_ADDRESS"), - list_to_integer(getenv("RABBITMQ_NODE_PORT"))} - catch _ -> - case application:get_env(rabbit, tcp_listeners) of - {ok, [{_IpAddy, _Port} = Listener]} -> - Listener; - {ok, []} -> - undefined; - {ok, Other} -> - throw({cannot_start_multiple_nodes, multiple_tcp_listeners, - Other}); - undefined -> - throw({missing_configuration, tcp_listeners}) - end - end. -- cgit v1.2.1 From b88381ac6a1ea6badf70f0eec7384f9beb7a09bf Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Wed, 19 Jan 2011 12:49:30 +0000 Subject: Sender-specified distribution for fanout exchanges --- src/rabbit_exchange.erl | 26 +++++++++++++++++++++++++- src/rabbit_exchange_type_direct.erl | 11 +++-------- src/rabbit_exchange_type_fanout.erl | 10 ++++++++-- src/rabbit_misc.erl | 11 +---------- src/rabbit_router.erl | 5 +++-- 5 files changed, 40 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange.erl b/src/rabbit_exchange.erl index a95cf0b1..d9e3431d 100644 --- a/src/rabbit_exchange.erl +++ b/src/rabbit_exchange.erl @@ -35,6 +35,7 @@ -export([recover/0, declare/6, lookup/1, lookup_or_die/1, list/1, info_keys/0, info/1, info/2, info_all/1, info_all/2, publish/2, delete/2]). +-export([header_routes/2]). %% this must be run inside a mnesia tx -export([maybe_auto_delete/1]). -export([assert_equivalence/6, assert_args_equivalence/2, check_type/1]). @@ -86,7 +87,8 @@ -spec(maybe_auto_delete/1:: (rabbit_types:exchange()) -> 'not_deleted' | {'deleted', rabbit_binding:deletions()}). - +-spec(header_routes/2 :: (rabbit_framing:amqp_table(), rabbit_types:vhost()) -> + [rabbit_types:r('queue')]). -endif. %%---------------------------------------------------------------------------- @@ -319,3 +321,25 @@ unconditional_delete(X = #exchange{name = XName}) -> ok = mnesia:delete({rabbit_exchange, XName}), Bindings = rabbit_binding:remove_for_source(XName), {deleted, X, Bindings, rabbit_binding:remove_for_destination(XName)}. + +header_routes(undefined, _VHost) -> + []; +header_routes(Headers, VHost) -> + [rabbit_misc:r(VHost, queue, RKey) || + RKey <- lists:flatten([routing_keys(Headers, Header) || + Header <- ?ROUTING_HEADERS])]. + +routing_keys(HeadersTable, Key) -> + case rabbit_misc:table_lookup(HeadersTable, Key) of + {longstr, Route} -> [Route]; + {array, Routes} -> rkeys(Routes, []); + _ -> [] + end. + +rkeys([{longstr, BinVal} | Rest], RKeys) -> + rkeys(Rest, [BinVal | RKeys]); +rkeys([{_, _} | Rest], RKeys) -> + rkeys(Rest, RKeys); +rkeys(_, RKeys) -> + RKeys. + diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index ab688853..9547117c 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -55,14 +55,9 @@ route(#exchange{name = #resource{virtual_host = VHost} = Name}, #delivery{message = #basic_message{routing_key = RoutingKey, content = Content}}) -> BindingRoutes = rabbit_router:match_routing_key(Name, RoutingKey), - HeaderRKeys = - case (Content#content.properties)#'P_basic'.headers of - undefined -> []; - Headers -> rabbit_misc:table_lookup(Headers, <<"CC">>, <<0>>) ++ - rabbit_misc:table_lookup(Headers, <<"BCC">>, <<0>>) - end, - HeaderRoutes = [rabbit_misc:r(VHost, queue, RKey) || RKey <- HeaderRKeys], - lists:usort(BindingRoutes ++ HeaderRoutes). + HeaderRoutes = rabbit_exchange:header_routes( + (Content#content.properties)#'P_basic'.headers, VHost), + BindingRoutes ++ HeaderRoutes. validate(_X) -> ok. create(_X) -> ok. diff --git a/src/rabbit_exchange_type_fanout.erl b/src/rabbit_exchange_type_fanout.erl index e7f75464..e9faf0a2 100644 --- a/src/rabbit_exchange_type_fanout.erl +++ b/src/rabbit_exchange_type_fanout.erl @@ -31,6 +31,7 @@ -module(rabbit_exchange_type_fanout). -include("rabbit.hrl"). +-include("rabbit_framing.hrl"). -behaviour(rabbit_exchange_type). @@ -50,8 +51,13 @@ description() -> [{name, <<"fanout">>}, {description, <<"AMQP fanout exchange, as per the AMQP specification">>}]. -route(#exchange{name = Name}, _Delivery) -> - rabbit_router:match_routing_key(Name, '_'). +route(#exchange{name = #resource{virtual_host = VHost} = Name}, + #delivery{message = #basic_message{content = Content}}) -> + BindingRoutes = rabbit_router:match_routing_key(Name, '_'), + HeaderRoutes = rabbit_exchange:header_routes( + (Content#content.properties)#'P_basic'.headers, VHost), + BindingRoutes ++ HeaderRoutes. + validate(_X) -> ok. create(_X) -> ok. diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index 604346ed..15ba787a 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -40,7 +40,7 @@ protocol_error/3, protocol_error/4, protocol_error/1]). -export([not_found/1, assert_args_equivalence/4]). -export([dirty_read/1]). --export([table_lookup/3, table_lookup/2]). +-export([table_lookup/2]). -export([r/3, r/2, r_arg/4, rs/1]). -export([enable_cover/0, report_cover/0]). -export([enable_cover/1, report_cover/1]). @@ -112,8 +112,6 @@ 'ok' | rabbit_types:connection_exit()). -spec(dirty_read/1 :: ({atom(), any()}) -> rabbit_types:ok_or_error2(any(), 'not_found')). --spec(table_lookup/3 :: - (rabbit_framing:amqp_table(), binary(), binary()) -> [binary()]). -spec(table_lookup/2 :: (rabbit_framing:amqp_table(), binary()) -> 'undefined' | {rabbit_framing:amqp_field_type(), any()}). @@ -255,13 +253,6 @@ dirty_read(ReadSpec) -> [] -> {error, not_found} end. -table_lookup(Table, Key, Separator) -> - case table_lookup(Table, Key) of - undefined -> []; - {longstr, BinVal} -> binary:split(BinVal, Separator, [global]); - _ -> [] - end. - table_lookup(Table, Key) -> case lists:keysearch(Key, 1, Table) of {value, {_, TypeBin, ValueBin}} -> {TypeBin, ValueBin}; diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index 2f556df7..7f9b823e 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -69,7 +69,7 @@ deliver(QNames, Delivery = #delivery{mandatory = false, %% is preserved. This scales much better than the non-immediate %% case below. QPids = lookup_qpids(QNames), - ModifiedDelivery = strip_header(Delivery, <<"BCC">>), + ModifiedDelivery = strip_header(Delivery, ?DELETED_HEADER), delegate:invoke_no_result( QPids, fun (Pid) -> rabbit_amqqueue:deliver(Pid, ModifiedDelivery) end), {routed, QPids}; @@ -77,7 +77,7 @@ deliver(QNames, Delivery = #delivery{mandatory = false, deliver(QNames, Delivery = #delivery{mandatory = Mandatory, immediate = Immediate}) -> QPids = lookup_qpids(QNames), - ModifiedDelivery = strip_header(Delivery, <<"BCC">>), + ModifiedDelivery = strip_header(Delivery, ?DELETED_HEADER), {Success, _} = delegate:invoke(QPids, fun (Pid) -> @@ -87,6 +87,7 @@ deliver(QNames, Delivery = #delivery{mandatory = Mandatory, lists:foldl(fun fold_deliveries/2, {false, []}, Success), check_delivery(Mandatory, Immediate, {Routed, Handled}). +%% This breaks the spec rule forbidding message modification strip_header(Delivery = #delivery{message = Message = #basic_message{ content = Content = #content{ properties = Props = #'P_basic'{headers = Headers}}}}, -- cgit v1.2.1 From 69f35fc60d84b1ffe4424dfe7d47f909bec8e423 Mon Sep 17 00:00:00 2001 From: Alexandru Scvortov Date: Wed, 19 Jan 2011 14:38:43 +0000 Subject: replace the sort with a gb_tree Instead of creating a list and sorting it, insert the MsgSeqNos into a gb_tree. Dicts and orddicts are slower. --- src/rabbit_amqqueue_process.erl | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl index 38b83117..b0aea012 100644 --- a/src/rabbit_amqqueue_process.erl +++ b/src/rabbit_amqqueue_process.erl @@ -431,27 +431,22 @@ confirm_messages(Guids, State = #q{guid_to_channel = GTC}) -> fun(Guid, {CMs, GTC0}) -> case dict:find(Guid, GTC0) of {ok, {ChPid, MsgSeqNo}} -> - {[{ChPid, MsgSeqNo} | CMs], dict:erase(Guid, GTC0)}; + {gb_trees_cons(ChPid, MsgSeqNo, CMs), + dict:erase(Guid, GTC0)}; _ -> {CMs, GTC0} end - end, {[], GTC}, Guids), - case lists:usort(CMs) of - [{Ch, MsgSeqNo} | CMs1] -> - [rabbit_channel:confirm(ChPid, MsgSeqNos) || - {ChPid, MsgSeqNos} <- group_confirms_by_channel( - CMs1, [{Ch, [MsgSeqNo]}])]; - [] -> - ok - end, + end, {gb_trees:empty(), GTC}, Guids), + gb_trees:map(fun(ChPid, MsgSeqNos) -> + rabbit_channel:confirm(ChPid, MsgSeqNos) + end, CMs), State#q{guid_to_channel = GTC1}. -group_confirms_by_channel([], Acc) -> - Acc; -group_confirms_by_channel([{Ch, Msg1} | CMs], [{Ch, Msgs} | Acc]) -> - group_confirms_by_channel(CMs, [{Ch, [Msg1 | Msgs]} | Acc]); -group_confirms_by_channel([{Ch, Msg1} | CMs], Acc) -> - group_confirms_by_channel(CMs, [{Ch, [Msg1]} | Acc]). +gb_trees_cons(Key, Value, Tree) -> + case gb_trees:lookup(Key, Tree) of + {value, Values} -> gb_trees:update(Key, [Value | Values], Tree); + none -> gb_trees:insert(Key, [Value], Tree) + end. record_confirm_message(#delivery{msg_seq_no = undefined}, State) -> {no_confirm, State}; -- cgit v1.2.1 From 73da7e693ef652858bc348bd340bc2df9d3440ef Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Wed, 19 Jan 2011 18:25:31 +0000 Subject: fixing merge conflicts; some stylistic adjustments --- src/rabbit_exchange_type_topic.erl | 149 +++++++++++++++++++------------------ src/rabbit_tests.erl | 76 +++++++++++-------- 2 files changed, 121 insertions(+), 104 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 2da3f3ee..2e181f1d 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -48,27 +48,28 @@ validate(_X) -> ok. create(_Tx, _X) -> ok. recover(_X, _Bs) -> ok. -delete(_Tx, #exchange{name = X}, _Bs) -> - rabbit_misc:execute_mnesia_transaction(fun () -> trie_remove_all_edges(X), - trie_remove_all_bindings(X) - end), +delete(true, #exchange{name = X}, _Bs) -> + trie_remove_all_edges(X), + trie_remove_all_bindings(X), + ok; +delete(false, _Exchange, _Bs) -> ok. -add_binding(_Tx, _Exchange, #binding{source = X, key = K, destination = D}) -> - rabbit_misc:execute_mnesia_transaction( - fun () -> FinalNode = follow_down_create(X, split_topic_key(K)), - trie_add_binding(X, FinalNode, D) - end), +add_binding(true, _Exchange, #binding{source = X, key = K, destination = D}) -> + FinalNode = follow_down_create(X, split_topic_key(K)), + trie_add_binding(X, FinalNode, D), + ok; +add_binding(false, _Exchange, _Binding) -> ok. -remove_bindings(_Tx, _X, Bs) -> - rabbit_misc:execute_mnesia_transaction( - fun () -> lists:foreach(fun remove_binding/1, Bs) end), +remove_bindings(true, _X, Bs) -> + lists:foreach(fun remove_binding/1, Bs), + ok; +remove_bindings(false, _X, _Bs) -> ok. remove_binding(#binding{source = X, key = K, destination = D}) -> - Path = follow_down_get_path(X, split_topic_key(K)), - {FinalNode, _} = hd(Path), + Path = [{FinalNode, _} | _] = follow_down_get_path(X, split_topic_key(K)), trie_remove_binding(X, FinalNode, D), remove_path_if_empty(X, Path), ok. @@ -108,19 +109,8 @@ trie_match_skip_any(X, Node, []) -> trie_match_skip_any(X, Node, [_ | RestW] = Words) -> trie_match(X, Node, Words) ++ trie_match_skip_any(X, Node, RestW). -follow_down(X, Words) -> - follow_down(X, root, Words). - -follow_down(_X, CurNode, []) -> - {ok, CurNode}; -follow_down(X, CurNode, [W | RestW]) -> - case trie_child(X, CurNode, W) of - {ok, NextNode} -> follow_down(X, NextNode, RestW); - error -> {error, CurNode, [W | RestW]} - end. - follow_down_create(X, Words) -> - case follow_down(X, Words) of + case follow_down_last_node(X, Words) of {ok, FinalNode} -> FinalNode; {error, Node, RestW} -> lists:foldl( fun (W, CurNode) -> @@ -130,14 +120,26 @@ follow_down_create(X, Words) -> end, Node, RestW) end. +follow_down_last_node(X, Words) -> + follow_down(X, fun (_, Node, _) -> Node end, root, Words). + follow_down_get_path(X, Words) -> - follow_down_get_path(X, root, Words, [{root, none}]). + {ok, Path} = + follow_down(X, fun (W, Node, PathAcc) -> [{Node, W} | PathAcc] end, + [{root, none}], Words), + Path. + +follow_down(X, AccFun, Acc0, Words) -> + follow_down(X, root, AccFun, Acc0, Words). -follow_down_get_path(_, _, [], PathAcc) -> - PathAcc; -follow_down_get_path(X, CurNode, [W | RestW], PathAcc) -> - {ok, NextNode} = trie_child(X, CurNode, W), - follow_down_get_path(X, NextNode, RestW, [{NextNode, W} | PathAcc]). +follow_down(_X, _CurNode, _AccFun, Acc, []) -> + {ok, Acc}; +follow_down(X, CurNode, AccFun, Acc, Words = [W | RestW]) -> + case trie_child(X, CurNode, W) of + {ok, NextNode} -> follow_down(X, NextNode, AccFun, + AccFun(W, NextNode, Acc), RestW); + error -> {error, Acc, Words} + end. remove_path_if_empty(_, [{root, none}]) -> ok; @@ -149,9 +151,10 @@ remove_path_if_empty(X, [{Node, W} | [{Parent, _} | _] = RestPath]) -> end. trie_child(X, Node, Word) -> - case mnesia:read(rabbit_topic_trie_edge, #trie_edge{exchange_name = X, - node_id = Node, - word = Word}) of + case mnesia:read(rabbit_topic_trie_edge, + #trie_edge{exchange_name = X, + node_id = Node, + word = Word}) of [#topic_trie_edge{node_id = NextNode}] -> {ok, NextNode}; [] -> error end. @@ -159,8 +162,8 @@ trie_child(X, Node, Word) -> trie_bindings(X, Node) -> MatchHead = #topic_trie_binding{ trie_binding = #trie_binding{exchange_name = X, - node_id = Node, - destination = '$1'}}, + node_id = Node, + destination = '$1'}}, mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$1']}]). trie_add_edge(X, FromNode, ToNode, W) -> @@ -172,9 +175,9 @@ trie_remove_edge(X, FromNode, ToNode, W) -> trie_edge_op(X, FromNode, ToNode, W, Op) -> ok = Op(rabbit_topic_trie_edge, #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - node_id = FromNode, - word = W}, - node_id = ToNode}, + node_id = FromNode, + word = W}, + node_id = ToNode}, write). trie_add_binding(X, Node, D) -> @@ -185,28 +188,41 @@ trie_remove_binding(X, Node, D) -> trie_binding_op(X, Node, D, Op) -> ok = Op(rabbit_topic_trie_binding, - #topic_trie_binding{trie_binding = #trie_binding{exchange_name = X, - node_id = Node, - destination = D}}, + #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X, + node_id = Node, + destination = D}}, write). trie_has_any_children(X, Node) -> - MatchHead = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - node_id = Node, - _='_'}, - _='_'}, - Select = mnesia:select(rabbit_topic_trie_edge, - [{MatchHead, [], ['$_']}], 1, read), - select_while_no_result(Select) /= '$end_of_table'. + has_any(rabbit_topic_trie_edge, + #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + node_id = Node, + _ = '_'}, + _ = '_'}). trie_has_any_bindings(X, Node) -> - MatchHead = #topic_trie_binding{ - trie_binding = #trie_binding{exchange_name = X, - node_id = Node, - _='_'}, - _='_'}, - Select = mnesia:select(rabbit_topic_trie_binding, - [{MatchHead, [], ['$_']}], 1, read), + has_any(rabbit_topic_trie_binding, + #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X, + node_id = Node, + _ = '_'}, + _ = '_'}). + +trie_remove_all_edges(X) -> + remove_all(rabbit_topic_trie_edge, + #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, + _ = '_'}, + _ = '_'}). + +trie_remove_all_bindings(X) -> + remove_all(rabbit_topic_trie_binding, + #topic_trie_binding{ + trie_binding = #trie_binding{exchange_name = X, _ = '_'}, + _ = '_'}). + +has_any(Table, MatchHead) -> + Select = mnesia:select(Table, [{MatchHead, [], ['$_']}], 1, read), select_while_no_result(Select) /= '$end_of_table'. select_while_no_result({[], Cont}) -> @@ -214,21 +230,9 @@ select_while_no_result({[], Cont}) -> select_while_no_result(Other) -> Other. -trie_remove_all_edges(X) -> - Pattern = #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - _='_'}, - _='_'}, - lists:foreach( - fun (R) -> mnesia:delete_object(rabbit_topic_trie_edge, R, write) end, - mnesia:match_object(rabbit_topic_trie_edge, Pattern, write)). - -trie_remove_all_bindings(X) -> - Pattern = #topic_trie_binding{trie_binding = #trie_binding{exchange_name =X, - _='_'}, - _='_'}, - lists:foreach( - fun (R) -> mnesia:delete_object(rabbit_topic_trie_binding, R, write) end, - mnesia:match_object(rabbit_topic_trie_binding, Pattern, write)). +remove_all(Table, Pattern) -> + lists:foreach(fun (R) -> mnesia:delete_object(Table, R, write) end, + mnesia:match_object(Table, Pattern, write)). new_node_id() -> rabbit_guid:guid(). @@ -244,3 +248,4 @@ split_topic_key(<<$., Rest/binary>>, RevWordAcc, RevResAcc) -> split_topic_key(Rest, [], [lists:reverse(RevWordAcc) | RevResAcc]); split_topic_key(<>, RevWordAcc, RevResAcc) -> split_topic_key(Rest, [C | RevWordAcc], RevResAcc). + diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index b80f3692..32cdaa52 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -588,7 +588,7 @@ test_topic_matching() -> auto_delete = false, arguments = []}, %% create rabbit_exchange_type_topic:validate(X), - rabbit_exchange_type_topic:create(X), + exchange_op_callback(X, create, []), %% add some bindings Bindings = lists:map( @@ -624,60 +624,72 @@ test_topic_matching() -> {"#.#.#", "t24"}, {"*", "t25"}, {"#.b.#", "t26"}]), - lists:foreach(fun (B) -> rabbit_exchange_type_topic:add_binding(X, B) end, + lists:foreach(fun (B) -> exchange_op_callback(X, add_binding, [B]) end, Bindings), %% test some matches test_topic_expect_match(X, - [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", "t18", "t20", - "t21", "t22", "t23", "t24", "t26"]}, - {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", "t12", "t15", - "t21", "t22", "t23", "t24", "t26"]}, - {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", "t18", "t21", - "t22", "t23", "t24", "t26"]}, - {"", ["t5", "t6", "t17", "t24"]}, - {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", "t24", "t26"]}, - {"a.a.a.a.a", ["t5", "t6", "t11", "t12", "t21", "t22", "t23", "t24"]}, - {"vodka.gin", ["t5", "t6", "t8", "t21", "t22", "t23", "t24"]}, - {"vodka.martini", ["t5", "t6", "t8", "t19", "t21", "t22", "t23", - "t24"]}, - {"b.b.c", ["t5", "t6", "t10", "t13", "t18", "t21", "t22", "t23", - "t24", "t26"]}, + [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", + "t18", "t20", "t21", "t22", "t23", "t24", + "t26"]}, + {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", + "t12", "t15", "t21", "t22", "t23", "t24", + "t26"]}, + {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", + "t18", "t21", "t22", "t23", "t24", "t26"]}, + {"", ["t5", "t6", "t17", "t24"]}, + {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", "t24", + "t26"]}, + {"a.a.a.a.a", ["t5", "t6", "t11", "t12", "t21", "t22", "t23", + "t24"]}, + {"vodka.gin", ["t5", "t6", "t8", "t21", "t22", "t23", + "t24"]}, + {"vodka.martini", ["t5", "t6", "t8", "t19", "t21", "t22", "t23", + "t24"]}, + {"b.b.c", ["t5", "t6", "t10", "t13", "t18", "t21", "t22", + "t23", "t24", "t26"]}, {"nothing.here.at.all", ["t5", "t6", "t21", "t22", "t23", "t24"]}, - {"oneword", ["t5", "t6", "t21", "t22", "t23", "t24", "t25"]}]), + {"oneword", ["t5", "t6", "t21", "t22", "t23", "t24", + "t25"]}]), %% remove some bindings RemovedBindings = [lists:nth(1, Bindings), lists:nth(5, Bindings), lists:nth(11, Bindings), lists:nth(19, Bindings), lists:nth(21, Bindings)], - rabbit_exchange_type_topic:remove_bindings(X, RemovedBindings), + exchange_op_callback(X, remove_bindings, [RemovedBindings]), RemainingBindings = ordsets:to_list( ordsets:subtract(ordsets:from_list(Bindings), ordsets:from_list(RemovedBindings))), %% test some matches test_topic_expect_match(X, - [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20", "t22", "t23", - "t24", "t26"]}, - {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15", "t22", "t23", - "t24", "t26"]}, - {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18", "t22", "t23", - "t24", "t26"]}, - {"", ["t6", "t17", "t24"]}, - {"b.c.c", ["t6", "t18", "t22", "t23", "t24", "t26"]}, - {"a.a.a.a.a", ["t6", "t12", "t22", "t23", "t24"]}, - {"vodka.gin", ["t6", "t8", "t22", "t23", "t24"]}, - {"vodka.martini", ["t6", "t8", "t22", "t23", "t24"]}, - {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", "t24", "t26"]}, + [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20", "t22", + "t23", "t24", "t26"]}, + {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15", + "t22", "t23", "t24", "t26"]}, + {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18", "t22", + "t23", "t24", "t26"]}, + {"", ["t6", "t17", "t24"]}, + {"b.c.c", ["t6", "t18", "t22", "t23", "t24", "t26"]}, + {"a.a.a.a.a", ["t6", "t12", "t22", "t23", "t24"]}, + {"vodka.gin", ["t6", "t8", "t22", "t23", "t24"]}, + {"vodka.martini", ["t6", "t8", "t22", "t23", "t24"]}, + {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", + "t24", "t26"]}, {"nothing.here.at.all", ["t6", "t22", "t23", "t24"]}, - {"oneword", ["t6", "t22", "t23", "t24", "t25"]}]), + {"oneword", ["t6", "t22", "t23", "t24", "t25"]}]), %% remove the entire exchange - rabbit_exchange_type_topic:delete(X, RemainingBindings), + exchange_op_callback(X, delete, [RemainingBindings]), %% none should match now test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), passed. +exchange_op_callback(X, Fun, ExtraArgs) -> + rabbit_misc:execute_mnesia_transaction( + fun () -> rabbit_exchange:callback(X, Fun, [true, X] ++ ExtraArgs) end), + rabbit_exchange:callback(X, Fun, [false, X] ++ ExtraArgs). + test_topic_expect_match(X, List) -> lists:foreach( fun ({Key, Expected}) -> -- cgit v1.2.1 From 0b35d977d92af97c5c0d36ef890f2a4ac9a48881 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 21 Jan 2011 12:06:49 +0000 Subject: Adding gm related files, plucked from branch bug23554 --- src/gm.erl | 1308 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/gm_test.erl | 126 ++++++ 2 files changed, 1434 insertions(+) create mode 100644 src/gm.erl create mode 100644 src/gm_test.erl (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl new file mode 100644 index 00000000..baf46471 --- /dev/null +++ b/src/gm.erl @@ -0,0 +1,1308 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2010 VMware, Inc. All rights reserved. +%% + +-module(gm). + +%% Guaranteed Multicast +%% ==================== +%% +%% This module provides the ability to create named groups of +%% processes to which members can be dynamically added and removed, +%% and for messages to be broadcast within the group that are +%% guaranteed to reach all members of the group during the lifetime of +%% the message. The lifetime of a message is defined as being, at a +%% minimum, the time from which the message is first sent to any +%% member of the group, up until the time at which it is known by the +%% member who published the message that the message has reached all +%% group members. +%% +%% The guarantee given is that provided a message, once sent, makes it +%% to members who do not all leave the group, the message will +%% continue to propagate to all group members. +%% +%% Another way of stating the guarantee is that if member P publishes +%% messages m and m', then for all members P', if P' is a member of +%% the group prior to the publication of m, and P' receives m', then +%% P' will receive m. +%% +%% Note that only local-ordering is enforced: i.e. if member P sends +%% message m and then message m', then for-all members P', if P' +%% receives m and m', then they will receive m' after m. Causality +%% ordering is _not_ enforced. I.e. if member P receives message m +%% and as a result publishes message m', there is no guarantee that +%% other members P' will receive m before m'. +%% +%% +%% API Use +%% ------- +%% +%% Mnesia must be started. Use the idempotent create_tables/0 function +%% to create the tables required. +%% +%% start_link/3 +%% Provide the group name, the callback module name, and a list of any +%% arguments you wish to be passed into the callback module's +%% functions. The joined/1 will be called when we have joined the +%% group, and the list of arguments will have appended to it a list of +%% the current members of the group. See the comments in +%% behaviour_info/1 below for further details of the callback +%% functions. +%% +%% leave/1 +%% Provide the Pid. Removes the Pid from the group. The callback +%% terminate/1 function will be called. +%% +%% broadcast/2 +%% Provide the Pid and a Message. The message will be sent to all +%% members of the group as per the guarantees given above. This is a +%% cast and the function call will return immediately. There is no +%% guarantee that the message will reach any member of the group. +%% +%% confirmed_broadcast/2 +%% Provide the Pid and a Message. As per broadcast/2 except that this +%% is a call, not a cast, and only returns 'ok' once the Message has +%% reached every member of the group. Do not call +%% confirmed_broadcast/2 directly from the callback module otherwise +%% you will deadlock the entire group. +%% +%% group_members/1 +%% Provide the Pid. Returns a list of the current group members. +%% +%% +%% Implementation Overview +%% ----------------------- +%% +%% One possible means of implementation would be a fan-out from the +%% sender to every member of the group. This would require that the +%% group is fully connected, and, in the event that the original +%% sender of the message disappears from the group before the message +%% has made it to every member of the group, raises questions as to +%% who is responsible for sending on the message to new group members. +%% In particular, the issue is with [ Pid ! Msg || Pid <- Members ] - +%% if the sender dies part way through, who is responsible for +%% ensuring that the remaining Members receive the Msg? In the event +%% that within the group, messages sent are broadcast from a subset of +%% the members, the fan-out arrangement has the potential to +%% substantially impact the CPU and network workload of such members, +%% as such members would have to accommodate the cost of sending each +%% message to every group member. +%% +%% Instead, if the members of the group are arranged in a chain, then +%% it becomes easier to reason about who within the group has received +%% each message and who has not. It eases issues of responsibility: in +%% the event of a group member disappearing, the nearest upstream +%% member of the chain is responsible for ensuring that messages +%% continue to propagate down the chain. It also results in equal +%% distribution of sending and receiving workload, even if all +%% messages are being sent from just a single group member. This +%% configuration has the further advantage that it is not necessary +%% for every group member to know of every other group member, and +%% even that a group member does not have to be accessible from all +%% other group members. +%% +%% Performance is kept high by permitting pipelining and all +%% communication between joined group members is asynchronous. In the +%% chain A -> B -> C -> D, if A sends a message to the group, it will +%% not directly contact C or D. However, it must know that D receives +%% the message (in addition to B and C) before it can consider the +%% message fully sent. A simplistic implementation would require that +%% D replies to C, C replies to B and B then replies to A. This would +%% result in a propagation delay of twice the length of the chain. It +%% would also require, in the event of the failure of C, that D knows +%% to directly contact B and issue the necessary replies. Instead, the +%% chain forms a ring: D sends the message on to A: D does not +%% distinguish A as the sender, merely as the next member (downstream) +%% within the chain (which has now become a ring). When A receives +%% from D messages that A sent, it knows that all members have +%% received the message. However, the message is not dead yet: if C +%% died as B was sending to C, then B would need to detect the death +%% of C and forward the message on to D instead: thus every node has +%% to remember every message published until it is told that it can +%% forget about the message. This is essential not just for dealing +%% with failure of members, but also for the addition of new members. +%% +%% Thus once A receives the message back again, it then sends to B an +%% acknowledgement for the message, indicating that B can now forget +%% about the message. B does so, and forwards the ack to C. C forgets +%% the message, and forwards the ack to D, which forgets the message +%% and finally forwards the ack back to A. At this point, A takes no +%% further action: the message and its acknowledgement have made it to +%% every member of the group. The message is now dead, and any new +%% member joining the group at this point will not receive the +%% message. +%% +%% We therefore have two roles: +%% +%% 1. The sender, who upon receiving their own messages back, must +%% then send out acknowledgements, and upon receiving their own +%% acknowledgements back perform no further action. +%% +%% 2. The other group members who upon receiving messages and +%% acknowledgements must update their own internal state accordingly +%% (the sending member must also do this in order to be able to +%% accommodate failures), and forwards messages on to their downstream +%% neighbours. +%% +%% +%% Implementation: It gets trickier +%% -------------------------------- +%% +%% Chain A -> B -> C -> D +%% +%% A publishes a message which B receives. A now dies. B and D will +%% detect the death of A, and will link up, thus the chain is now B -> +%% C -> D. B forwards A's message on to C, who forwards it to D, who +%% forwards it to B. Thus B is now responsible for A's messages - both +%% publications and acknowledgements that were in flight at the point +%% at which A died. Even worse is that this is transitive: after B +%% forwards A's message to C, B dies as well. Now C is not only +%% responsible for B's in-flight messages, but is also responsible for +%% A's in-flight messages. +%% +%% Lemma 1: A member can only determine which dead members they have +%% inherited responsibility for if there is a total ordering on the +%% conflicting additions and subtractions of members from the group. +%% +%% Consider the simultaneous death of B and addition of B' that +%% transitions a chain from A -> B -> C to A -> B' -> C. Either B' or +%% C is responsible for in-flight messages from B. It is easy to +%% ensure that at least one of them thinks they have inherited B, but +%% if we do not ensure that exactly one of them inherits B, then we +%% could have B' converting publishes to acks, which then will crash C +%% as C does not believe it has issued acks for those messages. +%% +%% More complex scenarios are easy to concoct: A -> B -> C -> D -> E +%% becoming A -> C' -> E. Who has inherited which of B, C and D? +%% +%% However, for non-conflicting membership changes, only a partial +%% ordering is required. For example, A -> B -> C becoming A -> A' -> +%% B. The addition of A', between A and B can have no conflicts with +%% the death of C: it is clear that A has inherited C's messages. +%% +%% For ease of implementation, we adopt the simple solution, of +%% imposing a total order on all membership changes. +%% +%% On the death of a member, it is ensured the dead member's +%% neighbours become aware of the death, and the upstream neighbour +%% now sends to its new downstream neighbour its state, including the +%% messages pending acknowledgement. The downstream neighbour can then +%% use this to calculate which publishes and acknowledgements it has +%% missed out on, due to the death of its old upstream. Thus the +%% downstream can catch up, and continues the propagation of messages +%% through the group. +%% +%% Lemma 2: When a member is joining, it must synchronously +%% communicate with its upstream member in order to receive its +%% starting state atomically with its addition to the group. +%% +%% New members must start with the same state as their nearest +%% upstream neighbour. This ensures that it is not surprised by +%% acknowledgements they are sent, and that should their downstream +%% neighbour die, they are able to send the correct state to their new +%% downstream neighbour to ensure it can catch up. Thus in the +%% transition A -> B -> C becomes A -> A' -> B -> C becomes A -> A' -> +%% C, A' must start with the state of A, so that it can send C the +%% correct state when B dies, allowing C to detect any missed +%% messages. +%% +%% If A' starts by adding itself to the group membership, A could then +%% die, without A' having received the necessary state from A. This +%% would leave A' responsible for in-flight messages from A, but +%% having the least knowledge of all, of those messages. Thus A' must +%% start by synchronously calling A, which then immediately sends A' +%% back its state. A then adds A' to the group. If A dies at this +%% point then A' will be able to see this (as A' will fail to appear +%% in the group membership), and thus A' will ignore the state it +%% receives from A, and will simply repeat the process, trying to now +%% join downstream from some other member. This ensures that should +%% the upstream die as soon as the new member has been joined, the new +%% member is guaranteed to receive the correct state, allowing it to +%% correctly process messages inherited due to the death of its +%% upstream neighbour. +%% +%% The canonical definition of the group membership is held by a +%% distributed database. Whilst this allows the total ordering of +%% changes to be achieved, it is nevertheless undesirable to have to +%% query this database for the current view, upon receiving each +%% message. Instead, we wish for members to be able to cache a view of +%% the group membership, which then requires a cache invalidation +%% mechanism. Each member maintains its own view of the group +%% membership. Thus when the group's membership changes, members may +%% need to become aware of such changes in order to be able to +%% accurately process messages they receive. Because of the +%% requirement of a total ordering of conflicting membership changes, +%% it is not possible to use the guaranteed broadcast mechanism to +%% communicate these changes: to achieve the necessary ordering, it +%% would be necessary for such messages to be published by exactly one +%% member, which can not be guaranteed given that such a member could +%% die. +%% +%% The total ordering we enforce on membership changes gives rise to a +%% view version number: every change to the membership creates a +%% different view, and the total ordering permits a simple +%% monotonically increasing view version number. +%% +%% Lemma 3: If a message is sent from a member that holds view version +%% N, it can be correctly processed by any member receiving the +%% message with a view version >= N. +%% +%% Initially, let us suppose that each view contains the ordering of +%% every member that was ever part of the group. Dead members are +%% marked as such. Thus we have a ring of members, some of which are +%% dead, and are thus inherited by the nearest alive downstream +%% member. +%% +%% In the chain A -> B -> C, all three members initially have view +%% version 1, which reflects reality. B publishes a message, which is +%% forward by C to A. B now dies, which A notices very quickly. Thus A +%% updates the view, creating version 2. It now forwards B's +%% publication, sending that message to its new downstream neighbour, +%% C. This happens before C is aware of the death of B. C must become +%% aware of the view change before it interprets the message its +%% received, otherwise it will fail to learn of the death of B, and +%% thus will not realise it has inherited B's messages (and will +%% likely crash). +%% +%% Thus very simply, we have that each subsequent view contains more +%% information than the preceding view. +%% +%% However, to avoid the views growing indefinitely, we need to be +%% able to delete members which have died _and_ for which no messages +%% are in-flight. This requires that upon inheriting a dead member, we +%% know the last publication sent by the dead member (this is easy: we +%% inherit a member because we are the nearest downstream member which +%% implies that we know at least as much than everyone else about the +%% publications of the dead member), and we know the earliest message +%% for which the acknowledgement is still in flight. +%% +%% In the chain A -> B -> C, when B dies, A will send to C its state +%% (as C is the new downstream from A), allowing C to calculate which +%% messages it has missed out on (described above). At this point, C +%% also inherits B's messages. If that state from A also includes the +%% last message published by B for which an acknowledgement has been +%% seen, then C knows exactly which further acknowledgements it must +%% receive (also including issuing acknowledgements for publications +%% still in-flight that it receives), after which it is known there +%% are no more messages in flight for B, thus all evidence that B was +%% ever part of the group can be safely removed from the canonical +%% group membership. +%% +%% Thus, for every message that a member sends, it includes with that +%% message its view version. When a member receives a message it will +%% update its view from the canonical copy, should its view be older +%% than the view version included in the message it has received. +%% +%% The state held by each member therefore includes the messages from +%% each publisher pending acknowledgement, the last publication seen +%% from that publisher, and the last acknowledgement from that +%% publisher. In the case of the member's own publications or +%% inherited members, this last acknowledgement seen state indicates +%% the last acknowledgement retired, rather than sent. +%% +%% +%% Proof sketch +%% ------------ +%% +%% We need to prove that with the provided operational semantics, we +%% can never reach a state that is not well formed from a well-formed +%% starting state. +%% +%% Operational semantics (small step): straight-forward message +%% sending, process monitoring, state updates. +%% +%% Well formed state: dead members inherited by exactly one non-dead +%% member; for every entry in anyone's pending-acks, either (the +%% publication of the message is in-flight downstream from the member +%% and upstream from the publisher) or (the acknowledgement of the +%% message is in-flight downstream from the publisher and upstream +%% from the member). +%% +%% Proof by induction on the applicable operational semantics. +%% +%% +%% Related work +%% ------------ +%% +%% The ring configuration and double traversal of messages around the +%% ring is similar (though developed independently) to the LCR +%% protocol by [Levy 2008]. However, LCR differs in several +%% ways. Firstly, by using vector clocks, it enforces a total order of +%% message delivery, which is unnecessary for our purposes. More +%% significantly, it is built on top of a "group communication system" +%% which performs the group management functions, taking +%% responsibility away from the protocol as to how to cope with safely +%% adding and removing members. When membership changes do occur, the +%% protocol stipulates that every member must perform communication +%% with every other member of the group, to ensure all outstanding +%% deliveries complete, before the entire group transitions to the new +%% view. This, in total, requires two sets of all-to-all synchronous +%% communications. +%% +%% This is not only rather inefficient, but also does not explain what +%% happens upon the failure of a member during this process. It does +%% though entirely avoid the need for inheritance of responsibility of +%% dead members that our protocol incorporates. +%% +%% In [Marandi et al 2010], a Paxos-based protocol is described. This +%% work explicitly focuses on the efficiency of communication. LCR +%% (and our protocol too) are more efficient, but at the cost of +%% higher latency. The Ring-Paxos protocol is itself built on top of +%% IP-multicast, which rules it out for many applications where +%% point-to-point communication is all that can be required. They also +%% have an excellent related work section which I really ought to +%% read... +%% +%% +%% [Levy 2008] The Complexity of Reliable Distributed Storage, 2008. +%% [Marandi et al 2010] Ring Paxos: A High-Throughput Atomic Broadcast +%% Protocol + + +-behaviour(gen_server2). + +-export([create_tables/0, start_link/3, leave/1, broadcast/2, + confirmed_broadcast/2, group_members/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3, prioritise_info/2]). + +-export([behaviour_info/1]). + +-export([table_definitions/0]). + +-define(GROUP_TABLE, gm_group). +-define(HIBERNATE_AFTER_MIN, 1000). +-define(DESIRED_HIBERNATE, 10000). +-define(SETS, ordsets). +-define(DICT, orddict). + +-record(state, + { self, + left, + right, + group_name, + module, + view, + pub_count, + members_state, + callback_args, + confirms + }). + +-record(gm_group, { name, version, members }). + +-record(view_member, { id, aliases, left, right }). + +-record(member, { pending_ack, last_pub, last_ack }). + +-define(TABLE, {?GROUP_TABLE, [{record_name, gm_group}, + {attributes, record_info(fields, gm_group)}]}). +-define(TABLE_MATCH, {match, #gm_group { _ = '_' }}). + +-define(TAG, '$gm'). + +-ifdef(use_specs). + +-export_type([group_name/0]). + +-type(group_name() :: any()). + +-spec(create_tables/0 :: () -> 'ok'). +-spec(start_link/3 :: (group_name(), atom(), [any()]) -> + {'ok', pid()} | {'error', any()}). +-spec(leave/1 :: (pid()) -> 'ok'). +-spec(broadcast/2 :: (pid(), any()) -> 'ok'). +-spec(confirmed_broadcast/2 :: (pid(), any()) -> 'ok'). +-spec(group_members/1 :: (pid()) -> [pid()]). + +-endif. + +behaviour_info(callbacks) -> + [ + %% Called when we've successfully joined the group. Supplied with + %% Args provided in start_link, plus current group members. + {joined, 2}, + + %% Supplied with Args provided in start_link, the list of new + %% members and the list of members previously known to us that + %% have since died. Note that if a member joins and dies very + %% quickly, it's possible that we will never see that member + %% appear in either births or deaths. However we are guaranteed + %% that (1) we will see a member joining either in the births + %% here, or in the members passed to joined/1 before receiving + %% any messages from it; and (2) we will not see members die that + %% we have not seen born (or supplied in the members to + %% joined/1). + {members_changed, 3}, + + %% Supplied with Args provided in start_link, the sender, and the + %% message. This does get called for messages injected by this + %% member, however, in such cases, there is no special + %% significance of this call: it does not indicate that the + %% message has made it to any other members, let alone all other + %% members. + {handle_msg, 3}, + + %% Called on gm member termination as per rules in gen_server, + %% with the Args provided in start_link plus the termination + %% Reason. + {terminate, 2} + ]; +behaviour_info(_Other) -> + undefined. + +create_tables() -> + create_tables([?TABLE]). + +create_tables([]) -> + ok; +create_tables([{Table, Attributes} | Tables]) -> + case mnesia:create_table(Table, Attributes) of + {atomic, ok} -> create_tables(Tables); + {aborted, {already_exists, gm_group}} -> create_tables(Tables); + Err -> Err + end. + +table_definitions() -> + {Name, Attributes} = ?TABLE, + [{Name, [?TABLE_MATCH | Attributes]}]. + +start_link(GroupName, Module, Args) -> + gen_server2:start_link(?MODULE, [GroupName, Module, Args], []). + +leave(Server) -> + gen_server2:cast(Server, leave). + +broadcast(Server, Msg) -> + gen_server2:cast(Server, {broadcast, Msg}). + +confirmed_broadcast(Server, Msg) -> + gen_server2:call(Server, {confirmed_broadcast, Msg}, infinity). + +group_members(Server) -> + gen_server2:call(Server, group_members, infinity). + + +init([GroupName, Module, Args]) -> + random:seed(now()), + gen_server2:cast(self(), join), + Self = self(), + {ok, #state { self = Self, + left = {Self, undefined}, + right = {Self, undefined}, + group_name = GroupName, + module = Module, + view = undefined, + pub_count = 0, + members_state = undefined, + callback_args = Args, + confirms = queue:new() }, hibernate, + {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. + + +handle_call({confirmed_broadcast, _Msg}, _From, + State = #state { members_state = undefined }) -> + reply(not_joined, State); + +handle_call({confirmed_broadcast, Msg}, _From, + State = #state { self = Self, + right = {Self, undefined}, + module = Module, + callback_args = Args }) -> + handle_callback_result({Module:handle_msg(Args, Self, Msg), ok, State}); + +handle_call({confirmed_broadcast, Msg}, From, State) -> + internal_broadcast(Msg, From, State); + +handle_call(group_members, _From, + State = #state { members_state = undefined }) -> + reply(not_joined, State); + +handle_call(group_members, _From, State = #state { view = View }) -> + reply(alive_view_members(View), State); + +handle_call({add_on_right, _NewMember}, _From, + State = #state { members_state = undefined }) -> + reply(not_ready, State); + +handle_call({add_on_right, NewMember}, _From, + State = #state { self = Self, + group_name = GroupName, + view = View, + members_state = MembersState, + module = Module, + callback_args = Args }) -> + Group = record_new_member_in_group( + GroupName, Self, NewMember, + fun (Group1) -> + View1 = group_to_view(Group1), + ok = send_right(NewMember, View1, + {catchup, Self, prepare_members_state( + MembersState)}) + end), + View2 = group_to_view(Group), + State1 = check_neighbours(State #state { view = View2 }), + Result = callback_view_changed(Args, Module, View, View2), + handle_callback_result({Result, {ok, Group}, State1}). + + +handle_cast({?TAG, ReqVer, Msg}, + State = #state { view = View, + group_name = GroupName, + module = Module, + callback_args = Args }) -> + {Result, State1} = + case needs_view_update(ReqVer, View) of + true -> + View1 = group_to_view(read_group(GroupName)), + {callback_view_changed(Args, Module, View, View1), + check_neighbours(State #state { view = View1 })}; + false -> + {ok, State} + end, + handle_callback_result( + if_callback_success( + Result, fun handle_msg_true/3, fun handle_msg_false/3, Msg, State1)); + +handle_cast({broadcast, _Msg}, State = #state { members_state = undefined }) -> + noreply(State); + +handle_cast({broadcast, Msg}, + State = #state { self = Self, + right = {Self, undefined}, + module = Module, + callback_args = Args }) -> + handle_callback_result({Module:handle_msg(Args, Self, Msg), State}); + +handle_cast({broadcast, Msg}, State) -> + internal_broadcast(Msg, none, State); + +handle_cast(join, State = #state { self = Self, + group_name = GroupName, + members_state = undefined, + module = Module, + callback_args = Args }) -> + View = join_group(Self, GroupName), + MembersState = + case alive_view_members(View) of + [Self] -> blank_member_state(); + _ -> undefined + end, + State1 = check_neighbours(State #state { view = View, + members_state = MembersState }), + handle_callback_result( + {Module:joined(Args, all_known_members(View)), State1}); + +handle_cast(leave, State) -> + {stop, normal, State}. + + +handle_info({'DOWN', MRef, process, _Pid, _Reason}, + State = #state { self = Self, + left = Left, + right = Right, + group_name = GroupName, + view = View, + module = Module, + callback_args = Args, + confirms = Confirms }) -> + Member = case {Left, Right} of + {{Member1, MRef}, _} -> Member1; + {_, {Member1, MRef}} -> Member1; + _ -> undefined + end, + case Member of + undefined -> + noreply(State); + _ -> + View1 = + group_to_view(record_dead_member_in_group(Member, GroupName)), + State1 = State #state { view = View1 }, + {Result, State2} = + case alive_view_members(View1) of + [Self] -> + maybe_erase_aliases( + State1 #state { + members_state = blank_member_state(), + confirms = purge_confirms(Confirms) }); + _ -> + %% here we won't be pointing out any deaths: + %% the concern is that there maybe births + %% which we'd otherwise miss. + {callback_view_changed(Args, Module, View, View1), + State1} + end, + handle_callback_result({Result, check_neighbours(State2)}) + end. + + +terminate(Reason, #state { module = Module, + callback_args = Args }) -> + Module:terminate(Args, Reason). + + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +prioritise_info({'DOWN', _MRef, process, _Pid, _Reason}, _State) -> 1; +prioritise_info(_ , _State) -> 0. + + +handle_msg(check_neighbours, State) -> + %% no-op - it's already been done by the calling handle_cast + {ok, State}; + +handle_msg({catchup, Left, MembersStateLeft}, + State = #state { self = Self, + left = {Left, _MRefL}, + right = {Right, _MRefR}, + view = View, + members_state = undefined }) -> + ok = send_right(Right, View, {catchup, Self, MembersStateLeft}), + MembersStateLeft1 = build_members_state(MembersStateLeft), + {ok, State #state { members_state = MembersStateLeft1 }}; + +handle_msg({catchup, Left, MembersStateLeft}, + State = #state { self = Self, + left = {Left, _MRefL}, + view = View, + members_state = MembersState }) + when MembersState =/= undefined -> + MembersStateLeft1 = build_members_state(MembersStateLeft), + AllMembers = lists:usort(?DICT:fetch_keys(MembersState) ++ + ?DICT:fetch_keys(MembersStateLeft1)), + {MembersState1, Activity} = + lists:foldl( + fun (Id, MembersStateActivity) -> + #member { pending_ack = PALeft, last_ack = LA } = + find_member_or_blank(Id, MembersStateLeft1), + with_member_acc( + fun (#member { pending_ack = PA } = Member, Activity1) -> + case is_member_alias(Id, Self, View) of + true -> + {_AcksInFlight, Pubs, _PA1} = + find_prefix_common_suffix(PALeft, PA), + {Member #member { last_ack = LA }, + activity_cons(Id, pubs_from_queue(Pubs), + [], Activity1)}; + false -> + {Acks, _Common, Pubs} = + find_prefix_common_suffix(PA, PALeft), + {Member, + activity_cons(Id, pubs_from_queue(Pubs), + acks_from_queue(Acks), + Activity1)} + end + end, Id, MembersStateActivity) + end, {MembersState, activity_nil()}, AllMembers), + handle_msg({activity, Left, activity_finalise(Activity)}, + State #state { members_state = MembersState1 }); + +handle_msg({catchup, _NotLeft, _MembersState}, State) -> + {ok, State}; + +handle_msg({activity, Left, Activity}, + State = #state { self = Self, + left = {Left, _MRefL}, + view = View, + members_state = MembersState, + confirms = Confirms }) + when MembersState =/= undefined -> + {MembersState1, {Confirms1, Activity1}} = + lists:foldl( + fun ({Id, Pubs, Acks}, MembersStateConfirmsActivity) -> + with_member_acc( + fun (Member = #member { pending_ack = PA, + last_pub = LP, + last_ack = LA }, + {Confirms2, Activity2}) -> + case is_member_alias(Id, Self, View) of + true -> + {ToAck, PA1} = + find_common(queue_from_pubs(Pubs), PA, + queue:new()), + LA1 = last_ack(Acks, LA), + AckNums = acks_from_queue(ToAck), + Confirms3 = maybe_confirm( + Self, Id, Confirms2, AckNums), + {Member #member { pending_ack = PA1, + last_ack = LA1 }, + {Confirms3, + activity_cons( + Id, [], AckNums, Activity2)}}; + false -> + PA1 = apply_acks(Acks, join_pubs(PA, Pubs)), + LA1 = last_ack(Acks, LA), + LP1 = last_pub(Pubs, LP), + {Member #member { pending_ack = PA1, + last_pub = LP1, + last_ack = LA1 }, + {Confirms2, + activity_cons(Id, Pubs, Acks, Activity2)}} + end + end, Id, MembersStateConfirmsActivity) + end, {MembersState, {Confirms, activity_nil()}}, Activity), + State1 = State #state { members_state = MembersState1, + confirms = Confirms1 }, + Activity3 = activity_finalise(Activity1), + {Result, State2} = maybe_erase_aliases(State1), + ok = maybe_send_activity(Activity3, State2), + if_callback_success( + Result, fun activity_true/3, fun activity_false/3, Activity3, State2); + +handle_msg({activity, _NotLeft, _Activity}, State) -> + {ok, State}. + + +noreply(State) -> + {noreply, State, hibernate}. + +reply(Reply, State) -> + {reply, Reply, State, hibernate}. + +internal_broadcast(Msg, From, State = #state { self = Self, + pub_count = PubCount, + members_state = MembersState, + module = Module, + confirms = Confirms, + callback_args = Args }) -> + PubMsg = {PubCount, Msg}, + Activity = activity_cons(Self, [PubMsg], [], activity_nil()), + ok = maybe_send_activity(activity_finalise(Activity), State), + MembersState1 = + with_member( + fun (Member = #member { pending_ack = PA }) -> + Member #member { pending_ack = queue:in(PubMsg, PA) } + end, Self, MembersState), + Confirms1 = case From of + none -> Confirms; + _ -> queue:in({PubCount, From}, Confirms) + end, + handle_callback_result({Module:handle_msg(Args, Self, Msg), + State #state { pub_count = PubCount + 1, + members_state = MembersState1, + confirms = Confirms1 }}). + + +%% --------------------------------------------------------------------------- +%% View construction and inspection +%% --------------------------------------------------------------------------- + +needs_view_update(ReqVer, {Ver, _View}) -> + Ver < ReqVer. + +view_version({Ver, _View}) -> + Ver. + +is_member_alive({dead, _Member}) -> false; +is_member_alive(_) -> true. + +is_member_alias(Self, Self, _View) -> + true; +is_member_alias(Member, Self, View) -> + ?SETS:is_element(Member, + ((fetch_view_member(Self, View)) #view_member.aliases)). + +dead_member_id({dead, Member}) -> Member. + +store_view_member(VMember = #view_member { id = Id }, {Ver, View}) -> + {Ver, ?DICT:store(Id, VMember, View)}. + +with_view_member(Fun, View, Id) -> + store_view_member(Fun(fetch_view_member(Id, View)), View). + +fetch_view_member(Id, {_Ver, View}) -> + ?DICT:fetch(Id, View). + +find_view_member(Id, {_Ver, View}) -> + ?DICT:find(Id, View). + +blank_view(Ver) -> + {Ver, ?DICT:new()}. + +alive_view_members({_Ver, View}) -> + ?DICT:fetch_keys(View). + +all_known_members({_Ver, View}) -> + ?DICT:fold( + fun (Member, #view_member { aliases = Aliases }, Acc) -> + ?SETS:to_list(Aliases) ++ [Member | Acc] + end, [], View). + +group_to_view(#gm_group { members = Members, version = Ver }) -> + Alive = lists:filter(fun is_member_alive/1, Members), + [_|_] = Alive, %% ASSERTION - can't have all dead members + add_aliases(link_view(Alive ++ Alive ++ Alive, blank_view(Ver)), Members). + +link_view([Left, Middle, Right | Rest], View) -> + case find_view_member(Middle, View) of + error -> + link_view( + [Middle, Right | Rest], + store_view_member(#view_member { id = Middle, + aliases = ?SETS:new(), + left = Left, + right = Right }, View)); + {ok, _} -> + View + end; +link_view(_, View) -> + View. + +add_aliases(View, Members) -> + Members1 = ensure_alive_suffix(Members), + {EmptyDeadSet, View1} = + lists:foldl( + fun (Member, {DeadAcc, ViewAcc}) -> + case is_member_alive(Member) of + true -> + {?SETS:new(), + with_view_member( + fun (VMember = + #view_member { aliases = Aliases }) -> + VMember #view_member { + aliases = ?SETS:union(Aliases, DeadAcc) } + end, ViewAcc, Member)}; + false -> + {?SETS:add_element(dead_member_id(Member), DeadAcc), + ViewAcc} + end + end, {?SETS:new(), View}, Members1), + 0 = ?SETS:size(EmptyDeadSet), %% ASSERTION + View1. + +ensure_alive_suffix(Members) -> + queue:to_list(ensure_alive_suffix1(queue:from_list(Members))). + +ensure_alive_suffix1(MembersQ) -> + {{value, Member}, MembersQ1} = queue:out_r(MembersQ), + case is_member_alive(Member) of + true -> MembersQ; + false -> ensure_alive_suffix1(queue:in_r(Member, MembersQ1)) + end. + + +%% --------------------------------------------------------------------------- +%% View modification +%% --------------------------------------------------------------------------- + +join_group(Self, GroupName) -> + join_group(Self, GroupName, read_group(GroupName)). + +join_group(Self, GroupName, {error, not_found}) -> + join_group(Self, GroupName, prune_or_create_group(Self, GroupName)); +join_group(Self, _GroupName, #gm_group { members = [Self] } = Group) -> + group_to_view(Group); +join_group(Self, GroupName, #gm_group { members = Members } = Group) -> + case lists:member(Self, Members) of + true -> + group_to_view(Group); + false -> + case lists:filter(fun is_member_alive/1, Members) of + [] -> + join_group(Self, GroupName, + prune_or_create_group(Self, GroupName)); + Alive -> + Left = lists:nth(random:uniform(length(Alive)), Alive), + try + case gen_server2:call( + Left, {add_on_right, Self}, infinity) of + {ok, Group1} -> group_to_view(Group1); + not_ready -> join_group(Self, GroupName) + end + catch + exit:{R, _} + when R =:= noproc; R =:= normal; R =:= shutdown -> + join_group( + Self, GroupName, + record_dead_member_in_group(Left, GroupName)) + end + end + end. + +read_group(GroupName) -> + case mnesia:dirty_read(?GROUP_TABLE, GroupName) of + [] -> {error, not_found}; + [Group] -> Group + end. + +prune_or_create_group(Self, GroupName) -> + {atomic, Group} = + mnesia:sync_transaction( + fun () -> GroupNew = #gm_group { name = GroupName, + members = [Self], + version = 0 }, + case mnesia:read(?GROUP_TABLE, GroupName) of + [] -> + mnesia:write(GroupNew), + GroupNew; + [Group1 = #gm_group { members = Members }] -> + case lists:any(fun is_member_alive/1, Members) of + true -> Group1; + false -> mnesia:write(GroupNew), + GroupNew + end + end + end), + Group. + +record_dead_member_in_group(Member, GroupName) -> + {atomic, Group} = + mnesia:sync_transaction( + fun () -> [Group1 = #gm_group { members = Members, version = Ver }] = + mnesia:read(?GROUP_TABLE, GroupName), + case lists:splitwith( + fun (Member1) -> Member1 =/= Member end, Members) of + {_Members1, []} -> %% not found - already recorded dead + Group1; + {Members1, [Member | Members2]} -> + Members3 = Members1 ++ [{dead, Member} | Members2], + Group2 = Group1 #gm_group { members = Members3, + version = Ver + 1 }, + mnesia:write(Group2), + Group2 + end + end), + Group. + +record_new_member_in_group(GroupName, Left, NewMember, Fun) -> + {atomic, Group} = + mnesia:sync_transaction( + fun () -> + [#gm_group { members = Members, version = Ver } = Group1] = + mnesia:read(?GROUP_TABLE, GroupName), + {Prefix, [Left | Suffix]} = + lists:splitwith(fun (M) -> M =/= Left end, Members), + Members1 = Prefix ++ [Left, NewMember | Suffix], + Group2 = Group1 #gm_group { members = Members1, + version = Ver + 1 }, + ok = Fun(Group2), + mnesia:write(Group2), + Group2 + end), + Group. + +erase_members_in_group(Members, GroupName) -> + DeadMembers = [{dead, Id} || Id <- Members], + {atomic, Group} = + mnesia:sync_transaction( + fun () -> + [Group1 = #gm_group { members = [_|_] = Members1, + version = Ver }] = + mnesia:read(?GROUP_TABLE, GroupName), + case Members1 -- DeadMembers of + Members1 -> Group1; + Members2 -> Group2 = + Group1 #gm_group { members = Members2, + version = Ver + 1 }, + mnesia:write(Group2), + Group2 + end + end), + Group. + +maybe_erase_aliases(State = #state { self = Self, + group_name = GroupName, + view = View, + members_state = MembersState, + module = Module, + callback_args = Args }) -> + #view_member { aliases = Aliases } = fetch_view_member(Self, View), + {Erasable, MembersState1} + = ?SETS:fold( + fun (Id, {ErasableAcc, MembersStateAcc} = Acc) -> + #member { last_pub = LP, last_ack = LA } = + find_member_or_blank(Id, MembersState), + case can_erase_view_member(Self, Id, LA, LP) of + true -> {[Id | ErasableAcc], + erase_member(Id, MembersStateAcc)}; + false -> Acc + end + end, {[], MembersState}, Aliases), + State1 = State #state { members_state = MembersState1 }, + case Erasable of + [] -> {ok, State1}; + _ -> View1 = group_to_view( + erase_members_in_group(Erasable, GroupName)), + {callback_view_changed(Args, Module, View, View1), + State1 #state { view = View1 }} + end. + +can_erase_view_member(Self, Self, _LA, _LP) -> false; +can_erase_view_member(_Self, _Id, N, N) -> true; +can_erase_view_member(_Self, _Id, _LA, _LP) -> false. + + +%% --------------------------------------------------------------------------- +%% View monitoring and maintanence +%% --------------------------------------------------------------------------- + +ensure_neighbour(_Ver, Self, {Self, undefined}, Self) -> + {Self, undefined}; +ensure_neighbour(Ver, Self, {Self, undefined}, RealNeighbour) -> + ok = gen_server2:cast(RealNeighbour, {?TAG, Ver, check_neighbours}), + {RealNeighbour, maybe_monitor(RealNeighbour, Self)}; +ensure_neighbour(_Ver, _Self, {RealNeighbour, MRef}, RealNeighbour) -> + {RealNeighbour, MRef}; +ensure_neighbour(Ver, Self, {RealNeighbour, MRef}, Neighbour) -> + true = erlang:demonitor(MRef), + Msg = {?TAG, Ver, check_neighbours}, + ok = gen_server2:cast(RealNeighbour, Msg), + ok = case Neighbour of + Self -> ok; + _ -> gen_server2:cast(Neighbour, Msg) + end, + {Neighbour, maybe_monitor(Neighbour, Self)}. + +maybe_monitor(Self, Self) -> + undefined; +maybe_monitor(Other, _Self) -> + erlang:monitor(process, Other). + +check_neighbours(State = #state { self = Self, + left = Left, + right = Right, + view = View }) -> + #view_member { left = VLeft, right = VRight } + = fetch_view_member(Self, View), + Ver = view_version(View), + Left1 = ensure_neighbour(Ver, Self, Left, VLeft), + Right1 = ensure_neighbour(Ver, Self, Right, VRight), + State1 = State #state { left = Left1, right = Right1 }, + ok = maybe_send_catchup(Right, State1), + State1. + +maybe_send_catchup(Right, #state { right = Right }) -> + ok; +maybe_send_catchup(_Right, #state { self = Self, + right = {Self, undefined} }) -> + ok; +maybe_send_catchup(_Right, #state { members_state = undefined }) -> + ok; +maybe_send_catchup(_Right, #state { self = Self, + right = {Right, _MRef}, + view = View, + members_state = MembersState }) -> + send_right(Right, View, + {catchup, Self, prepare_members_state(MembersState)}). + + +%% --------------------------------------------------------------------------- +%% Catch_up delta detection +%% --------------------------------------------------------------------------- + +find_prefix_common_suffix(A, B) -> + {Prefix, A1} = find_prefix(A, B, queue:new()), + {Common, Suffix} = find_common(A1, B, queue:new()), + {Prefix, Common, Suffix}. + +%% Returns the elements of A that occur before the first element of B, +%% plus the remainder of A. +find_prefix(A, B, Prefix) -> + case {queue:out(A), queue:out(B)} of + {{{value, Val}, _A1}, {{value, Val}, _B1}} -> + {Prefix, A}; + {{empty, A1}, {{value, _A}, _B1}} -> + {Prefix, A1}; + {{{value, {NumA, _MsgA} = Val}, A1}, + {{value, {NumB, _MsgB}}, _B1}} when NumA < NumB -> + find_prefix(A1, B, queue:in(Val, Prefix)); + {_, {empty, _B1}} -> + {A, Prefix} %% Prefix well be empty here + end. + +%% A should be a prefix of B. Returns the commonality plus the +%% remainder of B. +find_common(A, B, Common) -> + case {queue:out(A), queue:out(B)} of + {{{value, Val}, A1}, {{value, Val}, B1}} -> + find_common(A1, B1, queue:in(Val, Common)); + {{empty, _A}, _} -> + {Common, B} + end. + + +%% --------------------------------------------------------------------------- +%% Members helpers +%% --------------------------------------------------------------------------- + +with_member(Fun, Id, MembersState) -> + store_member( + Id, Fun(find_member_or_blank(Id, MembersState)), MembersState). + +with_member_acc(Fun, Id, {MembersState, Acc}) -> + {MemberState, Acc1} = Fun(find_member_or_blank(Id, MembersState), Acc), + {store_member(Id, MemberState, MembersState), Acc1}. + +find_member_or_blank(Id, MembersState) -> + case ?DICT:find(Id, MembersState) of + {ok, Result} -> Result; + error -> blank_member() + end. + +erase_member(Id, MembersState) -> + ?DICT:erase(Id, MembersState). + +blank_member() -> + #member { pending_ack = queue:new(), last_pub = -1, last_ack = -1 }. + +blank_member_state() -> + ?DICT:new(). + +store_member(Id, MemberState, MembersState) -> + ?DICT:store(Id, MemberState, MembersState). + +prepare_members_state(MembersState) -> + ?DICT:to_list(MembersState). + +build_members_state(MembersStateList) -> + ?DICT:from_list(MembersStateList). + + +%% --------------------------------------------------------------------------- +%% Activity assembly +%% --------------------------------------------------------------------------- + +activity_nil() -> + queue:new(). + +activity_cons(_Id, [], [], Tail) -> + Tail; +activity_cons(Sender, Pubs, Acks, Tail) -> + queue:in({Sender, Pubs, Acks}, Tail). + +activity_finalise(Activity) -> + queue:to_list(Activity). + +maybe_send_activity([], _State) -> + ok; +maybe_send_activity(Activity, #state { self = Self, + right = {Right, _MRefR}, + view = View }) -> + send_right(Right, View, {activity, Self, Activity}). + +send_right(Right, View, Msg) -> + ok = gen_server2:cast(Right, {?TAG, view_version(View), Msg}). + +callback(Args, Module, Activity) -> + lists:foldl( + fun ({Id, Pubs, _Acks}, ok) -> + lists:foldl(fun ({_PubNum, Pub}, ok) -> + Module:handle_msg(Args, Id, Pub); + (_, Error) -> + Error + end, ok, Pubs); + (_, Error) -> + Error + end, ok, Activity). + +callback_view_changed(Args, Module, OldView, NewView) -> + OldMembers = all_known_members(OldView), + NewMembers = all_known_members(NewView), + Births = NewMembers -- OldMembers, + Deaths = OldMembers -- NewMembers, + case {Births, Deaths} of + {[], []} -> ok; + _ -> Module:members_changed(Args, Births, Deaths) + end. + +handle_callback_result({Result, State}) -> + if_callback_success( + Result, fun no_reply_true/3, fun no_reply_false/3, undefined, State); +handle_callback_result({Result, Reply, State}) -> + if_callback_success( + Result, fun reply_true/3, fun reply_false/3, Reply, State). + +no_reply_true (_Result, _Undefined, State) -> noreply(State). +no_reply_false({stop, Reason}, _Undefined, State) -> {stop, Reason, State}. + +reply_true (_Result, Reply, State) -> reply(Reply, State). +reply_false({stop, Reason}, Reply, State) -> {stop, Reason, Reply, State}. + +handle_msg_true (_Result, Msg, State) -> handle_msg(Msg, State). +handle_msg_false(Result, _Msg, State) -> {Result, State}. + +activity_true(_Result, Activity, State = #state { module = Module, + callback_args = Args }) -> + {callback(Args, Module, Activity), State}. +activity_false(Result, _Activity, State) -> + {Result, State}. + +if_callback_success(ok, True, _False, Arg, State) -> + True(ok, Arg, State); +if_callback_success( + {become, Module, Args} = Result, True, _False, Arg, State) -> + True(Result, Arg, State #state { module = Module, + callback_args = Args }); +if_callback_success({stop, _Reason} = Result, _True, False, Arg, State) -> + False(Result, Arg, State). + +maybe_confirm(_Self, _Id, Confirms, []) -> + Confirms; +maybe_confirm(Self, Self, Confirms, [PubNum | PubNums]) -> + case queue:out(Confirms) of + {empty, _Confirms} -> + Confirms; + {{value, {PubNum, From}}, Confirms1} -> + gen_server2:reply(From, ok), + maybe_confirm(Self, Self, Confirms1, PubNums); + {{value, {PubNum1, _From}}, _Confirms} when PubNum1 > PubNum -> + maybe_confirm(Self, Self, Confirms, PubNums) + end; +maybe_confirm(_Self, _Id, Confirms, _PubNums) -> + Confirms. + +purge_confirms(Confirms) -> + [gen_server2:reply(From, ok) || {_PubNum, From} <- queue:to_list(Confirms)], + queue:new(). + + +%% --------------------------------------------------------------------------- +%% Msg transformation +%% --------------------------------------------------------------------------- + +acks_from_queue(Q) -> + [PubNum || {PubNum, _Msg} <- queue:to_list(Q)]. + +pubs_from_queue(Q) -> + queue:to_list(Q). + +queue_from_pubs(Pubs) -> + queue:from_list(Pubs). + +apply_acks([], Pubs) -> + Pubs; +apply_acks(List, Pubs) -> + {_, Pubs1} = queue:split(length(List), Pubs), + Pubs1. + +join_pubs(Q, []) -> Q; +join_pubs(Q, Pubs) -> queue:join(Q, queue_from_pubs(Pubs)). + +last_ack([], LA) -> + LA; +last_ack(List, LA) -> + LA1 = lists:last(List), + true = LA1 > LA, %% ASSERTION + LA1. + +last_pub([], LP) -> + LP; +last_pub(List, LP) -> + {PubNum, _Msg} = lists:last(List), + true = PubNum > LP, %% ASSERTION + PubNum. diff --git a/src/gm_test.erl b/src/gm_test.erl new file mode 100644 index 00000000..e8f28598 --- /dev/null +++ b/src/gm_test.erl @@ -0,0 +1,126 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2010 VMware, Inc. All rights reserved. +%% + +-module(gm_test). + +-export([test/0]). +-export([joined/2, members_changed/3, handle_msg/3, terminate/2]). + +-behaviour(gm). + +-include("gm_specs.hrl"). + +get_state() -> + get(state). + +with_state(Fun) -> + put(state, Fun(get_state())). + +inc() -> + case 1 + get(count) of + 100000 -> Now = os:timestamp(), + Start = put(ts, Now), + Diff = timer:now_diff(Now, Start), + Rate = 100000 / (Diff / 1000000), + io:format("~p seeing ~p msgs/sec~n", [self(), Rate]), + put(count, 0); + N -> put(count, N) + end. + +joined([], Members) -> + io:format("Joined ~p (~p members)~n", [self(), length(Members)]), + put(state, dict:from_list([{Member, empty} || Member <- Members])), + put(count, 0), + put(ts, os:timestamp()), + ok. + +members_changed([], Births, Deaths) -> + with_state( + fun (State) -> + State1 = + lists:foldl( + fun (Born, StateN) -> + false = dict:is_key(Born, StateN), + dict:store(Born, empty, StateN) + end, State, Births), + lists:foldl( + fun (Died, StateN) -> + true = dict:is_key(Died, StateN), + dict:store(Died, died, StateN) + end, State1, Deaths) + end), + ok. + +handle_msg([], From, {test_msg, Num}) -> + inc(), + with_state( + fun (State) -> + ok = case dict:find(From, State) of + {ok, died} -> + exit({{from, From}, + {received_posthumous_delivery, Num}}); + {ok, empty} -> ok; + {ok, Num} -> ok; + {ok, Num1} when Num < Num1 -> + exit({{from, From}, + {duplicate_delivery_of, Num1}, + {expecting, Num}}); + {ok, Num1} -> + exit({{from, From}, + {missing_delivery_of, Num}, + {received_early, Num1}}); + error -> + exit({{from, From}, + {received_premature_delivery, Num}}) + end, + dict:store(From, Num + 1, State) + end), + ok. + +terminate([], Reason) -> + io:format("Left ~p (~p)~n", [self(), Reason]), + ok. + +spawn_member() -> + spawn_link( + fun () -> + random:seed(now()), + %% start up delay of no more than 10 seconds + timer:sleep(random:uniform(10000)), + {ok, Pid} = gm:start_link(?MODULE, ?MODULE, []), + Start = random:uniform(10000), + send_loop(Pid, Start, Start + random:uniform(10000)), + gm:leave(Pid), + spawn_more() + end). + +spawn_more() -> + [spawn_member() || _ <- lists:seq(1, 4 - random:uniform(4))]. + +send_loop(_Pid, Target, Target) -> + ok; +send_loop(Pid, Count, Target) when Target > Count -> + case random:uniform(3) of + 3 -> gm:confirmed_broadcast(Pid, {test_msg, Count}); + _ -> gm:broadcast(Pid, {test_msg, Count}) + end, + timer:sleep(random:uniform(5) - 1), %% sleep up to 4 ms + send_loop(Pid, Count + 1, Target). + +test() -> + ok = gm:create_tables(), + spawn_member(), + spawn_member(). -- cgit v1.2.1 From a29958797d402243f9b36083f7d2f317eb9ed40f Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 21 Jan 2011 12:10:41 +0000 Subject: bump year on copyrights --- src/gm.erl | 2 +- src/gm_test.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl index baf46471..8fea9196 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -11,7 +11,7 @@ %% The Original Code is RabbitMQ. %% %% The Initial Developer of the Original Code is VMware, Inc. -%% Copyright (c) 2007-2010 VMware, Inc. All rights reserved. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. %% -module(gm). diff --git a/src/gm_test.erl b/src/gm_test.erl index e8f28598..e0a92a0c 100644 --- a/src/gm_test.erl +++ b/src/gm_test.erl @@ -11,7 +11,7 @@ %% The Original Code is RabbitMQ. %% %% The Initial Developer of the Original Code is VMware, Inc. -%% Copyright (c) 2007-2010 VMware, Inc. All rights reserved. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. %% -module(gm_test). -- cgit v1.2.1 From d887a84c64321582266051b9a26ac9a9f1d1f6f7 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Mon, 24 Jan 2011 17:40:26 +0000 Subject: Treat sender-specified destinations as routing keys rather than queue names --- src/rabbit_exchange.erl | 13 +++++-------- src/rabbit_exchange_type_direct.erl | 10 +++++----- src/rabbit_exchange_type_fanout.erl | 10 ++-------- src/rabbit_exchange_type_topic.erl | 15 ++++++++++----- 4 files changed, 22 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange.erl b/src/rabbit_exchange.erl index 24079d22..a94e57f8 100644 --- a/src/rabbit_exchange.erl +++ b/src/rabbit_exchange.erl @@ -36,7 +36,7 @@ -export([recover/0, declare/6, lookup/1, lookup_or_die/1, list/1, info_keys/0, info/1, info/2, info_all/1, info_all/2, publish/2, delete/2]). -export([callback/3]). --export([header_routes/2]). +-export([header_routes/1]). %% this must be run inside a mnesia tx -export([maybe_auto_delete/1]). -export([assert_equivalence/6, assert_args_equivalence/2, check_type/1]). @@ -89,8 +89,7 @@ (rabbit_types:exchange()) -> 'not_deleted' | {'deleted', rabbit_binding:deletions()}). -spec(callback/3:: (rabbit_types:exchange(), atom(), [any()]) -> 'ok'). --spec(header_routes/2 :: (rabbit_framing:amqp_table(), rabbit_types:vhost()) -> - [rabbit_types:r('queue')]). +-spec(header_routes/1 :: (rabbit_framing:amqp_table()) -> [binary()]). -endif. %%---------------------------------------------------------------------------- @@ -326,12 +325,10 @@ unconditional_delete(X = #exchange{name = XName}) -> Bindings = rabbit_binding:remove_for_source(XName), {deleted, X, Bindings, rabbit_binding:remove_for_destination(XName)}. -header_routes(undefined, _VHost) -> +header_routes(undefined) -> []; -header_routes(Headers, VHost) -> - [rabbit_misc:r(VHost, queue, RKey) || - RKey <- lists:flatten([routing_keys(Headers, Header) || - Header <- ?ROUTING_HEADERS])]. +header_routes(Headers) -> + lists:flatten([routing_keys(Headers, Header) || Header <- ?ROUTING_HEADERS]). routing_keys(HeadersTable, Key) -> case rabbit_misc:table_lookup(HeadersTable, Key) of diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index ade57451..97988381 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -51,13 +51,13 @@ description() -> [{name, <<"direct">>}, {description, <<"AMQP direct exchange, as per the AMQP specification">>}]. -route(#exchange{name = #resource{virtual_host = VHost} = Name}, +route(#exchange{name = Name}, #delivery{message = #basic_message{routing_key = RoutingKey, content = Content}}) -> - BindingRoutes = rabbit_router:match_routing_key(Name, RoutingKey), - HeaderRoutes = rabbit_exchange:header_routes( - (Content#content.properties)#'P_basic'.headers, VHost), - BindingRoutes ++ HeaderRoutes. + HeaderKeys = rabbit_exchange:header_routes( + (Content#content.properties)#'P_basic'.headers), + lists:flatten([rabbit_router:match_routing_key(Name, RKey) || + RKey <- [RoutingKey | HeaderKeys]]). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_exchange_type_fanout.erl b/src/rabbit_exchange_type_fanout.erl index f3716141..5266dd87 100644 --- a/src/rabbit_exchange_type_fanout.erl +++ b/src/rabbit_exchange_type_fanout.erl @@ -31,7 +31,6 @@ -module(rabbit_exchange_type_fanout). -include("rabbit.hrl"). --include("rabbit_framing.hrl"). -behaviour(rabbit_exchange_type). @@ -51,13 +50,8 @@ description() -> [{name, <<"fanout">>}, {description, <<"AMQP fanout exchange, as per the AMQP specification">>}]. -route(#exchange{name = #resource{virtual_host = VHost} = Name}, - #delivery{message = #basic_message{content = Content}}) -> - BindingRoutes = rabbit_router:match_routing_key(Name, '_'), - HeaderRoutes = rabbit_exchange:header_routes( - (Content#content.properties)#'P_basic'.headers, VHost), - BindingRoutes ++ HeaderRoutes. - +route(#exchange{name = Name}, _Delivery) -> + rabbit_router:match_routing_key(Name, '_'). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 2f0d47a7..8f3c0550 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -30,6 +30,7 @@ %% -module(rabbit_exchange_type_topic). +-include("rabbit_framing.hrl"). -include("rabbit.hrl"). -behaviour(rabbit_exchange_type). @@ -59,11 +60,15 @@ description() -> {description, <<"AMQP topic exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, - #delivery{message = #basic_message{routing_key = RoutingKey}}) -> - rabbit_router:match_bindings(Name, - fun (#binding{key = BindingKey}) -> - topic_matches(BindingKey, RoutingKey) - end). + #delivery{message = #basic_message{routing_key = RoutingKey, + content = Content}}) -> + HeaderKeys = rabbit_exchange:header_routes( + (Content#content.properties)#'P_basic'.headers), + lists:flatten([rabbit_router:match_bindings( + Name, + fun (#binding{key = BindingKey}) -> + topic_matches(BindingKey, RKey) + end) || RKey <- [RoutingKey | HeaderKeys]]). split_topic_key(Key) -> string:tokens(binary_to_list(Key), "."). -- cgit v1.2.1 From cfca23b81c44262977f879ec53e6bfac0792c8b8 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 26 Jan 2011 16:12:13 +0000 Subject: rabbitmqctl status is not adequate to wait for the server as it can return successfully when the vm has started but not the app. The app can then fail. Therefore introduce a new command to wait for the app to start. Note that this subcommand contains a timeout to wait for the VM to start, but will wait indefinitely for the app to start once the VM has. --- src/rabbit_control.erl | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index 80483097..a7d07b0f 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -20,6 +20,7 @@ -export([start/0, stop/0, action/5, diagnostics/1]). -define(RPC_TIMEOUT, infinity). +-define(WAIT_FOR_VM_TIMEOUT, 5000). -define(QUIET_OPT, "-q"). -define(NODE_OPT, "-n"). @@ -297,7 +298,30 @@ action(list_permissions, Node, [], Opts, Inform) -> VHost = proplists:get_value(?VHOST_OPT, Opts), Inform("Listing permissions in vhost ~p", [VHost]), display_list(call(Node, {rabbit_auth_backend_internal, - list_vhost_permissions, [VHost]})). + list_vhost_permissions, [VHost]})); + +action(wait, Node, [], _Opts, Inform) -> + Inform("Waiting for ~p", [Node]), + wait_for_application(Node, ?WAIT_FOR_VM_TIMEOUT). + +wait_for_application(_Node, NodeTimeout) when NodeTimeout =< 0 -> + {badrpc, nodedown}; + +wait_for_application(Node, NodeTimeout) -> + case call(Node, {application, which_applications, []}) of + {badrpc, nodedown} -> wait_for_application0(Node, NodeTimeout - 1000); + {badrpc, _} = E -> E; + Apps -> case proplists:is_defined(rabbit, Apps) of + %% We've seen the node up; if it goes down + %% die immediately. + false -> wait_for_application0(Node, 0); + true -> ok + end + end. + +wait_for_application0(Node, NodeTimeout) -> + timer:sleep(1000), + wait_for_application(Node, NodeTimeout). default_if_empty(List, Default) when is_list(List) -> if List == [] -> -- cgit v1.2.1 From 934688ae55c393bc2ddc693cd1e141e5cd761fa4 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 26 Jan 2011 16:28:10 +0000 Subject: Treat all {badrpc, _}s the same. Use which_applications/1. --- src/rabbit_control.erl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index a7d07b0f..8a19dcfb 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -304,19 +304,19 @@ action(wait, Node, [], _Opts, Inform) -> Inform("Waiting for ~p", [Node]), wait_for_application(Node, ?WAIT_FOR_VM_TIMEOUT). -wait_for_application(_Node, NodeTimeout) when NodeTimeout =< 0 -> - {badrpc, nodedown}; - wait_for_application(Node, NodeTimeout) -> - case call(Node, {application, which_applications, []}) of - {badrpc, nodedown} -> wait_for_application0(Node, NodeTimeout - 1000); - {badrpc, _} = E -> E; - Apps -> case proplists:is_defined(rabbit, Apps) of - %% We've seen the node up; if it goes down - %% die immediately. - false -> wait_for_application0(Node, 0); - true -> ok - end + case call(Node, {application, which_applications, [infinity]}) of + {badrpc, _} = E -> NewTimeout = NodeTimeout - 1000, + case NewTimeout =< 0 of + true -> E; + false -> wait_for_application0(Node, NewTimeout) + end; + Apps -> case proplists:is_defined(rabbit, Apps) of + %% We've seen the node up; if it goes down + %% die immediately. + true -> ok; + false -> wait_for_application0(Node, 0) + end end. wait_for_application0(Node, NodeTimeout) -> -- cgit v1.2.1 From 19d9256812c4d8f51d87f96f0dc3c2b04e902d53 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 26 Jan 2011 16:39:08 +0000 Subject: Oops. --- src/rabbit_control.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index 8a19dcfb..a8903102 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -305,7 +305,7 @@ action(wait, Node, [], _Opts, Inform) -> wait_for_application(Node, ?WAIT_FOR_VM_TIMEOUT). wait_for_application(Node, NodeTimeout) -> - case call(Node, {application, which_applications, [infinity]}) of + case rpc_call(Node, application, which_applications, [infinity]) of {badrpc, _} = E -> NewTimeout = NodeTimeout - 1000, case NewTimeout =< 0 of true -> E; -- cgit v1.2.1 From cf8e92bad03ef05487c1e2aec557557a17977e0e Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 27 Jan 2011 18:11:44 +0000 Subject: cosmetic; using accumulator in trie_match --- src/rabbit_exchange_type_topic.erl | 49 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 28 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 2e181f1d..fdababe7 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -80,34 +80,27 @@ assert_args_equivalence(X, Args) -> %%---------------------------------------------------------------------------- trie_match(X, Words) -> - trie_match(X, root, Words). - -trie_match(X, Node, []) -> - FinalRes = trie_bindings(X, Node), - HashRes = case trie_child(X, Node, "#") of - {ok, HashNode} -> trie_match(X, HashNode, []); - error -> [] - end, - FinalRes ++ HashRes; -trie_match(X, Node, [W | RestW] = Words) -> - ExactRes = case trie_child(X, Node, W) of - {ok, NextNode} -> trie_match(X, NextNode, RestW); - error -> [] - end, - StarRes = case trie_child(X, Node, "*") of - {ok, StarNode} -> trie_match(X, StarNode, RestW); - error -> [] - end, - HashRes = case trie_child(X, Node, "#") of - {ok, HashNode} -> trie_match_skip_any(X, HashNode, Words); - error -> [] - end, - ExactRes ++ StarRes ++ HashRes. - -trie_match_skip_any(X, Node, []) -> - trie_match(X, Node, []); -trie_match_skip_any(X, Node, [_ | RestW] = Words) -> - trie_match(X, Node, Words) ++ trie_match_skip_any(X, Node, RestW). + trie_match(X, root, Words, []). + +trie_match(X, Node, [], ResAcc) -> + ResAcc1 = trie_bindings(X, Node) ++ ResAcc, + trie_match_part(X, Node, "#", fun trie_match_skip_any/4, [], ResAcc1); +trie_match(X, Node, [W | RestW] = Words, ResAcc) -> + ResAcc1 = trie_match_part(X, Node, W, fun trie_match/4, RestW, ResAcc), + ResAcc2 = trie_match_part(X, Node, "*", fun trie_match/4, RestW, ResAcc1), + trie_match_part(X, Node, "#", fun trie_match_skip_any/4, Words, ResAcc2). + +trie_match_part(X, Node, Search, MatchFun, RestW, ResAcc) -> + case trie_child(X, Node, Search) of + {ok, NextNode} -> MatchFun(X, NextNode, RestW, ResAcc); + error -> ResAcc + end. + +trie_match_skip_any(X, Node, [], ResAcc) -> + trie_match(X, Node, [], ResAcc); +trie_match_skip_any(X, Node, [_ | RestW] = Words, ResAcc) -> + ResAcc1 = trie_match(X, Node, Words, ResAcc), + trie_match_skip_any(X, Node, RestW, ResAcc1). follow_down_create(X, Words) -> case follow_down_last_node(X, Words) of -- cgit v1.2.1 From 18680bf95808fd395f7bdd955320920288e9af2c Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Thu, 27 Jan 2011 19:18:49 +0000 Subject: cosmetic --- src/rabbit_exchange_type_topic.erl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index fdababe7..0beaa714 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -83,12 +83,14 @@ trie_match(X, Words) -> trie_match(X, root, Words, []). trie_match(X, Node, [], ResAcc) -> - ResAcc1 = trie_bindings(X, Node) ++ ResAcc, - trie_match_part(X, Node, "#", fun trie_match_skip_any/4, [], ResAcc1); + trie_match_part(X, Node, "#", fun trie_match_skip_any/4, [], + trie_bindings(X, Node) ++ ResAcc); trie_match(X, Node, [W | RestW] = Words, ResAcc) -> - ResAcc1 = trie_match_part(X, Node, W, fun trie_match/4, RestW, ResAcc), - ResAcc2 = trie_match_part(X, Node, "*", fun trie_match/4, RestW, ResAcc1), - trie_match_part(X, Node, "#", fun trie_match_skip_any/4, Words, ResAcc2). + lists:foldl(fun ({WArg, MatchFun, RestWArg}, Acc) -> + trie_match_part(X, Node, WArg, MatchFun, RestWArg, Acc) + end, ResAcc, [{W, fun trie_match/4, RestW}, + {"*", fun trie_match/4, RestW}, + {"#", fun trie_match_skip_any/4, Words}]). trie_match_part(X, Node, Search, MatchFun, RestW, ResAcc) -> case trie_child(X, Node, Search) of @@ -99,8 +101,8 @@ trie_match_part(X, Node, Search, MatchFun, RestW, ResAcc) -> trie_match_skip_any(X, Node, [], ResAcc) -> trie_match(X, Node, [], ResAcc); trie_match_skip_any(X, Node, [_ | RestW] = Words, ResAcc) -> - ResAcc1 = trie_match(X, Node, Words, ResAcc), - trie_match_skip_any(X, Node, RestW, ResAcc1). + trie_match_skip_any(X, Node, RestW, + trie_match(X, Node, Words, ResAcc)). follow_down_create(X, Words) -> case follow_down_last_node(X, Words) of -- cgit v1.2.1 From 8304d8f8a8618b6e3aae73c18b4b2594d62fd67a Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Wed, 2 Feb 2011 13:41:24 +0000 Subject: Refactored sender-supplied routing keys --- src/rabbit_basic.erl | 69 +++++++++++++++++++++++++++---------- src/rabbit_channel.erl | 18 +--------- src/rabbit_exchange.erl | 60 +++++++------------------------- src/rabbit_exchange_type_direct.erl | 45 +++++++----------------- src/rabbit_exchange_type_topic.erl | 8 ++--- src/rabbit_router.erl | 59 +++++++------------------------ 6 files changed, 92 insertions(+), 167 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index 1ac39b65..c9d4808c 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -33,10 +33,9 @@ -include("rabbit.hrl"). -include("rabbit_framing.hrl"). --export([publish/1, message/4, properties/1, delivery/5]). +-export([publish/1, message/3, message/4, properties/1, delivery/5]). -export([publish/4, publish/7]). -export([build_content/2, from_content/1]). --export([is_message_persistent/1]). %%---------------------------------------------------------------------------- @@ -56,8 +55,10 @@ rabbit_types:delivery()). -spec(message/4 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - properties_input(), binary()) -> - (rabbit_types:message() | rabbit_types:error(any()))). + properties_input(), binary()) -> rabbit_types:message()). +-spec(message/3 :: + (rabbit_exchange:name(), rabbit_router:routing_key(), + rabbit_types:decoded_content()) -> rabbit_types:message()). -spec(properties/1 :: (properties_input()) -> rabbit_framing:amqp_property_record()). -spec(publish/4 :: @@ -71,9 +72,6 @@ rabbit_types:content()). -spec(from_content/1 :: (rabbit_types:content()) -> {rabbit_framing:amqp_property_record(), binary()}). --spec(is_message_persistent/1 :: (rabbit_types:decoded_content()) -> - (boolean() | - {'invalid', non_neg_integer()})). -endif. @@ -113,19 +111,33 @@ from_content(Content) -> rabbit_framing_amqp_0_9_1:method_id('basic.publish'), {Props, list_to_binary(lists:reverse(FragmentsRev))}. +%% This breaks the spec rule forbidding message modification +strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} = DecodedContent, + Key) when Headers =/= undefined -> + case lists:keyfind(Key, 1, Headers) of + false -> DecodedContent; + Tuple -> Headers0 = lists:delete(Tuple, Headers), + DecodedContent#content{ + properties_bin = none, + properties = Props#'P_basic'{headers = Headers0}} + end; +strip_header(DecodedContent, _Key) -> + DecodedContent. + +message(ExchangeName, RoutingKey, + #content{properties = Props} = DecodedContent) -> + #basic_message{ + exchange_name = ExchangeName, + routing_key = RoutingKey, + content = strip_header(DecodedContent, ?DELETED_HEADER), + guid = rabbit_guid:guid(), + is_persistent = is_message_persistent(DecodedContent), + route_list = [RoutingKey | header_routes(Props#'P_basic'.headers)]}. + message(ExchangeName, RoutingKeyBin, RawProperties, BodyBin) -> Properties = properties(RawProperties), Content = build_content(Properties, BodyBin), - case is_message_persistent(Content) of - {invalid, Other} -> - {error, {invalid_delivery_mode, Other}}; - IsPersistent when is_boolean(IsPersistent) -> - #basic_message{exchange_name = ExchangeName, - routing_key = RoutingKeyBin, - content = Content, - guid = rabbit_guid:guid(), - is_persistent = IsPersistent} - end. + message(ExchangeName, RoutingKeyBin, Content). properties(P = #'P_basic'{}) -> P; @@ -167,5 +179,26 @@ is_message_persistent(#content{properties = #'P_basic'{ 1 -> false; 2 -> true; undefined -> false; - Other -> {invalid, Other} + Other -> rabbit_log:warning("Unknown delivery mode ~p - " + "treating as 1, non-persistent~n", + [Other]), + false end. + +% Extract CC routes from headers +header_routes(undefined) -> + []; +header_routes(HeadersTable) -> + lists:flatten([case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of + {longstr, Route} -> Route; + {array, Routes} -> rkeys(Routes, []); + _ -> [] + end || HeaderKey <- ?ROUTING_HEADERS]). + +rkeys([{longstr, Route} | Rest], RKeys) -> + rkeys(Rest, [Route | RKeys]); +rkeys([_ | Rest], RKeys) -> + rkeys(Rest, RKeys); +rkeys(_, RKeys) -> + RKeys. + diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 5c900b0b..e818dd54 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -527,18 +527,13 @@ handle_method(#'basic.publish'{exchange = ExchangeNameBin, %% certain to want to look at delivery-mode and priority. DecodedContent = rabbit_binary_parser:ensure_content_decoded(Content), check_user_id_header(DecodedContent#content.properties, State), - IsPersistent = is_message_persistent(DecodedContent), {MsgSeqNo, State1} = case ConfirmEnabled of false -> {undefined, State}; true -> SeqNo = State#ch.publish_seqno, {SeqNo, State#ch{publish_seqno = SeqNo + 1}} end, - Message = #basic_message{exchange_name = ExchangeName, - routing_key = RoutingKey, - content = DecodedContent, - guid = rabbit_guid:guid(), - is_persistent = IsPersistent}, + Message = rabbit_basic:message(ExchangeName, RoutingKey, DecodedContent), {RoutingRes, DeliveredQPids} = rabbit_exchange:publish( Exchange, @@ -1200,17 +1195,6 @@ notify_limiter(LimiterPid, Acked) -> Count -> rabbit_limiter:ack(LimiterPid, Count) end. -is_message_persistent(Content) -> - case rabbit_basic:is_message_persistent(Content) of - {invalid, Other} -> - rabbit_log:warning("Unknown delivery mode ~p - " - "treating as 1, non-persistent~n", - [Other]), - false; - IsPersistent when is_boolean(IsPersistent) -> - IsPersistent - end. - process_routing_result(unroutable, _, MsgSeqNo, Message, State) -> ok = basic_return(Message, State#ch.writer_pid, no_route), send_confirms([MsgSeqNo], State); diff --git a/src/rabbit_exchange.erl b/src/rabbit_exchange.erl index a94e57f8..92259195 100644 --- a/src/rabbit_exchange.erl +++ b/src/rabbit_exchange.erl @@ -1,32 +1,17 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. %% -%% The Original Code is RabbitMQ. +%% The Original Code is RabbitMQ. %% -%% The Initial Developers of the Original Code are LShift Ltd, -%% Cohesive Financial Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created before 22-Nov-2008 00:00:00 GMT by LShift Ltd, -%% Cohesive Financial Technologies LLC, or Rabbit Technologies Ltd -%% are Copyright (C) 2007-2008 LShift Ltd, Cohesive Financial -%% Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created by LShift Ltd are Copyright (C) 2007-2010 LShift -%% Ltd. Portions created by Cohesive Financial Technologies LLC are -%% Copyright (C) 2007-2010 Cohesive Financial Technologies -%% LLC. Portions created by Rabbit Technologies Ltd are Copyright -%% (C) 2007-2010 Rabbit Technologies Ltd. -%% -%% All Rights Reserved. -%% -%% Contributor(s): ______________________________________. +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. %% -module(rabbit_exchange). @@ -36,7 +21,6 @@ -export([recover/0, declare/6, lookup/1, lookup_or_die/1, list/1, info_keys/0, info/1, info/2, info_all/1, info_all/2, publish/2, delete/2]). -export([callback/3]). --export([header_routes/1]). %% this must be run inside a mnesia tx -export([maybe_auto_delete/1]). -export([assert_equivalence/6, assert_args_equivalence/2, check_type/1]). @@ -89,7 +73,7 @@ (rabbit_types:exchange()) -> 'not_deleted' | {'deleted', rabbit_binding:deletions()}). -spec(callback/3:: (rabbit_types:exchange(), atom(), [any()]) -> 'ok'). --spec(header_routes/1 :: (rabbit_framing:amqp_table()) -> [binary()]). + -endif. %%---------------------------------------------------------------------------- @@ -324,23 +308,3 @@ unconditional_delete(X = #exchange{name = XName}) -> ok = mnesia:delete({rabbit_exchange, XName}), Bindings = rabbit_binding:remove_for_source(XName), {deleted, X, Bindings, rabbit_binding:remove_for_destination(XName)}. - -header_routes(undefined) -> - []; -header_routes(Headers) -> - lists:flatten([routing_keys(Headers, Header) || Header <- ?ROUTING_HEADERS]). - -routing_keys(HeadersTable, Key) -> - case rabbit_misc:table_lookup(HeadersTable, Key) of - {longstr, Route} -> [Route]; - {array, Routes} -> rkeys(Routes, []); - _ -> [] - end. - -rkeys([{longstr, BinVal} | Rest], RKeys) -> - rkeys(Rest, [BinVal | RKeys]); -rkeys([{_, _} | Rest], RKeys) -> - rkeys(Rest, RKeys); -rkeys(_, RKeys) -> - RKeys. - diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index 97988381..0baac1f8 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -1,37 +1,21 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. %% -%% The Original Code is RabbitMQ. +%% The Original Code is RabbitMQ. %% -%% The Initial Developers of the Original Code are LShift Ltd, -%% Cohesive Financial Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created before 22-Nov-2008 00:00:00 GMT by LShift Ltd, -%% Cohesive Financial Technologies LLC, or Rabbit Technologies Ltd -%% are Copyright (C) 2007-2008 LShift Ltd, Cohesive Financial -%% Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created by LShift Ltd are Copyright (C) 2007-2010 LShift -%% Ltd. Portions created by Cohesive Financial Technologies LLC are -%% Copyright (C) 2007-2010 Cohesive Financial Technologies -%% LLC. Portions created by Rabbit Technologies Ltd are Copyright -%% (C) 2007-2010 Rabbit Technologies Ltd. -%% -%% All Rights Reserved. -%% -%% Contributor(s): ______________________________________. +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. %% -module(rabbit_exchange_type_direct). -include("rabbit.hrl"). --include("rabbit_framing.hrl"). -behaviour(rabbit_exchange_type). @@ -52,12 +36,9 @@ description() -> {description, <<"AMQP direct exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, - #delivery{message = #basic_message{routing_key = RoutingKey, - content = Content}}) -> - HeaderKeys = rabbit_exchange:header_routes( - (Content#content.properties)#'P_basic'.headers), + #delivery{message = #basic_message{route_list = Routes}}) -> lists:flatten([rabbit_router:match_routing_key(Name, RKey) || - RKey <- [RoutingKey | HeaderKeys]]). + RKey <- Routes]). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 8f3c0550..97cf8ecf 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -30,7 +30,6 @@ %% -module(rabbit_exchange_type_topic). --include("rabbit_framing.hrl"). -include("rabbit.hrl"). -behaviour(rabbit_exchange_type). @@ -60,15 +59,12 @@ description() -> {description, <<"AMQP topic exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, - #delivery{message = #basic_message{routing_key = RoutingKey, - content = Content}}) -> - HeaderKeys = rabbit_exchange:header_routes( - (Content#content.properties)#'P_basic'.headers), + #delivery{message = #basic_message{route_list = Routes}}) -> lists:flatten([rabbit_router:match_bindings( Name, fun (#binding{key = BindingKey}) -> topic_matches(BindingKey, RKey) - end) || RKey <- [RoutingKey | HeaderKeys]]). + end) || RKey <- Routes]). split_topic_key(Key) -> string:tokens(binary_to_list(Key), "."). diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index 7f9b823e..692d2473 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -1,38 +1,22 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. %% -%% The Original Code is RabbitMQ. +%% The Original Code is RabbitMQ. %% -%% The Initial Developers of the Original Code are LShift Ltd, -%% Cohesive Financial Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created before 22-Nov-2008 00:00:00 GMT by LShift Ltd, -%% Cohesive Financial Technologies LLC, or Rabbit Technologies Ltd -%% are Copyright (C) 2007-2008 LShift Ltd, Cohesive Financial -%% Technologies LLC, and Rabbit Technologies Ltd. -%% -%% Portions created by LShift Ltd are Copyright (C) 2007-2010 LShift -%% Ltd. Portions created by Cohesive Financial Technologies LLC are -%% Copyright (C) 2007-2010 Cohesive Financial Technologies -%% LLC. Portions created by Rabbit Technologies Ltd are Copyright -%% (C) 2007-2010 Rabbit Technologies Ltd. -%% -%% All Rights Reserved. -%% -%% Contributor(s): ______________________________________. +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. %% -module(rabbit_router). -include_lib("stdlib/include/qlc.hrl"). -include("rabbit.hrl"). --include("rabbit_framing.hrl"). -export([deliver/2, match_bindings/2, match_routing_key/2]). @@ -69,39 +53,22 @@ deliver(QNames, Delivery = #delivery{mandatory = false, %% is preserved. This scales much better than the non-immediate %% case below. QPids = lookup_qpids(QNames), - ModifiedDelivery = strip_header(Delivery, ?DELETED_HEADER), delegate:invoke_no_result( - QPids, fun (Pid) -> rabbit_amqqueue:deliver(Pid, ModifiedDelivery) end), + QPids, fun (Pid) -> rabbit_amqqueue:deliver(Pid, Delivery) end), {routed, QPids}; deliver(QNames, Delivery = #delivery{mandatory = Mandatory, immediate = Immediate}) -> QPids = lookup_qpids(QNames), - ModifiedDelivery = strip_header(Delivery, ?DELETED_HEADER), {Success, _} = delegate:invoke(QPids, fun (Pid) -> - rabbit_amqqueue:deliver(Pid, ModifiedDelivery) + rabbit_amqqueue:deliver(Pid, Delivery) end), {Routed, Handled} = lists:foldl(fun fold_deliveries/2, {false, []}, Success), check_delivery(Mandatory, Immediate, {Routed, Handled}). -%% This breaks the spec rule forbidding message modification -strip_header(Delivery = #delivery{message = Message = #basic_message{ - content = Content = #content{ - properties = Props = #'P_basic'{headers = Headers}}}}, - Key) when Headers =/= undefined -> - case lists:keyfind(Key, 1, Headers) of - false -> Delivery; - Tuple -> Headers0 = lists:delete(Tuple, Headers), - Delivery#delivery{message = Message#basic_message{ - content = Content#content{ - properties_bin = none, - properties = Props#'P_basic'{headers = Headers0}}}} - end; -strip_header(Delivery, _Key) -> - Delivery. %% TODO: Maybe this should be handled by a cursor instead. %% TODO: This causes a full scan for each entry with the same source -- cgit v1.2.1 From c6e14cf23bcf5cebe1a9f2c3f44d1669d05cb961 Mon Sep 17 00:00:00 2001 From: Marek Majkowski Date: Fri, 4 Feb 2011 13:39:35 +0000 Subject: Treat basic_return immediate/mandatory differently --- src/rabbit_channel.erl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index f9c3c286..ebd8b15c 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1081,12 +1081,11 @@ binding_action(Fun, ExchangeNameBin, DestinationType, DestinationNameBin, basic_return(#basic_message{exchange_name = ExchangeName, routing_key = RoutingKey, content = Content}, - State, Reason) -> - maybe_incr_stats([{ExchangeName, 1}], return, State), + WriterPid, Reason) -> {_Close, ReplyCode, ReplyText} = rabbit_framing_amqp_0_9_1:lookup_amqp_exception(Reason), ok = rabbit_writer:send_command( - State#ch.writer_pid, + WriterPid, #'basic.return'{reply_code = ReplyCode, reply_text = ReplyText, exchange = ExchangeName#resource.name, @@ -1240,11 +1239,17 @@ is_message_persistent(Content) -> IsPersistent end. -process_routing_result(unroutable, _, XName, MsgSeqNo, Msg, State) -> +process_routing_result(unroutable, _, XName, MsgSeqNo, + Msg = #basic_message{exchange_name = ExchangeName}, + State) -> ok = basic_return(Msg, State#ch.writer_pid, no_route), + maybe_incr_stats([{ExchangeName, 1}], return_unroutable, State), record_confirm(MsgSeqNo, XName, State); -process_routing_result(not_delivered, _, XName, MsgSeqNo, Msg, State) -> +process_routing_result(not_delivered, _, XName, MsgSeqNo, + Msg = #basic_message{exchange_name = ExchangeName}, + State) -> ok = basic_return(Msg, State#ch.writer_pid, no_consumers), + maybe_incr_stats([{ExchangeName, 1}], return_not_delivered, State), record_confirm(MsgSeqNo, XName, State); process_routing_result(routed, [], XName, MsgSeqNo, _, State) -> record_confirm(MsgSeqNo, XName, State); -- cgit v1.2.1 From cd64ab0f9b9fe0689a74681fed4e65d7ce333b8f Mon Sep 17 00:00:00 2001 From: Marek Majkowski Date: Fri, 4 Feb 2011 13:42:51 +0000 Subject: cosmetic --- src/rabbit_channel.erl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index ebd8b15c..87357b89 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1239,17 +1239,15 @@ is_message_persistent(Content) -> IsPersistent end. -process_routing_result(unroutable, _, XName, MsgSeqNo, - Msg = #basic_message{exchange_name = ExchangeName}, - State) -> +process_routing_result(unroutable, _, XName, MsgSeqNo, Msg, State) -> ok = basic_return(Msg, State#ch.writer_pid, no_route), - maybe_incr_stats([{ExchangeName, 1}], return_unroutable, State), + maybe_incr_stats([{Msg#basic_message.exchange_name, 1}], + return_unroutable, State), record_confirm(MsgSeqNo, XName, State); -process_routing_result(not_delivered, _, XName, MsgSeqNo, - Msg = #basic_message{exchange_name = ExchangeName}, - State) -> +process_routing_result(not_delivered, _, XName, MsgSeqNo, Msg, State) -> ok = basic_return(Msg, State#ch.writer_pid, no_consumers), - maybe_incr_stats([{ExchangeName, 1}], return_not_delivered, State), + maybe_incr_stats([{Msg#basic_message.exchange_name, 1}], + return_not_delivered, State), record_confirm(MsgSeqNo, XName, State); process_routing_result(routed, [], XName, MsgSeqNo, _, State) -> record_confirm(MsgSeqNo, XName, State); -- cgit v1.2.1 From 5ecfe82f4886dee81d6de41e2811b6ab46c0297c Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Fri, 4 Feb 2011 14:18:19 +0000 Subject: Remove redundant try/catch from event notifier --- src/rabbit_event.erl | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rabbit_event.erl b/src/rabbit_event.erl index 40ade4b7..40651d36 100644 --- a/src/rabbit_event.erl +++ b/src/rabbit_event.erl @@ -130,15 +130,8 @@ notify_if(true, Type, Props) -> notify(Type, Props); notify_if(false, _Type, _Props) -> ok. notify(Type, Props) -> - try - %% TODO: switch to os:timestamp() when we drop support for - %% Erlang/OTP < R13B01 - gen_event:notify(rabbit_event, #event{type = Type, - props = Props, - timestamp = now()}) - catch error:badarg -> - %% badarg means rabbit_event is no longer registered. We never - %% unregister it so the great likelihood is that we're shutting - %% down the broker but some events were backed up. Ignore it. - ok - end. + %% TODO: switch to os:timestamp() when we drop support for + %% Erlang/OTP < R13B01 + gen_event:notify(rabbit_event, #event{type = Type, + props = Props, + timestamp = now()}). -- cgit v1.2.1 From 631e455ea25ea4202568c40ceb615c8cdeb94a16 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Mon, 7 Feb 2011 14:23:01 +0000 Subject: fixing binding recovery --- src/rabbit_exchange_type_topic.erl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index 0beaa714..c1741b30 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -46,7 +46,12 @@ route(#exchange{name = X}, validate(_X) -> ok. create(_Tx, _X) -> ok. -recover(_X, _Bs) -> ok. + +recover(_Exchange, Bs) -> + rabbit_misc:execute_mnesia_transaction( + fun () -> + lists:foreach(fun (B) -> internal_add_binding(B) end, Bs) + end). delete(true, #exchange{name = X}, _Bs) -> trie_remove_all_edges(X), @@ -55,10 +60,8 @@ delete(true, #exchange{name = X}, _Bs) -> delete(false, _Exchange, _Bs) -> ok. -add_binding(true, _Exchange, #binding{source = X, key = K, destination = D}) -> - FinalNode = follow_down_create(X, split_topic_key(K)), - trie_add_binding(X, FinalNode, D), - ok; +add_binding(true, _Exchange, Binding) -> + internal_add_binding(Binding); add_binding(false, _Exchange, _Binding) -> ok. @@ -79,6 +82,11 @@ assert_args_equivalence(X, Args) -> %%---------------------------------------------------------------------------- +internal_add_binding(#binding{source = X, key = K, destination = D}) -> + FinalNode = follow_down_create(X, split_topic_key(K)), + trie_add_binding(X, FinalNode, D), + ok. + trie_match(X, Words) -> trie_match(X, root, Words, []). -- cgit v1.2.1 From f315a8348de87819dc3b1fbd1987f94c176e8e01 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Wed, 9 Feb 2011 12:01:21 +0000 Subject: Sender-selected destinations - qa feedback --- src/rabbit_basic.erl | 38 +++++++++++++------------------------ src/rabbit_channel.erl | 6 +++--- src/rabbit_exchange_type_direct.erl | 6 +++--- src/rabbit_exchange_type_topic.erl | 12 ++++++------ src/rabbit_types.erl | 2 +- 5 files changed, 26 insertions(+), 38 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index a144124f..f1348d33 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -97,15 +97,15 @@ from_content(Content) -> {Props, list_to_binary(lists:reverse(FragmentsRev))}. %% This breaks the spec rule forbidding message modification -strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} = DecodedContent, - Key) when Headers =/= undefined -> - case lists:keyfind(Key, 1, Headers) of - false -> DecodedContent; - Tuple -> Headers0 = lists:delete(Tuple, Headers), +strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} + = DecodedContent, Key) when Headers =/= undefined -> + rabbit_binary_generator:clear_encoded_content( + case lists:keyfind(Key, 1, Headers) of + false -> DecodedContent; + Tuple -> Headers0 = lists:delete(Tuple, Headers), DecodedContent#content{ - properties_bin = none, properties = Props#'P_basic'{headers = Headers0}} - end; + end); strip_header(DecodedContent, _Key) -> DecodedContent. @@ -113,11 +113,10 @@ message(ExchangeName, RoutingKey, #content{properties = Props} = DecodedContent) -> #basic_message{ exchange_name = ExchangeName, - routing_key = RoutingKey, content = strip_header(DecodedContent, ?DELETED_HEADER), guid = rabbit_guid:guid(), is_persistent = is_message_persistent(DecodedContent), - route_list = [RoutingKey | header_routes(Props#'P_basic'.headers)]}. + routing_keys = [RoutingKey | header_routes(Props#'P_basic'.headers)]}. message(ExchangeName, RoutingKeyBin, RawProperties, BodyBin) -> Properties = properties(RawProperties), @@ -164,26 +163,15 @@ is_message_persistent(#content{properties = #'P_basic'{ 1 -> false; 2 -> true; undefined -> false; - Other -> rabbit_log:warning("Unknown delivery mode ~p - " - "treating as 1, non-persistent~n", - [Other]), - false + _ -> false end. % Extract CC routes from headers header_routes(undefined) -> []; header_routes(HeadersTable) -> - lists:flatten([case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of - {longstr, Route} -> Route; - {array, Routes} -> rkeys(Routes, []); - _ -> [] - end || HeaderKey <- ?ROUTING_HEADERS]). - -rkeys([{longstr, Route} | Rest], RKeys) -> - rkeys(Rest, [Route | RKeys]); -rkeys([_ | Rest], RKeys) -> - rkeys(Rest, RKeys); -rkeys(_, RKeys) -> - RKeys. + lists:append([case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of + {array, Routes} -> [Route || {longstr, Route} <- Routes]; + _ -> [] + end || HeaderKey <- ?ROUTING_HEADERS]). diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index be232bd2..16a3911d 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -243,7 +243,7 @@ handle_cast({command, Msg}, State = #ch{writer_pid = WriterPid}) -> handle_cast({deliver, ConsumerTag, AckRequired, Msg = {_QName, QPid, _MsgId, Redelivered, #basic_message{exchange_name = ExchangeName, - routing_key = RoutingKey, + routing_keys = [RoutingKey | _CcRoutes], content = Content}}}, State = #ch{writer_pid = WriterPid, next_tag = DeliveryTag}) -> @@ -609,7 +609,7 @@ handle_method(#'basic.get'{queue = QueueNameBin, {ok, MessageCount, Msg = {_QName, QPid, _MsgId, Redelivered, #basic_message{exchange_name = ExchangeName, - routing_key = RoutingKey, + routing_keys = [RoutingKey | _CcRoutes], content = Content}}} -> State1 = lock_message(not(NoAck), ack_record(DeliveryTag, none, Msg), @@ -1074,7 +1074,7 @@ binding_action(Fun, ExchangeNameBin, DestinationType, DestinationNameBin, end. basic_return(#basic_message{exchange_name = ExchangeName, - routing_key = RoutingKey, + routing_keys = [RoutingKey | _CcRoutes], content = Content}, WriterPid, Reason) -> {_Close, ReplyCode, ReplyText} = diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index 0baac1f8..82776c4a 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -36,9 +36,9 @@ description() -> {description, <<"AMQP direct exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, - #delivery{message = #basic_message{route_list = Routes}}) -> - lists:flatten([rabbit_router:match_routing_key(Name, RKey) || - RKey <- Routes]). + #delivery{message = #basic_message{routing_keys = Routes}}) -> + lists:append([rabbit_router:match_routing_key(Name, RKey) || + RKey <- Routes]). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl index beee4974..27251d12 100644 --- a/src/rabbit_exchange_type_topic.erl +++ b/src/rabbit_exchange_type_topic.erl @@ -44,12 +44,12 @@ description() -> {description, <<"AMQP topic exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, - #delivery{message = #basic_message{route_list = Routes}}) -> - lists:flatten([rabbit_router:match_bindings( - Name, - fun (#binding{key = BindingKey}) -> - topic_matches(BindingKey, RKey) - end) || RKey <- Routes]). + #delivery{message = #basic_message{routing_keys = Routes}}) -> + lists:append([rabbit_router:match_bindings( + Name, + fun (#binding{key = BindingKey}) -> + topic_matches(BindingKey, RKey) + end) || RKey <- Routes]). split_topic_key(Key) -> string:tokens(binary_to_list(Key), "."). diff --git a/src/rabbit_types.erl b/src/rabbit_types.erl index 3dbe740f..ab2300c0 100644 --- a/src/rabbit_types.erl +++ b/src/rabbit_types.erl @@ -64,7 +64,7 @@ -type(content() :: undecoded_content() | decoded_content()). -type(basic_message() :: #basic_message{exchange_name :: rabbit_exchange:name(), - routing_key :: rabbit_router:routing_key(), + routing_keys :: [rabbit_router:routing_key()], content :: content(), guid :: rabbit_guid:guid(), is_persistent :: boolean()}). -- cgit v1.2.1 From 340ae1fdefe6b7b9558292ca1e7ff43ecde06ac4 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Wed, 9 Feb 2011 12:16:20 +0000 Subject: Only clear encoded content when necessary --- src/rabbit_basic.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index f1348d33..5ea145d4 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -99,13 +99,13 @@ from_content(Content) -> %% This breaks the spec rule forbidding message modification strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} = DecodedContent, Key) when Headers =/= undefined -> - rabbit_binary_generator:clear_encoded_content( - case lists:keyfind(Key, 1, Headers) of - false -> DecodedContent; - Tuple -> Headers0 = lists:delete(Tuple, Headers), + case lists:keyfind(Key, 1, Headers) of + false -> DecodedContent; + Tuple -> Headers0 = lists:delete(Tuple, Headers), + rabbit_binary_generator:clear_encoded_content( DecodedContent#content{ - properties = Props#'P_basic'{headers = Headers0}} - end); + properties = Props#'P_basic'{headers = Headers0}}) + end; strip_header(DecodedContent, _Key) -> DecodedContent. -- cgit v1.2.1 From 3f2244546b15181c20fef174ec2d6fdd0d192221 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 9 Feb 2011 16:25:51 +0000 Subject: Correct the behaviour of /etc/init.d/rabbitmq-server status --- src/rabbit_control.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index a8903102..3f7fdf92 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -25,6 +25,7 @@ -define(QUIET_OPT, "-q"). -define(NODE_OPT, "-n"). -define(VHOST_OPT, "-p"). +-define(INHERENTLY_QUIET, [init_status]). %%---------------------------------------------------------------------------- @@ -62,7 +63,8 @@ start() -> end end, Opts), Command = list_to_atom(Command0), - Quiet = proplists:get_bool(?QUIET_OPT, Opts1), + Quiet = proplists:get_bool(?QUIET_OPT, Opts1) + orelse lists:member(Command, ?INHERENTLY_QUIET), Node = proplists:get_value(?NODE_OPT, Opts1), Inform = case Quiet of true -> fun (_Format, _Args1) -> ok end; @@ -79,6 +81,8 @@ start() -> false -> io:format("...done.~n") end, quit(0); + fail_silent -> + quit(1); {'EXIT', {function_clause, [{?MODULE, action, _} | _]}} -> print_error("invalid command '~s'", [string:join([atom_to_list(Command) | Args], " ")]), @@ -173,6 +177,14 @@ action(status, Node, [], _Opts, Inform) -> ok end; +action(init_status, Node, [], Opts, _) -> + case call(Node, {os, getpid, []}) of + {badrpc, _} -> io:format("~p is NOT running.", [Node]), + fail_silent; + Res -> io:format("~p is running (pid ~s).", [Node, Res]), + ok + end; + action(rotate_logs, Node, [], _Opts, Inform) -> Inform("Reopening logs for node ~p", [Node]), call(Node, {rabbit, rotate_logs, [""]}); -- cgit v1.2.1 From f4c47ea3cf0264003194a03dafe3ab1c1bf9be9f Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 9 Feb 2011 16:52:05 +0000 Subject: Undo 3b4956543c30 --- src/rabbit_control.erl | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index 3f7fdf92..a8903102 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -25,7 +25,6 @@ -define(QUIET_OPT, "-q"). -define(NODE_OPT, "-n"). -define(VHOST_OPT, "-p"). --define(INHERENTLY_QUIET, [init_status]). %%---------------------------------------------------------------------------- @@ -63,8 +62,7 @@ start() -> end end, Opts), Command = list_to_atom(Command0), - Quiet = proplists:get_bool(?QUIET_OPT, Opts1) - orelse lists:member(Command, ?INHERENTLY_QUIET), + Quiet = proplists:get_bool(?QUIET_OPT, Opts1), Node = proplists:get_value(?NODE_OPT, Opts1), Inform = case Quiet of true -> fun (_Format, _Args1) -> ok end; @@ -81,8 +79,6 @@ start() -> false -> io:format("...done.~n") end, quit(0); - fail_silent -> - quit(1); {'EXIT', {function_clause, [{?MODULE, action, _} | _]}} -> print_error("invalid command '~s'", [string:join([atom_to_list(Command) | Args], " ")]), @@ -177,14 +173,6 @@ action(status, Node, [], _Opts, Inform) -> ok end; -action(init_status, Node, [], Opts, _) -> - case call(Node, {os, getpid, []}) of - {badrpc, _} -> io:format("~p is NOT running.", [Node]), - fail_silent; - Res -> io:format("~p is running (pid ~s).", [Node, Res]), - ok - end; - action(rotate_logs, Node, [], _Opts, Inform) -> Inform("Reopening logs for node ~p", [Node]), call(Node, {rabbit, rotate_logs, [""]}); -- cgit v1.2.1 From 571f68ed98a25c014411f93106b58be010ceb515 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 9 Feb 2011 16:56:18 +0000 Subject: Add pid to rabbitmqctl status. --- src/rabbit.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit.erl b/src/rabbit.erl index 67e2e40f..5ff96f90 100644 --- a/src/rabbit.erl +++ b/src/rabbit.erl @@ -212,7 +212,8 @@ stop_and_halt() -> ok. status() -> - [{running_applications, application:which_applications()}] ++ + [{pid, list_to_integer(os:getpid())}, + {running_applications, application:which_applications()}] ++ rabbit_mnesia:status(). rotate_logs(BinarySuffix) -> -- cgit v1.2.1 From 315d950996ab6709e19e1253673540a6d53fe8a5 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 15:59:14 +0000 Subject: Pass protocol through to channel --- src/rabbit_channel.erl | 23 +++++++++++++---------- src/rabbit_channel_sup.erl | 11 ++++++----- 2 files changed, 19 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index a82e5eff..e80d8818 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -20,7 +20,7 @@ -behaviour(gen_server2). --export([start_link/7, do/2, do/3, flush/1, shutdown/1]). +-export([start_link/8, do/2, do/3, flush/1, shutdown/1]). -export([send_command/2, deliver/4, flushed/2, confirm/2]). -export([list/0, info_keys/0, info/1, info/2, info_all/0, info_all/1]). -export([emit_stats/1]). @@ -34,7 +34,8 @@ uncommitted_ack_q, unacked_message_q, user, virtual_host, most_recently_declared_queue, consumer_mapping, blocking, queue_collector_pid, stats_timer, - confirm_enabled, publish_seqno, unconfirmed, confirmed}). + confirm_enabled, publish_seqno, unconfirmed, confirmed, + protocol}). -define(MAX_PERMISSION_CACHE_SIZE, 12). @@ -66,9 +67,9 @@ -type(channel_number() :: non_neg_integer()). --spec(start_link/7 :: - (channel_number(), pid(), pid(), rabbit_types:user(), - rabbit_types:vhost(), pid(), +-spec(start_link/8 :: + (rabbit_types:protocol(), channel_number(), pid(), pid(), + rabbit_types:user(), rabbit_types:vhost(), pid(), fun ((non_neg_integer()) -> rabbit_types:ok(pid()))) -> rabbit_types:ok_pid_or_error()). -spec(do/2 :: (pid(), rabbit_framing:amqp_method_record()) -> 'ok'). @@ -94,10 +95,11 @@ %%---------------------------------------------------------------------------- -start_link(Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, +start_link(Protocol, Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, StartLimiterFun) -> - gen_server2:start_link(?MODULE, [Channel, ReaderPid, WriterPid, User, - VHost, CollectorPid, StartLimiterFun], []). + gen_server2:start_link(?MODULE, + [Protocol, Channel, ReaderPid, WriterPid, User, + VHost, CollectorPid, StartLimiterFun], []). do(Pid, Method) -> do(Pid, Method, none). @@ -148,7 +150,7 @@ emit_stats(Pid) -> %%--------------------------------------------------------------------------- -init([Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, +init([Protocol, Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, StartLimiterFun]) -> process_flag(trap_exit, true), ok = pg_local:join(rabbit_channels, self()), @@ -174,7 +176,8 @@ init([Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, confirm_enabled = false, publish_seqno = 1, unconfirmed = gb_trees:empty(), - confirmed = []}, + confirmed = [], + protocol = Protocol}, rabbit_event:notify(channel_created, infos(?CREATION_EVENT_KEYS, State)), rabbit_event:if_enabled(StatsTimer, fun() -> internal_emit_stats(State) end), diff --git a/src/rabbit_channel_sup.erl b/src/rabbit_channel_sup.erl index d21cfdb7..9bc0546c 100644 --- a/src/rabbit_channel_sup.erl +++ b/src/rabbit_channel_sup.erl @@ -34,8 +34,8 @@ {'tcp', rabbit_types:protocol(), rabbit_net:socket(), rabbit_channel:channel_number(), non_neg_integer(), pid(), rabbit_types:user(), rabbit_types:vhost(), pid()} | - {'direct', rabbit_channel:channel_number(), pid(), rabbit_types:user(), - rabbit_types:vhost(), pid()}). + {'direct', rabbit_types:protocol(), rabbit_channel:channel_number(), + pid(), rabbit_types:user(), rabbit_types:vhost(), pid()}). -spec(start_link/1 :: (start_link_args()) -> {'ok', pid(), {pid(), any()}}). @@ -56,18 +56,19 @@ start_link({tcp, Protocol, Sock, Channel, FrameMax, ReaderPid, User, VHost, supervisor2:start_child( SupPid, {channel, {rabbit_channel, start_link, - [Channel, ReaderPid, WriterPid, User, VHost, + [Protocol, Channel, ReaderPid, WriterPid, User, VHost, Collector, start_limiter_fun(SupPid)]}, intrinsic, ?MAX_WAIT, worker, [rabbit_channel]}), {ok, AState} = rabbit_command_assembler:init(Protocol), {ok, SupPid, {ChannelPid, AState}}; -start_link({direct, Channel, ClientChannelPid, User, VHost, Collector}) -> +start_link({direct, Protocol, Channel, ClientChannelPid, User, VHost, + Collector}) -> {ok, SupPid} = supervisor2:start_link(?MODULE, []), {ok, ChannelPid} = supervisor2:start_child( SupPid, {channel, {rabbit_channel, start_link, - [Channel, ClientChannelPid, ClientChannelPid, + [Protocol, Channel, ClientChannelPid, ClientChannelPid, User, VHost, Collector, start_limiter_fun(SupPid)]}, intrinsic, ?MAX_WAIT, worker, [rabbit_channel]}), {ok, SupPid, {ChannelPid, none}}. -- cgit v1.2.1 From 3f916148aeccabdb2237a34a26353e734759e69e Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:11:12 +0000 Subject: Make client-initiated channel.close work in new scheme --- src/rabbit_channel.erl | 19 +++++++++++-------- src/rabbit_reader.erl | 7 ++++--- src/rabbit_tests.erl | 12 ++++++------ 3 files changed, 21 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index e80d8818..d0a1e1f7 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -219,16 +219,12 @@ handle_cast({method, Method, Content}, State) -> ok = rabbit_writer:send_command(NewState#ch.writer_pid, Reply), noreply(NewState); {noreply, NewState} -> - noreply(NewState); - stop -> - {stop, normal, State#ch{state = terminating}} + noreply(NewState) catch exit:Reason = #amqp_error{} -> MethodName = rabbit_misc:method_record_type(Method), {stop, normal, terminating(Reason#amqp_error{method = MethodName}, State)}; - exit:normal -> - {stop, normal, State}; _:Reason -> {stop, {Reason, erlang:get_stacktrace()}, State} end; @@ -236,6 +232,10 @@ handle_cast({method, Method, Content}, State) -> handle_cast({flushed, QPid}, State) -> {noreply, queue_blocked(QPid, State), hibernate}; +handle_cast(closed, State = #ch{state = closing, writer_pid = WriterPid}) -> + ok = rabbit_writer:send_command_sync(WriterPid, #'channel.close_ok'{}), + {stop, normal, State}; + handle_cast(terminate, State) -> {stop, normal, State}; @@ -529,10 +529,13 @@ handle_method(#'channel.open'{}, _, _State) -> handle_method(_Method, _, #ch{state = starting}) -> rabbit_misc:protocol_error(channel_error, "expected 'channel.open'", []); -handle_method(#'channel.close'{}, _, State = #ch{writer_pid = WriterPid}) -> +handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, + channel = Channel}) -> + Self = self(), + ReaderPid ! {channel_closing, Channel, + fun () -> ok = gen_server2:cast(Self, closed) end}, ok = rollback_and_notify(State), - ok = rabbit_writer:send_command_sync(WriterPid, #'channel.close_ok'{}), - stop; + {noreply, State#ch{state = closing}}; handle_method(#'access.request'{},_, State) -> {reply, #'access.request_ok'{ticket = 1}, State}; diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 1781469a..1beb49d3 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -328,6 +328,10 @@ mainloop(Deb, State = #v1{parent = Parent, sock= Sock, recv_ref = Ref}) -> throw({inet_error, Reason}); {conserve_memory, Conserve} -> mainloop(Deb, internal_conserve_memory(Conserve, State)); + {channel_closing, Channel, Fun} -> + ok = Fun(), + erase({channel, Channel}), + mainloop(Deb, State); {'EXIT', Parent, Reason} -> terminate(io_lib:format("broker forced connection closure " "with reason '~w'", [Reason]), State), @@ -552,9 +556,6 @@ handle_frame(Type, Channel, Payload, Channel, ChPid, FramingState), put({channel, Channel}, {ChPid, NewAState}), case AnalyzedFrame of - {method, 'channel.close', _} -> - erase({channel, Channel}), - State; {method, MethodName, _} -> case (State#v1.connection_state =:= blocking andalso diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 58c369b5..82eb1beb 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -1019,9 +1019,9 @@ test_user_management() -> test_server_status() -> %% create a few things so there is some useful information to list Writer = spawn(fun () -> receive shutdown -> ok end end), - {ok, Ch} = rabbit_channel:start_link(1, self(), Writer, - user(<<"user">>), <<"/">>, self(), - fun (_) -> {ok, self()} end), + {ok, Ch} = rabbit_channel:start_link( + rabbit_framing_amqp_0_9_1, 1, self(), Writer, user(<<"user">>), + <<"/">>, self(), fun (_) -> {ok, self()} end), [Q, Q2] = [Queue || Name <- [<<"foo">>, <<"bar">>], {new, Queue = #amqqueue{}} <- [rabbit_amqqueue:declare( @@ -1079,9 +1079,9 @@ test_server_status() -> test_spawn(Receiver) -> Me = self(), Writer = spawn(fun () -> Receiver(Me) end), - {ok, Ch} = rabbit_channel:start_link(1, Me, Writer, - user(<<"guest">>), <<"/">>, self(), - fun (_) -> {ok, self()} end), + {ok, Ch} = rabbit_channel:start_link( + rabbit_framing_amqp_0_9_1, 1, Me, Writer, user(<<"guest">>), + <<"/">>, self(), fun (_) -> {ok, self()} end), ok = rabbit_channel:do(Ch, #'channel.open'{}), receive #'channel.open_ok'{} -> ok after 1000 -> throw(failed_to_receive_channel_open_ok) -- cgit v1.2.1 From 2b0f40bd868dbd712df84af2f4cbcd6c3eb8b082 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:17:00 +0000 Subject: Ignore methods to the channel if we're closing --- src/rabbit_channel.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index d0a1e1f7..1903ff42 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -529,6 +529,9 @@ handle_method(#'channel.open'{}, _, _State) -> handle_method(_Method, _, #ch{state = starting}) -> rabbit_misc:protocol_error(channel_error, "expected 'channel.open'", []); +handle_method(_Method, _, State = #ch{state = closing}) -> + {noreply, State}; + handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, channel = Channel}) -> Self = self(), -- cgit v1.2.1 From bee010e1e874ba13db35ece60e86db8723df7651 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:30:55 +0000 Subject: server initiated channel.close: keep channel alive until ch.close_ok is received. Make channel responsible for deciding on the severity of the error --- src/rabbit_channel.erl | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 1903ff42..9da29b37 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -219,12 +219,13 @@ handle_cast({method, Method, Content}, State) -> ok = rabbit_writer:send_command(NewState#ch.writer_pid, Reply), noreply(NewState); {noreply, NewState} -> - noreply(NewState) + noreply(NewState); + stop -> + {stop, normal, State} catch exit:Reason = #amqp_error{} -> MethodName = rabbit_misc:method_record_type(Method), - {stop, normal, terminating(Reason#amqp_error{method = MethodName}, - State)}; + send_exception(Reason#amqp_error{method = MethodName}, State); _:Reason -> {stop, {Reason, erlang:get_stacktrace()}, State} end; @@ -359,6 +360,21 @@ terminating(Reason, State = #ch{channel = Channel, reader_pid = Reader}) -> Reader ! {channel_exit, Channel, Reason}, State#ch{state = terminating}. +send_exception(Reason, State = #ch{channel = Channel, + writer_pid = WriterPid, + protocol = Protocol, + reader_pid = ReaderPid}) -> + {_ShouldClose, CloseChannel, CloseMethod} = + rabbit_binary_generator:map_exception(Channel, Reason, Protocol), + case CloseChannel of + Channel -> + ok = rabbit_writer:send_command(WriterPid, CloseMethod), + {noreply, State#ch{state = closing}}; + _ -> + ReaderPid ! {channel_exit, Channel, Reason}, + {stop, normal, State} + end. + return_queue_declare_ok(#resource{name = ActualName}, NoWait, MessageCount, ConsumerCount, State) -> return_ok(State#ch{most_recently_declared_queue = ActualName}, NoWait, @@ -529,6 +545,9 @@ handle_method(#'channel.open'{}, _, _State) -> handle_method(_Method, _, #ch{state = starting}) -> rabbit_misc:protocol_error(channel_error, "expected 'channel.open'", []); +handle_method(#'channel.close_ok'{}, _, #ch{state = closing}) -> + stop; + handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; -- cgit v1.2.1 From a7bbc02302c953556e7598482c6aa333b331ebed Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:34:08 +0000 Subject: If we receive a channel.close whilst we're closing the channel already (presumably client and server initiated close at the same time), then make sure we still send back the close_ok --- src/rabbit_channel.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 9da29b37..d5a0ca38 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -548,6 +548,9 @@ handle_method(_Method, _, #ch{state = starting}) -> handle_method(#'channel.close_ok'{}, _, #ch{state = closing}) -> stop; +handle_method(#'channel.close'{}, _, State = #ch{state = closing}) -> + {reply, #'channel.close_ok'{}, State}; + handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; -- cgit v1.2.1 From 6a0ba23e4ed77ba523937d4c0c4ab93904a587a5 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:51:32 +0000 Subject: Tidy channel termination (state=closing implies rollback_and_notify done). Erase both ch_pid and channel from reader state on DOWN (required for soft errors in which the channel issues the close, waits for the close_ok, and then stops normally) --- src/rabbit_channel.erl | 39 ++++++++++++++++++--------------------- src/rabbit_reader.erl | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index d5a0ca38..cc0192b0 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -306,18 +306,19 @@ handle_pre_hibernate(State = #ch{stats_timer = StatsTimer}) -> StatsTimer1 = rabbit_event:stop_stats_timer(StatsTimer), {hibernate, State#ch{stats_timer = StatsTimer1}}. -terminate(_Reason, State = #ch{state = terminating}) -> - terminate(State); - terminate(Reason, State) -> - Res = rollback_and_notify(State), - case Reason of - normal -> ok = Res; - shutdown -> ok = Res; - {shutdown, _Term} -> ok = Res; - _ -> ok + case State#ch.state of + closing -> ok; + _ -> Res = rollback_and_notify(State), + case Reason of + normal -> ok = Res; + shutdown -> ok = Res; + {shutdown, _Term} -> ok = Res; + _ -> ok + end end, - terminate(State). + pg_local:leave(rabbit_channels, self()), + rabbit_event:notify(channel_closed, [{pid, self()}]). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -355,24 +356,22 @@ return_ok(State, false, Msg) -> {reply, Msg, State}. ok_msg(true, _Msg) -> undefined; ok_msg(false, Msg) -> Msg. -terminating(Reason, State = #ch{channel = Channel, reader_pid = Reader}) -> - ok = rollback_and_notify(State), - Reader ! {channel_exit, Channel, Reason}, - State#ch{state = terminating}. - send_exception(Reason, State = #ch{channel = Channel, writer_pid = WriterPid, protocol = Protocol, reader_pid = ReaderPid}) -> {_ShouldClose, CloseChannel, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), + %% something bad's happened: rollback_and_notify make not be 'ok' + rollback_and_notify(State), + State1 = State#ch{state = closing}, case CloseChannel of Channel -> ok = rabbit_writer:send_command(WriterPid, CloseMethod), - {noreply, State#ch{state = closing}}; + {noreply, State1}; _ -> ReaderPid ! {channel_exit, Channel, Reason}, - {stop, normal, State} + {stop, normal, State1} end. return_queue_declare_ok(#resource{name = ActualName}, @@ -559,6 +558,8 @@ handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, Self = self(), ReaderPid ! {channel_closing, Channel, fun () -> ok = gen_server2:cast(Self, closed) end}, + %% no error, so rollback_and_notify should be 'ok'. Do in parallel + %% with the reader picking up our message and running our Fun. ok = rollback_and_notify(State), {noreply, State#ch{state = closing}}; @@ -1336,10 +1337,6 @@ coalesce_and_send(MsgSeqNos, MkMsgFun, WriterPid, MkMsgFun(SeqNo, false)) || SeqNo <- Ss], State. -terminate(_State) -> - pg_local:leave(rabbit_channels, self()), - rabbit_event:notify(channel_closed, [{pid, self()}]). - infos(Items, State) -> [{Item, i(Item, State)} || Item <- Items]. i(pid, _) -> self(); diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 1beb49d3..a267837d 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -446,7 +446,7 @@ close_channel(Channel, State) -> handle_dependent_exit(ChPid, Reason, State) -> case termination_kind(Reason) of controlled -> - erase({ch_pid, ChPid}), + channel_cleanup(ChPid), maybe_close(State); uncontrolled -> case channel_cleanup(ChPid) of -- cgit v1.2.1 From b3006d7808b2e29773c3a173f1202c6b514d2610 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:52:09 +0000 Subject: cosmetic --- src/rabbit_reader.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index a267837d..0cb4b5bf 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -519,8 +519,8 @@ maybe_close(State = #v1{connection_state = closing, maybe_close(State) -> State. -termination_kind(normal) -> controlled; -termination_kind(_) -> uncontrolled. +termination_kind(normal) -> controlled; +termination_kind(_) -> uncontrolled. handle_frame(Type, 0, Payload, State = #v1{connection_state = CS, -- cgit v1.2.1 From c6725f21f584b3e51dabb8cc8fa213d5e07ad261 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 16:56:10 +0000 Subject: Log soft channel errors --- src/rabbit_channel.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index cc0192b0..1598ac4d 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -367,6 +367,8 @@ send_exception(Reason, State = #ch{channel = Channel, State1 = State#ch{state = closing}, case CloseChannel of Channel -> + rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", + [ReaderPid, Channel, Reason]), ok = rabbit_writer:send_command(WriterPid, CloseMethod), {noreply, State1}; _ -> -- cgit v1.2.1 From 4a31066692b2254b310c4befb461363f0fb6c686 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 17:06:42 +0000 Subject: Always log the error from the channel; can now guarantee exceptions that turn up in the reader are hard exceptions --- src/rabbit_channel.erl | 4 ++-- src/rabbit_reader.erl | 23 +++++++---------------- 2 files changed, 9 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 1598ac4d..b830a2cb 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -362,13 +362,13 @@ send_exception(Reason, State = #ch{channel = Channel, reader_pid = ReaderPid}) -> {_ShouldClose, CloseChannel, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), + rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", + [ReaderPid, Channel, Reason]), %% something bad's happened: rollback_and_notify make not be 'ok' rollback_and_notify(State), State1 = State#ch{state = closing}, case CloseChannel of Channel -> - rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", - [ReaderPid, Channel, Reason]), ok = rabbit_writer:send_command(WriterPid, CloseMethod), {noreply, State1}; _ -> diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 0cb4b5bf..951cf439 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -976,29 +976,20 @@ process_channel_frame(Frame, ErrPid, Channel, ChPid, AState) -> AState end. -log_channel_error(ConnectionState, Channel, Reason) -> - rabbit_log:error("connection ~p (~p), channel ~p - error:~n~p~n", - [self(), ConnectionState, Channel, Reason]). - -handle_exception(State = #v1{connection_state = closed}, Channel, Reason) -> - log_channel_error(closed, Channel, Reason), +handle_exception(State = #v1{connection_state = closed}, _Channel, _Reason) -> State; -handle_exception(State = #v1{connection_state = CS}, Channel, Reason) -> - log_channel_error(CS, Channel, Reason), +handle_exception(State, Channel, Reason) -> send_exception(State, Channel, Reason). send_exception(State = #v1{connection = #connection{protocol = Protocol}}, Channel, Reason) -> - {ShouldClose, CloseChannel, CloseMethod} = + {true, 0, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), - NewState = case ShouldClose of - true -> terminate_channels(), - close_connection(State); - false -> close_channel(Channel, State) - end, + terminate_channels(), + State1 = close_connection(State), ok = rabbit_writer:internal_send_command( - NewState#v1.sock, CloseChannel, CloseMethod, Protocol), - NewState. + State1#v1.sock, 0, CloseMethod, Protocol), + State1. internal_emit_stats(State = #v1{stats_timer = StatsTimer}) -> rabbit_event:notify(connection_stats, infos(?STATISTICS_KEYS, State)), -- cgit v1.2.1 From a8d59a53adc6e9fdacd60b39ba15f666d0a3ffab Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 17:08:24 +0000 Subject: close_channel is no longer needed in reader, which means that closing as a channel state is no longer needed. --- src/rabbit_reader.erl | 23 ----------------------- 1 file changed, 23 deletions(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 951cf439..4d02657f 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -439,10 +439,6 @@ close_connection(State = #v1{queue_collector = Collector, erlang:send_after(TimeoutMillisec, self(), terminate_connection), State#v1{connection_state = closed}. -close_channel(Channel, State) -> - put({channel, Channel}, closing), - State. - handle_dependent_exit(ChPid, Reason, State) -> case termination_kind(Reason) of controlled -> @@ -566,25 +562,6 @@ handle_frame(Type, Channel, Payload, _ -> State end; - closing -> - %% According to the spec, after sending a - %% channel.close we must ignore all frames except - %% channel.close and channel.close_ok. In the - %% event of a channel.close, we should send back a - %% channel.close_ok. - case AnalyzedFrame of - {method, 'channel.close_ok', _} -> - erase({channel, Channel}); - {method, 'channel.close', _} -> - %% We're already closing this channel, so - %% there's no cleanup to do (notify - %% queues, etc.) - ok = rabbit_writer:internal_send_command( - State#v1.sock, Channel, - #'channel.close_ok'{}, Protocol); - _ -> ok - end, - State; undefined -> case ?IS_RUNNING(State) of true -> send_to_new_channel( -- cgit v1.2.1 From 5afd5e740eeb09dd32aa43d20a9ad4cb2d6fa316 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 17:16:28 +0000 Subject: The channel.close_ok from the client might be immediately followed by a channel.open. As such we must make sure that we send the channel.open to a new channel --- src/rabbit_reader.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 4d02657f..7fdd8ee6 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -552,6 +552,9 @@ handle_frame(Type, Channel, Payload, Channel, ChPid, FramingState), put({channel, Channel}, {ChPid, NewAState}), case AnalyzedFrame of + {method, #'channel.close_ok'{}, _} -> + erase({channel, Channel}), + State; {method, MethodName, _} -> case (State#v1.connection_state =:= blocking andalso -- cgit v1.2.1 From 0dee0f7e89ea5adf5cccfce608fbd6ca66111434 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 10 Feb 2011 17:55:35 +0000 Subject: Whoops --- src/rabbit_reader.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 7fdd8ee6..8f8b3fe6 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -552,7 +552,7 @@ handle_frame(Type, Channel, Payload, Channel, ChPid, FramingState), put({channel, Channel}, {ChPid, NewAState}), case AnalyzedFrame of - {method, #'channel.close_ok'{}, _} -> + {method, 'channel.close_ok', _} -> erase({channel, Channel}), State; {method, MethodName, _} -> -- cgit v1.2.1 From 0c1b04438e9d50e88ffaaf1d4307f637b94a1a1c Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 12:38:26 +0000 Subject: Remove annoying documentation --- src/rabbit_reader.erl | 86 --------------------------------------------------- 1 file changed, 86 deletions(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 8f8b3fe6..66a2fca3 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -57,92 +57,6 @@ -define(INFO_KEYS, ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [pid]). -%% connection lifecycle -%% -%% all state transitions and terminations are marked with *...* -%% -%% The lifecycle begins with: start handshake_timeout timer, *pre-init* -%% -%% all states, unless specified otherwise: -%% socket error -> *exit* -%% socket close -> *throw* -%% writer send failure -> *throw* -%% forced termination -> *exit* -%% handshake_timeout -> *throw* -%% pre-init: -%% receive protocol header -> send connection.start, *starting* -%% starting: -%% receive connection.start_ok -> *securing* -%% securing: -%% check authentication credentials -%% if authentication success -> send connection.tune, *tuning* -%% if more challenge needed -> send connection.secure, -%% receive connection.secure_ok *securing* -%% otherwise send close, *exit* -%% tuning: -%% receive connection.tune_ok -> start heartbeats, *opening* -%% opening: -%% receive connection.open -> send connection.open_ok, *running* -%% running: -%% receive connection.close -> -%% tell channels to terminate gracefully -%% if no channels then send connection.close_ok, start -%% terminate_connection timer, *closed* -%% else *closing* -%% forced termination -%% -> wait for channels to terminate forcefully, start -%% terminate_connection timer, send close, *exit* -%% channel exit with hard error -%% -> log error, wait for channels to terminate forcefully, start -%% terminate_connection timer, send close, *closed* -%% channel exit with soft error -%% -> log error, mark channel as closing, *running* -%% handshake_timeout -> ignore, *running* -%% heartbeat timeout -> *throw* -%% conserve_memory=true -> *blocking* -%% blocking: -%% conserve_memory=true -> *blocking* -%% conserve_memory=false -> *running* -%% receive a method frame for a content-bearing method -%% -> process, stop receiving, *blocked* -%% ...rest same as 'running' -%% blocked: -%% conserve_memory=true -> *blocked* -%% conserve_memory=false -> resume receiving, *running* -%% ...rest same as 'running' -%% closing: -%% socket close -> *terminate* -%% receive connection.close -> send connection.close_ok, -%% *closing* -%% receive frame -> ignore, *closing* -%% handshake_timeout -> ignore, *closing* -%% heartbeat timeout -> *throw* -%% channel exit with hard error -%% -> log error, wait for channels to terminate forcefully, start -%% terminate_connection timer, send close, *closed* -%% channel exit with soft error -%% -> log error, mark channel as closing -%% if last channel to exit then send connection.close_ok, -%% start terminate_connection timer, *closed* -%% else *closing* -%% channel exits normally -%% -> if last channel to exit then send connection.close_ok, -%% start terminate_connection timer, *closed* -%% closed: -%% socket close -> *terminate* -%% receive connection.close -> send connection.close_ok, -%% *closed* -%% receive connection.close_ok -> self() ! terminate_connection, -%% *closed* -%% receive frame -> ignore, *closed* -%% terminate_connection timeout -> *terminate* -%% handshake_timeout -> ignore, *closed* -%% heartbeat timeout -> *throw* -%% channel exit -> log error, *closed* -%% -%% -%% TODO: refactor the code so that the above is obvious - -define(IS_RUNNING(State), (State#v1.connection_state =:= running orelse State#v1.connection_state =:= blocking orelse -- cgit v1.2.1 From 99ac15fbc28d60adc0d38899a5a7f770530ca466 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Fri, 11 Feb 2011 12:51:50 +0000 Subject: Upgrade messages --- src/rabbit.erl | 1 + src/rabbit_msg_file.erl | 68 +++++++++++++++++++++++----------------- src/rabbit_msg_store.erl | 62 ++++++++++++++++++++++++++++++++++++ src/rabbit_upgrade_functions.erl | 19 +++++++++++ src/rabbit_variable_queue.erl | 15 ++++++++- 5 files changed, 135 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/src/rabbit.erl b/src/rabbit.erl index c6661d39..9e241e80 100644 --- a/src/rabbit.erl +++ b/src/rabbit.erl @@ -38,6 +38,7 @@ -rabbit_boot_step({database, [{mfa, {rabbit_mnesia, init, []}}, + {requires, file_handle_cache}, {enables, external_infrastructure}]}). -rabbit_boot_step({file_handle_cache, diff --git a/src/rabbit_msg_file.erl b/src/rabbit_msg_file.erl index cfea4982..ad87ee16 100644 --- a/src/rabbit_msg_file.erl +++ b/src/rabbit_msg_file.erl @@ -16,7 +16,7 @@ -module(rabbit_msg_file). --export([append/3, read/2, scan/2]). +-export([append/3, read/2, scan/2, scan/3]). %%---------------------------------------------------------------------------- @@ -48,6 +48,9 @@ -spec(scan/2 :: (io_device(), file_size()) -> {'ok', [{rabbit_guid:guid(), msg_size(), position()}], position()}). +-spec(scan/3 :: (io_device(), file_size(), + fun ((rabbit_guid:guid(), msg_size(), position(), binary()) -> any())) -> + {'ok', [any()], position()}). -endif. @@ -79,43 +82,50 @@ read(FileHdl, TotalSize) -> KO -> KO end. +scan_fun(Guid, TotalSize, Offset, _Msg) -> + {Guid, TotalSize, Offset}. + scan(FileHdl, FileSize) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, [], 0). + scan(FileHdl, FileSize, <<>>, 0, [], 0, fun scan_fun/4). + +scan(FileHdl, FileSize, Fun) when FileSize >= 0 -> + scan(FileHdl, FileSize, <<>>, 0, [], 0, Fun). -scan(_FileHdl, FileSize, _Data, FileSize, Acc, ScanOffset) -> +scan(_FileHdl, FileSize, _Data, FileSize, Acc, ScanOffset, _Fun) -> {ok, Acc, ScanOffset}; -scan(FileHdl, FileSize, Data, ReadOffset, Acc, ScanOffset) -> +scan(FileHdl, FileSize, Data, ReadOffset, Acc, ScanOffset, Fun) -> Read = lists:min([?SCAN_BLOCK_SIZE, (FileSize - ReadOffset)]), case file_handle_cache:read(FileHdl, Read) of {ok, Data1} -> {Data2, Acc1, ScanOffset1} = - scan(<>, Acc, ScanOffset), + scanner(<>, Acc, ScanOffset, Fun), ReadOffset1 = ReadOffset + size(Data1), - scan(FileHdl, FileSize, Data2, ReadOffset1, Acc1, ScanOffset1); + scan(FileHdl, FileSize, Data2, ReadOffset1, Acc1, ScanOffset1, Fun); _KO -> {ok, Acc, ScanOffset} end. -scan(<<>>, Acc, Offset) -> - {<<>>, Acc, Offset}; -scan(<<0:?INTEGER_SIZE_BITS, _Rest/binary>>, Acc, Offset) -> - {<<>>, Acc, Offset}; %% Nothing to do other than stop. -scan(<>, Acc, Offset) -> - TotalSize = Size + ?FILE_PACKING_ADJUSTMENT, - case WriteMarker of - ?WRITE_OK_MARKER -> - %% Here we take option 5 from - %% http://www.erlang.org/cgi-bin/ezmlm-cgi?2:mss:1569 in - %% which we read the Guid as a number, and then convert it - %% back to a binary in order to work around bugs in - %% Erlang's GC. - <> = - <>, - <> = <>, - scan(Rest, [{Guid, TotalSize, Offset} | Acc], Offset + TotalSize); - _ -> - scan(Rest, Acc, Offset + TotalSize) - end; -scan(Data, Acc, Offset) -> - {Data, Acc, Offset}. +scanner(<<>>, Acc, Offset, _Fun) -> + {<<>>, Acc, Offset}; +scanner(<<0:?INTEGER_SIZE_BITS, _Rest/binary>>, Acc, Offset, _Fun) -> + {<<>>, Acc, Offset}; %% Nothing to do other than stop. +scanner(<>, Acc, Offset, Fun) -> + TotalSize = Size + ?FILE_PACKING_ADJUSTMENT, + case WriteMarker of + ?WRITE_OK_MARKER -> + %% Here we take option 5 from + %% http://www.erlang.org/cgi-bin/ezmlm-cgi?2:mss:1569 in + %% which we read the Guid as a number, and then convert it + %% back to a binary in order to work around bugs in + %% Erlang's GC. + <> = + <>, + <> = <>, + scanner(Rest, [Fun(Guid, TotalSize, Offset, Msg) | Acc], + Offset + TotalSize, Fun); + _ -> + scanner(Rest, Acc, Offset + TotalSize, Fun) + end; +scanner(Data, Acc, Offset, _Fun) -> + {Data, Acc, Offset}. diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index e9c356e1..bd8d61e8 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -26,16 +26,20 @@ -export([sync/1, set_maximum_since_use/2, has_readers/2, combine_files/3, delete_file/2]). %% internal +-export([transform_dir/3, force_recovery/2]). %% upgrade + -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3, prioritise_call/3, prioritise_cast/2]). %%---------------------------------------------------------------------------- -include("rabbit_msg_store.hrl"). +-include_lib("kernel/include/file.hrl"). -define(SYNC_INTERVAL, 5). %% milliseconds -define(CLEAN_FILENAME, "clean.dot"). -define(FILE_SUMMARY_FILENAME, "file_summary.ets"). +-define(TRANSFORM_TMP, "transform_tmp"). -define(BINARY_MODE, [raw, binary]). -define(READ_MODE, [read]). @@ -160,6 +164,10 @@ -spec(combine_files/3 :: (non_neg_integer(), non_neg_integer(), gc_state()) -> deletion_thunk()). -spec(delete_file/2 :: (non_neg_integer(), gc_state()) -> deletion_thunk()). +-spec(force_recovery/2 :: (file:filename(), server()) -> 'ok'). +-spec(transform_dir/3 :: (file:filename(), server(), + fun ((binary())->({'ok', msg()} | {error, any()}))) -> + non_neg_integer()). -endif. @@ -1956,3 +1964,57 @@ copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, {got, FinalOffsetZ}, {destination, Destination}]} end. + +force_recovery(BaseDir, Server) -> + Dir = filename:join(BaseDir, atom_to_list(Server)), + file:delete(filename:join(Dir, ?CLEAN_FILENAME)), + [file:delete(filename:join(Dir, File)) || + File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], + ok. + +transform_dir(BaseDir, Server, TransformFun) -> + Dir = filename:join(BaseDir, atom_to_list(Server)), + TmpDir = filename:join(Dir, ?TRANSFORM_TMP), + case filelib:is_dir(TmpDir) of + true -> throw({error, previously_failed_transform}); + false -> + Count = lists:sum( + [transform_msg_file(filename:join(Dir, File), + filename:join(TmpDir, File), + TransformFun) || + File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)]), + [file:delete(filename:join(Dir, File)) || + File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)], + [file:copy(filename:join(TmpDir, File), filename:join(Dir, File)) || + File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], + [file:delete(filename:join(TmpDir, File)) || + File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], + ok = file:del_dir(TmpDir), + Count + end. + +transform_msg_file(FileOld, FileNew, TransformFun) -> + rabbit_misc:ensure_parent_dirs_exist(FileNew), + {ok, #file_info{size=Size}} = file:read_file_info(FileOld), + {ok, RefOld} = file_handle_cache:open(FileOld, [raw, binary, read], []), + {ok, RefNew} = file_handle_cache:open(FileNew, [raw, binary, write], + [{write_buffer, + ?HANDLE_CACHE_BUFFER_SIZE}]), + {ok, Acc, Size} = + rabbit_msg_file:scan( + RefOld, Size, + fun(Guid, _Size, _Offset, BinMsg) -> + case TransformFun(BinMsg) of + {ok, MsgNew} -> + rabbit_msg_file:append(RefNew, Guid, MsgNew), + 1; + {error, Reason} -> + error_logger:error_msg("Message transform failed: ~p~n", + [Reason]), + 0 + end + end), + file_handle_cache:close(RefOld), + file_handle_cache:close(RefNew), + lists:sum(Acc). + diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 68b88b3e..f4e27cc8 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -25,6 +25,7 @@ -rabbit_upgrade({add_ip_to_listener, []}). -rabbit_upgrade({internal_exchanges, []}). -rabbit_upgrade({user_to_internal_user, [hash_passwords]}). +-rabbit_upgrade({multiple_routing_keys, []}). %% ------------------------------------------------------------------- @@ -35,6 +36,7 @@ -spec(add_ip_to_listener/0 :: () -> 'ok'). -spec(internal_exchanges/0 :: () -> 'ok'). -spec(user_to_internal_user/0 :: () -> 'ok'). +-spec(multiple_routing_keys/0 :: () -> 'ok'). -endif. @@ -101,3 +103,20 @@ mnesia(TableName, Fun, FieldList, NewRecordName) -> {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList, NewRecordName), ok. + +%%-------------------------------------------------------------------- + +multiple_routing_keys() -> + _UpgradeMsgCount = rabbit_variable_queue:transform_storage( + fun (BinMsg) -> + case binary_to_term(BinMsg) of + {basic_message, ExchangeName, Routing_Key, Content, Guid, + Persistent} -> + {ok, {basic_message, ExchangeName, [Routing_Key], Content, + Guid, Persistent}}; + _ -> + {error, corrupt_message} + end + end), + ok. + diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index 7142d560..f2176c0e 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -22,7 +22,7 @@ requeue/3, len/1, is_empty/1, dropwhile/2, set_ram_duration_target/2, ram_duration/1, needs_idle_timeout/1, idle_timeout/1, handle_pre_hibernate/1, - status/1]). + status/1, transform_storage/1]). -export([start/1, stop/0]). @@ -1801,3 +1801,16 @@ push_betas_to_deltas(Generator, Limit, Q, Count, RamIndexCount, IndexState) -> push_betas_to_deltas( Generator, Limit, Qa, Count + 1, RamIndexCount1, IndexState1) end. + +%%---------------------------------------------------------------------------- +%% Upgrading +%%---------------------------------------------------------------------------- + +%% Assumes message store is not running +transform_storage(TransformFun) -> + transform_store(?PERSISTENT_MSG_STORE, TransformFun) + + transform_store(?TRANSIENT_MSG_STORE, TransformFun). + +transform_store(Store, TransformFun) -> + rabbit_msg_store:force_recovery(rabbit_mnesia:dir(), Store), + rabbit_msg_store:transform_dir(rabbit_mnesia:dir(), Store, TransformFun). -- cgit v1.2.1 From 9df99ec6d95280c27710c70d96e5c55a99a3f85e Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 13:04:21 +0000 Subject: Remove hardcoded protocol now channels have the real protocol --- src/rabbit_channel.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index b830a2cb..aeb84758 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1115,9 +1115,8 @@ binding_action(Fun, ExchangeNameBin, DestinationType, DestinationNameBin, basic_return(#basic_message{exchange_name = ExchangeName, routing_key = RoutingKey, content = Content}, - WriterPid, Reason) -> - {_Close, ReplyCode, ReplyText} = - rabbit_framing_amqp_0_9_1:lookup_amqp_exception(Reason), + #ch{protocol = Protocol, writer_pid = WriterPid}, Reason) -> + {_Close, ReplyCode, ReplyText} = Protocol:lookup_amqp_exception(Reason), ok = rabbit_writer:send_command( WriterPid, #'basic.return'{reply_code = ReplyCode, @@ -1274,10 +1273,10 @@ is_message_persistent(Content) -> end. process_routing_result(unroutable, _, XName, MsgSeqNo, Msg, State) -> - ok = basic_return(Msg, State#ch.writer_pid, no_route), + ok = basic_return(Msg, State, no_route), record_confirm(MsgSeqNo, XName, State); process_routing_result(not_delivered, _, XName, MsgSeqNo, Msg, State) -> - ok = basic_return(Msg, State#ch.writer_pid, no_consumers), + ok = basic_return(Msg, State, no_consumers), record_confirm(MsgSeqNo, XName, State); process_routing_result(routed, [], XName, MsgSeqNo, _, State) -> record_confirm(MsgSeqNo, XName, State); -- cgit v1.2.1 From d269b2f64c413128b2c66e94528d1eb99bf8235f Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 13:07:51 +0000 Subject: cosmetic - moving protocol field up. --- src/rabbit_channel.erl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index aeb84758..b2b78470 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -29,13 +29,12 @@ handle_info/2, handle_pre_hibernate/1, prioritise_call/3, prioritise_cast/2]). --record(ch, {state, channel, reader_pid, writer_pid, limiter_pid, +-record(ch, {state, protocol, channel, reader_pid, writer_pid, limiter_pid, start_limiter_fun, transaction_id, tx_participants, next_tag, uncommitted_ack_q, unacked_message_q, user, virtual_host, most_recently_declared_queue, consumer_mapping, blocking, queue_collector_pid, stats_timer, - confirm_enabled, publish_seqno, unconfirmed, confirmed, - protocol}). + confirm_enabled, publish_seqno, unconfirmed, confirmed}). -define(MAX_PERMISSION_CACHE_SIZE, 12). @@ -156,6 +155,7 @@ init([Protocol, Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, ok = pg_local:join(rabbit_channels, self()), StatsTimer = rabbit_event:init_stats_timer(), State = #ch{state = starting, + protocol = Protocol, channel = Channel, reader_pid = ReaderPid, writer_pid = WriterPid, @@ -176,8 +176,7 @@ init([Protocol, Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, confirm_enabled = false, publish_seqno = 1, unconfirmed = gb_trees:empty(), - confirmed = [], - protocol = Protocol}, + confirmed = []}, rabbit_event:notify(channel_created, infos(?CREATION_EVENT_KEYS, State)), rabbit_event:if_enabled(StatsTimer, fun() -> internal_emit_stats(State) end), @@ -356,9 +355,9 @@ return_ok(State, false, Msg) -> {reply, Msg, State}. ok_msg(true, _Msg) -> undefined; ok_msg(false, Msg) -> Msg. -send_exception(Reason, State = #ch{channel = Channel, +send_exception(Reason, State = #ch{protocol = Protocol, + channel = Channel, writer_pid = WriterPid, - protocol = Protocol, reader_pid = ReaderPid}) -> {_ShouldClose, CloseChannel, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), -- cgit v1.2.1 From bb2925295671a7c1a5b9526804a848eeacd63183 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 13:14:27 +0000 Subject: Remove boolean shouldClose from response of bin_gen:map_exception --- src/rabbit_binary_generator.erl | 26 +++++++++++--------------- src/rabbit_channel.erl | 2 +- src/rabbit_reader.erl | 2 +- 3 files changed, 13 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/rabbit_binary_generator.erl b/src/rabbit_binary_generator.erl index d67c7f58..dc81ace6 100644 --- a/src/rabbit_binary_generator.erl +++ b/src/rabbit_binary_generator.erl @@ -61,8 +61,7 @@ -spec(map_exception/3 :: (rabbit_channel:channel_number(), rabbit_types:amqp_error() | any(), rabbit_types:protocol()) -> - {boolean(), - rabbit_channel:channel_number(), + {rabbit_channel:channel_number(), rabbit_framing:amqp_method_record()}). -endif. @@ -301,24 +300,21 @@ clear_encoded_content(Content = #content{}) -> map_exception(Channel, Reason, Protocol) -> {SuggestedClose, ReplyCode, ReplyText, FailedMethod} = lookup_amqp_exception(Reason, Protocol), - ShouldClose = SuggestedClose orelse (Channel == 0), {ClassId, MethodId} = case FailedMethod of {_, _} -> FailedMethod; none -> {0, 0}; _ -> Protocol:method_id(FailedMethod) end, - {CloseChannel, CloseMethod} = - case ShouldClose of - true -> {0, #'connection.close'{reply_code = ReplyCode, - reply_text = ReplyText, - class_id = ClassId, - method_id = MethodId}}; - false -> {Channel, #'channel.close'{reply_code = ReplyCode, - reply_text = ReplyText, - class_id = ClassId, - method_id = MethodId}} - end, - {ShouldClose, CloseChannel, CloseMethod}. + case SuggestedClose orelse (Channel == 0) of + true -> {0, #'connection.close'{reply_code = ReplyCode, + reply_text = ReplyText, + class_id = ClassId, + method_id = MethodId}}; + false -> {Channel, #'channel.close'{reply_code = ReplyCode, + reply_text = ReplyText, + class_id = ClassId, + method_id = MethodId}} + end. lookup_amqp_exception(#amqp_error{name = Name, explanation = Expl, diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index b2b78470..cdf8015c 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -359,7 +359,7 @@ send_exception(Reason, State = #ch{protocol = Protocol, channel = Channel, writer_pid = WriterPid, reader_pid = ReaderPid}) -> - {_ShouldClose, CloseChannel, CloseMethod} = + {CloseChannel, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", [ReaderPid, Channel, Reason]), diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 66a2fca3..a45166af 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -877,7 +877,7 @@ handle_exception(State, Channel, Reason) -> send_exception(State = #v1{connection = #connection{protocol = Protocol}}, Channel, Reason) -> - {true, 0, CloseMethod} = + {0, CloseMethod} = rabbit_binary_generator:map_exception(Channel, Reason, Protocol), terminate_channels(), State1 = close_connection(State), -- cgit v1.2.1 From 7764d9347e2f9305d9135531f03861f7e740d2cd Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 13:18:54 +0000 Subject: Move the state modification into rollback_and_notify --- src/rabbit_channel.erl | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index cdf8015c..74a93f9f 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -306,15 +306,12 @@ handle_pre_hibernate(State = #ch{stats_timer = StatsTimer}) -> {hibernate, State#ch{stats_timer = StatsTimer1}}. terminate(Reason, State) -> - case State#ch.state of - closing -> ok; - _ -> Res = rollback_and_notify(State), - case Reason of - normal -> ok = Res; - shutdown -> ok = Res; - {shutdown, _Term} -> ok = Res; - _ -> ok - end + {Res, _State1} = rollback_and_notify(State), + case Reason of + normal -> ok = Res; + shutdown -> ok = Res; + {shutdown, _Term} -> ok = Res; + _ -> ok end, pg_local:leave(rabbit_channels, self()), rabbit_event:notify(channel_closed, [{pid, self()}]). @@ -364,8 +361,7 @@ send_exception(Reason, State = #ch{protocol = Protocol, rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", [ReaderPid, Channel, Reason]), %% something bad's happened: rollback_and_notify make not be 'ok' - rollback_and_notify(State), - State1 = State#ch{state = closing}, + {_Result, State1} = rollback_and_notify(State), case CloseChannel of Channel -> ok = rabbit_writer:send_command(WriterPid, CloseMethod), @@ -561,8 +557,8 @@ handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, fun () -> ok = gen_server2:cast(Self, closed) end}, %% no error, so rollback_and_notify should be 'ok'. Do in parallel %% with the reader picking up our message and running our Fun. - ok = rollback_and_notify(State), - {noreply, State#ch{state = closing}}; + {ok, State1} = rollback_and_notify(State), + {noreply, State1}; handle_method(#'access.request'{},_, State) -> {reply, #'access.request_ok'{ticket = 1}, State}; @@ -1202,10 +1198,13 @@ internal_rollback(State = #ch{transaction_id = TxnKey, NewUAMQ = queue:join(UAQ, UAMQ), new_tx(State#ch{unacked_message_q = NewUAMQ}). +rollback_and_notify(State = #ch{state = closing}) -> + {ok, State}; rollback_and_notify(State = #ch{transaction_id = none}) -> - notify_queues(State); + {notify_queues(State), State#ch{state = closing}}; rollback_and_notify(State) -> - notify_queues(internal_rollback(State)). + State1 = internal_rollback(State), + {notify_queues(State1), State1#ch{state = closing}}. fold_per_queue(F, Acc0, UAQ) -> D = rabbit_misc:queue_fold( -- cgit v1.2.1 From 80f6d15e7e9b27abd06c1c64f8bc12a638ef2aef Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 13:27:28 +0000 Subject: drop CPS in favour of explicit API --- src/rabbit_channel.erl | 13 ++++++++----- src/rabbit_reader.erl | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 74a93f9f..180e9393 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -23,7 +23,7 @@ -export([start_link/8, do/2, do/3, flush/1, shutdown/1]). -export([send_command/2, deliver/4, flushed/2, confirm/2]). -export([list/0, info_keys/0, info/1, info/2, info_all/0, info_all/1]). --export([emit_stats/1]). +-export([emit_stats/1, ready_for_close/1]). -export([init/1, terminate/2, code_change/3, handle_call/3, handle_cast/2, handle_info/2, handle_pre_hibernate/1, prioritise_call/3, @@ -89,6 +89,7 @@ -spec(info_all/0 :: () -> [rabbit_types:infos()]). -spec(info_all/1 :: (rabbit_types:info_keys()) -> [rabbit_types:infos()]). -spec(emit_stats/1 :: (pid()) -> 'ok'). +-spec(ready_for_close/1 :: (pid()) -> 'ok'). -endif. @@ -147,6 +148,9 @@ info_all(Items) -> emit_stats(Pid) -> gen_server2:cast(Pid, emit_stats). +ready_for_close(Pid) -> + gen_server2:cast(Pid, ready_for_close). + %%--------------------------------------------------------------------------- init([Protocol, Channel, ReaderPid, WriterPid, User, VHost, CollectorPid, @@ -232,7 +236,8 @@ handle_cast({method, Method, Content}, State) -> handle_cast({flushed, QPid}, State) -> {noreply, queue_blocked(QPid, State), hibernate}; -handle_cast(closed, State = #ch{state = closing, writer_pid = WriterPid}) -> +handle_cast(ready_for_close, State = #ch{state = closing, + writer_pid = WriterPid}) -> ok = rabbit_writer:send_command_sync(WriterPid, #'channel.close_ok'{}), {stop, normal, State}; @@ -552,9 +557,7 @@ handle_method(_Method, _, State = #ch{state = closing}) -> handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, channel = Channel}) -> - Self = self(), - ReaderPid ! {channel_closing, Channel, - fun () -> ok = gen_server2:cast(Self, closed) end}, + ReaderPid ! {channel_closing, Channel, self()}, %% no error, so rollback_and_notify should be 'ok'. Do in parallel %% with the reader picking up our message and running our Fun. {ok, State1} = rollback_and_notify(State), diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index a45166af..f950bb00 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -242,8 +242,8 @@ mainloop(Deb, State = #v1{parent = Parent, sock= Sock, recv_ref = Ref}) -> throw({inet_error, Reason}); {conserve_memory, Conserve} -> mainloop(Deb, internal_conserve_memory(Conserve, State)); - {channel_closing, Channel, Fun} -> - ok = Fun(), + {channel_closing, Channel, ChPid} -> + ok = rabbit_channel:ready_for_close(ChPid), erase({channel, Channel}), mainloop(Deb, State); {'EXIT', Parent, Reason} -> -- cgit v1.2.1 From 18277f7419b7b187a609147b96a1925a6280f8f8 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 14:52:32 +0000 Subject: Always erase both channel and ch_pid. This is lovely - it makes it a proper bidirectional mapping. All java tests now pass again. --- src/rabbit_channel.erl | 5 ++--- src/rabbit_reader.erl | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 180e9393..7d47eecd 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -555,9 +555,8 @@ handle_method(#'channel.close'{}, _, State = #ch{state = closing}) -> handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; -handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid, - channel = Channel}) -> - ReaderPid ! {channel_closing, Channel, self()}, +handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> + ReaderPid ! {channel_closing, self()}, %% no error, so rollback_and_notify should be 'ok'. Do in parallel %% with the reader picking up our message and running our Fun. {ok, State1} = rollback_and_notify(State), diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index f950bb00..a8abebd8 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -242,9 +242,9 @@ mainloop(Deb, State = #v1{parent = Parent, sock= Sock, recv_ref = Ref}) -> throw({inet_error, Reason}); {conserve_memory, Conserve} -> mainloop(Deb, internal_conserve_memory(Conserve, State)); - {channel_closing, Channel, ChPid} -> + {channel_closing, ChPid} -> ok = rabbit_channel:ready_for_close(ChPid), - erase({channel, Channel}), + channel_cleanup(ChPid), mainloop(Deb, State); {'EXIT', Parent, Reason} -> terminate(io_lib:format("broker forced connection closure " @@ -467,7 +467,7 @@ handle_frame(Type, Channel, Payload, put({channel, Channel}, {ChPid, NewAState}), case AnalyzedFrame of {method, 'channel.close_ok', _} -> - erase({channel, Channel}), + channel_cleanup(ChPid), State; {method, MethodName, _} -> case (State#v1.connection_state =:= blocking -- cgit v1.2.1 From 8bb9bd70f35fdfb56a434d0fb9f2422cc81c3ffa Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:06:28 +0000 Subject: Correct rabbit test for new channel exit sequence --- src/rabbit_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 82eb1beb..57a0459a 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -1305,7 +1305,7 @@ test_queue_cleanup(_SecondaryNode) -> rabbit_channel:do(Ch, #'queue.declare'{ passive = true, queue = ?CLEANUP_QUEUE_NAME }), receive - {channel_exit, 1, {amqp_error, not_found, _, _}} -> + #'channel.close'{reply_code = 404} = CC -> ok after 2000 -> throw(failed_to_receive_channel_exit) -- cgit v1.2.1 From 652ebe7b70d9547a6e05e2b6e33ef32001da8bb4 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:07:31 +0000 Subject: Remove needless variable --- src/rabbit_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 57a0459a..45a11766 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -1305,7 +1305,7 @@ test_queue_cleanup(_SecondaryNode) -> rabbit_channel:do(Ch, #'queue.declare'{ passive = true, queue = ?CLEANUP_QUEUE_NAME }), receive - #'channel.close'{reply_code = 404} = CC -> + #'channel.close'{reply_code = 404} -> ok after 2000 -> throw(failed_to_receive_channel_exit) -- cgit v1.2.1 From 68747b3f3be08b88cda074220ee6606cfae06b7f Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:19:29 +0000 Subject: Log in the case of an uncontrolled, unexpected channel exit --- src/rabbit_reader.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index a8abebd8..e5862a92 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -361,7 +361,10 @@ handle_dependent_exit(ChPid, Reason, State) -> uncontrolled -> case channel_cleanup(ChPid) of undefined -> exit({abnormal_dependent_exit, ChPid, Reason}); - Channel -> maybe_close( + Channel -> rabbit_log:error( + "connection ~p, channel ~p - error:~n~p~n", + [self(), Channel, Reason]), + maybe_close( handle_exception(State, Channel, Reason)) end end. -- cgit v1.2.1 From fa1066429b954cb7cc110cd068e127402b22d70a Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:22:13 +0000 Subject: Demonitor on channel_cleanup --- src/rabbit_reader.erl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index e5862a92..a9403105 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -371,13 +371,14 @@ handle_dependent_exit(ChPid, Reason, State) -> channel_cleanup(ChPid) -> case get({ch_pid, ChPid}) of - undefined -> undefined; - Channel -> erase({channel, Channel}), - erase({ch_pid, ChPid}), - Channel + undefined -> undefined; + {Channel, MRef} -> erase({channel, Channel}), + erase({ch_pid, ChPid}), + erlang:demonitor(MRef, [flush]), + Channel end. -all_channels() -> [ChPid || {{ch_pid, ChPid}, _Channel} <- get()]. +all_channels() -> [ChPid || {{ch_pid, ChPid}, _ChannelMRef} <- get()]. terminate_channels() -> NChannels = @@ -853,11 +854,11 @@ send_to_new_channel(Channel, AnalyzedFrame, State) -> rabbit_channel_sup_sup:start_channel( ChanSupSup, {tcp, Protocol, Sock, Channel, FrameMax, self(), User, VHost, Collector}), - erlang:monitor(process, ChPid), + MRef = erlang:monitor(process, ChPid), NewAState = process_channel_frame(AnalyzedFrame, self(), Channel, ChPid, AState), put({channel, Channel}, {ChPid, NewAState}), - put({ch_pid, ChPid}, Channel), + put({ch_pid, ChPid}, {Channel, MRef}), State. process_channel_frame(Frame, ErrPid, Channel, ChPid, AState) -> -- cgit v1.2.1 From 0ba0463ad0e8cc923b6f728dccdeec2a0a270d4c Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:23:37 +0000 Subject: correct comment --- src/rabbit_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 7d47eecd..df56ee49 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -558,7 +558,7 @@ handle_method(_Method, _, State = #ch{state = closing}) -> handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> ReaderPid ! {channel_closing, self()}, %% no error, so rollback_and_notify should be 'ok'. Do in parallel - %% with the reader picking up our message and running our Fun. + %% with the reader picking up our message and casting back to us. {ok, State1} = rollback_and_notify(State), {noreply, State1}; -- cgit v1.2.1 From 6b35f68bf47d7f7ef37d8d2696b1c20b5a19e2b3 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 11 Feb 2011 15:24:44 +0000 Subject: Do rollback first so that reader will catch DOWN if rollback errors --- src/rabbit_channel.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index df56ee49..b9d1baf0 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -556,10 +556,8 @@ handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> - ReaderPid ! {channel_closing, self()}, - %% no error, so rollback_and_notify should be 'ok'. Do in parallel - %% with the reader picking up our message and casting back to us. {ok, State1} = rollback_and_notify(State), + ReaderPid ! {channel_closing, self()}, {noreply, State1}; handle_method(#'access.request'{},_, State) -> -- cgit v1.2.1 From 131e0bcdad6b6ecaa82ae807ec033a289c937179 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Fri, 11 Feb 2011 17:28:13 +0000 Subject: rabbit_msg_file:scan/4 now looks a bit more like fold Also ignore garbage at the end of a message store --- src/rabbit_msg_file.erl | 20 ++++++++++---------- src/rabbit_msg_store.erl | 31 ++++++++++++++----------------- src/rabbit_upgrade_functions.erl | 2 +- src/rabbit_variable_queue.erl | 2 +- 4 files changed, 26 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_file.erl b/src/rabbit_msg_file.erl index ad87ee16..9d5953d5 100644 --- a/src/rabbit_msg_file.erl +++ b/src/rabbit_msg_file.erl @@ -16,7 +16,7 @@ -module(rabbit_msg_file). --export([append/3, read/2, scan/2, scan/3]). +-export([append/3, read/2, scan/2, scan/4]). %%---------------------------------------------------------------------------- @@ -48,9 +48,9 @@ -spec(scan/2 :: (io_device(), file_size()) -> {'ok', [{rabbit_guid:guid(), msg_size(), position()}], position()}). --spec(scan/3 :: (io_device(), file_size(), - fun ((rabbit_guid:guid(), msg_size(), position(), binary()) -> any())) -> - {'ok', [any()], position()}). +-spec(scan/4 :: (io_device(), file_size(), + fun (({rabbit_guid:guid(), msg_size(), position(), binary()}, A) -> A), + A) -> {'ok', A, position()}). -endif. @@ -82,14 +82,14 @@ read(FileHdl, TotalSize) -> KO -> KO end. -scan_fun(Guid, TotalSize, Offset, _Msg) -> - {Guid, TotalSize, Offset}. +scan_fun({Guid, TotalSize, Offset, _Msg}, Acc) -> + [{Guid, TotalSize, Offset} | Acc]. scan(FileHdl, FileSize) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, [], 0, fun scan_fun/4). + scan(FileHdl, FileSize, <<>>, 0, [], 0, fun scan_fun/2). -scan(FileHdl, FileSize, Fun) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, [], 0, Fun). +scan(FileHdl, FileSize, Fun, Acc) when FileSize >= 0 -> + scan(FileHdl, FileSize, <<>>, 0, Acc, 0, Fun). scan(_FileHdl, FileSize, _Data, FileSize, Acc, ScanOffset, _Fun) -> {ok, Acc, ScanOffset}; @@ -122,7 +122,7 @@ scanner(<> = <>, <> = <>, - scanner(Rest, [Fun(Guid, TotalSize, Offset, Msg) | Acc], + scanner(Rest, Fun({Guid, TotalSize, Offset, Msg}, Acc), Offset + TotalSize, Fun); _ -> scanner(Rest, Acc, Offset + TotalSize, Fun) diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index bd8d61e8..b827eba9 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -166,8 +166,7 @@ -spec(delete_file/2 :: (non_neg_integer(), gc_state()) -> deletion_thunk()). -spec(force_recovery/2 :: (file:filename(), server()) -> 'ok'). -spec(transform_dir/3 :: (file:filename(), server(), - fun ((binary())->({'ok', msg()} | {error, any()}))) -> - non_neg_integer()). + fun ((binary()) -> ({'ok', msg()} | {error, any()}))) -> 'ok'). -endif. @@ -1976,21 +1975,19 @@ transform_dir(BaseDir, Server, TransformFun) -> Dir = filename:join(BaseDir, atom_to_list(Server)), TmpDir = filename:join(Dir, ?TRANSFORM_TMP), case filelib:is_dir(TmpDir) of - true -> throw({error, previously_failed_transform}); + true -> throw({error, transform_failed_previously}); false -> - Count = lists:sum( - [transform_msg_file(filename:join(Dir, File), - filename:join(TmpDir, File), - TransformFun) || - File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)]), + [transform_msg_file(filename:join(Dir, File), + filename:join(TmpDir, File), + TransformFun) || + File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)], [file:delete(filename:join(Dir, File)) || File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)], [file:copy(filename:join(TmpDir, File), filename:join(Dir, File)) || File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], [file:delete(filename:join(TmpDir, File)) || File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], - ok = file:del_dir(TmpDir), - Count + ok = file:del_dir(TmpDir) end. transform_msg_file(FileOld, FileNew, TransformFun) -> @@ -2000,21 +1997,21 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> {ok, RefNew} = file_handle_cache:open(FileNew, [raw, binary, write], [{write_buffer, ?HANDLE_CACHE_BUFFER_SIZE}]), - {ok, Acc, Size} = + {ok, Acc, _IgnoreSize} = rabbit_msg_file:scan( RefOld, Size, - fun(Guid, _Size, _Offset, BinMsg) -> + fun({Guid, _Size, _Offset, BinMsg}, ok) -> case TransformFun(BinMsg) of {ok, MsgNew} -> - rabbit_msg_file:append(RefNew, Guid, MsgNew), - 1; + {ok, _} = rabbit_msg_file:append(RefNew, Guid, MsgNew), + ok; {error, Reason} -> error_logger:error_msg("Message transform failed: ~p~n", [Reason]), - 0 + ok end - end), + end, ok), file_handle_cache:close(RefOld), file_handle_cache:close(RefNew), - lists:sum(Acc). + ok = Acc. diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index f4e27cc8..73f59557 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -107,7 +107,7 @@ mnesia(TableName, Fun, FieldList, NewRecordName) -> %%-------------------------------------------------------------------- multiple_routing_keys() -> - _UpgradeMsgCount = rabbit_variable_queue:transform_storage( + rabbit_variable_queue:transform_storage( fun (BinMsg) -> case binary_to_term(BinMsg) of {basic_message, ExchangeName, Routing_Key, Content, Guid, diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index f2176c0e..dee6a8e5 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -1808,7 +1808,7 @@ push_betas_to_deltas(Generator, Limit, Q, Count, RamIndexCount, IndexState) -> %% Assumes message store is not running transform_storage(TransformFun) -> - transform_store(?PERSISTENT_MSG_STORE, TransformFun) + + transform_store(?PERSISTENT_MSG_STORE, TransformFun), transform_store(?TRANSIENT_MSG_STORE, TransformFun). transform_store(Store, TransformFun) -> -- cgit v1.2.1 From a62685c1495b2e95f2e127ab607ec1634a18cc62 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Fri, 11 Feb 2011 17:56:09 +0000 Subject: Remove rabbit_msg_file:scan/2 --- src/rabbit_msg_file.erl | 11 +---------- src/rabbit_msg_store.erl | 6 +++++- 2 files changed, 6 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_file.erl b/src/rabbit_msg_file.erl index 9d5953d5..81f2f07e 100644 --- a/src/rabbit_msg_file.erl +++ b/src/rabbit_msg_file.erl @@ -16,7 +16,7 @@ -module(rabbit_msg_file). --export([append/3, read/2, scan/2, scan/4]). +-export([append/3, read/2, scan/4]). %%---------------------------------------------------------------------------- @@ -45,9 +45,6 @@ -spec(read/2 :: (io_device(), msg_size()) -> rabbit_types:ok_or_error2({rabbit_guid:guid(), msg()}, any())). --spec(scan/2 :: (io_device(), file_size()) -> - {'ok', [{rabbit_guid:guid(), msg_size(), position()}], - position()}). -spec(scan/4 :: (io_device(), file_size(), fun (({rabbit_guid:guid(), msg_size(), position(), binary()}, A) -> A), A) -> {'ok', A, position()}). @@ -82,12 +79,6 @@ read(FileHdl, TotalSize) -> KO -> KO end. -scan_fun({Guid, TotalSize, Offset, _Msg}, Acc) -> - [{Guid, TotalSize, Offset} | Acc]. - -scan(FileHdl, FileSize) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, [], 0, fun scan_fun/2). - scan(FileHdl, FileSize, Fun, Acc) when FileSize >= 0 -> scan(FileHdl, FileSize, <<>>, 0, Acc, 0, Fun). diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index b827eba9..82fb1735 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1530,7 +1530,8 @@ scan_file_for_valid_messages(Dir, FileName) -> case open_file(Dir, FileName, ?READ_MODE) of {ok, Hdl} -> Valid = rabbit_msg_file:scan( Hdl, filelib:file_size( - form_filename(Dir, FileName))), + form_filename(Dir, FileName)), + fun scan_fun/2, []), %% if something really bad has happened, %% the close could fail, but ignore file_handle_cache:close(Hdl), @@ -1539,6 +1540,9 @@ scan_file_for_valid_messages(Dir, FileName) -> {error, Reason} -> {error, {unable_to_scan_file, FileName, Reason}} end. +scan_fun({Guid, TotalSize, Offset, _Msg}, Acc) -> + [{Guid, TotalSize, Offset} | Acc]. + %% Takes the list in *ascending* order (i.e. eldest message %% first). This is the opposite of what scan_file_for_valid_messages %% produces. The list of msgs that is produced is youngest first. -- cgit v1.2.1 From 2d91f7b8e01c19f8e1e81199eb9fedf9ef485333 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Sat, 12 Feb 2011 20:59:39 +0000 Subject: Added documentation for gm 'become' callback result --- src/gm.erl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl index 8fea9196..283b2431 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -432,6 +432,20 @@ behaviour_info(callbacks) -> [ + %% The joined, members_changed and handle_msg callbacks can all + %% return any of the following terms: + %% + %% 'ok' - the callback function returns normally + %% + %% {'stop', Reason} - the callback indicates the member should + %% stop with reason Reason and should leave the group. + %% + %% {'become', Module, Args} - the callback indicates that the + %% callback module should be changed to Module and that the + %% callback functions should now be passed the arguments + %% Args. This allows the callback module to be dynamically + %% changed. + %% Called when we've successfully joined the group. Supplied with %% Args provided in start_link, plus current group members. {joined, 2}, -- cgit v1.2.1 From 394c73b033ca71d98b0572317852b107abe97a38 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Mon, 14 Feb 2011 16:57:16 +0000 Subject: Sender-selected distribution updates --- src/rabbit_basic.erl | 2 +- src/rabbit_msg_store.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index 5ea145d4..7fa68882 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -101,7 +101,7 @@ strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} = DecodedContent, Key) when Headers =/= undefined -> case lists:keyfind(Key, 1, Headers) of false -> DecodedContent; - Tuple -> Headers0 = lists:delete(Tuple, Headers), + Found -> Headers0 = lists:delete(Found, Headers), rabbit_binary_generator:clear_encoded_content( DecodedContent#content{ properties = Props#'P_basic'{headers = Headers0}}) diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index 82fb1735..f7afbef5 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -2001,7 +2001,7 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> {ok, RefNew} = file_handle_cache:open(FileNew, [raw, binary, write], [{write_buffer, ?HANDLE_CACHE_BUFFER_SIZE}]), - {ok, Acc, _IgnoreSize} = + {ok, _Acc, _IgnoreSize} = rabbit_msg_file:scan( RefOld, Size, fun({Guid, _Size, _Offset, BinMsg}, ok) -> @@ -2017,5 +2017,5 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> end, ok), file_handle_cache:close(RefOld), file_handle_cache:close(RefNew), - ok = Acc. + ok. -- cgit v1.2.1 From 94910f541801e40958d81bf7dbee15923c1a4c2a Mon Sep 17 00:00:00 2001 From: Vlad Ionescu Date: Mon, 14 Feb 2011 11:38:21 -0600 Subject: fixing database upgrade --- src/rabbit_upgrade_functions.erl | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 68b88b3e..4f679483 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -25,6 +25,7 @@ -rabbit_upgrade({add_ip_to_listener, []}). -rabbit_upgrade({internal_exchanges, []}). -rabbit_upgrade({user_to_internal_user, [hash_passwords]}). +-rabbit_upgrade({topic_trie, []}). %% ------------------------------------------------------------------- @@ -47,7 +48,7 @@ %% point. remove_user_scope() -> - mnesia( + transform( rabbit_user_permission, fun ({user_permission, UV, {permission, _Scope, Conf, Write, Read}}) -> {user_permission, UV, {permission, Conf, Write, Read}} @@ -55,7 +56,7 @@ remove_user_scope() -> [user_vhost, permission]). hash_passwords() -> - mnesia( + transform( rabbit_user, fun ({user, Username, Password, IsAdmin}) -> Hash = rabbit_auth_backend_internal:hash_password(Password), @@ -64,7 +65,7 @@ hash_passwords() -> [username, password_hash, is_admin]). add_ip_to_listener() -> - mnesia( + transform( rabbit_listener, fun ({listener, Node, Protocol, Host, Port}) -> {listener, Node, Protocol, Host, {0,0,0,0}, Port} @@ -77,27 +78,41 @@ internal_exchanges() -> fun ({exchange, Name, Type, Durable, AutoDelete, Args}) -> {exchange, Name, Type, Durable, AutoDelete, false, Args} end, - [ ok = mnesia(T, + [ ok = transform(T, AddInternalFun, [name, type, durable, auto_delete, internal, arguments]) || T <- Tables ], ok. user_to_internal_user() -> - mnesia( + transform( rabbit_user, fun({user, Username, PasswordHash, IsAdmin}) -> {internal_user, Username, PasswordHash, IsAdmin} end, [username, password_hash, is_admin], internal_user). +topic_trie() -> + create(rabbit_topic_trie_edge, + [{record_name, topic_trie_edge}, + {attributes, {trie_edge, exchange_name, node_id, word}}, + {type, ordered_set}]), + create(rabbit_topic_trie_binding, + [{record_name, topic_trie_binding}, + {attributes, {trie_binding, exchange_name, node_id, destination}}, + {type, ordered_set}]). + %%-------------------------------------------------------------------- -mnesia(TableName, Fun, FieldList) -> +transform(TableName, Fun, FieldList) -> {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList), ok. -mnesia(TableName, Fun, FieldList, NewRecordName) -> +transform(TableName, Fun, FieldList, NewRecordName) -> {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList, NewRecordName), ok. + +create(Tab, TabDef) -> + {atomic, ok} = mnesia:create_table(Tab, TabDef), + ok. -- cgit v1.2.1 From f01d24451a6f53313eacf82d2971d6eebe83bb55 Mon Sep 17 00:00:00 2001 From: Vlad Ionescu Date: Mon, 14 Feb 2011 11:53:00 -0600 Subject: fixing field lists --- src/rabbit_upgrade_functions.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 4f679483..a1ccb121 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -36,6 +36,7 @@ -spec(add_ip_to_listener/0 :: () -> 'ok'). -spec(internal_exchanges/0 :: () -> 'ok'). -spec(user_to_internal_user/0 :: () -> 'ok'). +-spec(topic_trie/0 :: () -> 'ok'). -endif. @@ -79,8 +80,8 @@ internal_exchanges() -> {exchange, Name, Type, Durable, AutoDelete, false, Args} end, [ ok = transform(T, - AddInternalFun, - [name, type, durable, auto_delete, internal, arguments]) + AddInternalFun, + [name, type, durable, auto_delete, internal, arguments]) || T <- Tables ], ok. @@ -95,11 +96,11 @@ user_to_internal_user() -> topic_trie() -> create(rabbit_topic_trie_edge, [{record_name, topic_trie_edge}, - {attributes, {trie_edge, exchange_name, node_id, word}}, + {attributes, {topic_trie_edge, trie_edge, node_id}}, {type, ordered_set}]), create(rabbit_topic_trie_binding, [{record_name, topic_trie_binding}, - {attributes, {trie_binding, exchange_name, node_id, destination}}, + {attributes, {topic_trie_binding, trie_binding, value = const}}, {type, ordered_set}]). %%-------------------------------------------------------------------- -- cgit v1.2.1 From 91323e269a0e9f9fff5c5b4cbdeb43e3b6c9014c Mon Sep 17 00:00:00 2001 From: Vlad Ionescu Date: Mon, 14 Feb 2011 12:07:19 -0600 Subject: fixing attributes in rabbit_upgrade_functions:topic_trie/0 --- src/rabbit_upgrade_functions.erl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index a1ccb121..36d1f2dc 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -94,14 +94,12 @@ user_to_internal_user() -> [username, password_hash, is_admin], internal_user). topic_trie() -> - create(rabbit_topic_trie_edge, - [{record_name, topic_trie_edge}, - {attributes, {topic_trie_edge, trie_edge, node_id}}, - {type, ordered_set}]), - create(rabbit_topic_trie_binding, - [{record_name, topic_trie_binding}, - {attributes, {topic_trie_binding, trie_binding, value = const}}, - {type, ordered_set}]). + create(rabbit_topic_trie_edge, [{record_name, topic_trie_edge}, + {attributes, [trie_edge, node_id]}, + {type, ordered_set}]), + create(rabbit_topic_trie_binding, [{record_name, topic_trie_binding}, + {attributes, [trie_binding, value]}, + {type, ordered_set}]). %%-------------------------------------------------------------------- -- cgit v1.2.1 From 3f86e7afaf4bfe8f66e81ad96f491de48f159d84 Mon Sep 17 00:00:00 2001 From: Vlad Ionescu Date: Mon, 14 Feb 2011 12:49:43 -0600 Subject: fixing upgrade from schema with missing tables --- src/rabbit_mnesia.erl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 9bebae4b..c6441b68 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -552,9 +552,17 @@ create_local_table_copy(Tab, Type) -> end, ok. -wait_for_replicated_tables() -> wait_for_tables(replicated_table_names()). - -wait_for_tables() -> wait_for_tables(table_names()). +wait_for_replicated_tables() -> + AllTablesSet = ordsets:from_list(mnesia:system_info(tables)), + ReplicatedTablesSet = ordsets:from_list(replicated_table_names()), + wait_for_tables(ordsets:to_list(ordsets:intersection(AllTablesSet, + ReplicatedTablesSet))). + +wait_for_tables() -> + AllTablesSet = ordsets:from_list(mnesia:system_info(tables)), + RabbitTablesSet = ordsets:from_list(table_names()), + wait_for_tables(ordsets:to_list(ordsets:intersection(AllTablesSet, + RabbitTablesSet))). wait_for_tables(TableNames) -> case mnesia:wait_for_tables(TableNames, 30000) of -- cgit v1.2.1 From 3263bbd984306d328d7d1c1f0314bce56cc6c0da Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Mon, 14 Feb 2011 13:19:56 -0600 Subject: removing duplication and use of sets in rabbit_mnesia --- src/rabbit_mnesia.erl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index c6441b68..51b6c6a9 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -553,19 +553,14 @@ create_local_table_copy(Tab, Type) -> ok. wait_for_replicated_tables() -> - AllTablesSet = ordsets:from_list(mnesia:system_info(tables)), - ReplicatedTablesSet = ordsets:from_list(replicated_table_names()), - wait_for_tables(ordsets:to_list(ordsets:intersection(AllTablesSet, - ReplicatedTablesSet))). + wait_for_tables(replicated_table_names()). wait_for_tables() -> - AllTablesSet = ordsets:from_list(mnesia:system_info(tables)), - RabbitTablesSet = ordsets:from_list(table_names()), - wait_for_tables(ordsets:to_list(ordsets:intersection(AllTablesSet, - RabbitTablesSet))). + wait_for_tables(table_names()). wait_for_tables(TableNames) -> - case mnesia:wait_for_tables(TableNames, 30000) of + Inexistent = TableNames -- mnesia:system_info(tables), + case mnesia:wait_for_tables(TableNames -- Inexistent, 30000) of ok -> ok; {timeout, BadTabs} -> throw({error, {timeout_waiting_for_tables, BadTabs}}); -- cgit v1.2.1 From a46eb3687308253a9f1bb86cba8ff386c6d61aeb Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 14 Feb 2011 19:20:06 +0000 Subject: cosmetic --- src/rabbit_channel.erl | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index b9d1baf0..fbca8f56 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -365,15 +365,13 @@ send_exception(Reason, State = #ch{protocol = Protocol, rabbit_binary_generator:map_exception(Channel, Reason, Protocol), rabbit_log:error("connection ~p, channel ~p - error:~n~p~n", [ReaderPid, Channel, Reason]), - %% something bad's happened: rollback_and_notify make not be 'ok' + %% something bad's happened: rollback_and_notify may not be 'ok' {_Result, State1} = rollback_and_notify(State), case CloseChannel of - Channel -> - ok = rabbit_writer:send_command(WriterPid, CloseMethod), - {noreply, State1}; - _ -> - ReaderPid ! {channel_exit, Channel, Reason}, - {stop, normal, State1} + Channel -> ok = rabbit_writer:send_command(WriterPid, CloseMethod), + {noreply, State1}; + _ -> ReaderPid ! {channel_exit, Channel, Reason}, + {stop, normal, State1} end. return_queue_declare_ok(#resource{name = ActualName}, -- cgit v1.2.1 From 43515b69fabeab63c1261b5d36c49274a3413bcc Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Tue, 15 Feb 2011 15:10:03 +0000 Subject: Make the timeout code a bit clearer. --- src/rabbit_control.erl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl index a8903102..f0b4ced1 100644 --- a/src/rabbit_control.erl +++ b/src/rabbit_control.erl @@ -20,7 +20,7 @@ -export([start/0, stop/0, action/5, diagnostics/1]). -define(RPC_TIMEOUT, infinity). --define(WAIT_FOR_VM_TIMEOUT, 5000). +-define(WAIT_FOR_VM_ATTEMPTS, 5). -define(QUIET_OPT, "-q"). -define(NODE_OPT, "-n"). @@ -302,14 +302,14 @@ action(list_permissions, Node, [], Opts, Inform) -> action(wait, Node, [], _Opts, Inform) -> Inform("Waiting for ~p", [Node]), - wait_for_application(Node, ?WAIT_FOR_VM_TIMEOUT). + wait_for_application(Node, ?WAIT_FOR_VM_ATTEMPTS). -wait_for_application(Node, NodeTimeout) -> +wait_for_application(Node, Attempts) -> case rpc_call(Node, application, which_applications, [infinity]) of - {badrpc, _} = E -> NewTimeout = NodeTimeout - 1000, - case NewTimeout =< 0 of - true -> E; - false -> wait_for_application0(Node, NewTimeout) + {badrpc, _} = E -> NewAttempts = Attempts - 1, + case NewAttempts of + 0 -> E; + _ -> wait_for_application0(Node, NewAttempts) end; Apps -> case proplists:is_defined(rabbit, Apps) of %% We've seen the node up; if it goes down @@ -319,9 +319,9 @@ wait_for_application(Node, NodeTimeout) -> end end. -wait_for_application0(Node, NodeTimeout) -> +wait_for_application0(Node, Attempts) -> timer:sleep(1000), - wait_for_application(Node, NodeTimeout). + wait_for_application(Node, Attempts). default_if_empty(List, Default) when is_list(List) -> if List == [] -> -- cgit v1.2.1 From 545aa642f2ce2218948a9786d0638637a72a2768 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Tue, 15 Feb 2011 09:20:07 -0600 Subject: adding wait_for_tables after database upgrade --- src/rabbit_mnesia.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 51b6c6a9..eac7dd14 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -388,7 +388,8 @@ init_db(ClusterNodes, Force) -> %% True single disc node, attempt upgrade ok = wait_for_tables(), case rabbit_upgrade:maybe_upgrade() of - ok -> ensure_schema_ok(); + ok -> ok = wait_for_tables(), + ensure_schema_ok(); version_not_available -> schema_ok_or_move() end; {[], true, _} -> @@ -559,8 +560,8 @@ wait_for_tables() -> wait_for_tables(table_names()). wait_for_tables(TableNames) -> - Inexistent = TableNames -- mnesia:system_info(tables), - case mnesia:wait_for_tables(TableNames -- Inexistent, 30000) of + Nonexistent = TableNames -- mnesia:system_info(tables), + case mnesia:wait_for_tables(TableNames -- Nonexistent, 30000) of ok -> ok; {timeout, BadTabs} -> throw({error, {timeout_waiting_for_tables, BadTabs}}); -- cgit v1.2.1 From 17ebfb85ebc28c01dfc29e7089dbbf6d1688bc6c Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Tue, 15 Feb 2011 15:27:52 +0000 Subject: Sender-specified distribution updates --- src/rabbit_basic.erl | 41 +++++++++++++++++++++++++-------------- src/rabbit_channel.erl | 36 ++++++++++++++++++++-------------- src/rabbit_msg_store.erl | 42 ++++++++++++++++++++++++++-------------- src/rabbit_upgrade_functions.erl | 19 ------------------ src/rabbit_variable_queue.erl | 18 ++++++++++++++++- 5 files changed, 92 insertions(+), 64 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index 7fa68882..503f01bc 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -31,6 +31,7 @@ -type(publish_result() :: ({ok, rabbit_router:routing_result(), [pid()]} | rabbit_types:error('not_found'))). +-type(msg_or_error() :: {'ok', rabbit_types:message()} | {'error', any()}). -spec(publish/1 :: (rabbit_types:delivery()) -> publish_result()). @@ -40,10 +41,10 @@ rabbit_types:delivery()). -spec(message/4 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - properties_input(), binary()) -> rabbit_types:message()). + properties_input(), binary()) -> msg_or_error()). -spec(message/3 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - rabbit_types:decoded_content()) -> rabbit_types:message()). + rabbit_types:decoded_content()) -> msg_or_error()). -spec(properties/1 :: (properties_input()) -> rabbit_framing:amqp_property_record()). -spec(publish/4 :: @@ -111,17 +112,23 @@ strip_header(DecodedContent, _Key) -> message(ExchangeName, RoutingKey, #content{properties = Props} = DecodedContent) -> - #basic_message{ - exchange_name = ExchangeName, - content = strip_header(DecodedContent, ?DELETED_HEADER), - guid = rabbit_guid:guid(), - is_persistent = is_message_persistent(DecodedContent), - routing_keys = [RoutingKey | header_routes(Props#'P_basic'.headers)]}. - -message(ExchangeName, RoutingKeyBin, RawProperties, BodyBin) -> + try + {ok, #basic_message{ + exchange_name = ExchangeName, + content = strip_header(DecodedContent, ?DELETED_HEADER), + guid = rabbit_guid:guid(), + is_persistent = is_message_persistent(DecodedContent), + routing_keys = [RoutingKey | + header_routes(Props#'P_basic'.headers)]}} + catch + {error, _Reason} = Error -> Error + end. + +message(ExchangeName, RoutingKey, RawProperties, BodyBin) -> Properties = properties(RawProperties), Content = build_content(Properties, BodyBin), - message(ExchangeName, RoutingKeyBin, Content). + {ok, Msg} = message(ExchangeName, RoutingKey, Content), + Msg. properties(P = #'P_basic'{}) -> P; @@ -170,8 +177,12 @@ is_message_persistent(#content{properties = #'P_basic'{ header_routes(undefined) -> []; header_routes(HeadersTable) -> - lists:append([case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of - {array, Routes} -> [Route || {longstr, Route} <- Routes]; - _ -> [] - end || HeaderKey <- ?ROUTING_HEADERS]). + lists:append( + [case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of + {array, Routes} -> [Route || {longstr, Route} <- Routes]; + undefined -> []; + {Type, _Val} -> throw({error, {unacceptable_type_in_header, + Type, + binary_to_list(HeaderKey)}}) + end || HeaderKey <- ?ROUTING_HEADERS]). diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 16a3911d..162580ec 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -555,21 +555,27 @@ handle_method(#'basic.publish'{exchange = ExchangeNameBin, true -> SeqNo = State#ch.publish_seqno, {SeqNo, State#ch{publish_seqno = SeqNo + 1}} end, - Message = rabbit_basic:message(ExchangeName, RoutingKey, DecodedContent), - {RoutingRes, DeliveredQPids} = - rabbit_exchange:publish( - Exchange, - rabbit_basic:delivery(Mandatory, Immediate, TxnKey, Message, - MsgSeqNo)), - State2 = process_routing_result(RoutingRes, DeliveredQPids, ExchangeName, - MsgSeqNo, Message, State1), - maybe_incr_stats([{ExchangeName, 1} | - [{{QPid, ExchangeName}, 1} || - QPid <- DeliveredQPids]], publish, State2), - {noreply, case TxnKey of - none -> State2; - _ -> add_tx_participants(DeliveredQPids, State2) - end}; + case rabbit_basic:message(ExchangeName, RoutingKey, DecodedContent) of + {ok, Message} -> + {RoutingRes, DeliveredQPids} = + rabbit_exchange:publish( + Exchange, + rabbit_basic:delivery(Mandatory, Immediate, TxnKey, Message, + MsgSeqNo)), + State2 = process_routing_result(RoutingRes, DeliveredQPids, + ExchangeName, MsgSeqNo, Message, + State1), + maybe_incr_stats([{ExchangeName, 1} | + [{{QPid, ExchangeName}, 1} || + QPid <- DeliveredQPids]], publish, State2), + {noreply, case TxnKey of + none -> State2; + _ -> add_tx_participants(DeliveredQPids, State2) + end}; + {error, Reason} -> + rabbit_misc:protocol_error(precondition_failed, + "invalid message: ~p", [Reason]) + end; handle_method(#'basic.nack'{delivery_tag = DeliveryTag, multiple = Multiple, diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index f7afbef5..00c2ab18 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1968,29 +1968,43 @@ copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, {destination, Destination}]} end. -force_recovery(BaseDir, Server) -> - Dir = filename:join(BaseDir, atom_to_list(Server)), +force_recovery(BaseDir, Store) -> + Dir = filename:join(BaseDir, atom_to_list(Store)), file:delete(filename:join(Dir, ?CLEAN_FILENAME)), [file:delete(filename:join(Dir, File)) || File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], ok. -transform_dir(BaseDir, Server, TransformFun) -> - Dir = filename:join(BaseDir, atom_to_list(Server)), +for_each_file(Files, Fun) -> + [Fun(File) || File <- Files]. + +transform_dir(BaseDir, Store, TransformFun) -> + Dir = filename:join(BaseDir, atom_to_list(Store)), TmpDir = filename:join(Dir, ?TRANSFORM_TMP), case filelib:is_dir(TmpDir) of true -> throw({error, transform_failed_previously}); false -> - [transform_msg_file(filename:join(Dir, File), - filename:join(TmpDir, File), - TransformFun) || - File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)], - [file:delete(filename:join(Dir, File)) || - File <- list_sorted_file_names(Dir, ?FILE_EXTENSION)], - [file:copy(filename:join(TmpDir, File), filename:join(Dir, File)) || - File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], - [file:delete(filename:join(TmpDir, File)) || - File <- list_sorted_file_names(TmpDir, ?FILE_EXTENSION)], + OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), + for_each_file(OldFileList, + fun (File) -> + transform_msg_file(filename:join(Dir, File), + filename:join(TmpDir, File), + TransformFun) + end), + for_each_file(OldFileList, + fun (File) -> + file:delete(filename:join(Dir, File)) + end), + NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), + for_each_file(NewFileList, + fun (File) -> + file:copy(filename:join(TmpDir, File), + filename:join(Dir, File)) + end), + for_each_file(NewFileList, + fun (File) -> + file:delete(filename:join(TmpDir, File)) + end), ok = file:del_dir(TmpDir) end. diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 73f59557..68b88b3e 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -25,7 +25,6 @@ -rabbit_upgrade({add_ip_to_listener, []}). -rabbit_upgrade({internal_exchanges, []}). -rabbit_upgrade({user_to_internal_user, [hash_passwords]}). --rabbit_upgrade({multiple_routing_keys, []}). %% ------------------------------------------------------------------- @@ -36,7 +35,6 @@ -spec(add_ip_to_listener/0 :: () -> 'ok'). -spec(internal_exchanges/0 :: () -> 'ok'). -spec(user_to_internal_user/0 :: () -> 'ok'). --spec(multiple_routing_keys/0 :: () -> 'ok'). -endif. @@ -103,20 +101,3 @@ mnesia(TableName, Fun, FieldList, NewRecordName) -> {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList, NewRecordName), ok. - -%%-------------------------------------------------------------------- - -multiple_routing_keys() -> - rabbit_variable_queue:transform_storage( - fun (BinMsg) -> - case binary_to_term(BinMsg) of - {basic_message, ExchangeName, Routing_Key, Content, Guid, - Persistent} -> - {ok, {basic_message, ExchangeName, [Routing_Key], Content, - Guid, Persistent}}; - _ -> - {error, corrupt_message} - end - end), - ok. - diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index dee6a8e5..b0781f8f 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -22,7 +22,7 @@ requeue/3, len/1, is_empty/1, dropwhile/2, set_ram_duration_target/2, ram_duration/1, needs_idle_timeout/1, idle_timeout/1, handle_pre_hibernate/1, - status/1, transform_storage/1]). + status/1, multiple_routing_keys/0]). -export([start/1, stop/0]). @@ -294,6 +294,8 @@ %%---------------------------------------------------------------------------- +-rabbit_upgrade({multiple_routing_keys, []}). + -ifdef(use_specs). -type(timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}). @@ -1806,6 +1808,20 @@ push_betas_to_deltas(Generator, Limit, Q, Count, RamIndexCount, IndexState) -> %% Upgrading %%---------------------------------------------------------------------------- +multiple_routing_keys() -> + transform_storage( + fun (BinMsg) -> + case binary_to_term(BinMsg) of + {basic_message, ExchangeName, Routing_Key, Content, Guid, + Persistent} -> + {ok, {basic_message, ExchangeName, [Routing_Key], Content, + Guid, Persistent}}; + _ -> + {error, corrupt_message} + end + end), + ok. + %% Assumes message store is not running transform_storage(TransformFun) -> transform_store(?PERSISTENT_MSG_STORE, TransformFun), -- cgit v1.2.1 From 944de8b5e3aec103afc672666bbf6044e8379016 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Tue, 15 Feb 2011 15:56:04 +0000 Subject: Swapped helper function arguments --- src/rabbit_msg_store.erl | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index 00c2ab18..a9d1e210 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1975,7 +1975,7 @@ force_recovery(BaseDir, Store) -> File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], ok. -for_each_file(Files, Fun) -> +for_each_file(Fun, Files) -> [Fun(File) || File <- Files]. transform_dir(BaseDir, Store, TransformFun) -> @@ -1985,26 +1985,22 @@ transform_dir(BaseDir, Store, TransformFun) -> true -> throw({error, transform_failed_previously}); false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - for_each_file(OldFileList, - fun (File) -> + for_each_file(fun (File) -> transform_msg_file(filename:join(Dir, File), filename:join(TmpDir, File), TransformFun) - end), - for_each_file(OldFileList, - fun (File) -> + end, OldFileList), + for_each_file(fun (File) -> file:delete(filename:join(Dir, File)) - end), + end, OldFileList), NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - for_each_file(NewFileList, - fun (File) -> + for_each_file(fun (File) -> file:copy(filename:join(TmpDir, File), filename:join(Dir, File)) - end), - for_each_file(NewFileList, - fun (File) -> + end, NewFileList), + for_each_file(fun (File) -> file:delete(filename:join(TmpDir, File)) - end), + end, NewFileList), ok = file:del_dir(TmpDir) end. -- cgit v1.2.1 From f300f1594b4224a4c20e1f39a138f3471f6e469e Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 16 Feb 2011 11:25:59 +0000 Subject: Shorten transform_dir. --- src/rabbit_msg_store.erl | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index a9d1e210..5bfd48fb 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1975,32 +1975,27 @@ force_recovery(BaseDir, Store) -> File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], ok. -for_each_file(Fun, Files) -> - [Fun(File) || File <- Files]. +for_each_file(D) -> + fun(Fun, Files) -> [Fun(filename:join(D, File)) || File <- Files] end. + +for_each_file(D1, D2) -> + fun(Fun, Files) -> [Fun(filename:join(D1, File), + filename:join(D2, File)) || File <- Files] end. transform_dir(BaseDir, Store, TransformFun) -> Dir = filename:join(BaseDir, atom_to_list(Store)), TmpDir = filename:join(Dir, ?TRANSFORM_TMP), + TransformFile = fun (A, B) -> transform_msg_file(A, B, TransformFun) end, case filelib:is_dir(TmpDir) of - true -> throw({error, transform_failed_previously}); + true -> + throw({error, transform_failed_previously}); false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - for_each_file(fun (File) -> - transform_msg_file(filename:join(Dir, File), - filename:join(TmpDir, File), - TransformFun) - end, OldFileList), - for_each_file(fun (File) -> - file:delete(filename:join(Dir, File)) - end, OldFileList), + (for_each_file(Dir, TmpDir))(TransformFile, OldFileList), + (for_each_file(Dir) )(fun file:delete/1, OldFileList), NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - for_each_file(fun (File) -> - file:copy(filename:join(TmpDir, File), - filename:join(Dir, File)) - end, NewFileList), - for_each_file(fun (File) -> - file:delete(filename:join(TmpDir, File)) - end, NewFileList), + (for_each_file(TmpDir, Dir))(fun file:copy/2, NewFileList), + (for_each_file(TmpDir) )(fun file:delete/1, NewFileList), ok = file:del_dir(TmpDir) end. -- cgit v1.2.1 From 789f49a33719c34d11c4385e67e46ac6bc081617 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 16 Feb 2011 11:43:14 +0000 Subject: Matthias points out this does not need to be second order. --- src/rabbit_msg_store.erl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index 5bfd48fb..fd3027e9 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1975,12 +1975,11 @@ force_recovery(BaseDir, Store) -> File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], ok. -for_each_file(D) -> - fun(Fun, Files) -> [Fun(filename:join(D, File)) || File <- Files] end. +for_each_file(D, Fun, Files) -> + [Fun(filename:join(D, File)) || File <- Files]. -for_each_file(D1, D2) -> - fun(Fun, Files) -> [Fun(filename:join(D1, File), - filename:join(D2, File)) || File <- Files] end. +for_each_file(D1, D2, Fun, Files) -> + [Fun(filename:join(D1, File), filename:join(D2, File)) || File <- Files]. transform_dir(BaseDir, Store, TransformFun) -> Dir = filename:join(BaseDir, atom_to_list(Store)), @@ -1991,11 +1990,11 @@ transform_dir(BaseDir, Store, TransformFun) -> throw({error, transform_failed_previously}); false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - (for_each_file(Dir, TmpDir))(TransformFile, OldFileList), - (for_each_file(Dir) )(fun file:delete/1, OldFileList), + for_each_file(Dir, TmpDir, TransformFile, OldFileList), + for_each_file(Dir, fun file:delete/1, OldFileList), NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - (for_each_file(TmpDir, Dir))(fun file:copy/2, NewFileList), - (for_each_file(TmpDir) )(fun file:delete/1, NewFileList), + for_each_file(TmpDir, Dir, fun file:copy/2, NewFileList), + for_each_file(TmpDir, fun file:delete/1, NewFileList), ok = file:del_dir(TmpDir) end. -- cgit v1.2.1 From 46b79df30bb8eac1905dfd2cc73e760f753b99f6 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 16 Feb 2011 12:03:22 +0000 Subject: Aesthetics --- src/rabbit_msg_store.erl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index fd3027e9..3d7411a9 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1986,16 +1986,14 @@ transform_dir(BaseDir, Store, TransformFun) -> TmpDir = filename:join(Dir, ?TRANSFORM_TMP), TransformFile = fun (A, B) -> transform_msg_file(A, B, TransformFun) end, case filelib:is_dir(TmpDir) of - true -> - throw({error, transform_failed_previously}); - false -> - OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - for_each_file(Dir, TmpDir, TransformFile, OldFileList), - for_each_file(Dir, fun file:delete/1, OldFileList), - NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - for_each_file(TmpDir, Dir, fun file:copy/2, NewFileList), - for_each_file(TmpDir, fun file:delete/1, NewFileList), - ok = file:del_dir(TmpDir) + true -> throw({error, transform_failed_previously}); + false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), + for_each_file(Dir, TmpDir, TransformFile, OldFileList), + for_each_file(Dir, fun file:delete/1, OldFileList), + NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), + for_each_file(TmpDir, Dir, fun file:copy/2, NewFileList), + for_each_file(TmpDir, fun file:delete/1, NewFileList), + ok = file:del_dir(TmpDir) end. transform_msg_file(FileOld, FileNew, TransformFun) -> -- cgit v1.2.1 From 95f2121cc5e3da61960c06ab95258074a26b531b Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Sun, 20 Feb 2011 16:37:07 -0600 Subject: less wait_for_tables --- src/rabbit_mnesia.erl | 7 +++---- src/rabbit_upgrade.erl | 1 - src/rabbit_upgrade_functions.erl | 5 +++++ 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index eac7dd14..8b8f2f2d 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -20,7 +20,7 @@ -export([ensure_mnesia_dir/0, dir/0, status/0, init/0, is_db_empty/0, cluster/1, force_cluster/1, reset/0, force_reset/0, is_clustered/0, running_clustered_nodes/0, all_clustered_nodes/0, - empty_ram_only_tables/0, copy_db/1]). + empty_ram_only_tables/0, copy_db/1, wait_for_tables/1]). -export([table_names/0]). @@ -54,6 +54,7 @@ -spec(empty_ram_only_tables/0 :: () -> 'ok'). -spec(create_tables/0 :: () -> 'ok'). -spec(copy_db/1 :: (file:filename()) -> rabbit_types:ok_or_error(any())). +-spec(wait_for_tables/1 :: ([atom()]) -> 'ok'). -endif. @@ -386,7 +387,6 @@ init_db(ClusterNodes, Force) -> case {Nodes, mnesia:system_info(use_dir), all_clustered_nodes()} of {[], true, [_]} -> %% True single disc node, attempt upgrade - ok = wait_for_tables(), case rabbit_upgrade:maybe_upgrade() of ok -> ok = wait_for_tables(), ensure_schema_ok(); @@ -490,8 +490,7 @@ copy_db(Destination) -> mnesia:stop(), case rabbit_misc:recursive_copy(dir(), Destination) of ok -> - rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia), - ok = wait_for_tables(); + rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia); {error, E} -> {error, E} end. diff --git a/src/rabbit_upgrade.erl b/src/rabbit_upgrade.erl index b0a71523..89acc10c 100644 --- a/src/rabbit_upgrade.erl +++ b/src/rabbit_upgrade.erl @@ -98,7 +98,6 @@ vertices(Module, Steps) -> edges(_Module, Steps) -> [{Require, StepName} || {StepName, Requires} <- Steps, Require <- Requires]. - unknown_heads(Heads, G) -> [H || H <- Heads, digraph:vertex(G, H) =:= false]. diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index 36d1f2dc..d6a79590 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -49,6 +49,7 @@ %% point. remove_user_scope() -> + rabbit_mnesia:wait_for_tables([rabbit_user_permission]), transform( rabbit_user_permission, fun ({user_permission, UV, {permission, _Scope, Conf, Write, Read}}) -> @@ -57,6 +58,7 @@ remove_user_scope() -> [user_vhost, permission]). hash_passwords() -> + rabbit_mnesia:wait_for_tables([rabbit_user]), transform( rabbit_user, fun ({user, Username, Password, IsAdmin}) -> @@ -66,6 +68,7 @@ hash_passwords() -> [username, password_hash, is_admin]). add_ip_to_listener() -> + rabbit_mnesia:wait_for_tables([rabbit_listener]), transform( rabbit_listener, fun ({listener, Node, Protocol, Host, Port}) -> @@ -75,6 +78,7 @@ add_ip_to_listener() -> internal_exchanges() -> Tables = [rabbit_exchange, rabbit_durable_exchange], + rabbit_mnesia:wait_for_tables(Tables), AddInternalFun = fun ({exchange, Name, Type, Durable, AutoDelete, Args}) -> {exchange, Name, Type, Durable, AutoDelete, false, Args} @@ -86,6 +90,7 @@ internal_exchanges() -> ok. user_to_internal_user() -> + rabbit_mnesia:wait_for_tables([rabbit_user]), transform( rabbit_user, fun({user, Username, PasswordHash, IsAdmin}) -> -- cgit v1.2.1 From 5013d8833776d927e034b8dc270659bdc2fadd60 Mon Sep 17 00:00:00 2001 From: Vlad Alexandru Ionescu Date: Sun, 20 Feb 2011 16:44:56 -0600 Subject: moving wait_for_tables call in transform --- src/rabbit_upgrade_functions.erl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl index d6a79590..b9dbe418 100644 --- a/src/rabbit_upgrade_functions.erl +++ b/src/rabbit_upgrade_functions.erl @@ -49,7 +49,6 @@ %% point. remove_user_scope() -> - rabbit_mnesia:wait_for_tables([rabbit_user_permission]), transform( rabbit_user_permission, fun ({user_permission, UV, {permission, _Scope, Conf, Write, Read}}) -> @@ -58,7 +57,6 @@ remove_user_scope() -> [user_vhost, permission]). hash_passwords() -> - rabbit_mnesia:wait_for_tables([rabbit_user]), transform( rabbit_user, fun ({user, Username, Password, IsAdmin}) -> @@ -68,7 +66,6 @@ hash_passwords() -> [username, password_hash, is_admin]). add_ip_to_listener() -> - rabbit_mnesia:wait_for_tables([rabbit_listener]), transform( rabbit_listener, fun ({listener, Node, Protocol, Host, Port}) -> @@ -78,7 +75,6 @@ add_ip_to_listener() -> internal_exchanges() -> Tables = [rabbit_exchange, rabbit_durable_exchange], - rabbit_mnesia:wait_for_tables(Tables), AddInternalFun = fun ({exchange, Name, Type, Durable, AutoDelete, Args}) -> {exchange, Name, Type, Durable, AutoDelete, false, Args} @@ -90,7 +86,6 @@ internal_exchanges() -> ok. user_to_internal_user() -> - rabbit_mnesia:wait_for_tables([rabbit_user]), transform( rabbit_user, fun({user, Username, PasswordHash, IsAdmin}) -> @@ -109,10 +104,12 @@ topic_trie() -> %%-------------------------------------------------------------------- transform(TableName, Fun, FieldList) -> + rabbit_mnesia:wait_for_tables([TableName]), {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList), ok. transform(TableName, Fun, FieldList, NewRecordName) -> + rabbit_mnesia:wait_for_tables([TableName]), {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList, NewRecordName), ok. -- cgit v1.2.1 From f977d37da50bce8c8863a28d1a90534e8b486275 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Mon, 21 Feb 2011 11:21:09 +0000 Subject: whitespace --- src/file_handle_cache.erl | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/file_handle_cache.erl b/src/file_handle_cache.erl index 7c548bdc..f41815d0 100644 --- a/src/file_handle_cache.erl +++ b/src/file_handle_cache.erl @@ -889,13 +889,16 @@ handle_call({obtain, Pid}, From, State = #fhc_state { obtain_count = Count, false -> {noreply, run_pending_item(Item, State)} end; + handle_call({set_limit, Limit}, _From, State) -> {reply, ok, maybe_reduce( process_pending(State #fhc_state { limit = Limit, obtain_limit = obtain_limit(Limit) }))}; + handle_call(get_limit, _From, State = #fhc_state { limit = Limit }) -> {reply, Limit, State}; + handle_call({info, Items}, _From, State) -> {reply, infos(Items, State), State}. -- cgit v1.2.1 From 587dc8810fc2cc68572dafc53024c9f267b0ebe7 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 22 Feb 2011 14:13:45 +0000 Subject: remove redundant function --- src/rabbit_mnesia.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index a9b4e177..74fde4a2 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -371,7 +371,7 @@ init_db(ClusterNodes, Force) -> %% True single disc node, attempt upgrade ok = wait_for_tables(), case rabbit_upgrade:maybe_upgrade() of - ok -> ensure_schema_ok(); + ok -> ensure_schema_integrity(); version_not_available -> schema_ok_or_move() end; {[], true, _} -> @@ -379,7 +379,7 @@ init_db(ClusterNodes, Force) -> %% verify schema ok = wait_for_tables(), ensure_version_ok(rabbit_upgrade:read_version()), - ensure_schema_ok(); + ensure_schema_integrity(); {[], false, _} -> %% Nothing there at all, start from scratch ok = create_schema(); @@ -396,7 +396,7 @@ init_db(ClusterNodes, Force) -> true -> disc; false -> ram end), - ensure_schema_ok() + ensure_schema_integrity() end; {error, Reason} -> %% one reason we may end up here is if we try to join @@ -429,12 +429,6 @@ ensure_version_ok({ok, DiscVersion}) -> ensure_version_ok({error, _}) -> ok = rabbit_upgrade:write_version(). -ensure_schema_ok() -> - case check_schema_integrity() of - ok -> ok; - {error, Reason} -> throw({error, {schema_invalid, Reason}}) - end. - create_schema() -> mnesia:stop(), rabbit_misc:ensure_ok(mnesia:create_schema([node()]), -- cgit v1.2.1 From 4ed65ad1f0bec39722e5d91320c1f5208f804eb9 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 22 Feb 2011 14:35:34 +0000 Subject: remove redundant calls to wait_for_tables ensure_schema_integrity calls it indirectly anyway --- src/rabbit_mnesia.erl | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 74fde4a2..a30f7996 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -377,7 +377,6 @@ init_db(ClusterNodes, Force) -> {[], true, _} -> %% "Master" (i.e. without config) disc node in cluster, %% verify schema - ok = wait_for_tables(), ensure_version_ok(rabbit_upgrade:read_version()), ensure_schema_integrity(); {[], false, _} -> @@ -437,7 +436,6 @@ create_schema() -> cannot_start_mnesia), ok = create_tables(), ok = ensure_schema_integrity(), - ok = wait_for_tables(), ok = rabbit_upgrade:write_version(). move_db() -> -- cgit v1.2.1 From 5b70262f2421af39e76b29b57fef44375ea44c9b Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Tue, 22 Feb 2011 14:41:24 +0000 Subject: Revert d3fd719c5287 (Remove should_offer/1). --- src/rabbit_auth_mechanism.erl | 4 ++++ src/rabbit_auth_mechanism_amqplain.erl | 5 ++++- src/rabbit_auth_mechanism_cr_demo.erl | 5 ++++- src/rabbit_auth_mechanism_plain.erl | 5 ++++- src/rabbit_reader.erl | 18 +++++++++--------- 5 files changed, 25 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/rabbit_auth_mechanism.erl b/src/rabbit_auth_mechanism.erl index 1d14f9f0..897199ee 100644 --- a/src/rabbit_auth_mechanism.erl +++ b/src/rabbit_auth_mechanism.erl @@ -23,6 +23,10 @@ behaviour_info(callbacks) -> %% A description. {description, 0}, + %% If this mechanism is enabled, should it be offered for a given socket? + %% (primarily so EXTERNAL can be SSL-only) + {should_offer, 1}, + %% Called before authentication starts. Should create a state %% object to be passed through all the stages of authentication. {init, 1}, diff --git a/src/rabbit_auth_mechanism_amqplain.erl b/src/rabbit_auth_mechanism_amqplain.erl index 5e422eee..2168495d 100644 --- a/src/rabbit_auth_mechanism_amqplain.erl +++ b/src/rabbit_auth_mechanism_amqplain.erl @@ -19,7 +19,7 @@ -behaviour(rabbit_auth_mechanism). --export([description/0, init/1, handle_response/2]). +-export([description/0, should_offer/1, init/1, handle_response/2]). -include("rabbit_auth_mechanism_spec.hrl"). @@ -38,6 +38,9 @@ description() -> [{name, <<"AMQPLAIN">>}, {description, <<"QPid AMQPLAIN mechanism">>}]. +should_offer(_Sock) -> + true. + init(_Sock) -> []. diff --git a/src/rabbit_auth_mechanism_cr_demo.erl b/src/rabbit_auth_mechanism_cr_demo.erl index 7fd20f8b..77aa34ea 100644 --- a/src/rabbit_auth_mechanism_cr_demo.erl +++ b/src/rabbit_auth_mechanism_cr_demo.erl @@ -19,7 +19,7 @@ -behaviour(rabbit_auth_mechanism). --export([description/0, init/1, handle_response/2]). +-export([description/0, should_offer/1, init/1, handle_response/2]). -include("rabbit_auth_mechanism_spec.hrl"). @@ -43,6 +43,9 @@ description() -> {description, <<"RabbitMQ Demo challenge-response authentication " "mechanism">>}]. +should_offer(_Sock) -> + true. + init(_Sock) -> #state{}. diff --git a/src/rabbit_auth_mechanism_plain.erl b/src/rabbit_auth_mechanism_plain.erl index 1ca07018..e2f9bff9 100644 --- a/src/rabbit_auth_mechanism_plain.erl +++ b/src/rabbit_auth_mechanism_plain.erl @@ -19,7 +19,7 @@ -behaviour(rabbit_auth_mechanism). --export([description/0, init/1, handle_response/2]). +-export([description/0, should_offer/1, init/1, handle_response/2]). -include("rabbit_auth_mechanism_spec.hrl"). @@ -41,6 +41,9 @@ description() -> [{name, <<"PLAIN">>}, {description, <<"SASL PLAIN authentication mechanism">>}]. +should_offer(_Sock) -> + true. + init(_Sock) -> []. diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 3908b646..29321c60 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -564,7 +564,7 @@ start_connection({ProtocolMajor, ProtocolMinor, _ProtocolRevision}, version_major = ProtocolMajor, version_minor = ProtocolMinor, server_properties = server_properties(Protocol), - mechanisms = auth_mechanisms_binary(), + mechanisms = auth_mechanisms_binary(Sock), locales = <<"en_US">> }, ok = send_on_channel0(Sock, Start, Protocol), switch_callback(State#v1{connection = Connection#connection{ @@ -616,7 +616,7 @@ handle_method0(#'connection.start_ok'{mechanism = Mechanism, State0 = #v1{connection_state = starting, connection = Connection, sock = Sock}) -> - AuthMechanism = auth_mechanism_to_module(Mechanism), + AuthMechanism = auth_mechanism_to_module(Mechanism, Sock), Capabilities = case rabbit_misc:table_lookup(ClientProperties, <<"capabilities">>) of {table, Capabilities1} -> Capabilities1; @@ -709,14 +709,14 @@ handle_method0(_Method, #v1{connection_state = S}) -> send_on_channel0(Sock, Method, Protocol) -> ok = rabbit_writer:internal_send_command(Sock, 0, Method, Protocol). -auth_mechanism_to_module(TypeBin) -> +auth_mechanism_to_module(TypeBin, Sock) -> case rabbit_registry:binary_to_type(TypeBin) of {error, not_found} -> rabbit_misc:protocol_error( command_invalid, "unknown authentication mechanism '~s'", [TypeBin]); T -> - case {lists:member(T, auth_mechanisms()), + case {lists:member(T, auth_mechanisms(Sock)), rabbit_registry:lookup_module(auth_mechanism, T)} of {true, {ok, Module}} -> Module; @@ -727,15 +727,15 @@ auth_mechanism_to_module(TypeBin) -> end end. -auth_mechanisms() -> +auth_mechanisms(Sock) -> {ok, Configured} = application:get_env(auth_mechanisms), - [Name || {Name, _Module} <- rabbit_registry:lookup_all(auth_mechanism), - lists:member(Name, Configured)]. + [Name || {Name, Module} <- rabbit_registry:lookup_all(auth_mechanism), + Module:should_offer(Sock), lists:member(Name, Configured)]. -auth_mechanisms_binary() -> +auth_mechanisms_binary(Sock) -> list_to_binary( string:join( - [atom_to_list(A) || A <- auth_mechanisms()], " ")). + [atom_to_list(A) || A <- auth_mechanisms(Sock)], " ")). auth_phase(Response, State = #v1{auth_mechanism = AuthMechanism, -- cgit v1.2.1 From 102eb1221e34274c2fa54595d3c2fd258645f410 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Tue, 22 Feb 2011 14:43:08 +0000 Subject: Sender-specified destinations updates build on R12B3 reduce mnesia lookups --- src/rabbit_basic.erl | 31 ++++++++++++++-------------- src/rabbit_exchange_type_direct.erl | 3 +-- src/rabbit_exchange_type_fanout.erl | 2 +- src/rabbit_msg_file.erl | 24 +++++++++++----------- src/rabbit_msg_store.erl | 41 +++++++++++++++++++++++++++---------- src/rabbit_router.erl | 17 ++++++++++++--- src/rabbit_variable_queue.erl | 32 +++-------------------------- 7 files changed, 77 insertions(+), 73 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index 503f01bc..376a303e 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -31,7 +31,6 @@ -type(publish_result() :: ({ok, rabbit_router:routing_result(), [pid()]} | rabbit_types:error('not_found'))). --type(msg_or_error() :: {'ok', rabbit_types:message()} | {'error', any()}). -spec(publish/1 :: (rabbit_types:delivery()) -> publish_result()). @@ -41,10 +40,11 @@ rabbit_types:delivery()). -spec(message/4 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - properties_input(), binary()) -> msg_or_error()). + properties_input(), binary()) -> rabbit_types:message()). -spec(message/3 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - rabbit_types:decoded_content()) -> msg_or_error()). + rabbit_types:decoded_content()) -> {'ok', rabbit_types:message()} | + {'error', any()}). -spec(properties/1 :: (properties_input()) -> rabbit_framing:amqp_property_record()). -spec(publish/4 :: @@ -98,17 +98,19 @@ from_content(Content) -> {Props, list_to_binary(lists:reverse(FragmentsRev))}. %% This breaks the spec rule forbidding message modification +strip_header(#content{properties = #'P_basic'{headers = undefined}} + = DecodedContent, _Key) -> + DecodedContent; strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} - = DecodedContent, Key) when Headers =/= undefined -> - case lists:keyfind(Key, 1, Headers) of - false -> DecodedContent; - Found -> Headers0 = lists:delete(Found, Headers), - rabbit_binary_generator:clear_encoded_content( - DecodedContent#content{ - properties = Props#'P_basic'{headers = Headers0}}) - end; -strip_header(DecodedContent, _Key) -> - DecodedContent. + = DecodedContent, Key) -> + case lists:keysearch(Key, 1, Headers) of + false -> DecodedContent; + {value, Found} -> Headers0 = lists:delete(Found, Headers), + rabbit_binary_generator:clear_encoded_content( + DecodedContent#content{ + properties = Props#'P_basic'{ + headers = Headers0}}) + end. message(ExchangeName, RoutingKey, #content{properties = Props} = DecodedContent) -> @@ -170,7 +172,7 @@ is_message_persistent(#content{properties = #'P_basic'{ 1 -> false; 2 -> true; undefined -> false; - _ -> false + Other -> throw({error, {delivery_mode_unknown, Other}}) end. % Extract CC routes from headers @@ -185,4 +187,3 @@ header_routes(HeadersTable) -> Type, binary_to_list(HeaderKey)}}) end || HeaderKey <- ?ROUTING_HEADERS]). - diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl index 82776c4a..349c2f6e 100644 --- a/src/rabbit_exchange_type_direct.erl +++ b/src/rabbit_exchange_type_direct.erl @@ -37,8 +37,7 @@ description() -> route(#exchange{name = Name}, #delivery{message = #basic_message{routing_keys = Routes}}) -> - lists:append([rabbit_router:match_routing_key(Name, RKey) || - RKey <- Routes]). + rabbit_router:match_routing_key(Name, Routes). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_exchange_type_fanout.erl b/src/rabbit_exchange_type_fanout.erl index 382fb627..bc5293c8 100644 --- a/src/rabbit_exchange_type_fanout.erl +++ b/src/rabbit_exchange_type_fanout.erl @@ -36,7 +36,7 @@ description() -> {description, <<"AMQP fanout exchange, as per the AMQP specification">>}]. route(#exchange{name = Name}, _Delivery) -> - rabbit_router:match_routing_key(Name, '_'). + rabbit_router:match_routing_key(Name, ['_']). validate(_X) -> ok. create(_Tx, _X) -> ok. diff --git a/src/rabbit_msg_file.erl b/src/rabbit_msg_file.erl index 81f2f07e..55e6ac47 100644 --- a/src/rabbit_msg_file.erl +++ b/src/rabbit_msg_file.erl @@ -80,28 +80,28 @@ read(FileHdl, TotalSize) -> end. scan(FileHdl, FileSize, Fun, Acc) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, Acc, 0, Fun). + scan(FileHdl, FileSize, <<>>, 0, 0, Fun, Acc). -scan(_FileHdl, FileSize, _Data, FileSize, Acc, ScanOffset, _Fun) -> +scan(_FileHdl, FileSize, _Data, FileSize, ScanOffset, _Fun, Acc) -> {ok, Acc, ScanOffset}; -scan(FileHdl, FileSize, Data, ReadOffset, Acc, ScanOffset, Fun) -> +scan(FileHdl, FileSize, Data, ReadOffset, ScanOffset, Fun, Acc) -> Read = lists:min([?SCAN_BLOCK_SIZE, (FileSize - ReadOffset)]), case file_handle_cache:read(FileHdl, Read) of {ok, Data1} -> {Data2, Acc1, ScanOffset1} = - scanner(<>, Acc, ScanOffset, Fun), + scanner(<>, ScanOffset, Fun, Acc), ReadOffset1 = ReadOffset + size(Data1), - scan(FileHdl, FileSize, Data2, ReadOffset1, Acc1, ScanOffset1, Fun); + scan(FileHdl, FileSize, Data2, ReadOffset1, ScanOffset1, Fun, Acc1); _KO -> {ok, Acc, ScanOffset} end. -scanner(<<>>, Acc, Offset, _Fun) -> +scanner(<<>>, Offset, _Fun, Acc) -> {<<>>, Acc, Offset}; -scanner(<<0:?INTEGER_SIZE_BITS, _Rest/binary>>, Acc, Offset, _Fun) -> +scanner(<<0:?INTEGER_SIZE_BITS, _Rest/binary>>, Offset, _Fun, Acc) -> {<<>>, Acc, Offset}; %% Nothing to do other than stop. scanner(<>, Acc, Offset, Fun) -> + WriteMarker:?WRITE_OK_SIZE_BITS, Rest/binary>>, Offset, Fun, Acc) -> TotalSize = Size + ?FILE_PACKING_ADJUSTMENT, case WriteMarker of ?WRITE_OK_MARKER -> @@ -113,10 +113,10 @@ scanner(<> = <>, <> = <>, - scanner(Rest, Fun({Guid, TotalSize, Offset, Msg}, Acc), - Offset + TotalSize, Fun); + scanner(Rest, Offset + TotalSize, Fun, + Fun({Guid, TotalSize, Offset, Msg}, Acc)); _ -> - scanner(Rest, Acc, Offset + TotalSize, Fun) + scanner(Rest, Offset + TotalSize, Fun, Acc) end; -scanner(Data, Acc, Offset, _Fun) -> +scanner(Data, Offset, _Fun, Acc) -> {Data, Acc, Offset}. diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index a2f6d7e2..d798c4f7 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -26,7 +26,7 @@ -export([sync/1, set_maximum_since_use/2, has_readers/2, combine_files/3, delete_file/2]). %% internal --export([transform_dir/3, force_recovery/2]). %% upgrade +-export([multiple_routing_keys/0]). %% upgrade -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3, prioritise_call/3, prioritise_cast/2]). @@ -106,6 +106,8 @@ %%---------------------------------------------------------------------------- +-rabbit_upgrade({multiple_routing_keys, []}). + -ifdef(use_specs). -export_type([gc_state/0, file_num/0]). @@ -164,9 +166,7 @@ -spec(combine_files/3 :: (non_neg_integer(), non_neg_integer(), gc_state()) -> deletion_thunk()). -spec(delete_file/2 :: (non_neg_integer(), gc_state()) -> deletion_thunk()). --spec(force_recovery/2 :: (file:filename(), server()) -> 'ok'). --spec(transform_dir/3 :: (file:filename(), server(), - fun ((binary()) -> ({'ok', msg()} | {error, any()}))) -> 'ok'). +-spec(multiple_routing_keys/0 :: () -> 'ok'). -endif. @@ -1968,6 +1968,25 @@ copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, {destination, Destination}]} end. +%%---------------------------------------------------------------------------- +%% upgrade +%%---------------------------------------------------------------------------- + +multiple_routing_keys() -> + [transform_store( + fun ({basic_message, ExchangeName, Routing_Key, Content, + Guid, Persistent}) -> + {ok, {basic_message, ExchangeName, [Routing_Key], Content, + Guid, Persistent}}; + (_) -> {error, corrupt_message} + end, Store) || Store <- rabbit_variable_queue:store_names()], + ok. + +%% Assumes message store is not running +transform_store(TransformFun, Store) -> + force_recovery(rabbit_mnesia:dir(), Store), + transform_dir(rabbit_mnesia:dir(), Store, TransformFun). + force_recovery(BaseDir, Store) -> Dir = filename:join(BaseDir, atom_to_list(Store)), file:delete(filename:join(Dir, ?CLEAN_FILENAME)), @@ -1975,10 +1994,10 @@ force_recovery(BaseDir, Store) -> File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], ok. -for_each_file(D, Fun, Files) -> +foreach_file(D, Fun, Files) -> [Fun(filename:join(D, File)) || File <- Files]. -for_each_file(D1, D2, Fun, Files) -> +foreach_file(D1, D2, Fun, Files) -> [Fun(filename:join(D1, File), filename:join(D2, File)) || File <- Files]. transform_dir(BaseDir, Store, TransformFun) -> @@ -1988,11 +2007,11 @@ transform_dir(BaseDir, Store, TransformFun) -> case filelib:is_dir(TmpDir) of true -> throw({error, transform_failed_previously}); false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - for_each_file(Dir, TmpDir, TransformFile, OldFileList), - for_each_file(Dir, fun file:delete/1, OldFileList), + foreach_file(Dir, TmpDir, TransformFile, OldFileList), + foreach_file(Dir, fun file:delete/1, OldFileList), NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - for_each_file(TmpDir, Dir, fun file:copy/2, NewFileList), - for_each_file(TmpDir, fun file:delete/1, NewFileList), + foreach_file(TmpDir, Dir, fun file:copy/2, NewFileList), + foreach_file(TmpDir, fun file:delete/1, NewFileList), ok = file:del_dir(TmpDir) end. @@ -2007,7 +2026,7 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> rabbit_msg_file:scan( RefOld, Size, fun({Guid, _Size, _Offset, BinMsg}, ok) -> - case TransformFun(BinMsg) of + case TransformFun(binary_to_term(BinMsg)) of {ok, MsgNew} -> {ok, _} = rabbit_msg_file:append(RefNew, Guid, MsgNew), ok; diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl index 692d2473..53e707f4 100644 --- a/src/rabbit_router.erl +++ b/src/rabbit_router.erl @@ -37,7 +37,8 @@ fun ((rabbit_types:binding()) -> boolean())) -> match_result()). -spec(match_routing_key/2 :: (rabbit_types:binding_source(), - routing_key() | '_') -> match_result()). + [routing_key()] | ['_']) -> + match_result()). -endif. @@ -82,12 +83,22 @@ match_bindings(SrcName, Match) -> Match(Binding)]), mnesia:async_dirty(fun qlc:e/1, [Query]). -match_routing_key(SrcName, RoutingKey) -> +match_routing_key(SrcName, [RoutingKey]) -> MatchHead = #route{binding = #binding{source = SrcName, destination = '$1', key = RoutingKey, _ = '_'}}, - mnesia:dirty_select(rabbit_route, [{MatchHead, [], ['$1']}]). + mnesia:dirty_select(rabbit_route, [{MatchHead, [], ['$1']}]); +match_routing_key(SrcName, [_|_] = RoutingKeys) -> + Condition = list_to_tuple(['orelse' | [{'=:=', '$2', RKey} || + RKey <- RoutingKeys]]), + MatchHead = #route{binding = #binding{source = SrcName, + destination = '$1', + key = '$2', + _ = '_'}}, + mnesia:dirty_select(rabbit_route, [{MatchHead, [Condition], ['$1']}]). + + %%-------------------------------------------------------------------- diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index b0781f8f..4eb9c3b8 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -22,7 +22,7 @@ requeue/3, len/1, is_empty/1, dropwhile/2, set_ram_duration_target/2, ram_duration/1, needs_idle_timeout/1, idle_timeout/1, handle_pre_hibernate/1, - status/1, multiple_routing_keys/0]). + status/1, store_names/0]). -export([start/1, stop/0]). @@ -294,8 +294,6 @@ %%---------------------------------------------------------------------------- --rabbit_upgrade({multiple_routing_keys, []}). - -ifdef(use_specs). -type(timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}). @@ -1804,29 +1802,5 @@ push_betas_to_deltas(Generator, Limit, Q, Count, RamIndexCount, IndexState) -> Generator, Limit, Qa, Count + 1, RamIndexCount1, IndexState1) end. -%%---------------------------------------------------------------------------- -%% Upgrading -%%---------------------------------------------------------------------------- - -multiple_routing_keys() -> - transform_storage( - fun (BinMsg) -> - case binary_to_term(BinMsg) of - {basic_message, ExchangeName, Routing_Key, Content, Guid, - Persistent} -> - {ok, {basic_message, ExchangeName, [Routing_Key], Content, - Guid, Persistent}}; - _ -> - {error, corrupt_message} - end - end), - ok. - -%% Assumes message store is not running -transform_storage(TransformFun) -> - transform_store(?PERSISTENT_MSG_STORE, TransformFun), - transform_store(?TRANSIENT_MSG_STORE, TransformFun). - -transform_store(Store, TransformFun) -> - rabbit_msg_store:force_recovery(rabbit_mnesia:dir(), Store), - rabbit_msg_store:transform_dir(rabbit_mnesia:dir(), Store, TransformFun). +store_names() -> + [?PERSISTENT_MSG_STORE, ?TRANSIENT_MSG_STORE]. -- cgit v1.2.1 From cbcafda448298d83067c1c66536df1f49f52b7de Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 22 Feb 2011 15:55:37 +0000 Subject: better error reporting for failed table integrity checks --- src/rabbit_mnesia.erl | 55 +++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index a30f7996..42f7e3b2 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -264,45 +264,48 @@ ensure_schema_integrity() -> check_schema_integrity() -> Tables = mnesia:system_info(tables), - case [Error || {Tab, TabDef} <- table_definitions(), - case lists:member(Tab, Tables) of - false -> - Error = {table_missing, Tab}, - true; - true -> - {_, ExpAttrs} = proplists:lookup(attributes, TabDef), - Attrs = mnesia:table_info(Tab, attributes), - Error = {table_attributes_mismatch, Tab, - ExpAttrs, Attrs}, - Attrs /= ExpAttrs - end] of - [] -> check_table_integrity(); - Errors -> {error, Errors} + case check_tables(fun (Tab, TabDef) -> + case lists:member(Tab, Tables) of + false -> {error, {table_missing, Tab}}; + true -> check_table_attributes(Tab, TabDef) + end + end) of + ok -> ok = wait_for_tables(), + check_tables(fun check_table_integrity/2); + Other -> Other end. -check_table_integrity() -> - ok = wait_for_tables(), - case lists:all(fun ({Tab, TabDef}) -> - {_, Match} = proplists:lookup(match, TabDef), - read_test_table(Tab, Match) - end, table_definitions()) of - true -> ok; - false -> {error, invalid_table_content} +check_table_attributes(Tab, TabDef) -> + {_, ExpAttrs} = proplists:lookup(attributes, TabDef), + case mnesia:table_info(Tab, attributes) of + ExpAttrs -> ok; + Attrs -> {error, {table_attributes_mismatch, Tab, ExpAttrs, Attrs}} end. -read_test_table(Tab, Match) -> +check_table_integrity(Tab, TabDef) -> + {_, Match} = proplists:lookup(match, TabDef), case mnesia:dirty_first(Tab) of '$end_of_table' -> - true; + ok; Key -> ObjList = mnesia:dirty_read(Tab, Key), MatchComp = ets:match_spec_compile([{Match, [], ['$_']}]), case ets:match_spec_run(ObjList, MatchComp) of - ObjList -> true; - _ -> false + ObjList -> ok; + _ -> {error, {table_content_invalid, Tab, Match, ObjList}} end end. +check_tables(Fun) -> + case [Error || {Tab, TabDef} <- table_definitions(), + case Fun(Tab, TabDef) of + ok -> Error = none, false; + {error, Error} -> true + end] of + [] -> ok; + Errors -> {error, Errors} + end. + %% The cluster node config file contains some or all of the disk nodes %% that are members of the cluster this node is / should be a part of. %% -- cgit v1.2.1 From 102c4420102346c0a66ff992eacb23630bd2d3f5 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Tue, 22 Feb 2011 15:57:47 +0000 Subject: better name --- src/rabbit_mnesia.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 42f7e3b2..5e990d61 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -271,7 +271,7 @@ check_schema_integrity() -> end end) of ok -> ok = wait_for_tables(), - check_tables(fun check_table_integrity/2); + check_tables(fun check_table_content/2); Other -> Other end. @@ -282,7 +282,7 @@ check_table_attributes(Tab, TabDef) -> Attrs -> {error, {table_attributes_mismatch, Tab, ExpAttrs, Attrs}} end. -check_table_integrity(Tab, TabDef) -> +check_table_content(Tab, TabDef) -> {_, Match} = proplists:lookup(match, TabDef), case mnesia:dirty_first(Tab) of '$end_of_table' -> -- cgit v1.2.1 From 9d3eb1f0bd42cc23d3ad2474721d0a0a4b4fcf8e Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Tue, 22 Feb 2011 16:57:39 +0000 Subject: Revert re-arrangement of upgrade steps --- src/rabbit_basic.erl | 4 ++-- src/rabbit_msg_store.erl | 33 ++++++--------------------------- src/rabbit_variable_queue.erl | 29 ++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index 376a303e..f29cc805 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -43,8 +43,8 @@ properties_input(), binary()) -> rabbit_types:message()). -spec(message/3 :: (rabbit_exchange:name(), rabbit_router:routing_key(), - rabbit_types:decoded_content()) -> {'ok', rabbit_types:message()} | - {'error', any()}). + rabbit_types:decoded_content()) -> + rabbit_types:ok_or_error2(rabbit_types:message() | any())). -spec(properties/1 :: (properties_input()) -> rabbit_framing:amqp_property_record()). -spec(publish/4 :: diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index d798c4f7..ef0e2e0d 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -26,7 +26,7 @@ -export([sync/1, set_maximum_since_use/2, has_readers/2, combine_files/3, delete_file/2]). %% internal --export([multiple_routing_keys/0]). %% upgrade +-export([transform_dir/3, force_recovery/2]). %% upgrade -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3, prioritise_call/3, prioritise_cast/2]). @@ -34,9 +34,8 @@ %%---------------------------------------------------------------------------- -include("rabbit_msg_store.hrl"). --include_lib("kernel/include/file.hrl"). --define(SYNC_INTERVAL, 25). %% milliseconds +-define(SYNC_INTERVAL, 5). %% milliseconds -define(CLEAN_FILENAME, "clean.dot"). -define(FILE_SUMMARY_FILENAME, "file_summary.ets"). -define(TRANSFORM_TMP, "transform_tmp"). @@ -106,8 +105,6 @@ %%---------------------------------------------------------------------------- --rabbit_upgrade({multiple_routing_keys, []}). - -ifdef(use_specs). -export_type([gc_state/0, file_num/0]). @@ -166,7 +163,9 @@ -spec(combine_files/3 :: (non_neg_integer(), non_neg_integer(), gc_state()) -> deletion_thunk()). -spec(delete_file/2 :: (non_neg_integer(), gc_state()) -> deletion_thunk()). --spec(multiple_routing_keys/0 :: () -> 'ok'). +-spec(force_recovery/2 :: (file:filename(), server()) -> 'ok'). +-spec(transform_dir/3 :: (file:filename(), server(), + fun ((any()) -> (rabbit_types:ok_or_error2(msg(), any())))) -> 'ok'). -endif. @@ -1968,25 +1967,6 @@ copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, {destination, Destination}]} end. -%%---------------------------------------------------------------------------- -%% upgrade -%%---------------------------------------------------------------------------- - -multiple_routing_keys() -> - [transform_store( - fun ({basic_message, ExchangeName, Routing_Key, Content, - Guid, Persistent}) -> - {ok, {basic_message, ExchangeName, [Routing_Key], Content, - Guid, Persistent}}; - (_) -> {error, corrupt_message} - end, Store) || Store <- rabbit_variable_queue:store_names()], - ok. - -%% Assumes message store is not running -transform_store(TransformFun, Store) -> - force_recovery(rabbit_mnesia:dir(), Store), - transform_dir(rabbit_mnesia:dir(), Store, TransformFun). - force_recovery(BaseDir, Store) -> Dir = filename:join(BaseDir, atom_to_list(Store)), file:delete(filename:join(Dir, ?CLEAN_FILENAME)), @@ -2017,14 +1997,13 @@ transform_dir(BaseDir, Store, TransformFun) -> transform_msg_file(FileOld, FileNew, TransformFun) -> rabbit_misc:ensure_parent_dirs_exist(FileNew), - {ok, #file_info{size=Size}} = file:read_file_info(FileOld), {ok, RefOld} = file_handle_cache:open(FileOld, [raw, binary, read], []), {ok, RefNew} = file_handle_cache:open(FileNew, [raw, binary, write], [{write_buffer, ?HANDLE_CACHE_BUFFER_SIZE}]), {ok, _Acc, _IgnoreSize} = rabbit_msg_file:scan( - RefOld, Size, + RefOld, filelib:file_size(FileOld), fun({Guid, _Size, _Offset, BinMsg}, ok) -> case TransformFun(binary_to_term(BinMsg)) of {ok, MsgNew} -> diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index 4eb9c3b8..3ef76d15 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -22,7 +22,7 @@ requeue/3, len/1, is_empty/1, dropwhile/2, set_ram_duration_target/2, ram_duration/1, needs_idle_timeout/1, idle_timeout/1, handle_pre_hibernate/1, - status/1, store_names/0]). + status/1, multiple_routing_keys/0]). -export([start/1, stop/0]). @@ -294,6 +294,8 @@ %%---------------------------------------------------------------------------- +-rabbit_upgrade({multiple_routing_keys, []}). + -ifdef(use_specs). -type(timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}). @@ -1802,5 +1804,26 @@ push_betas_to_deltas(Generator, Limit, Q, Count, RamIndexCount, IndexState) -> Generator, Limit, Qa, Count + 1, RamIndexCount1, IndexState1) end. -store_names() -> - [?PERSISTENT_MSG_STORE, ?TRANSIENT_MSG_STORE]. +%%---------------------------------------------------------------------------- +%% Upgrading +%%---------------------------------------------------------------------------- + +multiple_routing_keys() -> + transform_storage( + fun ({basic_message, ExchangeName, Routing_Key, Content, + Guid, Persistent}) -> + {ok, {basic_message, ExchangeName, [Routing_Key], Content, + Guid, Persistent}}; + (_) -> {error, corrupt_message} + end), + ok. + + +%% Assumes message store is not running +transform_storage(TransformFun) -> + transform_store(?PERSISTENT_MSG_STORE, TransformFun), + transform_store(?TRANSIENT_MSG_STORE, TransformFun). + +transform_store(Store, TransformFun) -> + rabbit_msg_store:force_recovery(rabbit_mnesia:dir(), Store), + rabbit_msg_store:transform_dir(rabbit_mnesia:dir(), Store, TransformFun). -- cgit v1.2.1 From fd53e724c289b17eca48aa2252376231be51eb41 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Tue, 22 Feb 2011 17:22:37 +0000 Subject: Added functional tests --- src/gm_soak_test.erl | 130 ++++++++++++++++++++++++++++++++++++++++ src/gm_test.erl | 126 --------------------------------------- src/gm_tests.erl | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 126 deletions(-) create mode 100644 src/gm_soak_test.erl delete mode 100644 src/gm_test.erl create mode 100644 src/gm_tests.erl (limited to 'src') diff --git a/src/gm_soak_test.erl b/src/gm_soak_test.erl new file mode 100644 index 00000000..1f8832a6 --- /dev/null +++ b/src/gm_soak_test.erl @@ -0,0 +1,130 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. +%% + +-module(gm_soak_test). + +-export([test/0]). +-export([joined/2, members_changed/3, handle_msg/3, terminate/2]). + +-behaviour(gm). + +-include("gm_specs.hrl"). + +%% --------------------------------------------------------------------------- +%% Soak test +%% --------------------------------------------------------------------------- + +get_state() -> + get(state). + +with_state(Fun) -> + put(state, Fun(get_state())). + +inc() -> + case 1 + get(count) of + 100000 -> Now = os:timestamp(), + Start = put(ts, Now), + Diff = timer:now_diff(Now, Start), + Rate = 100000 / (Diff / 1000000), + io:format("~p seeing ~p msgs/sec~n", [self(), Rate]), + put(count, 0); + N -> put(count, N) + end. + +joined([], Members) -> + io:format("Joined ~p (~p members)~n", [self(), length(Members)]), + put(state, dict:from_list([{Member, empty} || Member <- Members])), + put(count, 0), + put(ts, os:timestamp()), + ok. + +members_changed([], Births, Deaths) -> + with_state( + fun (State) -> + State1 = + lists:foldl( + fun (Born, StateN) -> + false = dict:is_key(Born, StateN), + dict:store(Born, empty, StateN) + end, State, Births), + lists:foldl( + fun (Died, StateN) -> + true = dict:is_key(Died, StateN), + dict:store(Died, died, StateN) + end, State1, Deaths) + end), + ok. + +handle_msg([], From, {test_msg, Num}) -> + inc(), + with_state( + fun (State) -> + ok = case dict:find(From, State) of + {ok, died} -> + exit({{from, From}, + {received_posthumous_delivery, Num}}); + {ok, empty} -> ok; + {ok, Num} -> ok; + {ok, Num1} when Num < Num1 -> + exit({{from, From}, + {duplicate_delivery_of, Num1}, + {expecting, Num}}); + {ok, Num1} -> + exit({{from, From}, + {missing_delivery_of, Num}, + {received_early, Num1}}); + error -> + exit({{from, From}, + {received_premature_delivery, Num}}) + end, + dict:store(From, Num + 1, State) + end), + ok. + +terminate([], Reason) -> + io:format("Left ~p (~p)~n", [self(), Reason]), + ok. + +spawn_member() -> + spawn_link( + fun () -> + random:seed(now()), + %% start up delay of no more than 10 seconds + timer:sleep(random:uniform(10000)), + {ok, Pid} = gm:start_link(?MODULE, ?MODULE, []), + Start = random:uniform(10000), + send_loop(Pid, Start, Start + random:uniform(10000)), + gm:leave(Pid), + spawn_more() + end). + +spawn_more() -> + [spawn_member() || _ <- lists:seq(1, 4 - random:uniform(4))]. + +send_loop(_Pid, Target, Target) -> + ok; +send_loop(Pid, Count, Target) when Target > Count -> + case random:uniform(3) of + 3 -> gm:confirmed_broadcast(Pid, {test_msg, Count}); + _ -> gm:broadcast(Pid, {test_msg, Count}) + end, + timer:sleep(random:uniform(5) - 1), %% sleep up to 4 ms + send_loop(Pid, Count + 1, Target). + +test() -> + ok = gm:create_tables(), + spawn_member(), + spawn_member(). diff --git a/src/gm_test.erl b/src/gm_test.erl deleted file mode 100644 index e0a92a0c..00000000 --- a/src/gm_test.erl +++ /dev/null @@ -1,126 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License at -%% http://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the -%% License for the specific language governing rights and limitations -%% under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is VMware, Inc. -%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. -%% - --module(gm_test). - --export([test/0]). --export([joined/2, members_changed/3, handle_msg/3, terminate/2]). - --behaviour(gm). - --include("gm_specs.hrl"). - -get_state() -> - get(state). - -with_state(Fun) -> - put(state, Fun(get_state())). - -inc() -> - case 1 + get(count) of - 100000 -> Now = os:timestamp(), - Start = put(ts, Now), - Diff = timer:now_diff(Now, Start), - Rate = 100000 / (Diff / 1000000), - io:format("~p seeing ~p msgs/sec~n", [self(), Rate]), - put(count, 0); - N -> put(count, N) - end. - -joined([], Members) -> - io:format("Joined ~p (~p members)~n", [self(), length(Members)]), - put(state, dict:from_list([{Member, empty} || Member <- Members])), - put(count, 0), - put(ts, os:timestamp()), - ok. - -members_changed([], Births, Deaths) -> - with_state( - fun (State) -> - State1 = - lists:foldl( - fun (Born, StateN) -> - false = dict:is_key(Born, StateN), - dict:store(Born, empty, StateN) - end, State, Births), - lists:foldl( - fun (Died, StateN) -> - true = dict:is_key(Died, StateN), - dict:store(Died, died, StateN) - end, State1, Deaths) - end), - ok. - -handle_msg([], From, {test_msg, Num}) -> - inc(), - with_state( - fun (State) -> - ok = case dict:find(From, State) of - {ok, died} -> - exit({{from, From}, - {received_posthumous_delivery, Num}}); - {ok, empty} -> ok; - {ok, Num} -> ok; - {ok, Num1} when Num < Num1 -> - exit({{from, From}, - {duplicate_delivery_of, Num1}, - {expecting, Num}}); - {ok, Num1} -> - exit({{from, From}, - {missing_delivery_of, Num}, - {received_early, Num1}}); - error -> - exit({{from, From}, - {received_premature_delivery, Num}}) - end, - dict:store(From, Num + 1, State) - end), - ok. - -terminate([], Reason) -> - io:format("Left ~p (~p)~n", [self(), Reason]), - ok. - -spawn_member() -> - spawn_link( - fun () -> - random:seed(now()), - %% start up delay of no more than 10 seconds - timer:sleep(random:uniform(10000)), - {ok, Pid} = gm:start_link(?MODULE, ?MODULE, []), - Start = random:uniform(10000), - send_loop(Pid, Start, Start + random:uniform(10000)), - gm:leave(Pid), - spawn_more() - end). - -spawn_more() -> - [spawn_member() || _ <- lists:seq(1, 4 - random:uniform(4))]. - -send_loop(_Pid, Target, Target) -> - ok; -send_loop(Pid, Count, Target) when Target > Count -> - case random:uniform(3) of - 3 -> gm:confirmed_broadcast(Pid, {test_msg, Count}); - _ -> gm:broadcast(Pid, {test_msg, Count}) - end, - timer:sleep(random:uniform(5) - 1), %% sleep up to 4 ms - send_loop(Pid, Count + 1, Target). - -test() -> - ok = gm:create_tables(), - spawn_member(), - spawn_member(). diff --git a/src/gm_tests.erl b/src/gm_tests.erl new file mode 100644 index 00000000..38b3db2f --- /dev/null +++ b/src/gm_tests.erl @@ -0,0 +1,165 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is VMware, Inc. +%% Copyright (c) 2007-2011 VMware, Inc. All rights reserved. +%% + +-module(gm_tests). + +-export([test_join_leave/0, + test_broadcast/0, + test_confirmed_broadcast/0, + test_member_death/0, + all_tests/0]). +-export([joined/2, members_changed/3, handle_msg/3, terminate/2]). + +-behaviour(gm). + +-include("gm_specs.hrl"). + +joined(Pid, Members) -> + Pid ! {joined, self(), Members}, + ok. + +members_changed(Pid, Births, Deaths) -> + Pid ! {members_changed, self(), Births, Deaths}, + ok. + +handle_msg(Pid, From, Msg) -> + Pid ! {msg, self(), From, Msg}, + ok. + +terminate(Pid, Reason) -> + Pid ! {termination, self(), Reason}, + ok. + +%% --------------------------------------------------------------------------- +%% Functional tests +%% --------------------------------------------------------------------------- + +all_tests() -> + passed = test_join_leave(), + passed = test_broadcast(), + passed = test_confirmed_broadcast(), + passed = test_member_death(), + passed. + +test_join_leave() -> + with_two_members(fun (_Pid, _Pid2) -> passed end). + +test_broadcast() -> + test_broadcast(fun gm:broadcast/2). + +test_confirmed_broadcast() -> + test_broadcast(fun gm:confirmed_broadcast/2). + +test_member_death() -> + with_two_members( + fun (Pid, Pid2) -> + {ok, Pid3} = gm:start_link(?MODULE, ?MODULE, self()), + passed = receive_joined(Pid3, [Pid, Pid2, Pid3], + timeout_joining_gm_group_3), + passed = receive_birth(Pid, Pid3, timeout_waiting_for_birth_3_1), + passed = receive_birth(Pid2, Pid3, timeout_waiting_for_birth_3_2), + + unlink(Pid3), + exit(Pid3, kill), + + passed = (test_broadcast_fun(fun gm:confirmed_broadcast/2))( + Pid, Pid2), + + passed = receive_death(Pid, Pid3, timeout_waiting_for_death_3_1), + passed = receive_death(Pid2, Pid3, timeout_waiting_for_death_3_2), + + passed + end). + +test_broadcast(Fun) -> + with_two_members(test_broadcast_fun(Fun)). + +test_broadcast_fun(Fun) -> + fun (Pid, Pid2) -> + ok = Fun(Pid, magic_message), + passed = receive_or_throw({msg, Pid, Pid, magic_message}, + timeout_waiting_for_msg), + passed = receive_or_throw({msg, Pid2, Pid, magic_message}, + timeout_waiting_for_msg) + end. + +with_two_members(Fun) -> + ok = gm:create_tables(), + + {ok, Pid} = gm:start_link(?MODULE, ?MODULE, self()), + passed = receive_joined(Pid, [Pid], timeout_joining_gm_group_1), + + {ok, Pid2} = gm:start_link(?MODULE, ?MODULE, self()), + passed = receive_joined(Pid2, [Pid, Pid2], timeout_joining_gm_group_2), + + passed = receive_birth(Pid, Pid2, timeout_waiting_for_birth_2), + + passed = Fun(Pid, Pid2), + + ok = gm:leave(Pid), + passed = receive_death(Pid2, Pid, timeout_waiting_for_death_1), + passed = + receive_termination(Pid, normal, timeout_waiting_for_termination_1), + + ok = gm:leave(Pid2), + passed = + receive_termination(Pid2, normal, timeout_waiting_for_termination_2), + + receive X -> throw({unexpected_message, X}) + after 0 -> passed + end. + +receive_or_throw(Pattern, Error) -> + receive Pattern -> + passed + after 1000 -> + throw(Error) + end. + +receive_birth(From, Born, Error) -> + receive {members_changed, From, Birth, Death} -> + [Born] = Birth, + [] = Death, + passed + after 1000 -> + throw(Error) + end. + +receive_death(From, Died, Error) -> + receive {members_changed, From, Birth, Death} -> + [] = Birth, + [Died] = Death, + passed + after 1000 -> + throw(Error) + end. + +receive_joined(From, Members, Error) -> + Members1 = lists:usort(Members), + receive {joined, From, Members2} -> + Members1 = lists:usort(Members2), + passed + after 1000 -> + throw(Error) + end. + +receive_termination(From, Reason, Error) -> + receive {termination, From, Reason1} -> + Reason = Reason1, + passed + after 1000 -> + throw(Error) + end. -- cgit v1.2.1 From c3f44f9a82132f63ad9b1566874c054909c6733f Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Tue, 22 Feb 2011 22:54:12 +0000 Subject: Magic macroification --- src/gm_tests.erl | 53 +++++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/gm_tests.erl b/src/gm_tests.erl index 38b3db2f..bb92bc4c 100644 --- a/src/gm_tests.erl +++ b/src/gm_tests.erl @@ -27,6 +27,14 @@ -include("gm_specs.hrl"). +-define(RECEIVE_AFTER(Body, Bool, Error), + receive Body -> + true = Bool, + passed + after 1000 -> + throw(Error) + end). + joined(Pid, Members) -> Pid ! {joined, self(), Members}, ok. @@ -123,43 +131,24 @@ with_two_members(Fun) -> end. receive_or_throw(Pattern, Error) -> - receive Pattern -> - passed - after 1000 -> - throw(Error) - end. + ?RECEIVE_AFTER(Pattern, true, Error). receive_birth(From, Born, Error) -> - receive {members_changed, From, Birth, Death} -> - [Born] = Birth, - [] = Death, - passed - after 1000 -> - throw(Error) - end. + ?RECEIVE_AFTER({members_changed, From, Birth, Death}, + ([Born] == Birth) andalso ([] == Death), + Error). receive_death(From, Died, Error) -> - receive {members_changed, From, Birth, Death} -> - [] = Birth, - [Died] = Death, - passed - after 1000 -> - throw(Error) - end. + ?RECEIVE_AFTER({members_changed, From, Birth, Death}, + ([] == Birth) andalso ([Died] == Death), + Error). receive_joined(From, Members, Error) -> - Members1 = lists:usort(Members), - receive {joined, From, Members2} -> - Members1 = lists:usort(Members2), - passed - after 1000 -> - throw(Error) - end. + ?RECEIVE_AFTER({joined, From, Members2}, + lists:usort(Members) == lists:usort(Members2), + Error). receive_termination(From, Reason, Error) -> - receive {termination, From, Reason1} -> - Reason = Reason1, - passed - after 1000 -> - throw(Error) - end. + ?RECEIVE_AFTER({termination, From, Reason1}, + Reason == Reason1, + Error). -- cgit v1.2.1 From 8a6cb10fd4817ebf92303e397f797c1a3de6ed57 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Tue, 22 Feb 2011 22:55:24 +0000 Subject: consistency --- src/gm_tests.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/gm_tests.erl b/src/gm_tests.erl index bb92bc4c..fd9a6487 100644 --- a/src/gm_tests.erl +++ b/src/gm_tests.erl @@ -144,8 +144,8 @@ receive_death(From, Died, Error) -> Error). receive_joined(From, Members, Error) -> - ?RECEIVE_AFTER({joined, From, Members2}, - lists:usort(Members) == lists:usort(Members2), + ?RECEIVE_AFTER({joined, From, Members1}, + lists:usort(Members) == lists:usort(Members1), Error). receive_termination(From, Reason, Error) -> -- cgit v1.2.1 From 5597c0f213da52331b090c1f2f954ccf155dd0cd Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Tue, 22 Feb 2011 22:57:01 +0000 Subject: rename --- src/gm_tests.erl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/gm_tests.erl b/src/gm_tests.erl index fd9a6487..87244153 100644 --- a/src/gm_tests.erl +++ b/src/gm_tests.erl @@ -27,7 +27,7 @@ -include("gm_specs.hrl"). --define(RECEIVE_AFTER(Body, Bool, Error), +-define(RECEIVE_OR_THROW(Body, Bool, Error), receive Body -> true = Bool, passed @@ -131,24 +131,24 @@ with_two_members(Fun) -> end. receive_or_throw(Pattern, Error) -> - ?RECEIVE_AFTER(Pattern, true, Error). + ?RECEIVE_OR_THROW(Pattern, true, Error). receive_birth(From, Born, Error) -> - ?RECEIVE_AFTER({members_changed, From, Birth, Death}, - ([Born] == Birth) andalso ([] == Death), - Error). + ?RECEIVE_OR_THROW({members_changed, From, Birth, Death}, + ([Born] == Birth) andalso ([] == Death), + Error). receive_death(From, Died, Error) -> - ?RECEIVE_AFTER({members_changed, From, Birth, Death}, - ([] == Birth) andalso ([Died] == Death), - Error). + ?RECEIVE_OR_THROW({members_changed, From, Birth, Death}, + ([] == Birth) andalso ([Died] == Death), + Error). receive_joined(From, Members, Error) -> - ?RECEIVE_AFTER({joined, From, Members1}, - lists:usort(Members) == lists:usort(Members1), - Error). + ?RECEIVE_OR_THROW({joined, From, Members1}, + lists:usort(Members) == lists:usort(Members1), + Error). receive_termination(From, Reason, Error) -> - ?RECEIVE_AFTER({termination, From, Reason1}, - Reason == Reason1, - Error). + ?RECEIVE_OR_THROW({termination, From, Reason1}, + Reason == Reason1, + Error). -- cgit v1.2.1 From a74602a5813a6915f3be26719e84a637fea337f5 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Wed, 23 Feb 2011 12:52:55 +0000 Subject: Added test to assert receiving messages in the order they're sent. Other cosmetics --- src/gm_tests.erl | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/gm_tests.erl b/src/gm_tests.erl index 87244153..65e9cff0 100644 --- a/src/gm_tests.erl +++ b/src/gm_tests.erl @@ -20,6 +20,7 @@ test_broadcast/0, test_confirmed_broadcast/0, test_member_death/0, + test_receive_in_order/0, all_tests/0]). -export([joined/2, members_changed/3, handle_msg/3, terminate/2]). @@ -60,6 +61,7 @@ all_tests() -> passed = test_broadcast(), passed = test_confirmed_broadcast(), passed = test_member_death(), + passed = test_receive_in_order(), passed. test_join_leave() -> @@ -83,6 +85,8 @@ test_member_death() -> unlink(Pid3), exit(Pid3, kill), + %% Have to do some broadcasts to ensure that all members + %% find out about the death. passed = (test_broadcast_fun(fun gm:confirmed_broadcast/2))( Pid, Pid2), @@ -92,6 +96,23 @@ test_member_death() -> passed end). +test_receive_in_order() -> + with_two_members( + fun (Pid, Pid2) -> + Numbers = lists:seq(1,1000), + [begin ok = gm:broadcast(Pid, N), ok = gm:broadcast(Pid2, N) end + || N <- Numbers], + passed = receive_numbers( + Pid, Pid, {timeout_for_msgs, Pid, Pid}, Numbers), + passed = receive_numbers( + Pid, Pid2, {timeout_for_msgs, Pid, Pid2}, Numbers), + passed = receive_numbers( + Pid2, Pid, {timeout_for_msgs, Pid2, Pid}, Numbers), + passed = receive_numbers( + Pid2, Pid2, {timeout_for_msgs, Pid2, Pid2}, Numbers), + passed + end). + test_broadcast(Fun) -> with_two_members(test_broadcast_fun(Fun)). @@ -112,7 +133,6 @@ with_two_members(Fun) -> {ok, Pid2} = gm:start_link(?MODULE, ?MODULE, self()), passed = receive_joined(Pid2, [Pid, Pid2], timeout_joining_gm_group_2), - passed = receive_birth(Pid, Pid2, timeout_waiting_for_birth_2), passed = Fun(Pid, Pid2), @@ -152,3 +172,11 @@ receive_termination(From, Reason, Error) -> ?RECEIVE_OR_THROW({termination, From, Reason1}, Reason == Reason1, Error). + +receive_numbers(_Pid, _Sender, _Error, []) -> + passed; +receive_numbers(Pid, Sender, Error, [N | Numbers]) -> + ?RECEIVE_OR_THROW({msg, Pid, Sender, M}, + M == N, + Error), + receive_numbers(Pid, Sender, Error, Numbers). -- cgit v1.2.1 From eccf06819029cc5c72b0d8b166dca929ba42e620 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Wed, 23 Feb 2011 12:54:40 +0000 Subject: Wire in gm_tests to rabbit tests --- src/rabbit_tests.erl | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl index 49b09508..644c4f96 100644 --- a/src/rabbit_tests.erl +++ b/src/rabbit_tests.erl @@ -34,6 +34,7 @@ test_content_prop_roundtrip(Datum, Binary) -> Binary = rabbit_binary_generator:encode_properties(Types, Values). %% assertion all_tests() -> + passed = gm_tests:all_tests(), application:set_env(rabbit, file_handles_high_watermark, 10, infinity), ok = file_handle_cache:set_limit(10), passed = test_file_handle_cache(), -- cgit v1.2.1 From 1ee22ba19d1cdfab15811b75d6a4b7a3020eb38d Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Wed, 23 Feb 2011 13:09:16 +0000 Subject: correction of specs --- src/gm.erl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl index 283b2431..b3fb7eca 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -53,13 +53,12 @@ %% to create the tables required. %% %% start_link/3 -%% Provide the group name, the callback module name, and a list of any -%% arguments you wish to be passed into the callback module's -%% functions. The joined/1 will be called when we have joined the -%% group, and the list of arguments will have appended to it a list of -%% the current members of the group. See the comments in -%% behaviour_info/1 below for further details of the callback -%% functions. +%% Provide the group name, the callback module name, and any arguments +%% you wish to be passed into the callback module's functions. The +%% joined/1 will be called when we have joined the group, and the list +%% of arguments will have appended to it a list of the current members +%% of the group. See the comments in behaviour_info/1 below for +%% further details of the callback functions. %% %% leave/1 %% Provide the Pid. Removes the Pid from the group. The callback @@ -421,7 +420,7 @@ -type(group_name() :: any()). -spec(create_tables/0 :: () -> 'ok'). --spec(start_link/3 :: (group_name(), atom(), [any()]) -> +-spec(start_link/3 :: (group_name(), atom(), any()) -> {'ok', pid()} | {'error', any()}). -spec(leave/1 :: (pid()) -> 'ok'). -spec(broadcast/2 :: (pid(), any()) -> 'ok'). -- cgit v1.2.1 From fff7752e4df43bdefecee6a9700b5d34df3097e5 Mon Sep 17 00:00:00 2001 From: Rob Harrop Date: Wed, 23 Feb 2011 13:40:15 +0000 Subject: Fixed incorrect binding pattern in rabbit_mnesia --- src/rabbit_mnesia.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 25767a55..93e20381 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -232,8 +232,8 @@ trie_edge_match() -> #trie_edge{exchange_name = exchange_name_match(), _='_'}. trie_binding_match() -> - #trie_edge{exchange_name = exchange_name_match(), - _='_'}. + #trie_binding{exchange_name = exchange_name_match(), + _='_'}. exchange_name_match() -> resource_match(exchange). queue_name_match() -> -- cgit v1.2.1 From d86469a2af5cc68da909a4698a0ee634f2e8aa8b Mon Sep 17 00:00:00 2001 From: Rob Harrop Date: Wed, 23 Feb 2011 14:43:00 +0000 Subject: Removed table name intersection in wait_for_tables and cleaned up whitespace changes --- src/rabbit_mnesia.erl | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index 93e20381..f2d23dad 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -388,8 +388,7 @@ init_db(ClusterNodes, Force) -> {[], true, [_]} -> %% True single disc node, attempt upgrade case rabbit_upgrade:maybe_upgrade() of - ok -> ok = wait_for_tables(), - ensure_schema_integrity(); + ok -> ensure_schema_integrity(); version_not_available -> schema_ok_or_move() end; {[], true, _} -> @@ -544,17 +543,15 @@ create_local_table_copy(Tab, Type) -> end, ok. -wait_for_replicated_tables() -> - wait_for_tables(replicated_table_names()). +wait_for_replicated_tables() -> wait_for_tables(replicated_table_names()). -wait_for_tables() -> - wait_for_tables(table_names()). +wait_for_tables() -> wait_for_tables(table_names()). wait_for_tables(TableNames) -> - Nonexistent = TableNames -- mnesia:system_info(tables), - case mnesia:wait_for_tables(TableNames -- Nonexistent, 30000) of - ok -> ok; - {timeout, BadTabs} -> + case mnesia:wait_for_tables(TableNames, 30000) of + ok -> + ok; + {timeout, BadTabs} -> throw({error, {timeout_waiting_for_tables, BadTabs}}); {error, Reason} -> throw({error, {failed_waiting_for_tables, Reason}}) -- cgit v1.2.1 From d2199eccd9ecbf0c50666fe793d780cdbbf23ef3 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Wed, 23 Feb 2011 15:12:38 +0000 Subject: cosmetic --- src/rabbit_mnesia.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index f2d23dad..d3cb492e 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -551,7 +551,7 @@ wait_for_tables(TableNames) -> case mnesia:wait_for_tables(TableNames, 30000) of ok -> ok; - {timeout, BadTabs} -> + {timeout, BadTabs} -> throw({error, {timeout_waiting_for_tables, BadTabs}}); {error, Reason} -> throw({error, {failed_waiting_for_tables, Reason}}) -- cgit v1.2.1 From 4d36462a0eb49acca8190c9aa6e5b54a59fc5d18 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 24 Feb 2011 14:25:06 +0000 Subject: English, not American --- src/rabbit_reader.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 29321c60..b172db56 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -158,7 +158,7 @@ server_properties(Protocol) -> {copyright, ?COPYRIGHT_MESSAGE}, {information, ?INFORMATION_MESSAGE}]]], - %% Filter duplicated properties in favor of config file provided values + %% Filter duplicated properties in favour of config file provided values lists:usort(fun ({K1,_,_}, {K2,_,_}) -> K1 =< K2 end, NormalizedConfigServerProps). -- cgit v1.2.1 From a64a627af2739a5556f00064c9b02443bd0c4215 Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Thu, 24 Feb 2011 15:14:26 +0000 Subject: Dialyzer typo --- src/rabbit_basic.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl index f29cc805..57aad808 100644 --- a/src/rabbit_basic.erl +++ b/src/rabbit_basic.erl @@ -44,7 +44,7 @@ -spec(message/3 :: (rabbit_exchange:name(), rabbit_router:routing_key(), rabbit_types:decoded_content()) -> - rabbit_types:ok_or_error2(rabbit_types:message() | any())). + rabbit_types:ok_or_error2(rabbit_types:message(), any())). -spec(properties/1 :: (properties_input()) -> rabbit_framing:amqp_property_record()). -spec(publish/4 :: -- cgit v1.2.1 From 6c854337b76061d06fb1dd5e9db4976fc5b9e6f4 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 25 Feb 2011 12:25:16 +0000 Subject: Make documentation accurate for current API... --- src/gm.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl index b3fb7eca..b21217f6 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -55,14 +55,14 @@ %% start_link/3 %% Provide the group name, the callback module name, and any arguments %% you wish to be passed into the callback module's functions. The -%% joined/1 will be called when we have joined the group, and the list -%% of arguments will have appended to it a list of the current members -%% of the group. See the comments in behaviour_info/1 below for -%% further details of the callback functions. +%% joined/2 will be called when we have joined the group, with the +%% arguments passed to start_link and a list of the current members of +%% the group. See the comments in behaviour_info/1 below for further +%% details of the callback functions. %% %% leave/1 %% Provide the Pid. Removes the Pid from the group. The callback -%% terminate/1 function will be called. +%% terminate/2 function will be called. %% %% broadcast/2 %% Provide the Pid and a Message. The message will be sent to all @@ -455,16 +455,16 @@ behaviour_info(callbacks) -> %% quickly, it's possible that we will never see that member %% appear in either births or deaths. However we are guaranteed %% that (1) we will see a member joining either in the births - %% here, or in the members passed to joined/1 before receiving + %% here, or in the members passed to joined/2 before receiving %% any messages from it; and (2) we will not see members die that %% we have not seen born (or supplied in the members to - %% joined/1). + %% joined/2). {members_changed, 3}, %% Supplied with Args provided in start_link, the sender, and the %% message. This does get called for messages injected by this %% member, however, in such cases, there is no special - %% significance of this call: it does not indicate that the + %% significance of this invocation: it does not indicate that the %% message has made it to any other members, let alone all other %% members. {handle_msg, 3}, -- cgit v1.2.1 From f4c23c93527e9bd37243ee883b552b478427c7c2 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 25 Feb 2011 13:05:21 +0000 Subject: Additional word --- src/gm.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/gm.erl b/src/gm.erl index b21217f6..70633a08 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -55,10 +55,10 @@ %% start_link/3 %% Provide the group name, the callback module name, and any arguments %% you wish to be passed into the callback module's functions. The -%% joined/2 will be called when we have joined the group, with the -%% arguments passed to start_link and a list of the current members of -%% the group. See the comments in behaviour_info/1 below for further -%% details of the callback functions. +%% joined/2 function will be called when we have joined the group, +%% with the arguments passed to start_link and a list of the current +%% members of the group. See the comments in behaviour_info/1 below +%% for further details of the callback functions. %% %% leave/1 %% Provide the Pid. Removes the Pid from the group. The callback -- cgit v1.2.1 From 1633fd03f06b5b43006ef83833d5a0c9f28c510f Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 25 Feb 2011 14:45:45 +0000 Subject: multiple_routing_keys/0 is not part of the backing_queue --- src/rabbit_variable_queue.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index 3ef76d15..13fe9fda 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -353,6 +353,8 @@ -include("rabbit_backing_queue_spec.hrl"). +-spec(multiple_routing_keys/0 :: () -> 'ok'). + -endif. -define(BLANK_DELTA, #delta { start_seq_id = undefined, -- cgit v1.2.1 From c62cfd0cea0a4691d3b7806d0353eaeca8d7a375 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Fri, 25 Feb 2011 14:46:30 +0000 Subject: remove blank trailing line --- src/rabbit_msg_store.erl | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index ef0e2e0d..907f567b 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -2018,4 +2018,3 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> file_handle_cache:close(RefOld), file_handle_cache:close(RefNew), ok. - -- cgit v1.2.1 From fcb9a05d24be5a256de6539b0208371cf17aae8f Mon Sep 17 00:00:00 2001 From: Emile Joubert Date: Fri, 25 Feb 2011 16:12:21 +0000 Subject: Stricter msg store upgrade --- src/rabbit_msg_store.erl | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl index 907f567b..9e65e442 100644 --- a/src/rabbit_msg_store.erl +++ b/src/rabbit_msg_store.erl @@ -1970,8 +1970,7 @@ copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, force_recovery(BaseDir, Store) -> Dir = filename:join(BaseDir, atom_to_list(Store)), file:delete(filename:join(Dir, ?CLEAN_FILENAME)), - [file:delete(filename:join(Dir, File)) || - File <- list_sorted_file_names(Dir, ?FILE_EXTENSION_TMP)], + recover_crashed_compactions(BaseDir), ok. foreach_file(D, Fun, Files) -> @@ -1986,12 +1985,11 @@ transform_dir(BaseDir, Store, TransformFun) -> TransformFile = fun (A, B) -> transform_msg_file(A, B, TransformFun) end, case filelib:is_dir(TmpDir) of true -> throw({error, transform_failed_previously}); - false -> OldFileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), - foreach_file(Dir, TmpDir, TransformFile, OldFileList), - foreach_file(Dir, fun file:delete/1, OldFileList), - NewFileList = list_sorted_file_names(TmpDir, ?FILE_EXTENSION), - foreach_file(TmpDir, Dir, fun file:copy/2, NewFileList), - foreach_file(TmpDir, fun file:delete/1, NewFileList), + false -> FileList = list_sorted_file_names(Dir, ?FILE_EXTENSION), + foreach_file(Dir, TmpDir, TransformFile, FileList), + foreach_file(Dir, fun file:delete/1, FileList), + foreach_file(TmpDir, Dir, fun file:copy/2, FileList), + foreach_file(TmpDir, fun file:delete/1, FileList), ok = file:del_dir(TmpDir) end. @@ -2005,15 +2003,9 @@ transform_msg_file(FileOld, FileNew, TransformFun) -> rabbit_msg_file:scan( RefOld, filelib:file_size(FileOld), fun({Guid, _Size, _Offset, BinMsg}, ok) -> - case TransformFun(binary_to_term(BinMsg)) of - {ok, MsgNew} -> - {ok, _} = rabbit_msg_file:append(RefNew, Guid, MsgNew), - ok; - {error, Reason} -> - error_logger:error_msg("Message transform failed: ~p~n", - [Reason]), - ok - end + {ok, MsgNew} = TransformFun(binary_to_term(BinMsg)), + {ok, _} = rabbit_msg_file:append(RefNew, Guid, MsgNew), + ok end, ok), file_handle_cache:close(RefOld), file_handle_cache:close(RefNew), -- cgit v1.2.1 From bbc9fcbcb631404e46259a606649a6bb5648db57 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Mon, 28 Feb 2011 11:02:29 +0000 Subject: ...and untabify. --- src/rabbit_channel.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index d8a332f3..7dc07e5a 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1288,12 +1288,12 @@ is_message_persistent(Content) -> process_routing_result(unroutable, _, XName, MsgSeqNo, Msg, State) -> ok = basic_return(Msg, State, no_route), maybe_incr_stats([{Msg#basic_message.exchange_name, 1}], - return_unroutable, State), + return_unroutable, State), record_confirm(MsgSeqNo, XName, State); process_routing_result(not_delivered, _, XName, MsgSeqNo, Msg, State) -> ok = basic_return(Msg, State, no_consumers), maybe_incr_stats([{Msg#basic_message.exchange_name, 1}], - return_not_delivered, State), + return_not_delivered, State), record_confirm(MsgSeqNo, XName, State); process_routing_result(routed, [], XName, MsgSeqNo, _, State) -> record_confirm(MsgSeqNo, XName, State); -- cgit v1.2.1 From 75b306010463265a291e84d91f9e13ebbd470714 Mon Sep 17 00:00:00 2001 From: Alexandru Scvortov Date: Wed, 2 Mar 2011 13:32:59 +0000 Subject: only confirm delivered messages that need confirming --- src/rabbit_variable_queue.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index d1307b85..d0c984cb 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -510,8 +510,13 @@ publish(Msg, MsgProps, State) -> a(reduce_memory_use(State1)). publish_delivered(false, #basic_message { guid = Guid }, - _MsgProps, State = #vqstate { len = 0 }) -> - blind_confirm(self(), gb_sets:singleton(Guid)), + MsgProps = #message_properties { + needs_confirming = NeedsConfirming }, + State = #vqstate { len = 0 }) -> + case NeedsConfirming of + true -> blind_confirm(self(), gb_sets:singleton(Guid)); + false -> ok + end, {undefined, a(State)}; publish_delivered(true, Msg = #basic_message { is_persistent = IsPersistent, guid = Guid }, -- cgit v1.2.1 From 5ac968c2f7a20f0b7b9da54c0ec72057b36abfd7 Mon Sep 17 00:00:00 2001 From: Matthew Sackman Date: Thu, 3 Mar 2011 15:05:41 +0000 Subject: Remove unused var --- src/rabbit_variable_queue.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl index d0c984cb..58a28d32 100644 --- a/src/rabbit_variable_queue.erl +++ b/src/rabbit_variable_queue.erl @@ -510,7 +510,7 @@ publish(Msg, MsgProps, State) -> a(reduce_memory_use(State1)). publish_delivered(false, #basic_message { guid = Guid }, - MsgProps = #message_properties { + #message_properties { needs_confirming = NeedsConfirming }, State = #vqstate { len = 0 }) -> case NeedsConfirming of -- cgit v1.2.1