From 05544ad98014efecd237a42af07958f82703867b Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Mon, 18 Feb 2013 16:47:40 +0000 Subject: cluster_cp_mode --- src/rabbit_mnesia.erl | 6 ++++++ src/rabbit_node_monitor.erl | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index c39e898c..ecb03f54 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -26,6 +26,7 @@ status/0, is_clustered/0, + majority/0, cluster_nodes/1, node_type/0, dir/0, @@ -67,6 +68,7 @@ -spec(status/0 :: () -> [{'nodes', [{node_type(), [node()]}]} | {'running_nodes', [node()]} | {'partitions', [{node(), [node()]}]}]). +-spec(majority/0 :: () -> boolean()). -spec(is_clustered/0 :: () -> boolean()). -spec(cluster_nodes/1 :: ('all' | 'disc' | 'ram' | 'running') -> [node()]). -spec(node_type/0 :: () -> node_type()). @@ -338,6 +340,10 @@ status() -> false -> [] end. +majority() -> + ensure_mnesia_running(), + (length(cluster_nodes(running)) / length(cluster_nodes(all))) > 0.5. + mnesia_partitions(Nodes) -> {Replies, _BadNodes} = rpc:multicall( Nodes, rabbit_node_monitor, partitions, []), diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index 71c2c80a..7b7fed5c 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -250,6 +250,10 @@ handle_info({mnesia_system_event, ordsets:add_element(Node, ordsets:from_list(Partitions))), {noreply, State#state{partitions = Partitions1}}; +handle_info({mnesia_system_event, {mnesia_down, _Node}}, State) -> + handle_dead_according_to_mnesia_rabbit(), + {noreply, State}; + handle_info(_Info, State) -> {noreply, State}. @@ -272,6 +276,28 @@ handle_dead_rabbit(Node) -> ok = rabbit_alarm:on_node_down(Node), ok = rabbit_mnesia:on_node_down(Node). +%% Since we will be introspecting the cluster in response to this, we +%% must only do so based on Mnesia having noticed the other node being +%% down - otherwise we have a race. +handle_dead_according_to_mnesia_rabbit() -> + case application:get_env(rabbit, cluster_cp_mode) of + {ok, true} -> case rabbit_mnesia:majority() of + true -> ok; + false -> stop_and_halt() + end; + {ok, false} -> ok + end, + ok. + +stop_and_halt() -> + rabbit_log:warning("Cluster minority status detected - stopping~n", []), + spawn(fun () -> + %% If our group leader is inside an application we are about + %% to stop, application:stop/1 does not return. + group_leader(whereis(init), self()), + rabbit:stop_and_halt() + end). + handle_live_rabbit(Node) -> ok = rabbit_alarm:on_node_up(Node), ok = rabbit_mnesia:on_node_up(Node). -- cgit v1.2.1 From 2e9879c6cde8d540cd8cd3e56223f0a3bbe22556 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 27 Feb 2013 14:20:07 +0000 Subject: When we lose majority, stop the applications and wait for the cluster to come back. --- src/rabbit_node_monitor.erl | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index 7b7fed5c..42df6e5d 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -283,21 +283,34 @@ handle_dead_according_to_mnesia_rabbit() -> case application:get_env(rabbit, cluster_cp_mode) of {ok, true} -> case rabbit_mnesia:majority() of true -> ok; - false -> stop_and_halt() + false -> await_cluster_recovery() end; {ok, false} -> ok end, ok. -stop_and_halt() -> - rabbit_log:warning("Cluster minority status detected - stopping~n", []), +await_cluster_recovery() -> + rabbit_log:warning("Cluster minority status detected - awaiting recovery~n", + []), + Nodes = rabbit_mnesia:cluster_nodes(all), spawn(fun () -> %% If our group leader is inside an application we are about %% to stop, application:stop/1 does not return. group_leader(whereis(init), self()), - rabbit:stop_and_halt() + rabbit:stop(), + wait_for_cluster_recovery(Nodes) end). +wait_for_cluster_recovery(Nodes) -> + [erlang:disconnect_node(Node) || Node <- Nodes], + mnesia:start(), + case rabbit_mnesia:majority() of + true -> rabbit:start(); + false -> mnesia:stop(), + timer:sleep(1000), + wait_for_cluster_recovery(Nodes) + end. + handle_live_rabbit(Node) -> ok = rabbit_alarm:on_node_up(Node), ok = rabbit_mnesia:on_node_up(Node). -- cgit v1.2.1 From fad1b961ea4d53670381caa4b701add21ee406b4 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 27 Feb 2013 14:25:12 +0000 Subject: Filter out all those events that look like: =INFO REPORT==== 27-Feb-2013::14:17:46 === application: mnesia exited: stopped type: temporary since they are not very interesting and this bug makes them appear to a highly verbose extent. --- src/rabbit_error_logger_file_h.erl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/rabbit_error_logger_file_h.erl b/src/rabbit_error_logger_file_h.erl index 3efc9c0c..c00c1df9 100644 --- a/src/rabbit_error_logger_file_h.erl +++ b/src/rabbit_error_logger_file_h.erl @@ -76,6 +76,8 @@ init_file(File, PrevHandler) -> Error -> Error end. +handle_event({info_report, _, {_, std_info, _}}, State) -> + ok; %% filter out "application: foo; exited: stopped; type: temporary" handle_event(Event, State) -> error_logger_file_h:handle_event(Event, State). -- cgit v1.2.1 From 9142ce00fec31cb22cb0351a4b4a257cdffbdfc9 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 27 Feb 2013 14:43:44 +0000 Subject: Base the whole thing off net_adm:ping/1 - because we might see other nodes come back but also be waiting (in the no-majority case, and RAM nodes). Better to detect they exist and come back than to stay stuck because they don't happen to be running Mnesia. --- src/rabbit_mnesia.erl | 6 ------ src/rabbit_node_monitor.erl | 15 ++++++++++----- 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl index ecb03f54..c39e898c 100644 --- a/src/rabbit_mnesia.erl +++ b/src/rabbit_mnesia.erl @@ -26,7 +26,6 @@ status/0, is_clustered/0, - majority/0, cluster_nodes/1, node_type/0, dir/0, @@ -68,7 +67,6 @@ -spec(status/0 :: () -> [{'nodes', [{node_type(), [node()]}]} | {'running_nodes', [node()]} | {'partitions', [{node(), [node()]}]}]). --spec(majority/0 :: () -> boolean()). -spec(is_clustered/0 :: () -> boolean()). -spec(cluster_nodes/1 :: ('all' | 'disc' | 'ram' | 'running') -> [node()]). -spec(node_type/0 :: () -> node_type()). @@ -340,10 +338,6 @@ status() -> false -> [] end. -majority() -> - ensure_mnesia_running(), - (length(cluster_nodes(running)) / length(cluster_nodes(all))) > 0.5. - mnesia_partitions(Nodes) -> {Replies, _BadNodes} = rpc:multicall( Nodes, rabbit_node_monitor, partitions, []), diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index 42df6e5d..249c17a4 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -281,7 +281,7 @@ handle_dead_rabbit(Node) -> %% down - otherwise we have a race. handle_dead_according_to_mnesia_rabbit() -> case application:get_env(rabbit, cluster_cp_mode) of - {ok, true} -> case rabbit_mnesia:majority() of + {ok, true} -> case majority() of true -> ok; false -> await_cluster_recovery() end; @@ -289,6 +289,13 @@ handle_dead_according_to_mnesia_rabbit() -> end, ok. +majority() -> + Nodes = rabbit_mnesia:cluster_nodes(all), + Alive = [Status || N <- Nodes, + Status <- [net_adm:ping(N)], + Status =:= pong], + length(Alive) / length(Nodes) > 0.5. + await_cluster_recovery() -> rabbit_log:warning("Cluster minority status detected - awaiting recovery~n", []), @@ -303,11 +310,9 @@ await_cluster_recovery() -> wait_for_cluster_recovery(Nodes) -> [erlang:disconnect_node(Node) || Node <- Nodes], - mnesia:start(), - case rabbit_mnesia:majority() of + case majority() of true -> rabbit:start(); - false -> mnesia:stop(), - timer:sleep(1000), + false -> timer:sleep(1000), wait_for_cluster_recovery(Nodes) end. -- cgit v1.2.1 From cb97ff80e3a876bfce7e22d213be66fa780032e8 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 27 Feb 2013 14:47:03 +0000 Subject: Simplify --- src/rabbit_node_monitor.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index 249c17a4..fd8080bc 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -291,9 +291,7 @@ handle_dead_according_to_mnesia_rabbit() -> majority() -> Nodes = rabbit_mnesia:cluster_nodes(all), - Alive = [Status || N <- Nodes, - Status <- [net_adm:ping(N)], - Status =:= pong], + Alive = [N || N <- Nodes, pong =:= net_adm:ping(N)], length(Alive) / length(Nodes) > 0.5. await_cluster_recovery() -> -- cgit v1.2.1 From 2279502946e436f40368d66477edafe5c6616f60 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Wed, 27 Feb 2013 15:30:43 +0000 Subject: We no longer need two different death detectors since we no longer look at Mnesia for majorityness. --- src/rabbit_node_monitor.erl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'src') diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index fd8080bc..ad2003a5 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -250,10 +250,6 @@ handle_info({mnesia_system_event, ordsets:add_element(Node, ordsets:from_list(Partitions))), {noreply, State#state{partitions = Partitions1}}; -handle_info({mnesia_system_event, {mnesia_down, _Node}}, State) -> - handle_dead_according_to_mnesia_rabbit(), - {noreply, State}; - handle_info(_Info, State) -> {noreply, State}. @@ -274,12 +270,7 @@ handle_dead_rabbit(Node) -> ok = rabbit_networking:on_node_down(Node), ok = rabbit_amqqueue:on_node_down(Node), ok = rabbit_alarm:on_node_down(Node), - ok = rabbit_mnesia:on_node_down(Node). - -%% Since we will be introspecting the cluster in response to this, we -%% must only do so based on Mnesia having noticed the other node being -%% down - otherwise we have a race. -handle_dead_according_to_mnesia_rabbit() -> + ok = rabbit_mnesia:on_node_down(Node), case application:get_env(rabbit, cluster_cp_mode) of {ok, true} -> case majority() of true -> ok; -- cgit v1.2.1 From 5ae47d98f0fe2f67503646316d7a0df186be80b6 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Fri, 1 Mar 2013 14:47:47 +0000 Subject: Rename this thing, to make space for bug 25471 --- src/rabbit_node_monitor.erl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index ad2003a5..5d587977 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -271,12 +271,18 @@ handle_dead_rabbit(Node) -> ok = rabbit_amqqueue:on_node_down(Node), ok = rabbit_alarm:on_node_down(Node), ok = rabbit_mnesia:on_node_down(Node), - case application:get_env(rabbit, cluster_cp_mode) of - {ok, true} -> case majority() of - true -> ok; - false -> await_cluster_recovery() - end; - {ok, false} -> ok + case application:get_env(rabbit, cluster_partition_handling) of + {ok, pause_minority} -> + case majority() of + true -> ok; + false -> await_cluster_recovery() + end; + {ok, ignore} -> + ok; + {ok, Term} -> + rabbit_log:warning("cluster_partition_handling ~p unrecognised, " + "assuming 'ignore'~n", [Term]), + ok end, ok. -- cgit v1.2.1