diff options
author | Rickard Green <rickard@erlang.org> | 2022-02-06 23:50:35 +0100 |
---|---|---|
committer | Rickard Green <rickard@erlang.org> | 2022-02-06 23:50:35 +0100 |
commit | 20c9e3428bb7f902447b755289e09eecbeac73dc (patch) | |
tree | 5234a3a31c8e59e8a99f94fdb5f0d19546b35b82 | |
parent | df48c260e74c3e9058ff8681ce9f554e6fa0fe34 (diff) | |
parent | 78c9e20ed86831c6d9618735c295475a7b9fedc4 (diff) | |
download | erlang-20c9e3428bb7f902447b755289e09eecbeac73dc.tar.gz |
Merge branch 'rickard/prevent-overlapping-partitions/23.3.4/ERIERL-732/OTP-17843' into rickard/prevent-overlapping-partitions/24.2/ERIERL-732/OTP-17843
* rickard/prevent-overlapping-partitions/23.3.4/ERIERL-732/OTP-17843:
global: Preventing overlapping partitions fix
global: Propagate and save version between all nodes
-rw-r--r-- | bootstrap/lib/kernel/ebin/global.beam | bin | 28340 -> 32700 bytes | |||
-rw-r--r-- | bootstrap/lib/kernel/ebin/global_group.beam | bin | 15772 -> 15808 bytes | |||
-rw-r--r-- | erts/emulator/test/distribution_SUITE.erl | 83 | ||||
-rw-r--r-- | erts/preloaded/ebin/init.beam | bin | 60364 -> 61088 bytes | |||
-rw-r--r-- | erts/preloaded/src/erts.app.src | 2 | ||||
-rw-r--r-- | erts/preloaded/src/init.erl | 20 | ||||
-rw-r--r-- | lib/kernel/doc/src/global.xml | 33 | ||||
-rw-r--r-- | lib/kernel/doc/src/kernel_app.xml | 13 | ||||
-rw-r--r-- | lib/kernel/src/global.erl | 528 | ||||
-rw-r--r-- | lib/kernel/src/global_group.erl | 26 | ||||
-rw-r--r-- | lib/kernel/src/kernel.app.src | 5 | ||||
-rw-r--r-- | lib/kernel/test/erl_distribution_SUITE.erl | 19 | ||||
-rw-r--r-- | lib/kernel/test/erpc_SUITE.erl | 22 | ||||
-rw-r--r-- | lib/kernel/test/global_SUITE.erl | 998 | ||||
-rw-r--r-- | lib/kernel/test/pg_SUITE.erl | 2 | ||||
-rw-r--r-- | lib/kernel/test/rpc_SUITE.erl | 68 |
16 files changed, 1228 insertions, 591 deletions
diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex 56ba635c60..845fb5a468 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/global_group.beam b/bootstrap/lib/kernel/ebin/global_group.beam Binary files differindex 5acce8de7c..6d000d094b 100644 --- a/bootstrap/lib/kernel/ebin/global_group.beam +++ b/bootstrap/lib/kernel/ebin/global_group.beam diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 8f3c1f3903..82c0e846a2 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1082,21 +1082,24 @@ dist_auto_connect_relay(Parent) -> dist_parallel_send(Config) when is_list(Config) -> - {ok, RNode} = start_node(dist_parallel_receiver), - {ok, SNode} = start_node(dist_parallel_sender), - WatchDog = spawn_link( - fun () -> - TRef = erlang:start_timer((2*60*1000), self(), oops), - receive - {timeout, TRef, _ } -> - spawn(SNode, fun () -> abort(timeout) end), - spawn(RNode, fun () -> abort(timeout) end) - %% rpc:cast(SNode, erlang, halt, - %% ["Timetrap (sender)"]), - %% rpc:cast(RNode, erlang, halt, - %% ["Timetrap (receiver)"]) - end - end), + %% Disabled "connect all" so global wont interfere... + {ok, RNode} = start_node(dist_parallel_receiver, "-connect_all false"), + {ok, SNode} = start_node(dist_parallel_sender, "-connect_all false"), + + %% WatchDog = spawn_link( + %% fun () -> + %% TRef = erlang:start_timer((2*60*1000), self(), oops), + %% receive + %% {timeout, TRef, _ } -> + %% spawn(SNode, fun () -> abort(timeout) end), + %% spawn(RNode, fun () -> abort(timeout) end) + %% %% rpc:cast(SNode, erlang, halt, + %% %% ["Timetrap (sender)"]), + %% %% rpc:cast(RNode, erlang, halt, + %% %% ["Timetrap (receiver)"]) + %% end + %% end), + MkSndrs = fun (Receiver) -> lists:map(fun (_) -> spawn_link(SNode, @@ -1105,18 +1108,23 @@ dist_parallel_send(Config) when is_list(Config) -> [self(), Receiver, 1000]) end, lists:seq(1, 64)) end, + Parent = self(), SndrsStart = fun (Sndrs) -> - Parent = self(), spawn_link(SNode, fun () -> lists:foreach(fun (P) -> P ! {go, Parent} - end, Sndrs) + end, Sndrs), + unlink(Parent) end) end, SndrsWait = fun (Sndrs) -> lists:foreach(fun (P) -> - receive {P, done} -> ok end + receive + {P, done} -> + unlink(P), + ok + end end, Sndrs) end, DPR = spawn_link(RNode, ?MODULE, dist_parallel_receiver, []), @@ -1133,8 +1141,8 @@ dist_parallel_send(Config) when is_list(Config) -> unlink(DEPR), exit(DEPR, bang), - unlink(WatchDog), - exit(WatchDog, bang), + %% unlink(WatchDog), + %% exit(WatchDog, bang), stop_node(RNode), stop_node(SNode), @@ -1785,8 +1793,9 @@ start_link(Offender,P) -> bad_dist_structure(Config) when is_list(Config) -> ct:timetrap({seconds, 15}), - {ok, Offender} = start_node(bad_dist_structure_offender), - {ok, Victim} = start_node(bad_dist_structure_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_structure_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_structure_victim, "-connect_all false"), start_node_monitors([Offender,Victim]), Parent = self(), P = spawn(Victim, @@ -1880,8 +1889,9 @@ bad_dist_structure(Config) when is_list(Config) -> bad_dist_fragments(Config) when is_list(Config) -> ct:timetrap({seconds, 15}), - {ok, Offender} = start_node(bad_dist_fragment_offender), - {ok, Victim} = start_node(bad_dist_fragment_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_fragment_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_fragment_victim, "-connect_all false"), Msg = iolist_to_binary(dmsg_ext(lists:duplicate(255,255))), @@ -2035,8 +2045,9 @@ send_bad_fragments(Offender,VictimNode,Victim,Ctrl,WhereToPutSelf,Fragments) -> end. bad_dist_ext_receive(Config) when is_list(Config) -> - {ok, Offender} = start_node(bad_dist_ext_receive_offender), - {ok, Victim} = start_node(bad_dist_ext_receive_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_ext_receive_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_ext_receive_victim, "-connect_all false"), start_node_monitors([Offender,Victim]), Parent = self(), @@ -2107,8 +2118,9 @@ bad_dist_ext_receive(Config) when is_list(Config) -> bad_dist_ext_process_info(Config) when is_list(Config) -> - {ok, Offender} = start_node(bad_dist_ext_process_info_offender), - {ok, Victim} = start_node(bad_dist_ext_process_info_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_ext_process_info_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_ext_process_info_victim, "-connect_all false"), start_node_monitors([Offender,Victim]), Parent = self(), @@ -2167,8 +2179,9 @@ bad_dist_ext_process_info(Config) when is_list(Config) -> stop_node(Victim). bad_dist_ext_control(Config) when is_list(Config) -> - {ok, Offender} = start_node(bad_dist_ext_control_offender), - {ok, Victim} = start_node(bad_dist_ext_control_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_ext_control_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_ext_control_victim, "-connect_all false"), start_node_monitors([Offender,Victim]), pong = rpc:call(Victim, net_adm, ping, [Offender]), @@ -2186,8 +2199,9 @@ bad_dist_ext_control(Config) when is_list(Config) -> stop_node(Victim). bad_dist_ext_connection_id(Config) when is_list(Config) -> - {ok, Offender} = start_node(bad_dist_ext_connection_id_offender), - {ok, Victim} = start_node(bad_dist_ext_connection_id_victim), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_ext_connection_id_offender, "-connect_all false"), + {ok, Victim} = start_node(bad_dist_ext_connection_id_victim, "-connect_all false"), start_node_monitors([Offender,Victim]), Parent = self(), @@ -2253,10 +2267,11 @@ bad_dist_ext_connection_id(Config) when is_list(Config) -> %% OTP-14661: Bad message is discovered by erts_msg_attached_data_size bad_dist_ext_size(Config) when is_list(Config) -> - {ok, Offender} = start_node(bad_dist_ext_process_info_offender), + %% Disabled "connect all" so global wont interfere... + {ok, Offender} = start_node(bad_dist_ext_process_info_offender, "-connect_all false"), %%Prog = "Prog=/home/uabseri/src/otp_new3/bin/cerl -rr -debug", Prog = [], - {ok, Victim} = start_node(bad_dist_ext_process_info_victim, [], Prog), + {ok, Victim} = start_node(bad_dist_ext_process_info_victim, "-connect_all false", Prog), start_node_monitors([Offender,Victim]), Parent = self(), diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex a2ab66db81..6948e1e483 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 73abe85b39..7c05c91b2d 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -42,7 +42,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.13", "kernel-8.0", "sasl-3.3"]} + {runtime_dependencies, ["stdlib-3.13", "kernel-@OTP-17843@", "sasl-3.3"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 5b59a9d65f..44a2661bc2 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -664,11 +664,31 @@ stop_heart(State) -> shutdown_pids(Heart,Logger,BootPid,State) -> Timer = shutdown_timer(State#state.flags), + global_prepare_shutdown(), catch shutdown(State#state.kernel,BootPid,Timer,State), kill_all_pids(Heart,Logger), % Even the shutdown timer. kill_all_ports(Heart), % Logger has no ports flush_timout(Timer). +global_prepare_shutdown() -> + %% Inform global that we are shutting down, so it wont + %% send 'lost_connection' messages when connections + %% goes down... + case whereis(global_name_server) of + undefined -> + ok; + Pid -> + Mon = erlang:monitor(process, Pid), + Pid ! {prepare_shutdown, self(), Mon}, + receive + {Mon, ok} -> + erlang:demonitor(Mon, [flush]), + ok; + {'DOWN', Mon, process, Pid, _Reason} -> + ok + end + end. + get_heart(Kernel) -> get_kernelpid(heart,Kernel). diff --git a/lib/kernel/doc/src/global.xml b/lib/kernel/doc/src/global.xml index 541d4b14da..041b08105c 100644 --- a/lib/kernel/doc/src/global.xml +++ b/lib/kernel/doc/src/global.xml @@ -37,6 +37,39 @@ <item>Global locks</item> <item>Maintenance of the fully connected network</item> </list> + <marker id="prevent_overlapping_partitions"/> + <warning> + <p> + By default <c>global</c> does <i>not</i> take any actions to restore + a fully connected network when connections are lost due to network + issues. This is problematic for all applications expecting a fully + connected network to be provided, such as for example <c>mnesia</c>, + but also for <c>global</c> itself. A network of overlapping partitions + might cause the internal state of <c>global</c> to become inconsistent. + Such an inconsistency can remain even after such partitions have been + brought together to form a fully connected network again. The effect + on other applications that expects that a fully connected network is + maintained may vary, but they might misbehave in very subtle hard to + detect ways during such a partitioning. + </p> + <p> + In order to prevent such issues, we have introduced a <i>prevent + overlapping partitions</i> fix which can be enabled using the + <seeapp marker="kernel_app#prevent_overlapping_partitions"> + <c>prevent_overlapping_partitions</c></seeapp> <c>kernel(6)</c> + parameter. When this fix has been enabled, <c>global</c> will actively + disconnect from nodes that reports that they have lost connections to + other nodes. This will cause fully connected partitions to form + instead of leaving the network in a state with overlapping partitions. + Note that this fix <i>has</i> to be enabled on <i>all</i> nodes in the + network in order to work properly. Since this quite substantially + changes the behavior, this fix is currently disabled by default. Since + you might get hard to detect issues without this fix you are, however, + <i>strongly</i> advised to enable this fix in order to avoid issues + such as the ones described above. As of OTP 25 this fix will become + enabled by default. + </p> + </warning> <p>These services are controlled through the process <c>global_name_server</c> that exists on every node. The global name server starts automatically when a node is started. diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index c17b327f32..3e76a0f4c8 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -348,6 +348,19 @@ MaxT = TickTime + TickTime / 4</code> <p>Normally, a terminating node is detected immediately by the transport protocol (like TCP/IP).</p> </item> + <tag><marker id="prevent_overlapping_partitions"/> + <c>prevent_overlapping_partitions = true | false</c></tag> + <item> + <p> + If enabled (<c>true</c>), <c>global</c> will actively prevent + overlapping partitions from forming when connections are lost + between nodes. This fix is, however, currently disabled by + default. See the + <seeerl marker="kernel:global#prevent_overlapping_partitions"> + <c>global(3)</c></seeerl> documentation for more + information. + </p> + </item> <tag><c>shutdown_timeout = integer() | infinity</c></tag> <item> <p>Specifies the time <c>application_controller</c> waits diff --git a/lib/kernel/src/global.erl b/lib/kernel/src/global.erl index 03f6c0aa94..32ad20bc61 100644 --- a/lib/kernel/src/global.erl +++ b/lib/kernel/src/global.erl @@ -32,7 +32,7 @@ whereis_name/1, register_name/2, register_name/3, register_name_external/2, register_name_external/3, unregister_name_external/1,re_register_name/2, re_register_name/3, - unregister_name/1, registered_names/0, send/2, node_disconnected/1, + unregister_name/1, registered_names/0, send/2, set_lock/1, set_lock/2, set_lock/3, del_lock/1, del_lock/2, trans/2, trans/3, trans/4, @@ -54,6 +54,10 @@ -define(N_CONNECT_RETRIES, global_connect_retries). -define(DEFAULT_N_CONNECT_RETRIES, 0). +%% Time that we keep information about multicasted lost_connection +%% messages... +-define(lost_conn_info_cleanup_time, 60*60*1000). + %%% In certain places in the server, calling io:format hangs everything, %%% so we'd better use erlang:display/1. %%% my_tracer is used in testsuites @@ -70,6 +74,9 @@ -define(trace(_), ok). -endif. +-define(MAX_64BIT_SMALL_INT, ((1 bsl 59) - 1)). +-define(MIN_64BIT_SMALL_INT, (-(1 bsl 59))). + %% These are the protocol versions: %% Vsn 1 is the original protocol. %% Vsn 2 is enhanced with code to take care of registration of names from @@ -82,15 +89,26 @@ %% nodes in the own partition. (R11B-) %% Vsn 6 does not send any message between locker processes on different %% nodes, but uses the server as a proxy. +%% Vsn 7 - propagate global versions between nodes, so we always know +%% versions of known nodes +%% - optional "prevent overlapping partitions" fix supported %% Current version of global does not support vsn 4 or earlier. --define(vsn, 6). +-define(vsn, 7). + +%% Version when the "prevent overlapping partitions" fix was introduced. +-define(pop_vsn, 7). +%% Version when the "propagate global protocol versions" feature +%% was introduced. +-define(pgpv_vsn, 7). %%----------------------------------------------------------------- %% connect_all = boolean() - true if we are supposed to set up a %% fully connected net -%% known = [Node] - all nodes known to us +%% known = #{} - Map of known nodes including protocol version +%% as well as some other information. See state +%% record declaration below for more info. %% synced = [Node] - all nodes that have the same names as us %% resolvers = [{Node, MyTag, Resolver}] - %% the tag separating different synch sessions, @@ -112,8 +130,21 @@ %% {sync_tag_his, Node} = The Node's tag, used at synchronization %% {lock_id, Node} = The resource locking the partitions %%----------------------------------------------------------------- --record(state, {connect_all :: boolean(), - known = [] :: [node()], +-record(conf, {connect_all :: boolean(), + prevent_over_part :: boolean() + }). + +-record(state, {conf = #conf{} :: #conf{}, + known = #{} :: #{ + %% Known connected node with protocol + %% version as value + node() => non_neg_integer(), + %% Connecting node, not yet known, with + %% protocol version as value + {pending, node()} => non_neg_integer(), + %% Node currently being removed + {removing, node()} => yes + }, synced = [] :: [node()], resolvers = [], syncers = [] :: [pid()], @@ -142,6 +173,12 @@ %%% Resources locked by Pid. %%% ref() is the same ref() as in global_locks. %%% +%%% global_lost_connections (set): +%%% {{NodeA, NodeB}, {ExtendedCreationA, OpIdA, Timer} +%%% Information about lost connections (reported by NodeA) used by +%%% the "prevent overlapping partitions" fix. The timer is used is +%%% used to remove the entry when not needed anymore. +%%% %%% global_pid_names is a 'bag' for backward compatibility. %%% (Before vsn 5 more than one name could be registered for a process.) %%% @@ -207,9 +244,6 @@ send(Name, Msg) -> whereis_name(Name) -> where(Name). -node_disconnected(Node) -> - global_name_server ! {nodedown, Node}. - %%----------------------------------------------------------------- %% Method = function(Name, Pid1, Pid2) -> Pid | Pid2 | none %% Method is called if a name conflict is detected when two nodes @@ -452,6 +486,7 @@ init([]) -> _ = ets:new(global_pid_names, [bag, named_table, protected]), _ = ets:new(global_pid_ids, [bag, named_table, protected]), + _ = ets:new(global_lost_connections, [set, named_table, protected]), %% This is for troubleshooting only. DoTrace = os:getenv("GLOBAL_HIGH_LEVEL_TRACE") =:= "TRUE", @@ -469,10 +504,28 @@ init([]) -> _ -> true end, + POP = case application:get_env(kernel, + prevent_overlapping_partitions) of + {ok, Bool} when Bool == true; Bool == false -> + Bool; + {ok, Invalid} -> + error({invalid_parameter_value, + prevent_overlapping_partitions, + Invalid}); + undefined -> + false + end, S = #state{the_locker = start_the_locker(DoTrace), trace = T0, the_registrar = start_the_registrar(), - connect_all = Ca}, + conf = #conf{connect_all = Ca, + prevent_over_part = POP}}, + _ = rand:seed(default, + (erlang:monotonic_time(nanosecond) rem 1000000000) + + (erlang:system_time(nanosecond) rem 1000000000)), + CreX = ((rand:uniform(?MAX_64BIT_SMALL_INT - ?MIN_64BIT_SMALL_INT) + - 1) band (bnot ((1 bsl 32) -1))), + put(creation_extension, CreX), {ok, trace_message(S, {init, node()}, [])}. %%----------------------------------------------------------------- @@ -593,6 +646,51 @@ init([]) -> %% sent by each node to all new nodes (Node becomes known to them) %%----------------------------------------------------------------- +%% ---------------------------------------------------------------- +%% Prevent Overlapping Partitions Algorithm +%% ======================================== +%% +%% 1. When a node lose connection to another node it sends a +%% {lost_connection, LostConnNode, OtherNode} message to all +%% other nodes that it knows of. +%% 2. When a lost_connection message is received the receiver +%% first checks if it has seen this message before. If so, it +%% just ignores it. If it has not seen it before, it sends the +%% message to all nodes it knows of. This in order to ensure +%% that all connected nodes will receive this message. It then +%% sends a {remove_connection, LostConnRecvNode} message (where +%% LostConnRecvNode is its own node name) to OtherNode and +%% clear all information about OtherNode so OtherNode wont be +%% part of ReceiverNode's cluster anymore. When this information +%% has been cleared, no lost_connection will be triggered when +%% a nodedown message for the connection to OtherNode is +%% received. +%% 3. When a {remove_connection, LostConnRecvNode} message is +%% received, the receiver node takes down the connection to +%% LostConnRecvNode and clears its information about +%% LostConnRecvNode so it is not part of its cluster anymore. +%% Both nodes will receive a nodedown message due to the +%% connection being closed, but none of them will send +%% lost_connection messages since they have cleared information +%% about the other node. +%% +%% This will take down more connections than the minimum amount +%% of connections to remove in order to form fully connected +%% partitions. For example, if the user takes down a connection +%% between two nodes, the rest of the nodes will disconnect from +%% both of these nodes instead of just one. This is due to: +%% * We do not want to partition a remaining network when a node +%% has halted. When you receive a nodedown and/or lost_connection +%% messages you don't know if the corresponding node has halted +%% or if there are network issues. +%% * We need to decide which connection to take down as soon as +%% we receive a lost_connection message in order to prevent +%% inconsistencies entering global's state. +%% * All nodes need to make the same choices independent of +%% each other. +%% +%% ---------------------------------------------------------------- + -spec handle_call(term(), {pid(), term()}, state()) -> {'noreply', state()} | {'reply', term(), state()} | @@ -625,7 +723,7 @@ handle_call({del_lock, Lock}, {Pid, _Tag}, S0) -> {reply, true, S}; handle_call(get_known, _From, S) -> - {reply, S#state.known, S}; + {reply, mk_known_list(0, S), S}; handle_call(get_synced, _From, S) -> {reply, S#state.synced, S}; @@ -676,27 +774,28 @@ handle_call(Request, From, S) -> -spec handle_cast(term(), state()) -> {'noreply', state()}. -handle_cast({init_connect, Vsn, Node, InitMsg}, S) -> +handle_cast({init_connect, Vsn, Node, InitMsg}, S0) -> %% Sent from global_name_server at Node. ?trace({'####', init_connect, {vsn, Vsn}, {node,Node},{initmsg,InitMsg}}), - case Vsn of - %% It is always the responsibility of newer versions to understand - %% older versions of the protocol. - {HisVsn, HisTag} when HisVsn > ?vsn -> - init_connect(?vsn, Node, InitMsg, HisTag, S#state.resolvers, S); - {HisVsn, HisTag} -> - init_connect(HisVsn, Node, InitMsg, HisTag, S#state.resolvers, S); - %% To be future compatible - Tuple when is_tuple(Tuple) -> - List = tuple_to_list(Tuple), - [_HisVsn, HisTag | _] = List, - %% use own version handling if his is newer. - init_connect(?vsn, Node, InitMsg, HisTag, S#state.resolvers, S); - _ -> - Txt = io_lib:format("Illegal global protocol version ~p Node: ~p\n", - [Vsn, Node]), - error_logger:info_report(lists:flatten(Txt)) - end, + S = case Vsn of + %% It is always the responsibility of newer versions to understand + %% older versions of the protocol. + {HisVsn, HisTag} when HisVsn > ?vsn -> + init_connect(?vsn, Node, InitMsg, HisTag, HisVsn, S0); + {HisVsn, HisTag} -> + init_connect(HisVsn, Node, InitMsg, HisTag, HisVsn, S0); + %% To be future compatible + Tuple when is_tuple(Tuple) -> + List = tuple_to_list(Tuple), + [HisVsn, HisTag | _] = List, + %% use own version handling if his is newer. + init_connect(?vsn, Node, InitMsg, HisTag, HisVsn, S0); + _ -> + Txt = io_lib:format("Illegal global protocol version ~p Node: ~p\n", + [Vsn, Node]), + error_logger:info_report(lists:flatten(Txt)), + S0 + end, {noreply, S}; %%======================================================================= @@ -745,10 +844,10 @@ handle_cast({exchange_ops, Node, MyTag, Ops, Resolved}, S0) -> S = trace_message(S0, {exit_resolver, Node}, [MyTag]), case get({sync_tag_my, Node}) of MyTag -> - Known = S#state.known, + Known = mk_known_list(node_vsn(Node, S), S), gen_server:cast({global_name_server, Node}, {resolved, node(), Resolved, Known, - Known,get_names_ext(),get({sync_tag_his,Node})}), + unused,get_names_ext(),get({sync_tag_his,Node})}), case get({save_ops, Node}) of {resolved, HisKnown, Names_ext, HisResolved} -> put({save_ops, Node}, Ops), @@ -768,7 +867,7 @@ handle_cast({exchange_ops, Node, MyTag, Ops, Resolved}, S0) -> %% %% Here the name clashes are resolved. %%======================================================================== -handle_cast({resolved, Node, HisResolved, HisKnown, _HisKnown_v2, +handle_cast({resolved, Node, HisResolved, HisKnown, _Unused, Names_ext, MyTag}, S) -> %% Sent from global_name_server at Node. ?trace({'####', resolved, {his_resolved,HisResolved}, {node,Node}}), @@ -805,7 +904,7 @@ handle_cast({new_nodes, Node, Ops, Names_ext, Nodes, ExtraInfo}, S) -> %% %% We are in sync with this node (from the other node's known world). %%======================================================================== -handle_cast({in_sync, Node, _IsKnown}, S) -> +handle_cast({in_sync, Node, _IsKnown}, #state{known = Known} = S) -> %% Sent from global_name_server at Node (in the other partition). ?trace({'####', in_sync, {Node, _IsKnown}}), lists:foreach(fun(Pid) -> Pid ! {synced, [Node]} end, S#state.syncers), @@ -815,7 +914,8 @@ handle_cast({in_sync, Node, _IsKnown}, S) -> true -> Synced; false -> [Node | Synced] end, - {noreply, NewS#state{synced = NSynced}}; + {noreply, NewS#state{known = maps:remove({pending, Node}, Known), + synced = NSynced}}; %% Called when Pid on other node crashed handle_cast({async_del_name, _Name, _Pid}, S) -> @@ -868,13 +968,20 @@ handle_info({nodedown, Node}, S) when Node =:= S#state.node_name -> handle_info({nodedown, Node}, S0) -> ?trace({'####', nodedown, {node,Node}}), S1 = trace_message(S0, {nodedown, Node}, []), - S = handle_nodedown(Node, S1), + S = handle_nodedown(Node, S1, disconnected), {noreply, S}; handle_info({extra_nodedown, Node}, S0) -> ?trace({'####', extra_nodedown, {node,Node}}), S1 = trace_message(S0, {extra_nodedown, Node}, []), - S = handle_nodedown(Node, S1), + S = handle_nodedown(Node, S1, disconnected), + {noreply, S}; + +handle_info({ignore_node, Node}, S0) -> + %% global_group wants us to ignore this node... + ?trace({'####', ignore_node, {node,Node}}), + S1 = trace_message(S0, {ignore_node, Node}, []), + S = handle_nodedown(Node, S1, ignore_node), {noreply, S}; handle_info({nodeup, Node}, S) when Node =:= node() -> @@ -883,11 +990,12 @@ handle_info({nodeup, Node}, S) when Node =:= node() -> %% references to old node name ('nonode@nohost') to Node. {noreply, change_our_node_name(Node, S)}; -handle_info({nodeup, _Node}, S) when not S#state.connect_all -> +handle_info({nodeup, _Node}, + #state{conf = #conf{connect_all = false}} = S) -> {noreply, S}; -handle_info({nodeup, Node}, S0) when S0#state.connect_all -> - IsKnown = lists:member(Node, S0#state.known) or +handle_info({nodeup, Node}, S0) -> + IsKnown = maps:is_key(Node, S0#state.known) or %% This one is only for double nodeups (shouldn't occur!) lists:keymember(Node, 1, S0#state.resolvers), ?trace({'####', nodeup, {node,Node}, {isknown,IsKnown}}), @@ -928,6 +1036,110 @@ handle_info({whereis, Name, From}, S) -> _ = do_whereis(Name, From), {noreply, S}; +handle_info({lost_connection, NodeA, XCreationA, OpIdA, NodeB} = Msg, + #state{conf = #conf{connect_all = true, + prevent_over_part = true}} = S0) -> + %% Message introduced in protocol version ?pop_vsn + %% + %% NodeA reports that it lost connection to NodeB. If we got a + %% connection to NodeB, we need to disconnect it in order to + %% prevent overlapping partitions... + LcKey = {NodeA, NodeB}, + S1 = case get_lost_connection_info(LcKey) of + {XCreationA, OpId, _Tmr} when OpIdA =< OpId -> + %% We have already handled this lost connection + %% message... + S0; + {_, _, Tmr} -> + %% Inform all other nodes we know of about this as well. This + %% in order to prevent that some nodes in this cluster wont + %% get the original message from NodeA. This ensures that all + %% nodes will know of this connection loss, even if some nodes + %% lose connection to NodeA while NodeA is multicasting that + %% it lost the connection to NodeB. + gns_volatile_multicast(Msg, NodeA, ?pop_vsn, true, S0), + + %% Save info about this lost_connection message... + save_lost_connection_info(LcKey, XCreationA, OpIdA, Tmr), + + RmNode = case node() == NodeB of + false -> + NodeB; + true -> + %% This toghether with NodeA being known by + %% us probably is unusal, but can happen + %% since lost_connection messages are + %% reapeted by receiving nodes. All other + %% nodes will remove us, so there is no + %% need to make them remove NodeA as well. + %% We therefore request removal from NodeA + %% and wait for the nodedown which + %% eventually will come since NodeA reported + %% this... + NodeA + end, + + case is_node_potentially_known(RmNode, S0) of + false -> + S0; + true -> + case node_vsn(RmNode, S0) of + Vsn when Vsn < ?pop_vsn -> + erlang:disconnect_node(RmNode), + error_logger:warning_msg( + "'global' at node ~p disconnected old " + "node ~p in order to prevent overlapping " + "partitions", + [node(), RmNode]), + ok; + _Vsn -> + gns_volatile_send(RmNode, + {remove_connection, node()}), + error_logger:warning_msg( + "'global' at node ~p requested disconnect " + "from node ~p in order to prevent " + "overlapping partitions", + [node(), RmNode]), + ok + end, + handle_nodedown(RmNode, S0, remove_connection) + end + end, + + {noreply, S1}; + +handle_info({lost_connection, _NodeA, _XCreationA, _OpIdA, _NodeB}, S) -> + %% Message introduced in protocol version ?pop_vsn + {noreply, S}; + +handle_info({timeout, _, _} = TmoMsg, S) -> + %% Instance specific message + remove_lost_connection_info(TmoMsg), + {noreply, S}; + +handle_info({remove_connection, Node}, S0) -> + %% Message introduced in protocol version ?pop_vsn + S2 = case is_node_potentially_known(Node, S0) of + false -> + S0; + true -> + erlang:disconnect_node(Node), + S1 = handle_nodedown(Node, S0, remove_connection), + error_logger:warning_msg( + "'global' at node ~p disconnected node ~p in order to " + "prevent overlapping partitions", [node(), Node]), + S1 + end, + {noreply, S2}; + +handle_info({prepare_shutdown, From, Ref}, S0) -> + %% Prevent lost_connection messages being sent due to + %% connections being taken down during the shutdown... + S1 = S0#state{conf = #conf{connect_all = false, + prevent_over_part = false}}, + From ! {Ref, ok}, + {noreply, S1}; + handle_info(known, S) -> io:format(">>>> ~p\n",[S#state.known]), {noreply, S}; @@ -1067,7 +1279,8 @@ check_replies([], _Id, _Replies) -> %% Another node wants to synchronize its registered names with us. %% Both nodes must have a lock before they are allowed to continue. %%======================================================================== -init_connect(Vsn, Node, InitMsg, HisTag, Resolvers, S) -> +init_connect(Vsn, Node, InitMsg, HisTag, HisVsn, + #state{resolvers = Resolvers, known = Known} = S) -> %% It is always the responsibility of newer versions to understand %% older versions of the protocol. put({prot_vsn, Node}, Vsn), @@ -1083,7 +1296,9 @@ init_connect(Vsn, Node, InitMsg, HisTag, Resolvers, S) -> false -> ?trace({init_connect,{pre_connect,Node},{histag,HisTag}}), put({pre_connect, Node}, {Vsn, InitMsg, HisTag}) - end. + end, + S#state{known = maps:put({pending, Node}, HisVsn, Known)}. + %%======================================================================== %% In the simple case, we'll get lock_is_set before we get exchange, @@ -1129,8 +1344,9 @@ resolved(Node, HisResolved, HisKnown, Names_ext, S0) -> %% Known may have shrunk since the lock was taken (due to nodedowns). Known = S0#state.known, Synced = S0#state.synced, - NewNodes = [Node | HisKnown], - sync_others(HisKnown), + NewNodes = make_node_vsn_list([Node | HisKnown], S0), + HisKnownNodes = node_list(HisKnown), + sync_others(HisKnownNodes), ExtraInfo = [{vsn,get({prot_vsn, Node})}, {lock, get({lock_id, Node})}], S = do_ops(Ops, node(), Names_ext, ExtraInfo, S0), %% I am synced with Node, but not with HisKnown yet @@ -1138,47 +1354,54 @@ resolved(Node, HisResolved, HisKnown, Names_ext, S0) -> S3 = lists:foldl(fun(Node1, S1) -> F = fun(Tag) -> cancel_locker(Node1,S1,Tag) end, cancel_resolved_locker(Node1, F) - end, S, HisKnown), + end, S, HisKnownNodes), %% The locker that took the lock is asked to send %% the {new_nodes, ...} message. This ensures that %% {del_lock, ...} is received after {new_nodes, ...} - %% (except when abcast spawns process(es)...). + NewNodesMsg = {new_nodes, node(), Ops, Names_ext, NewNodes, ExtraInfo}, NewNodesF = fun() -> - gen_server:abcast(Known, global_name_server, - {new_nodes, node(), Ops, Names_ext, - NewNodes, ExtraInfo}) + maps:foreach( + fun (N, V) when is_atom(N), V >= ?pgpv_vsn -> + gen_server:cast({global_name_server, N}, + NewNodesMsg); + (N, _OldV) when is_atom(N) -> + gen_server:cast({global_name_server, N}, + {new_nodes, node(), Ops, + Names_ext, node_list(NewNodes), + ExtraInfo}); + (_, _) -> + ok + end, + Known) end, F = fun(Tag) -> cancel_locker(Node, S3, Tag, NewNodesF) end, S4 = cancel_resolved_locker(Node, F), %% See (*) below... we're node b in that description - AddedNodes = (NewNodes -- Known), - NewKnown = Known ++ AddedNodes, - S4#state.the_locker ! {add_to_known, AddedNodes}, - NewS = trace_message(S4, {added, AddedNodes}, - [{new_nodes, NewNodes}, {abcast, Known}, {ops,Ops}]), - NewS#state{known = NewKnown, synced = [Node | Synced]}. + {AddedNodes, S5} = add_to_known(NewNodes, S4), + S5#state.the_locker ! {add_to_known, AddedNodes}, + S6 = trace_message(S5, {added, AddedNodes}, + [{new_nodes, NewNodes}, {abcast, Known}, {ops,Ops}]), + S6#state{synced = [Node | Synced]}. cancel_resolved_locker(Node, CancelFun) -> Tag = get({sync_tag_my, Node}), ?trace({calling_cancel_locker,Tag,get()}), S = CancelFun(Tag), reset_node_state(Node), - S. + S#state{known = maps:remove({pending, Node}, S#state.known)}. new_nodes(Ops, ConnNode, Names_ext, Nodes, ExtraInfo, S0) -> - Known = S0#state.known, %% (*) This one requires some thought... %% We're node a, other nodes b and c: %% The problem is that {in_sync, a} may arrive before {resolved, [a]} to %% b from c, leading to b sending {new_nodes, [a]} to us (node a). %% Therefore, we make sure we never get duplicates in Known. - AddedNodes = lists:delete(node(), Nodes -- Known), + {AddedNodes, S1} = add_to_known(Nodes, S0), sync_others(AddedNodes), - S = do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S0), + S2 = do_ops(Ops, ConnNode, Names_ext, ExtraInfo, S1), ?trace({added_nodes_in_sync,{added_nodes,AddedNodes}}), - S#state.the_locker ! {add_to_known, AddedNodes}, - S1 = trace_message(S, {added, AddedNodes}, [{ops,Ops}]), - S1#state{known = Known ++ AddedNodes}. + S2#state.the_locker ! {add_to_known, AddedNodes}, + trace_message(S2, {added, AddedNodes}, [{ops,Ops}]). do_whereis(Name, From) -> case is_global_lock_set() of @@ -2070,15 +2293,192 @@ pid_locks(Ref) -> ref_is_locking(Ref, PidRefs) -> lists:keyfind(Ref, 2, PidRefs) =/= false. -handle_nodedown(Node, S) -> +handle_nodedown(Node, #state{synced = Syncs, + known = Known0} = S, What) -> %% DOWN signals from monitors have removed locks and registered names. - #state{known = Known, synced = Syncs} = S, NewS = cancel_locker(Node, S, get({sync_tag_my, Node})), NewS#state.the_locker ! {remove_from_known, Node}, reset_node_state(Node), - NewS#state{known = lists:delete(Node, Known), + Known1 = case What of + remove_connection -> + maps:put({removing, Node}, yes, Known0); + ignore_node -> + maps:remove({removing, Node}, Known0); + disconnected -> + case maps:get({removing, Node}, Known0, no) of + yes -> + maps:remove({removing, Node}, Known0); + no -> + inform_connection_loss(Node, S), + Known0 + end + end, + NewS#state{known = maps:remove(Node, + maps:remove({pending, Node}, Known1)), synced = lists:delete(Node, Syncs)}. +inform_connection_loss(Node, + #state{conf = #conf{connect_all = true, + prevent_over_part = true}} = S) -> + Msg = {lost_connection, + node(), + (get(creation_extension) + + erlang:system_info(creation) + - ?MIN_64BIT_SMALL_INT), + erlang:unique_integer([monotonic]), + Node}, + gns_volatile_multicast(Msg, Node, ?pop_vsn, true, S); +inform_connection_loss(_Node, #state{}) -> + ok. + +%% +%% Volatile send (does not bring up connections and does not +%% preserve signal order) of Msg to global name server at Node... +%% +gns_volatile_send(Node, Msg) -> + To = {global_name_server, Node}, + case erlang:send(To, Msg, [nosuspend, noconnect]) of + ok -> + ok; + noconnect -> + ok; + nosuspend -> + _ = spawn(fun () -> + _ = erlang:send(To, Msg, [noconnect]) + end), + ok + end. + +%% +%% Volatile multicast of Msg to all global name servers on known nodes +%% (and pending known nodes if AlsoPend is true) using protocol version +%% MinVer or larger... +%% +gns_volatile_multicast(Msg, IgnoreNode, MinVer, + AlsoPend, #state{known = Known}) -> + maps:foreach(fun (Node, Ver) when is_atom(Node), + Node =/= IgnoreNode, + Ver >= MinVer -> + gns_volatile_send(Node, Msg); + ({pending, Node}, Ver) when AlsoPend == true, + Node =/= IgnoreNode, + Ver >= MinVer -> + gns_volatile_send(Node, Msg); + (_, _) -> + ok + end, Known). + +is_node_potentially_known(Node, #state{known = Known}) -> + maps:is_key(Node, Known) orelse maps:is_key({pending, Node}, Known). + +node_vsn(Node, #state{}) when node() == Node -> + ?vsn; +node_vsn(Node, #state{known = Known}) -> + case maps:find(Node, Known) of + {ok, Ver} -> + Ver; + error -> + case maps:find({pending, Node}, Known) of + {ok, Ver} -> + Ver; + error -> + 0 + end + end. + +node_list(NList) -> + lists:map(fun (N) when is_atom(N) -> + N; + ({N, _V}) when is_atom(N) -> + N + end, NList). + + +make_node_vsn_list(NList, #state{} = S) -> + lists:map(fun ({N, 0}) when is_atom(N) -> + {N, node_vsn(N, S)}; + (N) when is_atom(N) -> + {N, node_vsn(N, S)}; + ({N, V} = NV) when is_atom(N), + is_integer(V) -> + NV + end, NList). + +mk_known_list(Vsn, #state{known = Known}) when Vsn < ?pgpv_vsn -> + lists:foldl(fun ({N, _V}, Ns) when is_atom(N) -> + [N | Ns]; + (_, Ns) -> + Ns + end, + [], + maps:to_list(Known)); +mk_known_list(_Vsn, #state{known = Known}) -> + lists:foldl(fun ({N, _V} = NV, NVs) when is_atom(N) -> + [NV | NVs]; + (_, Ns) -> + Ns + end, + [], + maps:to_list(Known)). + +add_to_known(AddKnown, #state{known = Known} = S) -> + Fun = fun (N, Acc) when N == node() -> + Acc; + ({N, _V}, Acc) when N == node() -> + Acc; + (N, {A, K} = Acc) when is_atom(N) -> + case maps:is_key(N, K) of + true -> Acc; + false -> {[N|A], maps:put(N, 0, K)} + end; + ({N, V}, {A, K} = Acc) -> + case maps:find(N, K) of + error -> + {[N|A], maps:put(N, V, K)}; + {ok, NV} when NV >= 0 -> + Acc; + {ok, _UnknownVsn} -> + %% Update version, but don't count + %% it as an added node... + {A, maps:put(N, V, K)} + end + end, + + {Added, NewKnown} = lists:foldl(Fun, {[], Known}, AddKnown), + {Added, S#state{known = NewKnown}}. + +get_lost_connection_info(LcKey) -> + case ets:lookup(global_lost_connections, LcKey) of + [{LcKey, LcValue}] -> + LcValue; + _ -> + {undefined, undefined, undefined} + end. + +save_lost_connection_info(LcKey, XCre, OpId, undefined) -> + %% Make sure we clean up old unused information about + %% lost connections... + Tmr = erlang:start_timer(?lost_conn_info_cleanup_time, + self(), {lost_connection, LcKey}), + Value = {XCre, OpId, Tmr}, + _ = ets:insert(global_lost_connections, {LcKey, Value}), + ok; +save_lost_connection_info(LcKey, XCre, OpId, OldTmr) -> + %% Cancel lost connection info cleanup timer for info being replaced... + _ = erlang:cancel_timer(OldTmr, [{async, true}, {info, false}]), + save_lost_connection_info(LcKey, XCre, OpId, undefined). + +remove_lost_connection_info({timeout, Tmr, {lost_connection, LcKey}}) -> + case ets:lookup(global_lost_connections, LcKey) of + [{LcKey, {_, _, Tmr}}] -> + _ = ets:delete(global_lost_connections, LcKey), + ok; + _ -> + ok + end; +remove_lost_connection_info(_) -> + ok. + get_names() -> ets:select(global_names, ets:fun2ms(fun({Name, Pid, Method, _Ref}) -> diff --git a/lib/kernel/src/global_group.erl b/lib/kernel/src/global_group.erl index 77ed46de5f..8811f41adb 100644 --- a/lib/kernel/src/global_group.erl +++ b/lib/kernel/src/global_group.erl @@ -925,15 +925,20 @@ handle_info({nodedown, Node}, S) when S#state.sync_state =:= no_conf -> handle_info({nodedown, Node}, S) -> % io:format("~p>>>>> nodedown, Node ~p ~n",[node(), Node]), send_monitor(S#state.monitor, {nodedown, Node}, S#state.sync_state), - global_name_server ! {nodedown, Node}, NN = lists:delete(Node, S#state.nodes), NSE = lists:delete(Node, S#state.sync_error), - NNC = case {lists:member(Node, get_own_nodes()), - lists:member(Node, S#state.no_contact)} of - {true, false} -> - [Node | S#state.no_contact]; - _ -> - S#state.no_contact + NNC = case lists:member(Node, get_own_nodes()) of + false -> + global_name_server ! {ignore_node, Node}, + S#state.no_contact; + true -> + global_name_server ! {nodedown, Node}, + case lists:member(Node, S#state.no_contact) of + false -> + [Node | S#state.no_contact]; + true -> + S#state.no_contact + end end, {noreply, S#state{nodes = NN, no_contact = NNC, sync_error = NSE}}; @@ -950,8 +955,7 @@ handle_info({disconnect_node, Node}, S) -> _ -> cont end, - global_name_server ! {nodedown, Node}, %% nodedown is used to inform global of the - %% disconnected node + global_name_server ! {ignore_node, Node}, NN = lists:delete(Node, S#state.nodes), NNC = lists:delete(Node, S#state.no_contact), NSE = lists:delete(Node, S#state.sync_error), @@ -1264,7 +1268,7 @@ kill_global_group_check() -> disconnect_nodes(DisconnectNodes) -> lists:foreach(fun(Node) -> {global_group, Node} ! {disconnect_node, node()}, - global:node_disconnected(Node) + global_name_server ! {ignore_node, Node} end, DisconnectNodes). @@ -1275,7 +1279,7 @@ disconnect_nodes(DisconnectNodes) -> force_nodedown(DisconnectNodes) -> lists:foreach(fun(Node) -> erlang:disconnect_node(Node), - global:node_disconnected(Node) + global_name_server ! {ignore_node, Node} end, DisconnectNodes). diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index ce714e506f..142003ecca 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -152,9 +152,10 @@ {applications, []}, {env, [{logger_level, notice}, {logger_sasl_compatible, false}, - {shell_docs_ansi,auto} + {shell_docs_ansi,auto}, + {prevent_overlapping_partitions, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-12.2", "stdlib-3.13", "sasl-3.0", "crypto-5.0"]} + {runtime_dependencies, ["erts-@OTP-17843@", "stdlib-3.13", "sasl-3.0", "crypto-5.0"]} ] }. diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index aa1a6d6a27..45adad09ff 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -157,6 +157,11 @@ tick(Config) when is_list(Config) -> run_dist_configs(fun tick/2, Config). tick(DCfg, _Config) -> + %% + %% This test case use disabled "connect all" so that + %% global wont interfere... + %% + %% First check that the normal case is OK! [Name1, Name2] = get_nodenames(2, dist_test), {ok, Node} = start_node(DCfg, Name1), @@ -184,11 +189,11 @@ tick(DCfg, _Config) -> %% node doesn't tick the client node within the interval ... {ok, ServNode} = start_node(DCfg, Name2, - "-kernel net_ticktime 100"), + "-kernel net_ticktime 100 -connect_all false"), rpc:call(ServNode, erl_distribution_SUITE, tick_serv_test, [Node, node()]), {ok, Node} = start_node(DCfg, Name1, - "-kernel net_ticktime 12"), + "-kernel net_ticktime 12 -connect_all false"), rpc:call(Node, erl_distribution_SUITE, tick_cli_test, [ServNode]), spawn_link(erl_distribution_SUITE, keep_conn, [Node]), @@ -758,6 +763,10 @@ tick_change(Config) when is_list(Config) -> run_dist_configs(fun tick_change/2, Config). tick_change(DCfg, _Config) -> + %% + %% This test case use disabled "connect all" so that + %% global wont interfere... + %% [BN, CN] = get_nodenames(2, tick_change), DefaultTT = net_kernel:get_net_ticktime(), unchanged = net_kernel:set_net_ticktime(DefaultTT, 60), @@ -774,7 +783,7 @@ tick_change(DCfg, _Config) -> end, wait_until(fun () -> 10 == net_kernel:get_net_ticktime() end), - {ok, B} = start_node(DCfg, BN, "-kernel net_ticktime 10"), + {ok, B} = start_node(DCfg, BN, "-kernel net_ticktime 10 -connect_all false"), {ok, C} = start_node(DCfg, CN, "-kernel net_ticktime 10 -hidden"), OTE = process_flag(trap_exit, true), @@ -836,7 +845,7 @@ run_tick_change_test(DCfg, B, C, PrevTT, TT) -> wait_for_nodedowns(Tester, Ref) end, - {ok, D} = start_node(DCfg, DN, "-kernel net_ticktime " + {ok, D} = start_node(DCfg, DN, "-connect_all false -kernel net_ticktime " ++ integer_to_list(PrevTT)), NMA = spawn_link(fun () -> MonitorNodes([B, C, D]) end), @@ -870,7 +879,7 @@ run_tick_change_test(DCfg, B, C, PrevTT, TT) -> sleep(7), change_initiated = rpc:call(C,net_kernel,set_net_ticktime,[TT,10]), - {ok, E} = start_node(DCfg, EN, "-kernel net_ticktime " + {ok, E} = start_node(DCfg, EN, "-connect_all false -kernel net_ticktime " ++ integer_to_list(TT)), NME = spawn_link(E, fun () -> MonitorNodes([node(), B, C, D]) end), NMA2 = spawn_link(fun () -> MonitorNodes([E]) end), diff --git a/lib/kernel/test/erpc_SUITE.erl b/lib/kernel/test/erpc_SUITE.erl index 8cfaa47184..012df04779 100644 --- a/lib/kernel/test/erpc_SUITE.erl +++ b/lib/kernel/test/erpc_SUITE.erl @@ -762,20 +762,22 @@ send_request_against_old_node(Config) when is_list(Config) -> end. multicall(Config) -> - {ok, Node1} = start_node(Config), - {ok, Node2} = start_node(Config), + {ok, Node1} = start_peer_node(Config), + {ok, Node2} = start_peer_node(Config), {Node3, Node3Res} = case start_22_node(Config) of {ok, N3} -> {N3, {error, {erpc, notsup}}}; _ -> - {ok, N3} = start_node(Config), + {ok, N3} = start_peer_node(Config), stop_node(N3), {N3, {error, {erpc, noconnection}}} end, - {ok, Node4} = start_node(Config), - {ok, Node5} = start_node(Config), + {ok, Node4} = start_peer_node(Config), + {ok, Node5} = start_peer_node(Config), stop_node(Node2), - + io:format("Node1=~p~nNode2=~p~nNode3=~p~nNode4=~p~nNode5=~p~n", + [Node1, Node2, Node3, Node4, Node5]), + ThisNode = node(), Nodes = [ThisNode, Node1, Node2, Node3, Node4, Node5], @@ -1341,6 +1343,14 @@ start_node(Config) -> Pa = filename:dirname(code:which(?MODULE)), test_server:start_node(Name, slave, [{args, "-pa " ++ Pa}]). +start_peer_node(Config) -> + Name = list_to_atom(atom_to_list(?MODULE) + ++ "-" ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" ++ integer_to_list(erlang:system_time(second)) + ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), + Pa = filename:dirname(code:which(?MODULE)), + test_server:start_node(Name, peer, [{args, "-pa " ++ Pa}]). + start_22_node(Config) -> Rel = "22_latest", case test_server:is_release_available(Rel) of diff --git a/lib/kernel/test/global_SUITE.erl b/lib/kernel/test/global_SUITE.erl index 0c3e56741b..c5ce2557f6 100644 --- a/lib/kernel/test/global_SUITE.erl +++ b/lib/kernel/test/global_SUITE.erl @@ -44,12 +44,13 @@ lost_unregister/1, mass_death/1, garbage_messages/1, - ring_line/1 + ring_line/1, + lost_connection/1, + lost_connection2/1 ]). %% Not used --export([simple_dis/4, - config_dc/4, +-export([config_dc/4, w/2, check_same/2, check_same/1, @@ -72,16 +73,8 @@ init_mass_spawn/1, %% Called via rpc_cast - part_2_2/4, - part1/4, - part1_5/4, - part2/10, - part3/10, crash/1, - single_node/3, - part_ext/4, - isolated_node/4, - mk_part_node/3, + single_node/2, alone/2, global_load/3, @@ -141,7 +134,8 @@ all() -> simple_resolve2, simple_resolve3, leftover_name, re_register_name, name_exit, external_nodes, many_nodes, sync_0, global_groups_change, register_1, both_known_1, - lost_unregister, mass_death, garbage_messages + lost_unregister, mass_death, garbage_messages, + lost_connection, lost_connection2 ] end. @@ -312,6 +306,14 @@ lock_global(Parent, Config) -> %%% to obtain a lock for 'global' on node 3, which would keep the %%% name registry from ever becoming consistent again. both_known_1(Config) when is_list(Config) -> + case prevent_overlapping_partitions() of + true -> + {skipped, "Prevent overlapping partitions enabled"}; + false -> + both_known_1_test(Config) + end. + +both_known_1_test(Config) when is_list(Config) -> Timeout = 30, ct:timetrap({seconds,Timeout}), init_high_level_trace(Timeout), @@ -394,6 +396,14 @@ both_known_1(Config) when is_list(Config) -> %% OTP-6428. An unregistered name reappears. lost_unregister(Config) when is_list(Config) -> + case prevent_overlapping_partitions() of + true -> + {skipped, "Prevent overlapping partitions enabled"}; + false -> + lost_unregister_test(Config) + end. + +lost_unregister_test(Config) when is_list(Config) -> Timeout = 30, ct:timetrap({seconds,Timeout}), init_high_level_trace(Timeout), @@ -434,6 +444,119 @@ lost_unregister(Config) when is_list(Config) -> init_condition(Config), ok. +lost_connection(Config) when is_list(Config) -> + case prevent_overlapping_partitions() of + true -> + lost_connection_test(Config); + false -> + {skipped, "Prevent overlapping partitions disabled"} + end. + +lost_connection_test(Config) when is_list(Config) -> + %% OTP-17843: Registered names could become inconsistent due to + %% overlapping partitions. This has been solved by + %% global actively disconnecting nodes to prevent + %% overlapping partitions. + ct:timetrap({seconds, 15}), + + [Cp1, Cp2] = start_nodes([cp1, cp2], peer, Config), + + PartCtrlr = setup_partitions(Config, [[node(), Cp1, Cp2]]), + + wait_for_ready_net(Config), + + {Gurka, yes} = start_proc_basic(gurka), + + check_everywhere([node(), Cp1, Cp2], gurka, Config), + + Gurka = global:whereis_name(gurka), + + erlang:disconnect_node(Cp2), %% lost connection previously causing issues... + + erpc:call( + PartCtrlr, + fun () -> + erpc:call( + Cp2, + fun () -> + {AltGurka, yes} = start_proc_basic(gurka), + AltGurka = global:whereis_name(gurka) + end), + timer:sleep(1000), + erpc:cast(Cp2, erlang, halt, []), + wait_until(fun () -> not lists:member(Cp2, nodes(hidden)) end) + end), + + Reconnected = case lists:member(Cp1, nodes()) of + true -> + false; + false -> + erlang:display("reconnecting Cp1"), + pong = net_adm:ping(Cp1), + timer:sleep(500), + true + end, + + check_everywhere([node(), Cp1], gurka, Config), + + Gurka = global:whereis_name(gurka), + + ok = global:unregister_name(gurka), + + stop_node(Cp1), + stop_partition_controller(PartCtrlr), + + {comment, case Reconnected of + true -> "Re-connected Cp1"; + false -> "No re-connection of Cp1 needed" + end}. + +lost_connection2(Config) when is_list(Config) -> + case prevent_overlapping_partitions() of + true -> + lost_connection2_test(Config); + false -> + {skipped, "Prevent overlapping partitions disabled"} + end. + +lost_connection2_test(Config) when is_list(Config) -> + %% OTP-17843: Registered names could become inconsistent due to + %% overlapping partitions. This has been solved by + %% global actively disconnecting nodes to prevent + %% overlapping partitions. + ct:timetrap({seconds, 15}), + + [Cp1, Cp2, Cp3, Cp4] = start_nodes([cp1, cp2, cp3, cp4], peer, Config), + + PartCtrlr = setup_partitions(Config, [[node(), Cp1, Cp2, Cp3, Cp4]]), + + wait_for_ready_net(Config), + + {Gurka, yes} = start_proc_basic(gurka), + + check_everywhere([node(), Cp1, Cp2], gurka, Config), + + Gurka = global:whereis_name(gurka), + + disconnect_nodes(PartCtrlr, Cp3, Cp4), + + Nodes = nodes(), + true = lists:member(Cp1, Nodes), + true = lists:member(Cp2, Nodes), + false = lists:member(Cp3, Nodes), + false = lists:member(Cp4, Nodes), + + pong = net_adm:ping(Cp3), + pong = net_adm:ping(Cp4), + + stop_node(Cp1), + stop_node(Cp2), + stop_node(Cp3), + stop_node(Cp4), + stop_partition_controller(PartCtrlr), + + ok. + -define(UNTIL_LOOP, 300). -define(end_tag, 'end at'). @@ -1103,12 +1226,9 @@ name_die(Config) when is_list(Config) -> T1 = node(), Part1 = [T1], Part2 = [Cp1], - rpc_cast(Cp1, - ?MODULE, part_2_2, [Config, - Part1, - Part2, - []]), - ?UNTIL(is_ready_partition(Config)), + + PartCtrlr = setup_partitions(Config, [Part1, Part2]), + ?UNTIL(undefined =:= global:whereis_name(Name)), yes = global:register_name(Name, Pid), @@ -1126,12 +1246,9 @@ name_die(Config) when is_list(Config) -> KillFile = filename:join([Dir, "kill.txt"]), file:delete(KillFile), erlang:spawn(Cp1, fun() -> kill_pid(Pid2, KillFile, Config) end), - rpc_cast(Cp1, - ?MODULE, part_2_2, [Config, - Part1, - Part2, - []]), - ?UNTIL(is_ready_partition(Config)), + + setup_partitions(PartCtrlr, [Part1, Part2]), + ?UNTIL(undefined =:= global:whereis_name(Name)), yes = global:register_name(Name, Pid2), touch(KillFile, "kill"), @@ -1141,6 +1258,7 @@ name_die(Config) when is_list(Config) -> ?UNTIL(OrigNames =:= global:registered_names()), write_high_level_trace(Config), stop_nodes(Cps), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -1163,16 +1281,26 @@ basic_partition(Config) when is_list(Config) -> wait_for_ready_net(Config), %% make cp2 and cp3 connected, partitioned from us and cp1 - rpc_cast(Cp2, ?MODULE, part1, [Config, node(), Cp1, Cp3]), - ?UNTIL(is_ready_partition(Config)), + PCtrlr = setup_partitions(Config, [[node(), Cp1], [Cp2, Cp3]]), %% start different processes in both partitions {Pid, yes} = start_proc(test), + %% Reach into the other partition via PCtrlr... + ok = erpc:call( + PCtrlr, + fun () -> + {_, yes} = erpc:call(Cp2, ?MODULE, + start_proc, [test2]), + {_, yes} = erpc:call(Cp1, ?MODULE, + start_proc, [test4]), + ok + end), + %% connect to other partition pong = net_adm:ping(Cp2), pong = net_adm:ping(Cp3), - [Cp1, Cp2, Cp3] = lists:sort(nodes()), + wait_until(fun () -> [Cp1, Cp2, Cp3] == lists:sort(nodes()) end), %% check names ?UNTIL(Pid =:= rpc:call(Cp2, global, whereis_name, [test])), @@ -1199,6 +1327,7 @@ basic_partition(Config) when is_list(Config) -> stop_node(Cp1), stop_node(Cp2), stop_node(Cp3), + stop_node(PCtrlr), init_condition(Config), ok. @@ -1231,12 +1360,23 @@ basic_name_partition(Config) when is_list(Config) -> %% cp2: register name12 %% cp3: register name03 - rpc_cast(Cp2, ?MODULE, part1_5, [Config, node(), Cp1, Cp3]), - ?UNTIL(is_ready_partition(Config)), + PCtrlr = setup_partitions(Config, [[node(), Cp1], [Cp2, Cp3]]), %% start different processes in both partitions {_, yes} = start_proc_basic(name03), - {_, yes} = rpc:call(Cp1, ?MODULE, start_proc_basic, [name12]), + {_, yes} = erpc:call(Cp1, ?MODULE, start_proc_basic, [name12]), + + %% Reach into the other partition via PCtrlr... + ok = erpc:call( + PCtrlr, + fun () -> + {_, yes} = erpc:call(Cp2, ?MODULE, + start_proc_basic, [name12]), + {_, yes} = erpc:call(Cp3, ?MODULE, + start_proc_basic, [name03]), + ok + end), + ct:sleep(1000), %% connect to other partition @@ -1275,6 +1415,7 @@ basic_name_partition(Config) when is_list(Config) -> stop_node(Cp1), stop_node(Cp2), stop_node(Cp3), + stop_node(PCtrlr), init_condition(Config), ok. @@ -1300,24 +1441,35 @@ advanced_partition(Config) when is_list(Config) -> init_condition(Config), OrigNames = global:registered_names(), + Parent = self(), + [Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6] = start_nodes([cp0, cp1, cp2, cp3, cp4, cp5, cp6], peer, Config), Nodes = lists:sort([node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), wait_for_ready_net(Config), %% make cp3-cp6 connected, partitioned from us and cp0-cp2 - rpc_cast(Cp3, ?MODULE, part2, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), - ?UNTIL(is_ready_partition(Config)), + PCntrlr = setup_partitions(Config, [[node(), Cp0, Cp1, Cp2], [Cp3, Cp4, Cp5, Cp6]]), + + %% start processes in other partition (via partition controller)... + _ = erpc:call(PCntrlr, + fun () -> + erpc:call(Cp3, + fun () -> + start_procs(Parent, Cp4, Cp5, Cp6, Config) + end) + end), %% start different processes in this partition start_procs(self(), Cp0, Cp1, Cp2, Config), %% connect to other partition pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), - pong = net_adm:ping(Cp6), + + wait_until(fun () -> + CurrNodes = lists:sort(?NODES), + io:format("CurrNodes: ~p~n", [CurrNodes]), + Nodes == CurrNodes end), wait_for_ready_net(Config), @@ -1380,6 +1532,7 @@ advanced_partition(Config) when is_list(Config) -> stop_node(Cp4), stop_node(Cp5), stop_node(Cp6), + stop_node(PCntrlr), init_condition(Config), ok. @@ -1401,17 +1554,43 @@ stress_partition(Config) when is_list(Config) -> init_condition(Config), OrigNames = global:registered_names(), - [Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6] - = start_nodes([cp0, cp1, cp2, cp3, cp4, cp5, cp6], peer, Config), + Parent = self(), + + [Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6a, Cp6b] + = start_nodes([cp0, cp1, cp2, cp3, cp4, cp5, cp6a, cp6b], peer, Config), wait_for_ready_net(Config), %% make cp3-cp5 connected, partitioned from us and cp0-cp2 %% cp6 is alone (single node). cp6 pings cp0 and cp3 in 12 secs... - rpc_cast(Cp3, ?MODULE, part3, - [Config, self(), node(), Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6]), - ?UNTIL(is_ready_partition(Config)), - + PCntrlr = setup_partitions(Config, [[node(), Cp0, Cp1, Cp2], [Cp3, Cp4, Cp5, Cp6a], [Cp6b]]), + + %% start processes in other partition (via partition controller)... + _ = erpc:call(PCntrlr, + fun () -> + erpc:call(Cp3, + fun () -> + start_procs(Parent, Cp4, Cp5, Cp6a, Config) + end), + erpc:call(Cp6b, + fun () -> + Pid1 = start_proc3(test1), + assert_pid(Pid1), + Pid2 = start_proc3(test3), + assert_pid(Pid2), + yes = global:register_name( + test1, Pid1), + yes = global:register_name( + test3, Pid2, + fun global:random_notify_name/3) + end), + %% Make Cp5 crash + erpc:cast(Cp5, ?MODULE, crash, [12000]), + %% Make Cp6b alone + erpc:cast(Cp6b, ?MODULE, alone, [Cp0, Cp3]) + end), + + erlang:display(starting_test), %% start different processes in this partition start_procs(self(), Cp0, Cp1, Cp2, Config), @@ -1429,17 +1608,19 @@ stress_partition(Config) when is_list(Config) -> %% connect to other partition pong = net_adm:ping(Cp3), + pong = net_adm:ping(Cp4), rpc_cast(Cp2, ?MODULE, crash, [0]), %% Start new nodes {ok, Cp7} = start_peer_node(cp7, Config), {ok, Cp2_2} = start_peer_node(cp2, Config), - Nodes = lists:sort([node(), Cp0, Cp1, Cp2_2, Cp3, Cp4, Cp6, Cp7, Cp8]), + Nodes = lists:sort([node(), Cp0, Cp1, Cp2_2, Cp3, Cp4, Cp6a, Cp6b, Cp7, Cp8]), put(?nodes_tag, Nodes), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp6), - pong = net_adm:ping(Cp8), + %% We don't know how the crashes partitions the net, so + %% we cast trough all nodes so we get them all connected + %% again... + cast_line(Nodes), wait_for_ready_net(Nodes, Config), @@ -1472,9 +1653,11 @@ stress_partition(Config) when is_list(Config) -> stop_node(Cp3), stop_node(Cp4), stop_node(Cp5), - stop_node(Cp6), + stop_node(Cp6a), + stop_node(Cp6b), stop_node(Cp7), stop_node(Cp8), + stop_node(PCntrlr), init_condition(Config), ok. @@ -1513,40 +1696,28 @@ ring(Config) when is_list(Config) -> wait_for_ready_net(Config), - Time = msec() + 7000, - - rpc_cast(Cp0, ?MODULE, single_node, [Time, Cp8, Config]), - rpc_cast(Cp1, ?MODULE, single_node, [Time, Cp0, Config]), - rpc_cast(Cp2, ?MODULE, single_node, [Time, Cp1, Config]), - rpc_cast(Cp3, ?MODULE, single_node, [Time, Cp2, Config]), - rpc_cast(Cp4, ?MODULE, single_node, [Time, Cp3, Config]), - rpc_cast(Cp5, ?MODULE, single_node, [Time, Cp4, Config]), - rpc_cast(Cp6, ?MODULE, single_node, [Time, Cp5, Config]), - rpc_cast(Cp7, ?MODULE, single_node, [Time, Cp6, Config]), - rpc_cast(Cp8, ?MODULE, single_node, [Time, Cp7, Config]), - - %% sleep to make the partitioned net ready - sleep(Time - msec()), - - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), - pong = net_adm:ping(Cp6), - pong = net_adm:ping(Cp7), - pong = net_adm:ping(Cp8), + PartCtrlr = setup_partitions(Config, + [[node()], [Cp0], [Cp1], [Cp2], [Cp3], + [Cp4], [Cp5], [Cp6], [Cp7], [Cp8]]), + + Time = msec() + 1000, + + ok = erpc:call( + PartCtrlr, + fun () -> + erpc:cast(Cp0, ?MODULE, single_node, [Time, Cp8]), % ping ourself! + erpc:cast(Cp1, ?MODULE, single_node, [Time, Cp0]), + erpc:cast(Cp2, ?MODULE, single_node, [Time, Cp1]), + erpc:cast(Cp3, ?MODULE, single_node, [Time, Cp2]), + erpc:cast(Cp4, ?MODULE, single_node, [Time, Cp3]), + erpc:cast(Cp5, ?MODULE, single_node, [Time, Cp4]), + erpc:cast(Cp6, ?MODULE, single_node, [Time, Cp5]), + erpc:cast(Cp7, ?MODULE, single_node, [Time, Cp6]), + erpc:cast(Cp8, ?MODULE, single_node, [Time, Cp7]), + ok + end), pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), - pong = net_adm:ping(Cp6), - pong = net_adm:ping(Cp7), - pong = net_adm:ping(Cp8), wait_for_ready_net(Nodes, Config), @@ -1577,6 +1748,7 @@ ring(Config) when is_list(Config) -> stop_node(Cp6), stop_node(Cp7), stop_node(Cp8), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -1599,31 +1771,24 @@ simple_ring(Config) when is_list(Config) -> wait_for_ready_net(Config), - Time = msec() + 5000, - - rpc_cast(Cp0, ?MODULE, single_node, [Time, Cp5, Config]), - rpc_cast(Cp1, ?MODULE, single_node, [Time, Cp0, Config]), - rpc_cast(Cp2, ?MODULE, single_node, [Time, Cp1, Config]), - rpc_cast(Cp3, ?MODULE, single_node, [Time, Cp2, Config]), - rpc_cast(Cp4, ?MODULE, single_node, [Time, Cp3, Config]), - rpc_cast(Cp5, ?MODULE, single_node, [Time, Cp4, Config]), - - %% sleep to make the partitioned net ready - sleep(Time - msec()), - - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), + PartCtrlr = setup_partitions(Config, + [[node()], [Cp0], [Cp1], [Cp2], [Cp3], + [Cp4], [Cp5]]), + Time = msec() + 1000, + + ok = erpc:call( + PartCtrlr, + fun () -> + erpc:cast(Cp0, ?MODULE, single_node, [Time, Cp5]), % ping ourself! + erpc:cast(Cp1, ?MODULE, single_node, [Time, Cp0]), + erpc:cast(Cp2, ?MODULE, single_node, [Time, Cp1]), + erpc:cast(Cp3, ?MODULE, single_node, [Time, Cp2]), + erpc:cast(Cp4, ?MODULE, single_node, [Time, Cp3]), + erpc:cast(Cp5, ?MODULE, single_node, [Time, Cp4]), + ok + end), pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), wait_for_ready_net(Nodes, Config), @@ -1651,6 +1816,7 @@ simple_ring(Config) when is_list(Config) -> stop_node(Cp3), stop_node(Cp4), stop_node(Cp5), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -1671,41 +1837,27 @@ line(Config) when is_list(Config) -> wait_for_ready_net(Config), - Time = msec() + 7000, - - rpc_cast(Cp0, ?MODULE, single_node, - [Time, Cp0, Config]), % ping ourself! - rpc_cast(Cp1, ?MODULE, single_node, [Time, Cp0, Config]), - rpc_cast(Cp2, ?MODULE, single_node, [Time, Cp1, Config]), - rpc_cast(Cp3, ?MODULE, single_node, [Time, Cp2, Config]), - rpc_cast(Cp4, ?MODULE, single_node, [Time, Cp3, Config]), - rpc_cast(Cp5, ?MODULE, single_node, [Time, Cp4, Config]), - rpc_cast(Cp6, ?MODULE, single_node, [Time, Cp5, Config]), - rpc_cast(Cp7, ?MODULE, single_node, [Time, Cp6, Config]), - rpc_cast(Cp8, ?MODULE, single_node, [Time, Cp7, Config]), - - %% Sleep to make the partitioned net ready - sleep(Time - msec()), - - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), - pong = net_adm:ping(Cp6), - pong = net_adm:ping(Cp7), - pong = net_adm:ping(Cp8), + PartCtrlr = setup_partitions(Config, + [[node()], [Cp0], [Cp1], [Cp2], [Cp3], + [Cp4], [Cp5], [Cp6], [Cp7], [Cp8]]), + ThisNode = node(), - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), - pong = net_adm:ping(Cp6), - pong = net_adm:ping(Cp7), - pong = net_adm:ping(Cp8), + Time = msec() + 1000, + ok = erpc:call( + PartCtrlr, + fun () -> + erpc:cast(Cp0, ?MODULE, single_node, [Time, Cp0]), % ping ourself! + erpc:cast(Cp1, ?MODULE, single_node, [Time, Cp0]), + erpc:cast(Cp2, ?MODULE, single_node, [Time, Cp1]), + erpc:cast(Cp3, ?MODULE, single_node, [Time, Cp2]), + erpc:cast(Cp4, ?MODULE, single_node, [Time, Cp3]), + erpc:cast(Cp5, ?MODULE, single_node, [Time, Cp4]), + erpc:cast(Cp6, ?MODULE, single_node, [Time, Cp5]), + erpc:cast(Cp7, ?MODULE, single_node, [Time, Cp6]), + erpc:cast(Cp8, ?MODULE, single_node, [Time, Cp7]), + erpc:cast(ThisNode, ?MODULE, single_node, [Time, Cp8]), + ok + end), wait_for_ready_net(Nodes, Config), @@ -1736,6 +1888,7 @@ line(Config) when is_list(Config) -> stop_node(Cp6), stop_node(Cp7), stop_node(Cp8), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -1758,32 +1911,24 @@ simple_line(Config) when is_list(Config) -> wait_for_ready_net(Config), - Time = msec() + 5000, - - rpc_cast(Cp0, ?MODULE, single_node, - [Time, Cp0, Config]), % ping ourself! - rpc_cast(Cp1, ?MODULE, single_node, [Time, Cp0, Config]), - rpc_cast(Cp2, ?MODULE, single_node, [Time, Cp1, Config]), - rpc_cast(Cp3, ?MODULE, single_node, [Time, Cp2, Config]), - rpc_cast(Cp4, ?MODULE, single_node, [Time, Cp3, Config]), - rpc_cast(Cp5, ?MODULE, single_node, [Time, Cp4, Config]), - - %% sleep to make the partitioned net ready - sleep(Time - msec()), - - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), + PartCtrlr = setup_partitions(Config, + [[node()], [Cp0], [Cp1], [Cp2], [Cp3], + [Cp4], [Cp5]]), + ThisNode = node(), - pong = net_adm:ping(Cp0), - pong = net_adm:ping(Cp1), - pong = net_adm:ping(Cp2), - pong = net_adm:ping(Cp3), - pong = net_adm:ping(Cp4), - pong = net_adm:ping(Cp5), + Time = msec() + 1000, + ok = erpc:call( + PartCtrlr, + fun () -> + erpc:cast(Cp0, ?MODULE, single_node, [Time, Cp0]), % ping ourself! + erpc:cast(Cp1, ?MODULE, single_node, [Time, Cp0]), + erpc:cast(Cp2, ?MODULE, single_node, [Time, Cp1]), + erpc:cast(Cp3, ?MODULE, single_node, [Time, Cp2]), + erpc:cast(Cp4, ?MODULE, single_node, [Time, Cp3]), + erpc:cast(Cp5, ?MODULE, single_node, [Time, Cp4]), + erpc:cast(ThisNode, ?MODULE, single_node, [Time, Cp5]), + ok + end), wait_for_ready_net(Nodes, Config), @@ -1811,6 +1956,7 @@ simple_line(Config) when is_list(Config) -> stop_node(Cp3), stop_node(Cp4), stop_node(Cp5), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -1896,9 +2042,9 @@ otp_1849(Config) when is_list(Config) -> %% Test ticket: Deadlock in global. otp_3162(Config) when is_list(Config) -> StartFun = fun() -> - {ok, Cp1} = start_node(cp1, Config), - {ok, Cp2} = start_node(cp2, Config), - {ok, Cp3} = start_node(cp3, Config), + {ok, Cp1} = start_node(cp1, peer, Config), + {ok, Cp2} = start_node(cp2, peer, Config), + {ok, Cp3} = start_node(cp3, peer, Config), [Cp1, Cp2, Cp3] end, do_otp_3162(StartFun, Config). @@ -1912,6 +2058,10 @@ do_otp_3162(StartFun, Config) -> wait_for_ready_net(Config), + ThisNode = node(), + + PartCntrlr = setup_partitions(Config, [[ThisNode, Cp1, Cp2, Cp3]]), + %% start procs on each node Pid1 = rpc:call(Cp1, ?MODULE, start_proc4, [kalle]), assert_pid(Pid1), @@ -1920,47 +2070,29 @@ do_otp_3162(StartFun, Config) -> Pid3 = rpc:call(Cp3, ?MODULE, start_proc4, [vera]), assert_pid(Pid3), - rpc_disconnect_node(Cp1, Cp2, Config), - - ?UNTIL - ([Cp3] =:= lists:sort(rpc:call(Cp1, erlang, nodes, [])) -- [node()]), - - ?UNTIL([kalle, vera] =:= - lists:sort(rpc:call(Cp1, global, registered_names, []))), - ?UNTIL - ([Cp3] =:= lists:sort(rpc:call(Cp2, erlang, nodes, [])) -- [node()]), - ?UNTIL([stina, vera] =:= - lists:sort(rpc:call(Cp2, global, registered_names, []))), - ?UNTIL - ([Cp1, Cp2] =:= - lists:sort(rpc:call(Cp3, erlang, nodes, [])) -- [node()]), - ?UNTIL([kalle, stina, vera] =:= - lists:sort(rpc:call(Cp3, global, registered_names, []))), - - pong = rpc:call(Cp2, net_adm, ping, [Cp1]), - - ?UNTIL - ([Cp2, Cp3] =:= - lists:sort(rpc:call(Cp1, erlang, nodes, [])) -- [node()]), - ?UNTIL(begin - NN = lists:sort(rpc:call(Cp1, global, registered_names, [])), - [kalle, stina, vera] =:= NN - end), - ?UNTIL - ([Cp1, Cp3] =:= - lists:sort(rpc:call(Cp2, erlang, nodes, [])) -- [node()]), + disconnect_nodes(PartCntrlr, Cp1, Cp2), + %% Nowadays we do not know how the net have been partitioned after the + %% disconnect, so we cannot perform all the tests this testcase originally + %% did. We instead only test the end result is ok after we have reconnected + %% all nodes. + + erpc:call(PartCntrlr, + fun () -> + erpc:cast(Cp1, net_kernel, connect_node, [Cp3]), + erpc:cast(Cp2, net_kernel, connect_node, [Cp3]), + erpc:cast(ThisNode, net_kernel, connect_node, [Cp3]) + end), + + ?UNTIL(lists:sort([ThisNode, Cp1, Cp2]) =:= + lists:sort(erpc:call(Cp3, erlang, nodes, []))), ?UNTIL([kalle, stina, vera] =:= - lists:sort(rpc:call(Cp2, global, registered_names, []))), - ?UNTIL - ([Cp1, Cp2] =:= - lists:sort(rpc:call(Cp3, erlang, nodes, [])) -- [node()]), - ?UNTIL([kalle, stina, vera] =:= - lists:sort(rpc:call(Cp3, global, registered_names, []))), + lists:sort(erpc:call(Cp3, global, registered_names, []))), write_high_level_trace(Config), stop_node(Cp1), stop_node(Cp2), stop_node(Cp3), + stop_partition_controller(PartCntrlr), init_condition(Config), ok. @@ -2089,6 +2221,8 @@ simple_disconnect(Config) when is_list(Config) -> [Cp1, Cp2] = Cps = start_nodes([n_1, n_2], peer, Config), wait_for_ready_net(Config), + PartCtrlr = start_partition_controller(Config), + Nodes = lists:sort([node() | Cps]), lists:foreach(fun(N) -> rpc:call(N, ?MODULE, start_tracer, []) end,Nodes), @@ -2102,16 +2236,21 @@ simple_disconnect(Config) when is_list(Config) -> ct:sleep(100), %% Disconnect test_server and Cp2. - true = erlang:disconnect_node(Cp2), - ct:sleep(500), + disconnect_nodes(PartCtrlr, node(), Cp2), + %% We might have been disconnected from Cp1 as well. Ensure connected + %% to Cp1... + pong = net_adm:ping(Cp1), %% _Pid is registered on Cp1. The exchange of names between Cp2 and %% test_server sees two identical pids. pong = net_adm:ping(PingNode), + ?UNTIL(Cps =:= lists:sort(nodes())), {_, Trace0} = collect_tracers(Nodes), Resolvers = [P || {_Node,new_resolver,{pid,P}} <- Trace0], + check_everywhere(Nodes, Name, Config), + true = undefined /= global:whereis_name(Name), lists:foreach(fun(P) -> P ! die end, Resolvers), lists:foreach(fun(P) -> wait_for_exit(P) end, Resolvers), check_everywhere(Nodes, Name, Config), @@ -2129,30 +2268,10 @@ simple_disconnect(Config) when is_list(Config) -> OrigNames = global:registered_names(), write_high_level_trace(Config), stop_nodes(Cps), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. -%% Not used right now. -simple_dis(Nodes0, Name, Resolver, Config) -> - Nodes = [node() | Nodes0], - NN = lists:zip(Nodes, lists:seq(1, length(Nodes))), - [{_Node,Other} | Dis] = - [{N,[N1 || {N1,I1} <- NN, I1 > I + 1]} || {N,I} <- NN], - lists:foreach( - fun({Node, DisNodes}) -> - Args = [Node, DisNodes, Name, Resolver], - ok = rpc:call(Node, ?MODULE, simple_dis_node, Args) - end, Dis), - ok = simple_dis_node(node(), Other, Name, Resolver, Config). - -simple_dis_node(_Node, DisNodes, _Name, _Resolver, Config) -> - lists:foreach( - fun(OtherNode) -> _ = erlang:disconnect_node(OtherNode) end, DisNodes), - ?UNTIL(DisNodes -- nodes() =:= DisNodes), - ok. - - - %%%----------------------------------------------------------------- %%% Testing resolve of name. Many combinations with four nodes. %%%----------------------------------------------------------------- @@ -2163,7 +2282,8 @@ simple_dis_node(_Node, DisNodes, _Name, _Resolver, Config) -> n2, % node starting registered process in partition 2 nodes, % nodes expected to exist after ping n_res, % expected number of resolvers after ping - config + config, + ctrlr }). -define(RES(F), {F, fun ?MODULE:F/3}). @@ -2180,6 +2300,8 @@ simple_resolve(Config) when is_list(Config) -> Nodes = lists:sort([node() | Cps]), wait_for_ready_net(Config), + PartCtrlr = start_partition_controller(Config), + lists:foreach(fun(N) -> rpc:call(N, ?MODULE, start_tracer, []) end, Nodes), @@ -2189,7 +2311,8 @@ simple_resolve(Config) when is_list(Config) -> %% name 'link' remains... Cf = #cf{link = none, ping = A2, n1 = node(), n2 = A2, - nodes = [node(), N1, A2, Z2], n_res = 2, config = Config}, + nodes = [node(), N1, A2, Z2], n_res = 2, config = Config, + ctrlr = PartCtrlr}, %% There is no test with a resolver that deletes a pid (like %% global_exit_name does). The resulting DOWN signal just clears @@ -2297,6 +2420,7 @@ simple_resolve(Config) when is_list(Config) -> OrigNames = global:registered_names(), write_high_level_trace(Config), stop_nodes(Cps), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -2315,12 +2439,15 @@ simple_resolve2(Config) when is_list(Config) -> wait_for_ready_net(Config), Nodes = lists:sort([node() | Cps]), + PartCtrlr = start_partition_controller(Config), + lists:foreach(fun(N) -> rpc:call(N, ?MODULE, start_tracer, []) end, Nodes), Cf = #cf{link = none, ping = A2, n1 = node(), n2 = A2, - nodes = [node(), N1, A2, Z2], n_res = 2, config = Config}, + nodes = [node(), N1, A2, Z2], n_res = 2, config = Config, + ctrlr = PartCtrlr}, %% Halt z_2. res(?RES(halt_second), Cps, Cf#cf{link = N1, n1 = N1, n2 = Z2, ping = A2, @@ -2333,6 +2460,7 @@ simple_resolve2(Config) when is_list(Config) -> OrigNames = global:registered_names(), write_high_level_trace(Config), stop_nodes(Cps), % Not all nodes may be present, but it works anyway. + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -2350,12 +2478,15 @@ simple_resolve3(Config) when is_list(Config) -> wait_for_ready_net(Config), Nodes = lists:sort([node() | Cps]), + PartCtrlr = start_partition_controller(Config), + lists:foreach(fun(N) -> rpc:call(N, ?MODULE, start_tracer, []) end, Nodes), Cf = #cf{link = none, ping = A2, n1 = node(), n2 = A2, - nodes = [node(), N1, A2, Z2], n_res = 2, config = Config}, + nodes = [node(), N1, A2, Z2], n_res = 2, config = Config, + ctrlr = PartCtrlr}, %% Halt a_2. res(?RES(halt_second), Cps, Cf#cf{link = node(), n2 = A2, @@ -2368,13 +2499,14 @@ simple_resolve3(Config) when is_list(Config) -> OrigNames = global:registered_names(), write_high_level_trace(Config), stop_nodes(Cps), % Not all nodes may be present, but it works anyway. + stop_partition_controller(PartCtrlr), init_condition(Config), ok. res({Res,Resolver}, [N1, A2, Z2], Cf) -> %% Note: there are no links anymore, but monitors. #cf{link = LinkedNode, ping = PingNode, n1 = Res1, n2 = OtherNode, - nodes = Nodes0, n_res = NRes, config = Config} = Cf, + nodes = Nodes0, n_res = NRes, config = Config, ctrlr = PartCtrlr} = Cf, io:format("~n~nResolver: ~p", [Res]), io:format(" Registered on partition 1: ~p", [Res1]), io:format(" Registered on partition 2: ~p", [OtherNode]), @@ -2392,11 +2524,19 @@ res({Res,Resolver}, [N1, A2, Z2], Cf) -> %% expected monitors remain between registered processes and the %% global_name_server. - rpc_cast(OtherNode, - ?MODULE, - part_2_2, - [Config, Part1, Part2, [{Name, Resolver}]]), - ?UNTIL(is_ready_partition(Config)), + setup_partitions(PartCtrlr, [Part1, Part2]), + + erpc:call(PartCtrlr, + fun () -> + erpc:call(OtherNode, + fun () -> + {Pid2, yes} = start_resolver(Name, + Resolver), + trace_message({node(), part_2_2, + nodes(), {pid2,Pid2}}) + end) + end), + {_Pid1, yes} = rpc:call(Res1, ?MODULE, start_resolver, [Name, Resolver]), @@ -2459,15 +2599,6 @@ monitored_by_node(Trace, Servers) -> M <- ML, lists:member(M, Servers)]). -%% Runs on a node in Part2 -part_2_2(Config, Part1, Part2, NameResolvers) -> - make_partition(Config, Part1, Part2), - lists:foreach - (fun({Name, Resolver}) -> - {Pid2, yes} = start_resolver(Name, Resolver), - trace_message({node(), part_2_2, nodes(), {pid2,Pid2}}) - end, NameResolvers). - resolve_first(name, Pid1, _Pid2) -> Pid1. @@ -2538,7 +2669,7 @@ mon_by_servers(Proc) -> {monitored_by, ML} = process_info(Proc, monitored_by), {monitors_2levels, lists:append([ML | - [begin + [begin {monitored_by, MML} = rpc:call(node(M), erlang, process_info, @@ -2573,13 +2704,22 @@ leftover_name(Config) when is_list(Config) -> Part2 = [A2, Z2], NoResolver = {no_module, resolve_none}, Resolver = fun contact_a_2/3, - rpc_cast(A2, - ?MODULE, part_2_2, [Config, - Part1, - Part2, - [{Name, NoResolver}, - {ResName, Resolver}]]), - ?UNTIL(is_ready_partition(Config)), + + PartCtrlr = setup_partitions(Config, [Part1, Part2]), + + erpc:call( + PartCtrlr, + fun () -> + erpc:call( + A2, + fun () -> + lists:foreach( + fun({TheName, TheResolver}) -> + {Pid2, yes} = start_resolver(TheName, TheResolver), + trace_message({node(), part_2_2, nodes(), {pid2,Pid2}}) + end, [{Name, NoResolver}, {ResName, Resolver}]) + end) + end), %% resolved_name is resolved to run on a_2, an insert operation is %% sent to n_1. The resolver function halts a_2, but the nodedown @@ -2611,6 +2751,7 @@ leftover_name(Config) when is_list(Config) -> ?UNTIL(OrigNames =:= global:registered_names()), write_high_level_trace(Config), stop_nodes(Cps), + stop_partition_controller(PartCtrlr), init_condition(Config), ok. @@ -2776,9 +2917,21 @@ external_nodes(Config) when is_list(Config) -> %% Two partitions: [test_server] and [b, c]. %% c registers an external name on b - rpc_cast(NodeB, ?MODULE, part_ext, - [Config, node(), NodeC, Name]), - ?UNTIL(is_ready_partition(Config)), + + PartCtrlr = setup_partitions(Config, [[node()], [NodeB, NodeC]]), + + erpc:call( + PartCtrlr, + fun () -> + erpc:call( + NodeB, + fun () -> + Pid = spawn(NodeC, fun() -> cnode_proc(NodeB) end), + Pid ! {register, self(), Name}, + receive {Pid, Reply} -> yes = Reply end, + erpc:call(NodeC, erlang, register, [Name, Pid]) + end) + end), pong = net_adm:ping(NodeB), ?UNTIL([NodeB, NodeC] =:= lists:sort(nodes())), @@ -2842,6 +2995,7 @@ external_nodes(Config) when is_list(Config) -> ?UNTIL(length(get_ext_names()) =:= 1), stop_node(NodeC), + stop_partition_controller(PartCtrlr), ?UNTIL(length(get_ext_names()) =:= 0), init_condition(Config), @@ -2850,15 +3004,6 @@ external_nodes(Config) when is_list(Config) -> get_ext_names() -> gen_server:call(global_name_server, get_names_ext, infinity). -%% Runs at B -part_ext(Config, Main, C, Name) -> - make_partition(Config, [Main], [node(), C]), - ThisNode = node(), - Pid = erlang:spawn(C, fun() -> cnode_proc(ThisNode) end), - Pid ! {register, self(), Name}, - receive {Pid, Reply} -> yes = Reply end, - rpc:call(C, erlang, register, [Name, Pid]). - cnode_links(Pid) -> Pid ! {links, self()}, receive @@ -2916,25 +3061,29 @@ many_nodes(Config) when is_list(Config) -> Nodes = lists:sort(?NODES), wait_for_ready_net(Nodes, Config), - Dir = proplists:get_value(priv_dir, Config), - GoFile = filename:join([Dir, "go.txt"]), - file:delete(GoFile), - - CpsFiles = [{N, filename:join([Dir, atom_to_list(N)++".node"])} || - N <- Cps], - IsoFun = - fun({N, File}) -> - file:delete(File), - rpc_cast(N, ?MODULE, isolated_node, [File, GoFile, Cps, Config]) - end, - lists:foreach(IsoFun, CpsFiles), + %% All nodes isolated not connected to any other (visible) nodes... + Partitions = [[node()] | lists:map(fun (Node) -> [Node] end, Cps)], + + PartCtrlr = setup_partitions(Config, Partitions), - all_nodes_files(CpsFiles, "isolated", Config), Time = msec(), - sync_until(), - erlang:display(ready_to_go), - touch(GoFile, "go"), - all_nodes_files(CpsFiles, "done", Config), + + erpc:call( + PartCtrlr, + fun () -> + OkRes = lists:map(fun (_) -> {ok, ok} end, Cps), + OkRes = erpc:multicall( + Cps, + fun () -> + lists:foreach(fun(N) -> + _ = net_adm:ping(N) + end, shuffle(Cps)), + ?UNTIL((Cps -- get_known(node())) =:= []), + ok + end), + ok + end), + Time2 = msec(), lists:foreach(fun(N) -> pong = net_adm:ping(N) end, Cps), @@ -2943,12 +3092,10 @@ many_nodes(Config) when is_list(Config) -> write_high_level_trace(Config), % The test succeeded, but was it slow? - lists:foreach(fun({_N, File}) -> file:delete(File) end, CpsFiles), - file:delete(GoFile), - ?UNTIL(OrigNames =:= global:registered_names()), write_high_level_trace(Config), stop_nodes(Cps), + stop_partition_controller(PartCtrlr), init_condition(Config), Diff = Time2 - Time, Return = lists:flatten(io_lib:format("~w nodes took ~w ms", @@ -2971,29 +3118,6 @@ node_rel(From, To) -> 1 -> Last end} || N <- NodeNumbers]. -isolated_node(File, GoFile, Nodes, Config) -> - Ns = lists:sort(Nodes), - exit(erlang:whereis(user), kill), - touch(File, "start_isolated"), - NodesList = nodes(), - append_to_file(File, [{nodes,Nodes},{nodes_list,NodesList}]), - Replies = - lists:map(fun(N) -> _ = erlang:disconnect_node(N) end, NodesList), - append_to_file(File, {replies,Replies}), - ?UNTIL(begin - Known = get_known(node()), - append_to_file(File, {known,Known}), - Known =:= [node()] - end), - touch(File, "isolated"), - sync_until(File), - file_contents(GoFile, "go", Config, File), - touch(File, "got_go"), - lists:foreach(fun(N) -> _ = net_adm:ping(N) end, shuffle(Nodes)), - touch(File, "pinged"), - ?UNTIL((Ns -- get_known(node())) =:= []), - touch(File, "done"). - touch(File, List) -> ok = file:write_file(File, list_to_binary(List)). @@ -3002,11 +3126,6 @@ append_to_file(File, Term) -> ok = file:write(Fd, io_lib:format("~p.~n", [Term])), ok = file:close(Fd). -all_nodes_files(CpsFiles, ContentsList, Config) -> - lists:all(fun({_N,File}) -> - file_contents(File, ContentsList, Config) - end, CpsFiles). - file_contents(File, ContentsList, Config) -> file_contents(File, ContentsList, Config, no_log_file). @@ -3030,14 +3149,6 @@ file_contents(File, ContentsList, Config, LogFile) -> end end). -sync_until() -> - sync_until(no_log_file). - -sync_until(LogFile) -> - Time = ?UNTIL_LOOP - (msec(erlang:timestamp()) rem ?UNTIL_LOOP), - catch append_to_file(LogFile, {sync_until, Time}), - timer:sleep(Time). - shuffle(L) -> [E || {_, E} <- lists:keysort(1, [{rand:uniform(), E} || E <- L])]. @@ -3518,109 +3629,11 @@ from(_H, []) -> []. other(A, [A, _B]) -> A; other(_, [_A, B]) -> B. - -%% this one runs at cp2 -part1(Config, Main, Cp1, Cp3) -> - case catch begin - make_partition(Config, [Main, Cp1], [node(), Cp3]), - {_Pid, yes} = start_proc(test2), - {_Pid2, yes} = start_proc(test4) - end of - {_, yes} -> ok; % w("ok", []); - {'EXIT', _R} -> - ok - %% w("global_SUITE line:~w: ~p", [?LINE, _R]) - end. - -%% Runs at Cp2 -part1_5(Config, Main, Cp1, Cp3) -> - case catch begin - make_partition(Config, [Main, Cp1], [node(), Cp3]), - {_Pid1, yes} = start_proc_basic(name12), - {_Pid2, yes} = - rpc:call(Cp3, ?MODULE, start_proc_basic, [name03]) - end of - {_, yes} -> ok; % w("ok", []); - {'EXIT', _R} -> - ok - %% w("global_SUITE line:~w: ~p", [?LINE, _R]) - end. - w(X,Y) -> {ok, F} = file:open("cp2.log", [write]), io:format(F, X, Y), file:close(F). -%% this one runs on one node in Part2 -%% The partition is ready when is_ready_partition(Config) returns (true). -make_partition(Config, Part1, Part2) -> - Dir = proplists:get_value(priv_dir, Config), - Ns = [begin - Name = lists:concat([atom_to_list(N),"_",msec(),".part"]), - File = filename:join([Dir, Name]), - file:delete(File), - rpc_cast(N, ?MODULE, mk_part_node, [File, Part, Config], File), - {N, File} - end || Part <- [Part1, Part2], N <- Part], - all_nodes_files(Ns, "done", Config), - lists:foreach(fun({_N,File}) -> file:delete(File) end, Ns), - PartFile = make_partition_file(Config), - touch(PartFile, "done"). - -%% The node signals its success by touching a file. -mk_part_node(File, MyPart0, Config) -> - touch(File, "start"), % debug - MyPart = lists:sort(MyPart0), - ?UNTIL(is_node_in_part(File, MyPart)), - touch(File, "done"). - -%% The calls to append_to_file are for debugging. -is_node_in_part(File, MyPart) -> - lists:foreach(fun(N) -> - _ = erlang:disconnect_node(N) - end, nodes() -- MyPart), - case {(Known = get_known(node())) =:= MyPart, - (Nodes = lists:sort([node() | nodes()])) =:= MyPart} of - {true, true} -> - %% Make sure the resolvers have been terminated, - %% otherwise they may pop up and send some message. - %% (This check is probably unnecessary.) - case element(5, global:info()) of - [] -> - true; - Rs -> - erlang:display({is_node_in_part, resolvers, Rs}), - trace_message({node(), is_node_in_part, Rs}), - append_to_file(File, {erlang:timestamp(), Known, Nodes, Rs}), - false - end; - _ -> - append_to_file(File, {erlang:timestamp(), Known, Nodes}), - false - end. - -is_ready_partition(Config) -> - File = make_partition_file(Config), - file_contents(File, "done", Config), - file:delete(File), - true. - -make_partition_file(Config) -> - Dir = proplists:get_value(priv_dir, Config), - filename:join([Dir, atom_to_list(make_partition_done)]). - -%% this one runs at cp3 -part2(Config, Parent, Main, Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6) -> - make_partition(Config, [Main, Cp0, Cp1, Cp2], [Cp3, Cp4, Cp5, Cp6]), - start_procs(Parent, Cp4, Cp5, Cp6, Config). - -part3(Config, Parent, Main, Cp0, Cp1, Cp2, Cp3, Cp4, Cp5, Cp6) -> - make_partition(Config, [Main, Cp0, Cp1, Cp2], [Cp3, Cp4, Cp5, Cp6]), - start_procs(Parent, Cp4, Cp5, Cp6, Config), - %% Make Cp6 alone - rpc_cast(Cp5, ?MODULE, crash, [12000]), - rpc_cast(Cp6, ?MODULE, alone, [Cp0, Cp3]). - start_procs(Parent, N1, N2, N3, Config) -> S1 = lists:sort([N1, N2, N3]), ?UNTIL(begin @@ -3715,13 +3728,11 @@ init_proc_basic(Parent, Name) -> Parent ! {self(),X}, loop(). -single_node(Time, Node, Config) -> - exit(erlang:whereis(user), kill), - lists:foreach(fun(N) -> _ = erlang:disconnect_node(N) end, nodes()), - ?UNTIL(get_known(node()) =:= [node()]), +single_node(Time, Node) -> spawn(?MODULE, init_2, []), - sleep(Time - msec()), - net_adm:ping(Node). + timer:sleep(Time - msec()), + _ = net_adm:ping(Node), + ok. init_2() -> register(single_name, self()), @@ -3780,8 +3791,6 @@ sreq(Pid, Msg) -> receive {Ref, X} -> X end. alone(N1, N2) -> - lists:foreach(fun(Node) -> true = erlang:disconnect_node(Node) end, - nodes()), ct:sleep(12000), net_adm:ping(N1), net_adm:ping(N2), @@ -4212,11 +4221,6 @@ wait_for_exit_fast(Pid) -> ok end. -sleep(Time) when Time > 0 -> - ct:sleep(Time); -sleep(_Time) -> - ok. - check_everywhere(Nodes, Name, Config) -> ?UNTIL(begin case rpc:multicall(Nodes, global, whereis_name, [Name]) of @@ -4386,16 +4390,136 @@ rpc_cast(Node, Module, Function, Args) -> {_,pong,Node}= {node(),net_adm:ping(Node),Node}, rpc:cast(Node, Module, Function, Args). -rpc_cast(Node, Module, Function, Args, File) -> - case net_adm:ping(Node) of - pong -> - rpc:cast(Node, Module, Function, Args); - Else -> - append_to_file(File, {erlang:timestamp(), {rpc_cast, Node, Module, Function, - Args, Else}}) - %% Maybe we should crash, but it probably doesn't matter. +global_known(Node) -> + gen_server:call({global_name_server, Node}, get_known, infinity). + +global_known() -> + global_known(node()). + +disconnect(Node) -> + erlang:disconnect_node(Node), + wait_until(fun () -> not lists:member(Node, global_known()) end), + ok. + +disconnect_nodes(HiddenCtrlNode, NodeA, NodeB) -> + Nodes = [node()|nodes()], + ok = erpc:call(HiddenCtrlNode, + fun () -> + ok = erpc:call(NodeA, + fun () -> + disconnect(NodeB) + end), + ok = erpc:call(NodeB, + fun () -> + disconnect(NodeA) + end), + %% Try to ensure 'lost_connection' messages + %% have been handled (see comment in + %% create_partitions)... + lists:foreach(fun (N) -> + _ = global_known(N) + end, Nodes), + ok + end). + +create_partitions(PartitionsList) -> + AllNodes = lists:sort(lists:flatten(PartitionsList)), + + %% Take down all connections on all nodes... + AllOk = lists:map(fun (_) -> {ok, ok} end, AllNodes), + io:format("Disconnecting all nodes from eachother...", []), + AllOk = erpc:multicall( + AllNodes, + fun () -> + lists:foreach(fun (N) -> + erlang:disconnect_node(N) + end, nodes()), + wait_until(fun () -> [] == global_known() end), + ok + end, 5000), + %% Here we know that all 'lost_connection' messages that will be + %% sent by global name servers due to these disconnects have been + %% sent, but we don't know that all of them have been received and + %% handled. By communicating with all global name servers one more + %% time it is very likely that all of them have been received and + %% handled (however, not guaranteed). If 'lost_connection' messages + %% are received after we begin to set up the partitions, the + %% partitions may lose connection with some of its nodes. + lists:foreach(fun (N) -> [] = global_known(N) end, AllNodes), + + %% Set up fully connected partitions... + io:format("Connecting partitions...", []), + lists:foreach( + fun (Partition) -> + Part = lists:sort(Partition), + PartOk = lists:map(fun (_) -> {ok, ok} end, Part), + PartOk = erpc:multicall( + Part, + fun () -> + wait_until( + fun () -> + ConnNodes = Part -- [node()|nodes()], + if ConnNodes == [] -> + true; + true -> + lists:foreach( + fun (N) -> + net_kernel:connect_node(N) + end, ConnNodes), + false + end + end), + ok + + end, 5000) + end, PartitionsList), + ok. + +setup_partitions(PartCtrlr, PartList) when is_atom(PartCtrlr) -> + ok = erpc:call(PartCtrlr, fun () -> create_partitions(PartList) end), + io:format("Partitions successfully setup:~n", []), + lists:foreach(fun (Part) -> + io:format("~p~n", [Part]) + end, + PartList), + ok; +setup_partitions(Config, PartList) when is_list(Config) -> + PartCtrlr = start_partition_controller(Config), + setup_partitions(PartCtrlr, PartList), + PartCtrlr. + +start_partition_controller(Config) when is_list(Config) -> + {ok, PartCtrlr} = start_hidden_node(part_ctrlr, Config), + PartCtrlr. + +stop_partition_controller(PartCtrlr) -> + stop_node(PartCtrlr). + +prevent_overlapping_partitions() -> + case application:get_env(kernel, prevent_overlapping_partitions) of + {ok, true} -> + true; + _ -> + false end. +cast_line([]) -> + ok; +cast_line([N|Ns]) when N == node() -> + cast_line(Ns); +cast_line([N|Ns]) -> + erpc:cast(N, fun () -> cast_line(Ns) end). + +wait_until(F) -> + case catch F() of + true -> + ok; + _ -> + receive after 10 -> ok end, + wait_until(F) + end. + + %% The emulator now ensures that the node has been removed from %% nodes(). rpc_disconnect_node(Node, DisconnectedNode, Config) -> diff --git a/lib/kernel/test/pg_SUITE.erl b/lib/kernel/test/pg_SUITE.erl index 3a69e33bcf..9c84e51c43 100644 --- a/lib/kernel/test/pg_SUITE.erl +++ b/lib/kernel/test/pg_SUITE.erl @@ -721,7 +721,7 @@ controller(Name, Scope, Self) -> Pa = filename:dirname(code:which(?MODULE)), Pa2 = filename:dirname(code:which(pg)), Args = lists:concat(["-setcookie ", erlang:get_cookie(), - "-connect_all false -kernel dist_auto_connect never -noshell -pa ", Pa, " -pa ", Pa2]), + " -connect_all false -kernel dist_auto_connect never -noshell -pa ", Pa, " -pa ", Pa2]), {ok, Node} = test_server:start_node(Name, peer, [{args, Args}]), case rpc:call(Node, ?MODULE, control, [Scope], 5000) of {badrpc, nodedown} -> diff --git a/lib/kernel/test/rpc_SUITE.erl b/lib/kernel/test/rpc_SUITE.erl index 2cb27021da..6e17d4390b 100644 --- a/lib/kernel/test/rpc_SUITE.erl +++ b/lib/kernel/test/rpc_SUITE.erl @@ -92,13 +92,13 @@ call(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), %% Note. First part of nodename sets response delay in seconds {ok, N1} = test_server:start_node('3_rpc_SUITE_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N2} = test_server:start_node('1_rcp_SUITE_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N3} = test_server:start_node('4_rcp_SUITE_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N4} = test_server:start_node('8_rcp_SUITE_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), ok = io:format("~p~n", [[N1, N2, N3]]), {hej,_,N1} = rpc:call(N1, ?MODULE, f, []), {hej,_,N2} = rpc:call(N2, ?MODULE, f, [], 2000), @@ -170,13 +170,13 @@ block_call(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), %% Note. First part of nodename sets response delay in seconds {ok, N1} = test_server:start_node('3_rpc_SUITE_block_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N2} = test_server:start_node('1_rcp_SUITE_block_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N3} = test_server:start_node('4_rcp_SUITE_block_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N4} = test_server:start_node('8_rcp_SUITE_block_call', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), ok = io:format("~p~n", [[N1, N2, N3]]), {hej,_,N1} = rpc:block_call(N1, ?MODULE, f, []), {hej,_,N2} = rpc:block_call(N2, ?MODULE, f, [], 2000), @@ -196,9 +196,9 @@ multicall(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), %% Note. First part of nodename sets response delay in seconds {ok, N1} = test_server:start_node('3_rpc_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N2} = test_server:start_node('1_rcp_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), ok = io:format("~p~n", [[N1, N2]]), {[{hej,_,N1},{hej,_,N2}],[]} = rpc:multicall([N1, N2], ?MODULE, f, []), @@ -212,13 +212,13 @@ multicall_timeout(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), %% Note. First part of nodename sets response delay in seconds {ok, N1} = test_server:start_node('11_rpc_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N2} = test_server:start_node('8_rpc_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N3} = test_server:start_node('5_rpc_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N4} = test_server:start_node('2_rcp_SUITE_multicall', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), ok = io:format("~p~n", [[N1, N2]]), {[{hej,_,N3},{hej,_,N4}],[N1, N2]} = rpc:multicall([N3, N1, N2, N4], ?MODULE, f, [], 6000), @@ -258,9 +258,9 @@ multicall_reqtmo(Config) when is_list(Config) -> multicall_dies(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), {ok, N1} = test_server:start_node('rpc_SUITE_multicall_dies_1', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), {ok, N2} = test_server:start_node('rcp_SUITE_multicall_dies_2', slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), Nodes = [N1, N2], %% {[{badrpc, {'EXIT', normal}}, {badrpc, {'EXIT', normal}}], []} = @@ -312,10 +312,10 @@ multicall_node_dies(Config) when is_list(Config) -> do_multicall_2_nodes_dies(Mod, Func, Args) -> ok = io:format("~p:~p~p~n", [Mod, Func, Args]), PA = filename:dirname(code:which(?MODULE)), - {ok, N1} = test_server:start_node('rpc_SUITE_multicall_node_dies_1', slave, - [{args, "-pa " ++ PA}]), - {ok, N2} = test_server:start_node('rcp_SUITE_multicall_node_dies_2', slave, - [{args, "-pa " ++ PA}]), + {ok, N1} = test_server:start_node('rpc_SUITE_multicall_node_dies_1', peer, + [{args, "-connect_all false -pa " ++ PA}]), + {ok, N2} = test_server:start_node('rcp_SUITE_multicall_node_dies_2', peer, + [{args, "-connect_all false -pa " ++ PA}]), Nodes = [N1, N2], case {Mod, Func, rpc:multicall(Nodes, Mod, Func, Args)} of {_, _, {[], Nodes}} -> @@ -338,7 +338,7 @@ do_multicall_2_nodes_dies(Mod, Func, Args) -> called_dies(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), {ok, N} = test_server:start_node(rpc_SUITE_called_dies, slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), %% rep(fun (Tag, Call, Args) -> {Tag,{badrpc,{'EXIT',normal}}} = @@ -512,7 +512,7 @@ node_rep(Fun, Name, PA, M, F, A) -> node_rep_call(Tag, Call, Args, Fun, Name0, PA) -> Name = list_to_atom(Name0 ++ "_" ++ atom_to_list(Tag)), {ok, N} = test_server:start_node(Name, slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), Fun(Call, [N|Args]), catch test_server:stop_node(N), ok. @@ -522,7 +522,7 @@ called_throws(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), %% {ok, N} = test_server:start_node(rpc_SUITE_called_throws, slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), %% rep(fun (Tag, Call, Args) -> {Tag,up} = @@ -541,7 +541,7 @@ called_throws(Config) when is_list(Config) -> call_benchmark(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), {ok, Node} = test_server:start_node(rpc_SUITE_call_benchmark, slave, - [{args, "-pa " ++ PA}]), + [{args, "-connect_all false -pa " ++ PA}]), Iter = case erlang:system_info(modified_timing_level) of undefined -> 10000; _ -> 500 %Modified timing - spawn is slower @@ -569,7 +569,7 @@ do_call_benchmark(Node, I, M) -> async_call(Config) when is_list(Config) -> %% Note: First part of nodename sets response delay in seconds. PA = filename:dirname(code:which(?MODULE)), - NodeArgs = [{args,"-pa "++ PA}], + NodeArgs = [{args,"-connect_all false -pa "++ PA}], {ok,Node1} = test_server:start_node('1_rpc_SUITE_call', slave, NodeArgs), {ok,Node2} = test_server:start_node('10_rpc_SUITE_call', slave, NodeArgs), {ok,Node3} = test_server:start_node('20_rpc_SUITE_call', slave, NodeArgs), @@ -602,17 +602,17 @@ call_against_old_node(Config) -> end. multicall_mix(Config) -> - {ok, Node1} = start_node(Config), - {ok, Node2} = start_node(Config), + {ok, Node1} = start_peer_node(Config), + {ok, Node2} = start_peer_node(Config), {Node3, OldNodeTest} = case start_22_node(Config) of {ok, N3} -> {N3, true}; _ -> - {ok, N3} = start_node(Config), + {ok, N3} = start_peer_node(Config), {N3, false} end, - {ok, Node4} = start_node(Config), - {ok, Node5} = start_node(Config), + {ok, Node4} = start_peer_node(Config), + {ok, Node5} = start_peer_node(Config), stop_node(Node2), [] = flush([]), @@ -875,6 +875,14 @@ cast_old_against_new_test([Node22], [NodeCurr]) -> %%% Utility functions. %%% +start_peer_node(Config) -> + Name = list_to_atom(atom_to_list(?MODULE) + ++ "-" ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" ++ integer_to_list(erlang:system_time(second)) + ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), + Pa = filename:dirname(code:which(?MODULE)), + test_server:start_node(Name, peer, [{args, "-pa " ++ Pa}]). + start_node(Config) -> Name = list_to_atom(atom_to_list(?MODULE) ++ "-" ++ atom_to_list(proplists:get_value(testcase, Config)) |