diff options
Diffstat (limited to 'storage/ndb/test/ndbnet/testError.run')
| -rw-r--r-- | storage/ndb/test/ndbnet/testError.run | 266 |
1 files changed, 266 insertions, 0 deletions
diff --git a/storage/ndb/test/ndbnet/testError.run b/storage/ndb/test/ndbnet/testError.run new file mode 100644 index 00000000000..3cce489a3da --- /dev/null +++ b/storage/ndb/test/ndbnet/testError.run @@ -0,0 +1,266 @@ +# +# file : test/ndbnet/testError.run +# usage: perl testError.run +# +# you need to have $NDB_TOP/lib/perl5 on search path $PERL5LIB +# or else write perl -I$NDB_TOP/lib/perl5 test1.run +# +# The database is specified by the $NDB_DATABASE environment variable +# +# method names and argument style will change slightly. +# +# + +use strict; +use NDB::Run; + +my $env = NDB::Run->getenv; +my $log = $env->getlog; +$log->setpart(time => 1, line => 0); +$log->setprio("info"); + +my $database = $ENV{NDB_DATABASE}; +my $api_cmd = $ENV{API_CMD}; +$log->put("start test database=$database"); +$env->init or $log->push("init failed")->fatal; + +my $db = $env->getdb($database) or $log->push->fatal; +my $mgm = $db->getnode(1) or $log->push->fatal; +my $api = $db->getnode(4) or $log->push->fatal; + +my @dbnode = (); # array of db nodes indexed 2..3 +for my $i (2..3) { + $dbnode[$i] = $db->getnode($i) or $log->push->fatal; +} + +# list of db nodes and errors to insert +my @errors = ( # array of array refs + [ 2, 9998 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37017 ], + [ 2, 9998 ], + [ 3, 9998 ], + [ 3, 9998 ], + [ 2, 9998 ], + [ 3, 9998 ], + [ 3, 9998 ], + [ 2, 38002 ], + [ 2, 38002 ], + [ 2, 8002 ], + [ 3, 8029 ], + [ 2, 8030 ], + [ 2, 8031 ], + [ 3, 8020 ], + [ 2, 8021 ], + [ 3, 8022 ], + [ 2, 8023 ], + [ 3, 8025 ], + [ 2, 8027 ], + [ 2, 38002 ], + [ 3, 38029 ], + [ 2, 38030 ], + [ 2, 38031 ], + [ 3, 38020 ], + [ 2, 38021 ], + [ 3, 38022 ], + [ 2, 38023 ], + [ 3, 38025 ], + [ 2, 38027 ], + [ 2, 48002 ], + [ 3, 48029 ], + [ 2, 48030 ], + [ 2, 48031 ], + [ 3, 48020 ], + [ 2, 48021 ], + [ 3, 48022 ], + [ 2, 48023 ], + [ 3, 48025 ], + [ 2, 48027 ], + [ 2, 9999 ], + [ 3, 9999 ], + [ 3, 9999 ], + [ 2, 9998 ], + [ 3, 9998 ], + [ 3, 9998 ], + [ 2, 9998 ], + [ 3, 9998 ], + [ 3, 9998 ], + [ 3, 37000 ], + [ 2, 37001 ], + [ 2, 37002 ], + [ 2, 37003 ], + [ 2, 47005 ], + [ 2, 47006 ], + [ 2, 47007 ], + [ 2, 47008 ], + [ 2, 45000 ], + [ 2, 37005 ], + [ 2, 37006 ], + [ 2, 37007 ], + [ 2, 37008 ], + [ 2, 35000 ], + [ 2, 37009 ], + [ 2, 37010 ], + [ 2, 37013 ], + [ 2, 37014 ], + [ 2, 37015 ], + [ 2, 37016 ], + [ 2, 37017 ], + [ 2, 37018 ], + [ 2, 37019 ], + [ 2, 47020 ], + [ 2, 37020 ], + [ 2, 48000 ], + [ 2, 38000 ], + [ 2, 48001 ], + [ 2, 38001 ], + [ 2, 45001 ], + [ 2, 35001 ], +); + +$db->kill; +$db->start({init_rm=>1}) or $log->push->fatal; +sleep 10; + +# should be option (or default) in $db->start +sub wait_until_started { + my $local_cnt = 60; + while (--$local_cnt > 0) { + sleep 2; + my $ret = $mgm->write("all status", { wait => 3 }); + $ret or $log->fatal; + my $output = $ret->{output}; + if ($output =~ /\bstarted\b(.|\n)*started\b/i) { + $log->put("*** db is started ***")->info; + return; + } + if ($output =~ /\bno.contact\b(.|\n)*no.contact\b/i) { + print "NDBT_ProgramExit: 1 - test error\n"; + $log->put("*** db is dead ***")->fatal; + } + } + print "NDBT_ProgramExit: 1 - test error\n"; + $log->put("*** node recovery failed ***")->fatal; +} + +sub getdbstatus { + my $ret = $mgm->write("all status", { wait => 3 }); + $ret or return undef; + my $output = $ret->{output}; + my @status = (); # indexed 2..3 + for my $i (2..3) { + my $s = "Unknown"; + if ($output =~ /Node\s*$i\s*:\s*(\w+)/i) { + $s = $1; + }#if + $status[$i] = lc $s; + }#for + return \@status; +} + +# count elapsed time +my $lasttime; +sub settime { $lasttime = time }; +sub gettime { return time - $lasttime }; + +wait_until_started(); +$api->start({run=>$api_cmd}); +sleep 10; + +# loop over error inserts +for my $e (@errors) { +for my $loop (1..7) { + my $i = $e->[0]; # db node number 2..3 + my $c = $e->[1]; # error code + my $dead_node_id = $i; + my $dbnode = $dbnode[$i]; + my $two = 2; + my $three = 3; + my $kill_no = 9998; + + $log->put("insert error $c")->push($dbnode)->notice; + + # insert error + if ($c eq $kill_no) { + $dbnode->kill + or $log->put("Kill 1 failed")->fatal; + } else { + $mgm->write("$i error $c") + or $log->put("insert error fault")->fatal; + }#if + + # after a few seconds check that node is dead + settime(); + loop: { + gettime() <= 300 + or $log->put("db node $i refuses to die")->fatal; + my $status = getdbstatus() + or $log->put("getdbstatus error")->fatal; + + if (($status->[$two] eq 'no') && ($status->[$three] eq 'no')) { + print "NDBT_ProgramExit: 1 - test error\n"; + $log->put("*** db is dead ***")->fatal; + }#if + if ($c < 10000) { + if ($status->[$i] ne 'no') { # ...contact + sleep 2; + redo loop; + }#if + $dead_node_id = $i; + } else { + if (($status->[$two] eq 'no') && + ($status->[$three] eq 'started')) { + $dead_node_id = 2; + } else { + if (($status->[$three] eq 'no') && + ($status->[$two] eq 'started')) { + $dead_node_id = 3; + } else { + sleep 2; + redo loop; + }#if + }#if + }#if + }#loop + + my $dead_node = $dbnode[$dead_node_id]; + # have to even check the process is gone + sleep 5; + if ($dead_node->stat ne "down") { + $log->put("ndb did not die, kill it")->push($dead_node)->warn; + $dead_node->kill + or $log->put("Kill 2 failed")->fatal; + }#if + + $log->put("node $dead_node_id is dead")->notice; + + # start the failed node + $dead_node->start + or $log->put("Start ndb node failed")->fatal; + + wait_until_started(); + $log->put("node $dead_node_id is up again")->notice; + + # check test pgm is running + my $stat = $api->stat + or $log->put("api->stat failed")->fatal; + if ($stat ne "up") { + print "NDBT_ProgramExit: 1 - test error\n"; + $db->kill; + $log->put("flexBench has crashed")->fatal; + }#if +}#for +}#for +print "NDBT_ProgramExit: 0 - test ready\n"; +$db->kill; + +# vim: set sw=4: |