blob: f7a5eef3494fcd650a55dad0bc9553dcc9529ec6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
# Node recovery killing 1 node out of 4 at the time and waiting for recover
use strict;
use NDB::Run;
my $env = NDB::Run->getenv;
my $log = $env->getlog;
$log->setpart(time => 1, line => 0);
$log->setprio("info");
my $database = $ENV{NDB_DATABASE};
$log->put("start test database=$database");
$env->init or $log->push("init failed")->fatal;
my $db = $env->getdb($database) or $log->push->fatal;
my $mgm = $db->getnode(1) or $log->push->fatal;
my $db1 = $db->getnode(2) or $log->push->fatal;
my $db2 = $db->getnode(3) or $log->push->fatal;
my $db3 = $db->getnode(4) or $log->push->fatal;
my $db4 = $db->getnode(5) or $log->push->fatal;
my $api = $db->getnode(6) or $log->push->fatal;
$db->kill;
$db->start({init_rm=>1}) or $log->push->fatal;
sleep 10;
# should be option (or default) in $db->start
sub wait_until_started {
my $local_cnt = 100;
while (--$local_cnt > 0) {
sleep 2;
my $ret = $mgm->write("all status", { wait => 2 });
$ret or $log->fatal;
my $output = $ret->{output};
if ($output =~ /((.|\n)*\bstarted\b(.|\n)*){4}/i) {
# if all 4 nodes started
$log->put("*** db is started ***")->info;
return;
}
if ($output =~ /((.|\n)*\bno.contact\b(.|\n)*){4}/i) {
#if all 4 nodes no contact
print "NDBT_ProgramExit: 1 - test error\n";
$log->put("*** db is dead ***")->fatal;
}
}
print "NDBT_ProgramExit: 1 - test error\n";
$log->put("*** node recovery failed ***")->fatal;
}
my $cnt = 0;
wait_until_started();
$api->start({run=>"flexBench -t 32 -l 800"});
while (1) {
wait_until_started();
sleep 10;
my $id = (++$cnt % 4);
my $dbx = 0;
if ($id eq 0) {
$dbx = $db1;
}
elsif ($id eq 1) {
$dbx = $db2;
}
elsif ($id eq 2) {
$dbx = $db3;
}
else {
$dbx = $db4;
}
$dbx->kill or $log->fatal;
sleep 10;
$dbx->start; # start the node
}
$db->kill;
# vim: set sw=4:
|
