diff options
Diffstat (limited to 'mysql-test/lib')
| -rw-r--r-- | mysql-test/lib/mtr_cases.pl | 365 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_im.pl | 761 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_io.pl | 66 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_misc.pl | 40 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_process.pl | 675 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_report.pl | 137 | ||||
| -rw-r--r-- | mysql-test/lib/mtr_stress.pl | 178 |
7 files changed, 1883 insertions, 339 deletions
diff --git a/mysql-test/lib/mtr_cases.pl b/mysql-test/lib/mtr_cases.pl index 650fb79155d..bb92730444c 100644 --- a/mysql-test/lib/mtr_cases.pl +++ b/mysql-test/lib/mtr_cases.pl @@ -5,10 +5,13 @@ # same name. use File::Basename; +use IO::File(); use strict; sub collect_test_cases ($); -sub collect_one_test_case ($$$$$$); +sub collect_one_test_case ($$$$$$$); + +sub mtr_options_from_test_file($$); ############################################################################## # @@ -37,44 +40,114 @@ sub collect_test_cases ($) { opendir(TESTDIR, $testdir) or mtr_error("Can't open dir \"$testdir\": $!"); + # ---------------------------------------------------------------------- + # Disable some tests listed in disabled.def + # ---------------------------------------------------------------------- + my %disabled; + if ( open(DISABLED, "$testdir/disabled.def" ) ) + { + while ( <DISABLED> ) + { + chomp; + if ( /^\s*(\S+)\s*:\s*(.*?)\s*$/ ) + { + $disabled{$1}= $2; + } + } + close DISABLED; + } + if ( @::opt_cases ) { foreach my $tname ( @::opt_cases ) { # Run in specified order, no sort - $tname= basename($tname, ".test"); - my $elem= "$tname.test"; - if ( ! -f "$testdir/$elem") + my $elem= undef; + my $component_id= undef; + + # Get rid of directory part (path). Leave the extension since it is used + # to understand type of the test. + + $tname = basename($tname); + + # Check if the extenstion has been specified. + + if ( mtr_match_extension($tname, "test") ) + { + $elem= $tname; + $tname=~ s/\.test$//; + $component_id= 'mysqld'; + } + elsif ( mtr_match_extension($tname, "imtest") ) + { + $elem= $tname; + $tname =~ s/\.imtest$//; + $component_id= 'im'; + } + + # If target component is known, check that the specified test case + # exists. + # + # Otherwise, try to guess the target component. + + if ( $component_id ) + { + if ( ! -f "$testdir/$elem") + { + mtr_error("Test case $tname ($testdir/$elem) is not found"); + } + } + else { - mtr_error("Test case $tname ($testdir/$elem) is not found"); + my $mysqld_test_exists = -f "$testdir/$tname.test"; + my $im_test_exists = -f "$testdir/$tname.imtest"; + + if ( $mysqld_test_exists and $im_test_exists ) + { + mtr_error("Ambiguous test case name ($tname)"); + } + elsif ( ! $mysqld_test_exists and ! $im_test_exists ) + { + mtr_error("Test case $tname is not found"); + } + elsif ( $mysqld_test_exists ) + { + $elem= "$tname.test"; + $component_id= 'mysqld'; + } + elsif ( $im_test_exists ) + { + $elem= "$tname.imtest"; + $component_id= 'im'; + } } - collect_one_test_case($testdir,$resdir,$tname,$elem,$cases,{}); + + collect_one_test_case($testdir,$resdir,$tname,$elem,$cases,\%disabled, + $component_id); } closedir TESTDIR; } else { - # ---------------------------------------------------------------------- - # Disable some tests listed in disabled.def - # ---------------------------------------------------------------------- - my %disabled; - if ( open(DISABLED, "$testdir/disabled.def" ) ) - { - while ( <DISABLED> ) + foreach my $elem ( sort readdir(TESTDIR) ) { + my $component_id= undef; + my $tname= undef; + + if ($tname= mtr_match_extension($elem, 'test')) { - chomp; - if ( /^\s*(\S+)\s*:\s*(.*?)\s*$/ ) - { - $disabled{$1}= $2; - } + $component_id = 'mysqld'; } - close DISABLED; - } - - foreach my $elem ( sort readdir(TESTDIR) ) { - my $tname= mtr_match_extension($elem,"test"); - next if ! defined $tname; + elsif ($tname= mtr_match_extension($elem, 'imtest')) + { + $component_id = 'im'; + } + else + { + next; + } + next if $::opt_do_test and ! defined mtr_match_prefix($elem,$::opt_do_test); - collect_one_test_case($testdir,$resdir,$tname,$elem,$cases,\%disabled); + collect_one_test_case($testdir,$resdir,$tname,$elem,$cases,\%disabled, + $component_id); } closedir TESTDIR; } @@ -84,34 +157,38 @@ sub collect_test_cases ($) { if ( $::opt_reorder ) { - @$cases = sort { - if ( ! $a->{'master_restart'} and ! $b->{'master_restart'} ) - { - return $a->{'name'} cmp $b->{'name'}; - } - if ( $a->{'master_restart'} and $b->{'master_restart'} ) - { - my $cmp= mtr_cmp_opts($a->{'master_opt'}, $b->{'master_opt'}); - if ( $cmp == 0 ) - { - return $a->{'name'} cmp $b->{'name'}; - } - else - { - return $cmp; - } - } + my %sort_criteria; + my $tinfo; - if ( $a->{'master_restart'} ) - { - return 1; # Is greater - } - else - { - return -1; # Is less - } - } @$cases; + # Make a mapping of test name to a string that represents how that test + # should be sorted among the other tests. Put the most important criterion + # first, then a sub-criterion, then sub-sub-criterion, et c. + foreach $tinfo (@$cases) + { + my @this_criteria = (); + + # Append the criteria for sorting, in order of importance. + push(@this_criteria, join("!", sort @{$tinfo->{'master_opt'}}) . "~"); # Ending with "~" makes empty sort later than filled + push(@this_criteria, "ndb=" . ($tinfo->{'ndb_test'} ? "1" : "0")); + push(@this_criteria, "restart=" . ($tinfo->{'master_restart'} ? "1" : "0")); + push(@this_criteria, "big_test=" . ($tinfo->{'big_test'} ? "1" : "0")); + push(@this_criteria, join("|", sort keys %{$tinfo})); # Group similar things together. The values may differ substantially. FIXME? + push(@this_criteria, $tinfo->{'name'}); # Finally, order by the name + + $sort_criteria{$tinfo->{"name"}} = join(" ", @this_criteria); + } + + @$cases = sort { $sort_criteria{$a->{"name"}} cmp $sort_criteria{$b->{"name"}}; } @$cases; + +### For debugging the sort-order +# foreach $tinfo (@$cases) +# { +# print $sort_criteria{$tinfo->{"name"}}; +# print " -> \t"; +# print $tinfo->{"name"}; +# print "\n"; +# } } return $cases; @@ -125,13 +202,14 @@ sub collect_test_cases ($) { ############################################################################## -sub collect_one_test_case($$$$$$) { +sub collect_one_test_case($$$$$$$) { my $testdir= shift; my $resdir= shift; my $tname= shift; my $elem= shift; my $cases= shift; my $disabled=shift; + my $component_id= shift; my $path= "$testdir/$elem"; @@ -151,6 +229,7 @@ sub collect_one_test_case($$$$$$) { my $tinfo= {}; $tinfo->{'name'}= $tname; $tinfo->{'result_file'}= "$resdir/$tname.result"; + $tinfo->{'component_id'} = $component_id; push(@$cases, $tinfo); if ( $::opt_skip_test and defined mtr_match_prefix($tname,$::opt_skip_test) ) @@ -171,48 +250,54 @@ sub collect_one_test_case($$$$$$) { if ( $::opt_skip_rpl ) { $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No replication tests(--skip-rpl)"; return; } $tinfo->{'slave_num'}= 1; # Default, use one slave - # FIXME currently we always restart slaves - $tinfo->{'slave_restart'}= 1; - if ( $tname eq 'rpl_failsafe' or $tname eq 'rpl_chain_temp_table' ) { -# $tinfo->{'slave_num'}= 3; # Not 3 ? Check old code, strange + # $tinfo->{'slave_num'}= 3; # Not 3 ? Check old code, strange } } if ( defined mtr_match_prefix($tname,"federated") ) { - $tinfo->{'slave_num'}= 1; # Default, use one slave - - # FIXME currently we always restart slaves - $tinfo->{'slave_restart'}= 1; + # Default, federated uses the first slave as it's federated database + $tinfo->{'slave_num'}= 1; } - # Cluster is needed by test case if testname contains ndb - if ( defined mtr_match_substring($tname,"ndb") ) + if ( $::opt_with_ndbcluster or defined mtr_match_substring($tname,"ndb") ) { + # This is an ndb test or all tests should be run with ndb cluster started $tinfo->{'ndb_test'}= 1; if ( $::opt_skip_ndbcluster ) { - # Skip all ndb tests + # All ndb test's should be skipped $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No ndbcluster test(--skip-ndbcluster)"; return; } - if ( ! $::opt_with_ndbcluster ) + if ( ! $::opt_ndbcluster_supported ) { # Ndb is not supported, skip them $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No ndbcluster support"; return; } } else { + # This is not a ndb test $tinfo->{'ndb_test'}= 0; + if ( $::opt_with_ndbcluster_only ) + { + # Only the ndb test should be run, all other should be skipped + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Only ndbcluster tests(--with-ndbcluster-only)"; + return; + } } # FIXME what about embedded_server + ndbcluster, skip ?! @@ -223,6 +308,7 @@ sub collect_one_test_case($$$$$$) { my $master_sh= "$testdir/$tname-master.sh"; my $slave_sh= "$testdir/$tname-slave.sh"; my $disabled_file= "$testdir/$tname.disabled"; + my $im_opt_file= "$testdir/$tname-im.opt"; $tinfo->{'master_opt'}= []; $tinfo->{'slave_opt'}= []; @@ -303,6 +389,8 @@ sub collect_one_test_case($$$$$$) { if ( $::glob_win32_perl ) { $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No tests with sh scripts on Windows"; + return; } else { @@ -316,6 +404,8 @@ sub collect_one_test_case($$$$$$) { if ( $::glob_win32_perl ) { $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No tests with sh scripts on Windows"; + return; } else { @@ -324,29 +414,166 @@ sub collect_one_test_case($$$$$$) { } } + if ( -f $im_opt_file ) + { + $tinfo->{'im_opts'} = mtr_get_opts_from_file($im_opt_file); + } + else + { + $tinfo->{'im_opts'} = []; + } + # FIXME why this late? + my $marked_as_disabled= 0; if ( $disabled->{$tname} ) { - $tinfo->{'skip'}= 1; - $tinfo->{'disable'}= 1; # Sub type of 'skip' - $tinfo->{'comment'}= $disabled->{$tname} if $disabled->{$tname}; + $marked_as_disabled= 1; + $tinfo->{'comment'}= $disabled->{$tname}; } if ( -f $disabled_file ) { - $tinfo->{'skip'}= 1; - $tinfo->{'disable'}= 1; # Sub type of 'skip' + $marked_as_disabled= 1; $tinfo->{'comment'}= mtr_fromfile($disabled_file); } + # If test was marked as disabled, either opt_enable_disabled is off and then + # we skip this test, or it is on and then we run this test but warn + + if ( $marked_as_disabled ) + { + if ( $::opt_enable_disabled ) + { + $tinfo->{'dont_skip_though_disabled'}= 1; + } + else + { + $tinfo->{'skip'}= 1; + $tinfo->{'disable'}= 1; # Sub type of 'skip' + return; + } + } + + if ( $component_id eq 'im' ) + { + if ( $::glob_use_embedded_server ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No IM with embedded server"; + return; + } + elsif ( $::opt_ps_protocol ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No IM with --ps-protocol"; + return; + } + elsif ( $::opt_skip_im ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "No IM tests(--skip-im)"; + return; + } + } + else + { + mtr_options_from_test_file($tinfo,"$testdir/${tname}.test"); + + if ( $tinfo->{'big_test'} and ! $::opt_big_test ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Test need 'big-test' option"; + return; + } + + if ( $tinfo->{'ndb_extra'} and ! $::opt_ndb_extra_test ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Test need 'ndb_extra' option"; + return; + } + + if ( $tinfo->{'require_manager'} ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Test need the _old_ manager(to be removed)"; + return; + } + + if ( defined $tinfo->{'binlog_format'} and + ! ( $tinfo->{'binlog_format'} eq $::used_binlog_format ) ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Not running with binlog format '$tinfo->{'binlog_format'}'"; + return; + } + + if ( $tinfo->{'need_debug'} && ! $::debug_compiled_binaries ) + { + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Test need debug binaries"; + return; + } + } + # We can't restart a running server that may be in use if ( $::glob_use_running_server and ( $tinfo->{'master_restart'} or $tinfo->{'slave_restart'} ) ) { $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "Can't restart a running server"; + return; } + } +# List of tags in the .test files that if found should set +# the specified value in "tinfo" +our @tags= +( + ["include/have_innodb.inc", "innodb_test", 1], + ["include/have_binlog_format_row.inc", "binlog_format", "row"], + ["include/have_binlog_format_statement.inc", "binlog_format", "stmt"], + ["include/big_test.inc", "big_test", 1], + ["include/have_debug.inc", "need_debug", 1], + ["include/have_ndb_extra.inc", "ndb_extra", 1], + ["require_manager", "require_manager", 1], +); + +sub mtr_options_from_test_file($$) { + my $tinfo= shift; + my $file= shift; + #mtr_verbose("$file"); + my $F= IO::File->new($file) or mtr_error("can't open file \"$file\": $!"); + + while ( my $line= <$F> ) + { + next if ( $line !~ /^--/ ); + + # Match this line against tag in "tags" array + foreach my $tag (@tags) + { + if ( index($line, $tag->[0]) >= 0 ) + { + # Tag matched, assign value to "tinfo" + $tinfo->{"$tag->[1]"}= $tag->[2]; + } + } + + # If test sources another file, open it as well + if ( $line =~ /^\-\-([[:space:]]*)source(.*)$/ ) + { + my $value= $2; + $value =~ s/^\s+//; # Remove leading space + $value =~ s/[[:space:]]+$//; # Remove ending space + + my $sourced_file= "$::glob_mysql_test_dir/$value"; + mtr_options_from_test_file($tinfo, $sourced_file); + } + + } +} + 1; diff --git a/mysql-test/lib/mtr_im.pl b/mysql-test/lib/mtr_im.pl new file mode 100644 index 00000000000..ca17516278e --- /dev/null +++ b/mysql-test/lib/mtr_im.pl @@ -0,0 +1,761 @@ +# -*- cperl -*- + +# This is a library file used by the Perl version of mysql-test-run, +# and is part of the translation of the Bourne shell script with the +# same name. + +use strict; + +# Private IM-related operations. + +sub mtr_im_kill_process ($$$$); +sub mtr_im_load_pids ($); +sub mtr_im_terminate ($); +sub mtr_im_check_alive ($); +sub mtr_im_check_main_alive ($); +sub mtr_im_check_angel_alive ($); +sub mtr_im_check_mysqlds_alive ($); +sub mtr_im_check_mysqld_alive ($); +sub mtr_im_cleanup ($); +sub mtr_im_rm_file ($); +sub mtr_im_errlog ($); +sub mtr_im_kill ($); +sub mtr_im_wait_for_connection ($$$); +sub mtr_im_wait_for_mysqld($$$); + +# Public IM-related operations. + +sub mtr_im_start ($$); +sub mtr_im_stop ($); + +############################################################################## +# +# Private operations. +# +############################################################################## + +sub mtr_im_kill_process ($$$$) { + my $pid_lst= shift; + my $signal= shift; + my $total_retries= shift; + my $timeout= shift; + + my %pids; + + foreach my $pid ( @{$pid_lst} ) + { + $pids{$pid}= 1; + } + + for ( my $cur_attempt= 1; $cur_attempt <= $total_retries; ++$cur_attempt ) + { + foreach my $pid ( keys %pids ) + { + mtr_debug("Sending $signal to $pid..."); + + kill($signal, $pid); + + unless ( kill (0, $pid) ) + { + mtr_debug("Process $pid died."); + delete $pids{$pid}; + } + } + + return if scalar keys %pids == 0; + + mtr_debug("Sleeping $timeout second(s) waiting for processes to die..."); + + sleep($timeout); + } + + mtr_debug("Process(es) " . + join(' ', keys %pids) . + " is still alive after $total_retries " . + "of sending signal $signal."); +} + +########################################################################### + +sub mtr_im_load_pids($) { + my $im= shift; + + mtr_debug("Loading PID files..."); + + # Obtain mysqld-process pids. + + my $instances = $im->{'instances'}; + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + mtr_debug("IM-guarded mysqld[$idx] PID file: '" . + $instances->[$idx]->{'path_pid'} . "'."); + + my $mysqld_pid; + + if ( -r $instances->[$idx]->{'path_pid'} ) + { + $mysqld_pid= mtr_get_pid_from_file($instances->[$idx]->{'path_pid'}); + mtr_debug("IM-guarded mysqld[$idx] PID: $mysqld_pid."); + } + else + { + $mysqld_pid= undef; + mtr_debug("IM-guarded mysqld[$idx]: no PID file."); + } + + $instances->[$idx]->{'pid'}= $mysqld_pid; + } + + # Re-read Instance Manager PIDs from the file, since during tests Instance + # Manager could have been restarted, so its PIDs could have been changed. + + # - IM-main + + mtr_debug("IM-main PID file: '$im->{path_pid}'."); + + if ( -f $im->{'path_pid'} ) + { + $im->{'pid'} = + mtr_get_pid_from_file($im->{'path_pid'}); + + mtr_debug("IM-main PID: $im->{pid}."); + } + else + { + mtr_debug("IM-main: no PID file."); + $im->{'pid'}= undef; + } + + # - IM-angel + + mtr_debug("IM-angel PID file: '$im->{path_angel_pid}'."); + + if ( -f $im->{'path_angel_pid'} ) + { + $im->{'angel_pid'} = + mtr_get_pid_from_file($im->{'path_angel_pid'}); + + mtr_debug("IM-angel PID: $im->{'angel_pid'}."); + } + else + { + mtr_debug("IM-angel: no PID file."); + $im->{'angel_pid'} = undef; + } +} + +########################################################################### + +sub mtr_im_terminate($) { + my $im= shift; + + # Load pids from pid-files. We should do it first of all, because IM deletes + # them on shutdown. + + mtr_im_load_pids($im); + + mtr_debug("Shutting Instance Manager down..."); + + # Ignoring SIGCHLD so that all children could rest in peace. + + start_reap_all(); + + # Send SIGTERM to IM-main. + + if ( defined $im->{'pid'} ) + { + mtr_debug("IM-main pid: $im->{pid}."); + mtr_debug("Stopping IM-main..."); + + mtr_im_kill_process([ $im->{'pid'} ], 'TERM', 10, 1); + } + else + { + mtr_debug("IM-main pid: n/a."); + } + + # If IM-angel was alive, wait for it to die. + + if ( defined $im->{'angel_pid'} ) + { + mtr_debug("IM-angel pid: $im->{'angel_pid'}."); + mtr_debug("Waiting for IM-angel to die..."); + + my $total_attempts= 10; + + for ( my $cur_attempt=1; $cur_attempt <= $total_attempts; ++$cur_attempt ) + { + unless ( kill (0, $im->{'angel_pid'}) ) + { + mtr_debug("IM-angel died."); + last; + } + + sleep(1); + } + } + else + { + mtr_debug("IM-angel pid: n/a."); + } + + stop_reap_all(); + + # Re-load PIDs. + + mtr_im_load_pids($im); +} + +########################################################################### + +sub mtr_im_check_alive($) { + my $im= shift; + + mtr_debug("Checking whether IM-components are alive..."); + + return 1 if mtr_im_check_main_alive($im); + + return 1 if mtr_im_check_angel_alive($im); + + return 1 if mtr_im_check_mysqlds_alive($im); + + return 0; +} + +########################################################################### + +sub mtr_im_check_main_alive($) { + my $im= shift; + + # Check that the process, that we know to be IM's, is dead. + + if ( defined $im->{'pid'} ) + { + if ( kill (0, $im->{'pid'}) ) + { + mtr_debug("IM-main (PID: $im->{pid}) is alive."); + return 1; + } + else + { + mtr_debug("IM-main (PID: $im->{pid}) is dead."); + } + } + else + { + mtr_debug("No PID file for IM-main."); + } + + # Check that IM does not accept client connections. + + if ( mtr_ping_port($im->{'port'}) ) + { + mtr_debug("IM-main (port: $im->{port}) " . + "is accepting connections."); + + mtr_im_errlog("IM-main is accepting connections on port " . + "$im->{port}, but there is no " . + "process information."); + return 1; + } + else + { + mtr_debug("IM-main (port: $im->{port}) " . + "does not accept connections."); + return 0; + } +} + +########################################################################### + +sub mtr_im_check_angel_alive($) { + my $im= shift; + + # Check that the process, that we know to be the Angel, is dead. + + if ( defined $im->{'angel_pid'} ) + { + if ( kill (0, $im->{'angel_pid'}) ) + { + mtr_debug("IM-angel (PID: $im->{angel_pid}) is alive."); + return 1; + } + else + { + mtr_debug("IM-angel (PID: $im->{angel_pid}) is dead."); + return 0; + } + } + else + { + mtr_debug("No PID file for IM-angel."); + return 0; + } +} + +########################################################################### + +sub mtr_im_check_mysqlds_alive($) { + my $im= shift; + + mtr_debug("Checking for IM-guarded mysqld instances..."); + + my $instances = $im->{'instances'}; + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + mtr_debug("Checking mysqld[$idx]..."); + + return 1 + if mtr_im_check_mysqld_alive($instances->[$idx]); + } +} + +########################################################################### + +sub mtr_im_check_mysqld_alive($) { + my $mysqld_instance= shift; + + # Check that the process is dead. + + if ( defined $mysqld_instance->{'pid'} ) + { + if ( kill (0, $mysqld_instance->{'pid'}) ) + { + mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is alive."); + return 1; + } + else + { + mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is dead."); + } + } + else + { + mtr_debug("No PID file for mysqld instance."); + } + + # Check that mysqld does not accept client connections. + + if ( mtr_ping_port($mysqld_instance->{'port'}) ) + { + mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " . + "is accepting connections."); + + mtr_im_errlog("Mysqld is accepting connections on port " . + "$mysqld_instance->{port}, but there is no " . + "process information."); + return 1; + } + else + { + mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " . + "does not accept connections."); + return 0; + } +} + +########################################################################### + +sub mtr_im_cleanup($) { + my $im= shift; + + mtr_im_rm_file($im->{'path_pid'}); + mtr_im_rm_file($im->{'path_sock'}); + + mtr_im_rm_file($im->{'path_angel_pid'}); + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_pid'}); + mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_sock'}); + } +} + +########################################################################### + +sub mtr_im_rm_file($) +{ + my $file_path= shift; + + if ( -f $file_path ) + { + mtr_debug("Removing '$file_path'..."); + + unless ( unlink($file_path) ) + { + mtr_warning("Can not remove '$file_path'.") + } + } + else + { + mtr_debug("File '$file_path' does not exist already."); + } +} + +########################################################################### + +sub mtr_im_errlog($) { + my $msg= shift; + + # Complain in error log so that a warning will be shown. + # + # TODO: unless BUG#20761 is fixed, we will print the warning to stdout, so + # that it can be seen on console and does not produce pushbuild error. + + # my $errlog= "$opt_vardir/log/mysql-test-run.pl.err"; + # + # open (ERRLOG, ">>$errlog") || + # mtr_error("Can not open error log ($errlog)"); + # + # my $ts= localtime(); + # print ERRLOG + # "Warning: [$ts] $msg\n"; + # + # close ERRLOG; + + my $ts= localtime(); + print "Warning: [$ts] $msg\n"; +} + +########################################################################### + +sub mtr_im_kill($) { + my $im= shift; + + # Re-load PIDs. That can be useful because some processes could have been + # restarted. + + mtr_im_load_pids($im); + + # Ignoring SIGCHLD so that all children could rest in peace. + + start_reap_all(); + + # Kill IM-angel first of all. + + if ( defined $im->{'angel_pid'} ) + { + mtr_debug("Killing IM-angel (PID: $im->{angel_pid})..."); + mtr_im_kill_process([ $im->{'angel_pid'} ], 'KILL', 10, 1) + } + else + { + mtr_debug("IM-angel is dead."); + } + + # Re-load PIDs again. + + mtr_im_load_pids($im); + + # Kill IM-main. + + if ( defined $im->{'pid'} ) + { + mtr_debug("Killing IM-main (PID: $im->pid})..."); + mtr_im_kill_process([ $im->{'pid'} ], 'KILL', 10, 1); + } + else + { + mtr_debug("IM-main is dead."); + } + + # Re-load PIDs again. + + mtr_im_load_pids($im); + + # Kill guarded mysqld instances. + + my @mysqld_pids; + + mtr_debug("Collecting PIDs of mysqld instances to kill..."); + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + my $pid= $im->{'instances'}->[$idx]->{'pid'}; + + unless ( defined $pid ) + { + next; + } + + mtr_debug(" - IM-guarded mysqld[$idx] PID: $pid."); + + push (@mysqld_pids, $pid); + } + + if ( scalar @mysqld_pids > 0 ) + { + mtr_debug("Killing IM-guarded mysqld instances..."); + mtr_im_kill_process(\@mysqld_pids, 'KILL', 10, 1); + } + + # That's all. + + stop_reap_all(); +} + +############################################################################## + +sub mtr_im_wait_for_connection($$$) { + my $im= shift; + my $total_attempts= shift; + my $connect_timeout= shift; + + mtr_debug("Waiting for IM on port $im->{port} " . + "to start accepting connections..."); + + for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt ) + { + mtr_debug("Trying to connect to IM ($cur_attempt of $total_attempts)..."); + + if ( mtr_ping_port($im->{'port'}) ) + { + mtr_debug("IM is accepting connections " . + "on port $im->{port}."); + return 1; + } + + mtr_debug("Sleeping $connect_timeout..."); + sleep($connect_timeout); + } + + mtr_debug("IM does not accept connections " . + "on port $im->{port} after " . + ($total_attempts * $connect_timeout) . " seconds."); + + return 0; +} + +############################################################################## + +sub mtr_im_wait_for_mysqld($$$) { + my $mysqld= shift; + my $total_attempts= shift; + my $connect_timeout= shift; + + mtr_debug("Waiting for IM-guarded mysqld on port $mysqld->{port} " . + "to start accepting connections..."); + + for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt ) + { + mtr_debug("Trying to connect to mysqld " . + "($cur_attempt of $total_attempts)..."); + + if ( mtr_ping_port($mysqld->{'port'}) ) + { + mtr_debug("Mysqld is accepting connections " . + "on port $mysqld->{port}."); + return 1; + } + + mtr_debug("Sleeping $connect_timeout..."); + sleep($connect_timeout); + } + + mtr_debug("Mysqld does not accept connections " . + "on port $mysqld->{port} after " . + ($total_attempts * $connect_timeout) . " seconds."); + + return 0; +} + +############################################################################## +# +# Public operations. +# +############################################################################## + +sub mtr_im_start($$) { + my $im = shift; + my $opts = shift; + + mtr_debug("Starting Instance Manager..."); + + my $args; + mtr_init_args(\$args); + mtr_add_arg($args, "--defaults-file=%s", $im->{'defaults_file'}); + + foreach my $opt ( @{$opts} ) + { + mtr_add_arg($args, $opt); + } + + $im->{'pid'} = + mtr_spawn( + $::exe_im, # path to the executable + $args, # cmd-line args + '', # stdin + $im->{'path_log'}, # stdout + $im->{'path_err'}, # stderr + '', # pid file path (not used) + { append_log_file => 1 } # append log files + ); + + unless ( $im->{'pid'} ) + { + mtr_error('Could not start Instance Manager.') + } + + # Instance Manager can be run in daemon mode. In this case, it creates + # several processes and the parent process, created by mtr_spawn(), exits just + # after start. So, we have to obtain Instance Manager PID from the PID file. + + mtr_debug("Waiting for IM to create PID file (" . + "path: '$im->{path_pid}'; " . + "timeout: $im->{start_timeout})..."); + + unless ( sleep_until_file_created($im->{'path_pid'}, + $im->{'start_timeout'}, + -1) ) # real PID is still unknown + { + mtr_debug("IM has not created PID file in $im->{start_timeout} secs."); + mtr_debug("Aborting test suite..."); + + mtr_kill_leftovers(); + + mtr_report("IM has not created PID file in $im->{start_timeout} secs."); + return 0; + } + + $im->{'pid'}= mtr_get_pid_from_file($im->{'path_pid'}); + + mtr_debug("Instance Manager started. PID: $im->{pid}."); + + # Wait until we can connect to IM. + + my $IM_CONNECT_TIMEOUT= 30; + + unless ( mtr_im_wait_for_connection($im, + $IM_CONNECT_TIMEOUT, 1) ) + { + mtr_debug("Can not connect to Instance Manager " . + "in $IM_CONNECT_TIMEOUT seconds after start."); + mtr_debug("Aborting test suite..."); + + mtr_kill_leftovers(); + + mtr_report("Can not connect to Instance Manager " . + "in $IM_CONNECT_TIMEOUT seconds after start."); + return 0; + } + + # Wait for IM to start guarded instances: + # - wait for PID files; + + mtr_debug("Waiting for guarded mysqlds instances to create PID files..."); + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + my $mysqld= $im->{'instances'}->[$idx]; + + if ( exists $mysqld->{'nonguarded'} ) + { + next; + } + + mtr_debug("Waiting for mysqld[$idx] to create PID file (" . + "path: '$mysqld->{path_pid}'; " . + "timeout: $mysqld->{start_timeout})..."); + + unless ( sleep_until_file_created($mysqld->{'path_pid'}, + $mysqld->{'start_timeout'}, + -1) ) # real PID is still unknown + { + mtr_debug("mysqld[$idx] has not created PID file in " . + "$mysqld->{start_timeout} secs."); + mtr_debug("Aborting test suite..."); + + mtr_kill_leftovers(); + + mtr_report("mysqld[$idx] has not created PID file in " . + "$mysqld->{start_timeout} secs."); + return 0; + } + + mtr_debug("PID file for mysqld[$idx] ($mysqld->{path_pid} created."); + } + + # Wait until we can connect to guarded mysqld-instances + # (in other words -- wait for IM to start guarded instances). + + mtr_debug("Waiting for guarded mysqlds to start accepting connections..."); + + for ( my $idx= 0; $idx < 2; ++$idx ) + { + my $mysqld= $im->{'instances'}->[$idx]; + + if ( exists $mysqld->{'nonguarded'} ) + { + next; + } + + mtr_debug("Waiting for mysqld[$idx] to accept connection..."); + + unless ( mtr_im_wait_for_mysqld($mysqld, 30, 1) ) + { + mtr_debug("Can not connect to mysqld[$idx] " . + "in $IM_CONNECT_TIMEOUT seconds after start."); + mtr_debug("Aborting test suite..."); + + mtr_kill_leftovers(); + + mtr_report("Can not connect to mysqld[$idx] " . + "in $IM_CONNECT_TIMEOUT seconds after start."); + return 0; + } + + mtr_debug("mysqld[$idx] started."); + } + + mtr_debug("Instance Manager and its components are up and running."); + + return 1; +} + +############################################################################## + +sub mtr_im_stop($) { + my $im= shift; + + mtr_debug("Stopping Instance Manager..."); + + # Try graceful shutdown. + + mtr_im_terminate($im); + + # Check that all processes died. + + unless ( mtr_im_check_alive($im) ) + { + mtr_debug("Instance Manager has been stopped successfully."); + mtr_im_cleanup($im); + return 1; + } + + # Instance Manager don't want to die. We should kill it. + + mtr_im_errlog("Instance Manager did not shutdown gracefully."); + + mtr_im_kill($im); + + # Check again that all IM-related processes have been killed. + + my $im_is_alive= mtr_im_check_alive($im); + + mtr_im_cleanup($im); + + if ( $im_is_alive ) + { + mtr_debug("Can not kill Instance Manager or its children."); + return 0; + } + + mtr_debug("Instance Manager has been killed successfully."); + return 1; +} + +########################################################################### + +1; diff --git a/mysql-test/lib/mtr_io.pl b/mysql-test/lib/mtr_io.pl index b3da6d97664..8a7fc56269c 100644 --- a/mysql-test/lib/mtr_io.pl +++ b/mysql-test/lib/mtr_io.pl @@ -11,6 +11,8 @@ sub mtr_get_opts_from_file ($); sub mtr_fromfile ($); sub mtr_tofile ($@); sub mtr_tonewfile($@); +sub mtr_lastlinefromfile($); +sub mtr_appendfile_to_file ($$); ############################################################################## # @@ -19,13 +21,39 @@ sub mtr_tonewfile($@); ############################################################################## sub mtr_get_pid_from_file ($) { - my $file= shift; + my $pid_file_path= shift; + my $TOTAL_ATTEMPTS= 30; + my $timeout= 1; - open(FILE,"<",$file) or mtr_error("can't open file \"$file\": $!"); - my $pid= <FILE>; - chomp($pid); - close FILE; - return $pid; + # We should read from the file until we get correct pid. As it is + # stated in BUG#21884, pid file can be empty at some moment. So, we should + # read it until we get valid data. + + for (my $cur_attempt= 1; $cur_attempt <= $TOTAL_ATTEMPTS; ++$cur_attempt) + { + mtr_debug("Reading pid file '$pid_file_path' " . + "($cur_attempt of $TOTAL_ATTEMPTS)..."); + + open(FILE, '<', $pid_file_path) + or mtr_error("can't open file \"$pid_file_path\": $!"); + + my $pid= <FILE>; + + chomp($pid) if defined $pid; + + close FILE; + + return $pid if defined $pid && $pid ne ''; + + mtr_debug("Pid file '$pid_file_path' is empty. " . + "Sleeping $timeout second(s)..."); + + sleep(1); + } + + mtr_error("Pid file '$pid_file_path' is corrupted. " . + "Can not retrieve PID in " . + ($timeout * $TOTAL_ATTEMPTS) . " seconds."); } sub mtr_get_opts_from_file ($) { @@ -113,6 +141,20 @@ sub mtr_fromfile ($) { return $text; } +sub mtr_lastlinefromfile ($) { + my $file= shift; + my $text; + + open(FILE,"<",$file) or mtr_error("can't open file \"$file\": $!"); + while (my $line= <FILE>) + { + $text= $line; + } + close FILE; + return $text; +} + + sub mtr_tofile ($@) { my $file= shift; @@ -129,5 +171,17 @@ sub mtr_tonewfile ($@) { close FILE; } +sub mtr_appendfile_to_file ($$) { + my $from_file= shift; + my $to_file= shift; + + open(TOFILE,">>",$to_file) or mtr_error("can't open file \"$to_file\": $!"); + open(FROMFILE,">>",$from_file) + or mtr_error("can't open file \"$from_file\": $!"); + print TOFILE while (<FROMFILE>); + close FROMFILE; + close TOFILE; +} + 1; diff --git a/mysql-test/lib/mtr_misc.pl b/mysql-test/lib/mtr_misc.pl index 08c99e90906..dd9d24ebc8e 100644 --- a/mysql-test/lib/mtr_misc.pl +++ b/mysql-test/lib/mtr_misc.pl @@ -9,9 +9,10 @@ use strict; sub mtr_full_hostname (); sub mtr_short_hostname (); sub mtr_init_args ($); -sub mtr_add_arg ($$); +sub mtr_add_arg ($$@); sub mtr_path_exists(@); sub mtr_script_exists(@); +sub mtr_file_exists(@); sub mtr_exe_exists(@); sub mtr_copy_dir($$); sub mtr_same_opts($$); @@ -54,7 +55,7 @@ sub mtr_init_args ($) { $$args = []; # Empty list } -sub mtr_add_arg ($$) { +sub mtr_add_arg ($$@) { my $args= shift; my $format= shift; my @fargs = @_; @@ -101,6 +102,14 @@ sub mtr_script_exists (@) { } } +sub mtr_file_exists (@) { + foreach my $path ( @_ ) + { + return $path if -e $path; + } + return ""; +} + sub mtr_exe_exists (@) { my @path= @_; map {$_.= ".exe"} @path if $::glob_win32; @@ -125,19 +134,30 @@ sub mtr_exe_exists (@) { } } + sub mtr_copy_dir($$) { - my $srcdir= shift; - my $dstdir= shift; + my $from_dir= shift; + my $to_dir= shift; - # Create destination directory - mkpath($dstdir); - find(\&mtr_copy_one_file, $dstdir); -} +# mtr_verbose("Copying from $from_dir to $to_dir"); + + mkpath("$to_dir"); + opendir(DIR, "$from_dir") + or mtr_error("Can't find $from_dir$!"); + for(readdir(DIR)) { + next if "$_" eq "." or "$_" eq ".."; + if ( -d "$from_dir/$_" ) + { + mtr_copy_dir("$from_dir/$_", "$to_dir/$_"); + next; + } + copy("$from_dir/$_", "$to_dir/$_"); + } + closedir(DIR); -sub mtr_copy_one_file { - print $File::Find::name, "\n"; } + sub mtr_same_opts ($$) { my $l1= shift; my $l2= shift; diff --git a/mysql-test/lib/mtr_process.pl b/mysql-test/lib/mtr_process.pl index 662b70a4fee..5e21248790e 100644 --- a/mysql-test/lib/mtr_process.pl +++ b/mysql-test/lib/mtr_process.pl @@ -4,7 +4,6 @@ # and is part of the translation of the Bourne shell script with the # same name. -#use Carp qw(cluck); use Socket; use Errno; use strict; @@ -14,12 +13,17 @@ use POSIX 'WNOHANG'; sub mtr_run ($$$$$$;$); sub mtr_spawn ($$$$$$;$); -sub mtr_stop_mysqld_servers ($); +sub mtr_check_stop_servers ($); sub mtr_kill_leftovers (); +sub mtr_wait_blocking ($); sub mtr_record_dead_children (); +sub mtr_ndbmgm_start($$); +sub mtr_mysqladmin_start($$$); sub mtr_exit ($); sub sleep_until_file_created ($$$); sub mtr_kill_processes ($); +sub mtr_ping_with_timeout($); +sub mtr_ping_port ($); # static in C sub spawn_impl ($$$$$$$$); @@ -31,7 +35,6 @@ sub spawn_impl ($$$$$$$$); ############################################################################## # This function try to mimic the C version used in "netware/mysql_test_run.c" -# FIXME learn it to handle append mode as well, a "new" flag or a "append" sub mtr_run ($$$$$$;$) { my $path= shift; @@ -112,6 +115,9 @@ sub spawn_impl ($$$$$$$$) { print STDERR "#### ", "-" x 78, "\n"; } + mtr_error("Can't spawn with empty \"path\"") unless defined $path; + + FORK: { my $pid= fork(); @@ -144,17 +150,6 @@ sub spawn_impl ($$$$$$$$) { $SIG{INT}= 'DEFAULT'; # Parent do some stuff, we don't - if ( $::glob_cygwin_shell and $mode eq 'test' ) - { - # Programs started from mysqltest under Cygwin, are to - # execute them within Cygwin. Else simple things in test - # files like - # --system "echo 1 > file" - # will fail. - # FIXME not working :-( -# $ENV{'COMSPEC'}= "$::glob_cygwin_shell -c"; - } - my $log_file_open_mode = '>'; if ($spawn_opts and $spawn_opts->{'append_log_file'}) @@ -164,7 +159,15 @@ sub spawn_impl ($$$$$$$$) { if ( $output ) { - if ( ! open(STDOUT,$log_file_open_mode,$output) ) + if ( $::glob_win32_perl ) + { + # Don't redirect stdout on ActiveState perl since this is + # just another thread in the same process. + # Should be fixed so that the thread that is created with fork + # executes the exe in another process and wait's for it to return. + # In the meanwhile, we get all the output from mysqld's to screen + } + elsif ( ! open(STDOUT,$log_file_open_mode,$output) ) { mtr_child_error("can't redirect STDOUT to \"$output\": $!"); } @@ -216,8 +219,7 @@ sub spawn_parent_impl { { # Simple run of command, we wait for it to return my $ret_pid= waitpid($pid,0); - - if ( $ret_pid <= 0 ) + if ( $ret_pid != $pid ) { mtr_error("$path ($pid) got lost somehow"); } @@ -245,7 +247,6 @@ sub spawn_parent_impl { # Someone terminated, don't know who. Collect # status info first before $? is lost, # but not $exit_value, this is flagged from - # my $timer_name= mtr_timer_timeout($::glob_timers, $ret_pid); if ( $timer_name ) @@ -272,45 +273,22 @@ sub spawn_parent_impl { last; } - # If one of the mysqld processes died, we want to - # mark this, and kill the mysqltest process. - - foreach my $idx (0..1) - { - if ( $::master->[$idx]->{'pid'} eq $ret_pid ) - { - mtr_debug("child $ret_pid was master[$idx], " . - "exit during mysqltest run"); - $::master->[$idx]->{'pid'}= 0; - last; - } - } - - foreach my $idx (0..2) - { - if ( $::slave->[$idx]->{'pid'} eq $ret_pid ) - { - mtr_debug("child $ret_pid was slave[$idx], " . - "exit during mysqltest run"); - $::slave->[$idx]->{'pid'}= 0; - last; - } - } + # One of the child processes died, unless this was expected + # mysqltest should be killed and test aborted - mtr_debug("waitpid() catched exit of unknown child $ret_pid, " . - "exit during mysqltest run"); + check_expected_crash_and_restart($ret_pid); } if ( $ret_pid != $pid ) { # We terminated the waiting because a "mysqld" process died. # Kill the mysqltest process. - + mtr_verbose("Kill mysqltest because another process died"); kill(9,$pid); $ret_pid= waitpid($pid,0); - if ( $ret_pid == -1 ) + if ( $ret_pid != $pid ) { mtr_error("$path ($pid) got lost somehow"); } @@ -351,39 +329,101 @@ sub mtr_process_exit_status { # ############################################################################## -# We just "ping" on the ports, and if we can't do a socket connect -# we assume the server is dead. So we don't *really* know a server -# is dead, we just hope that it after letting the listen port go, -# it is dead enough for us to start a new server. +# Kill all processes(mysqld, ndbd, ndb_mgmd and im) that would conflict with +# this run +# Make sure to remove the PID file, if any. +# kill IM manager first, else it will restart the servers sub mtr_kill_leftovers () { - # First, kill all masters and slaves that would conflict with - # this run. Make sure to remove the PID file, if any. + mtr_report("Killing Possible Leftover Processes"); + mtr_debug("mtr_kill_leftovers(): started."); - my @args; + mkpath("$::opt_vardir/log"); # Needed for mysqladmin log - for ( my $idx; $idx < 2; $idx++ ) + # Stop or kill Instance Manager and all its children. If we failed to do + # that, we can only abort -- there is nothing left to do. + +# mtr_error("Failed to stop Instance Manager.") +# unless mtr_im_stop($::instance_manager); + + # Start shutdown of masters and slaves. Don't touch IM-managed mysqld + # instances -- they should be stopped by mtr_im_stop(). + + mtr_debug("Shutting down mysqld-instances..."); + + my @kill_pids; + my %admin_pids; + + foreach my $srv (@{$::master}, @{$::slave}) { - push(@args,{ - pid => 0, # We don't know the PID - pidfile => $::master->[$idx]->{'path_mypid'}, - sockfile => $::master->[$idx]->{'path_mysock'}, - port => $::master->[$idx]->{'path_myport'}, - }); + mtr_debug(" - mysqld " . + "(pid: $srv->{pid}; " . + "pid file: '$srv->{path_pid}'; " . + "socket: '$srv->{path_sock}'; ". + "port: $srv->{port})"); + + my $pid= mtr_mysqladmin_start($srv, "shutdown", 70); + + # Save the pid of the mysqladmin process + $admin_pids{$pid}= 1; + + push(@kill_pids,{ + pid => $srv->{'pid'}, + pidfile => $srv->{'path_pid'}, + sockfile => $srv->{'path_sock'}, + port => $srv->{'port'}, + }); + $srv->{'pid'}= 0; # Assume we are done with it } - for ( my $idx; $idx < 3; $idx++ ) + if ( ! $::opt_skip_ndbcluster ) { - push(@args,{ - pid => 0, # We don't know the PID - pidfile => $::slave->[$idx]->{'path_mypid'}, - sockfile => $::slave->[$idx]->{'path_mysock'}, - port => $::slave->[$idx]->{'path_myport'}, - }); + # Start shutdown of clusters. + mtr_debug("Shutting down cluster..."); + + foreach my $cluster (@{$::clusters}) + { + mtr_debug(" - cluster " . + "(pid: $cluster->{pid}; " . + "pid file: '$cluster->{path_pid})"); + + my $pid= mtr_ndbmgm_start($cluster, "shutdown"); + + # Save the pid of the ndb_mgm process + $admin_pids{$pid}= 1; + + push(@kill_pids,{ + pid => $cluster->{'pid'}, + pidfile => $cluster->{'path_pid'} + }); + + $cluster->{'pid'}= 0; # Assume we are done with it + + foreach my $ndbd (@{$cluster->{'ndbds'}}) + { + mtr_debug(" - ndbd " . + "(pid: $ndbd->{pid}; " . + "pid file: '$ndbd->{path_pid})"); + + push(@kill_pids,{ + pid => $ndbd->{'pid'}, + pidfile => $ndbd->{'path_pid'}, + }); + $ndbd->{'pid'}= 0; # Assume we are done with it + } + } } - mtr_mysqladmin_shutdown(\@args, 20); + # Wait for all the admin processes to complete + mtr_wait_blocking(\%admin_pids); + + # If we trusted "mysqladmin --shutdown_timeout= ..." we could just + # terminate now, but we don't (FIXME should be debugged). + # So we try again to ping and at least wait the same amount of time + # mysqladmin would for all to die. + + mtr_ping_with_timeout(\@kill_pids); # We now have tried to terminate nice. We have waited for the listen # port to be free, but can't really tell if the mysqld process died @@ -401,6 +441,8 @@ sub mtr_kill_leftovers () { # FIXME $path_run_dir or something my $rundir= "$::opt_vardir/run"; + mtr_debug("Processing PID files in directory '$rundir'..."); + if ( -d $rundir ) { opendir(RUNDIR, $rundir) @@ -414,26 +456,32 @@ sub mtr_kill_leftovers () { if ( -f $pidfile ) { - my $pid= mtr_get_pid_from_file($pidfile); + mtr_debug("Processing PID file: '$pidfile'..."); - # Race, could have been removed between I tested with -f - # and the unlink() below, so I better check again with -f + my $pid= mtr_get_pid_from_file($pidfile); - if ( ! unlink($pidfile) and -f $pidfile ) - { - mtr_error("can't remove $pidfile"); - } + mtr_debug("Got pid: $pid from file '$pidfile'"); if ( $::glob_cygwin_perl or kill(0, $pid) ) { + mtr_debug("There is process with pid $pid -- scheduling for kill."); push(@pids, $pid); # We know (cygwin guess) it exists } + else + { + mtr_debug("There is no process with pid $pid -- skipping."); + } } } closedir(RUNDIR); if ( @pids ) { + mtr_debug("Killing the following processes with PID files: " . + join(' ', @pids) . "..."); + + start_reap_all(); + if ( $::glob_cygwin_perl ) { # We have no (easy) way of knowing the Cygwin controlling @@ -447,8 +495,9 @@ sub mtr_kill_leftovers () { my $retries= 10; # 10 seconds do { + mtr_debug("Sending SIGKILL to pids: " . join(' ', @pids)); kill(9, @pids); - mtr_debug("Sleep 1 second waiting for processes to die"); + mtr_report("Sleep 1 second waiting for processes to die"); sleep(1) # Wait one second } while ( $retries-- and kill(0, @pids) ); @@ -457,56 +506,74 @@ sub mtr_kill_leftovers () { mtr_warning("can't kill process(es) " . join(" ", @pids)); } } + + stop_reap_all(); } } + else + { + mtr_debug("Directory for PID files ($rundir) does not exist."); + } - # We may have failed everything, bug we now check again if we have + # We may have failed everything, but we now check again if we have # the listen ports free to use, and if they are free, just go for it. - foreach my $srv ( @args ) + mtr_debug("Checking known mysqld servers..."); + + foreach my $srv ( @kill_pids ) { - if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) ) + if ( defined $srv->{'port'} and mtr_ping_port($srv->{'port'}) ) { - mtr_warning("can't kill old mysqld holding port $srv->{'port'}"); + mtr_warning("can't kill old process holding port $srv->{'port'}"); } } -} -############################################################################## -# -# Shut down mysqld servers we have started from this run of this script -# -############################################################################## + mtr_debug("mtr_kill_leftovers(): finished."); +} -# To speed things we kill servers in parallel. The argument is a list -# of 'ports', 'pids', 'pidfiles' and 'socketfiles'. +# Check that all processes in list are killed +# The argument is a list of 'ports', 'pids', 'pidfiles' and 'socketfiles' +# for which shutdown has been started. Make sure they all get killed +# in one way or the other. +# # FIXME On Cygwin, and maybe some other platforms, $srv->{'pid'} and -# $srv->{'pidfile'} will not be the same PID. We need to try to kill +# the pid in $srv->{'pidfile'} will not be the same PID. We need to try to kill # both I think. -sub mtr_stop_mysqld_servers ($) { +sub mtr_check_stop_servers ($) { my $spec= shift; - # ---------------------------------------------------------------------- - # First try nice normal shutdown using 'mysqladmin' - # ---------------------------------------------------------------------- + # Return if no processes are defined + return if ! @$spec; - # Shutdown time must be high as slave may be in reconnect - mtr_mysqladmin_shutdown($spec, 70); + #mtr_report("mtr_check_stop_servers"); + + mtr_ping_with_timeout(\@$spec); # ---------------------------------------------------------------------- # We loop with waitpid() nonblocking to see how many of the ones we - # are to kill, actually got killed by mtr_mysqladmin_shutdown(). - # Note that we don't rely on this, the mysqld server might have stop + # are to kill, actually got killed by mysqladmin or ndb_mgm + # + # Note that we don't rely on this, the mysqld server might have stopped # listening to the port, but still be alive. But it is a start. # ---------------------------------------------------------------------- foreach my $srv ( @$spec ) { - if ( $srv->{'pid'} and (waitpid($srv->{'pid'},&WNOHANG) == $srv->{'pid'}) ) + my $ret_pid; + if ( $srv->{'pid'} ) { - $srv->{'pid'}= 0; + $ret_pid= waitpid($srv->{'pid'},&WNOHANG); + if ($ret_pid == $srv->{'pid'}) + { + mtr_verbose("Caught exit of process $ret_pid"); + $srv->{'pid'}= 0; + } + else + { + # mtr_warning("caught exit of unknown child $ret_pid"); + } } } @@ -540,13 +607,12 @@ sub mtr_stop_mysqld_servers ($) { } # ---------------------------------------------------------------------- - # If the processes where started from this script, and we had no PIDS + # If all the processes in list already have been killed, # then we don't have to do anything. # ---------------------------------------------------------------------- if ( ! keys %mysqld_pids ) { - # cluck "This is how we got here!"; return; } @@ -595,139 +661,288 @@ sub mtr_stop_mysqld_servers ($) { foreach my $file ($srv->{'pidfile'}, $srv->{'sockfile'}) { # Know it is dead so should be no race, careful anyway - if ( -f $file and ! unlink($file) and -f $file ) + if ( defined $file and -f $file and ! unlink($file) and -f $file ) { $errors++; mtr_warning("couldn't delete $file"); } } + $srv->{'pid'}= 0; } } } if ( $errors ) { - # We are in trouble, just die.... - mtr_error("we could not kill or clean up all processes"); + # There where errors killing processes + # do one last attempt to ping the servers + # and if they can't be pinged, assume they are dead + if ( ! mtr_ping_with_timeout( \@$spec ) ) + { + mtr_error("we could not kill or clean up all processes"); + } + else + { + mtr_verbose("All ports were free, continuing"); + } } } # FIXME We just assume they are all dead, for Cygwin we are not # really sure - + } +# Wait for all the process in the list to terminate +sub mtr_wait_blocking($) { + my $admin_pids= shift; -############################################################################## -# -# Shut down mysqld servers using "mysqladmin ... shutdown". -# To speed this up, we start them in parallel and use waitpid() to -# catch their termination. Note that this doesn't say the servers -# are terminated, just that 'mysqladmin' is terminated. -# -# Note that mysqladmin will ask the server about what PID file it uses, -# and mysqladmin will wait for it to be removed before it terminates -# (unless passes timeout). -# -# This function will take at most about 20 seconds, and we still are not -# sure we killed them all. If none is responding to ping, we return 1, -# else we return 0. -# -############################################################################## -sub mtr_mysqladmin_shutdown { - my $spec= shift; + # Return if no processes defined + return if ! %$admin_pids; + + mtr_verbose("mtr_wait_blocking"); + + # Wait for all the started processes to exit + # As mysqladmin is such a simple program, we trust it to terminate itself. + # I.e. we wait blocking, and wait for them all before we go on. + foreach my $pid (keys %{$admin_pids}) + { + my $ret_pid= waitpid($pid,0); + + } +} + +# Start "mysqladmin shutdown" for a specific mysqld +sub mtr_mysqladmin_start($$$) { + my $srv= shift; + my $command= shift; my $adm_shutdown_tmo= shift; - my %mysql_admin_pids; - my @to_kill_specs; + my $args; + mtr_init_args(\$args); - foreach my $srv ( @$spec ) + mtr_add_arg($args, "--no-defaults"); + mtr_add_arg($args, "--user=%s", $::opt_user); + mtr_add_arg($args, "--password="); + mtr_add_arg($args, "--silent"); + if ( -e $srv->{'path_sock'} ) + { + mtr_add_arg($args, "--socket=%s", $srv->{'path_sock'}); + } + if ( $srv->{'port'} ) { - if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) ) + mtr_add_arg($args, "--port=%s", $srv->{'port'}); + } + if ( $srv->{'port'} and ! -e $srv->{'path_sock'} ) + { + mtr_add_arg($args, "--protocol=tcp"); # Needed if no --socket + } + mtr_add_arg($args, "--connect_timeout=5"); + + # Shutdown time must be high as slave may be in reconnect + mtr_add_arg($args, "--shutdown_timeout=$adm_shutdown_tmo"); + mtr_add_arg($args, "$command"); + my $path_mysqladmin_log= "$::opt_vardir/log/mysqladmin.log"; + my $pid= mtr_spawn($::exe_mysqladmin, $args, + "", $path_mysqladmin_log, $path_mysqladmin_log, "", + { append_log_file => 1 }); + mtr_verbose("mtr_mysqladmin_start, pid: $pid"); + return $pid; + +} + +# Start "ndb_mgm shutdown" for a specific cluster, it will +# shutdown all data nodes and leave the ndb_mgmd running +sub mtr_ndbmgm_start($$) { + my $cluster= shift; + my $command= shift; + + my $args; + + mtr_init_args(\$args); + + mtr_add_arg($args, "--no-defaults"); + mtr_add_arg($args, "--core"); + mtr_add_arg($args, "--try-reconnect=1"); + mtr_add_arg($args, "--ndb_connectstring=%s", $cluster->{'connect_string'}); + mtr_add_arg($args, "-e"); + mtr_add_arg($args, "$command"); + + my $pid= mtr_spawn($::exe_ndb_mgm, $args, + "", "/dev/null", "/dev/null", "", + {}); + mtr_verbose("mtr_ndbmgm_start, pid: $pid"); + return $pid; + +} + + +# Ping all servers in list, exit when none of them answers +# or when timeout has passed +sub mtr_ping_with_timeout($) { + my $spec= shift; + my $timeout= 200; # 20 seconds max + my $res= 1; # If we just fall through, we are done + # in the sense that the servers don't + # listen to their ports any longer + + mtr_debug("Waiting for mysqld servers to stop..."); + + TIME: + while ( $timeout-- ) + { + foreach my $srv ( @$spec ) { - push(@to_kill_specs, $srv); + $res= 1; # We are optimistic + if ( $srv->{'pid'} and defined $srv->{'port'} ) + { + if ( mtr_ping_port($srv->{'port'}) ) + { + mtr_verbose("waiting for process $srv->{'pid'} to stop ". + "using port $srv->{'port'}"); + + # Millisceond sleep emulated with select + select(undef, undef, undef, (0.1)); + $res= 0; + next TIME; + } + else + { + # Process was not using port + } + } } + last; # If we got here, we are done } - - foreach my $srv ( @to_kill_specs ) + if ($res) + { + mtr_debug("mtr_ping_with_timeout(): All mysqld instances are down."); + } + else { - # FIXME wrong log..... - # FIXME, stderr..... - # Shutdown time must be high as slave may be in reconnect - my $args; + mtr_report("mtr_ping_with_timeout(): At least one server is alive."); + } - mtr_init_args(\$args); + return $res; +} + + +# +# Loop through our list of processes and look for and entry +# with the provided pid +# Set the pid of that process to 0 if found +# +sub mark_process_dead($) +{ + my $ret_pid= shift; - mtr_add_arg($args, "--no-defaults"); - mtr_add_arg($args, "--user=%s", $::opt_user); - mtr_add_arg($args, "--password="); - if ( -e $srv->{'sockfile'} ) + foreach my $mysqld (@{$::master}, @{$::slave}) + { + if ( $mysqld->{'pid'} eq $ret_pid ) { - mtr_add_arg($args, "--socket=%s", $srv->{'sockfile'}); + mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid"); + $mysqld->{'pid'}= 0; + return; } - if ( $srv->{'port'} ) + } + + foreach my $cluster (@{$::clusters}) + { + if ( $cluster->{'pid'} eq $ret_pid ) { - mtr_add_arg($args, "--port=%s", $srv->{'port'}); + mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid"); + $cluster->{'pid'}= 0; + return; } - if ( $srv->{'port'} and ! -e $srv->{'sockfile'} ) + + foreach my $ndbd (@{$cluster->{'ndbds'}}) { - mtr_add_arg($args, "--protocol=tcp"); # Needed if no --socket + if ( $ndbd->{'pid'} eq $ret_pid ) + { + mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid"); + $ndbd->{'pid'}= 0; + return; + } } - mtr_add_arg($args, "--connect_timeout=5"); - mtr_add_arg($args, "--shutdown_timeout=$adm_shutdown_tmo"); - mtr_add_arg($args, "shutdown"); - # We don't wait for termination of mysqladmin - my $pid= mtr_spawn($::exe_mysqladmin, $args, - "", $::path_manager_log, $::path_manager_log, "", - { append_log_file => 1 }); - $mysql_admin_pids{$pid}= 1; } + mtr_warning("mark_process_dead couldn't find an entry for pid: $ret_pid"); + +} + +# +# Loop through our list of processes and look for and entry +# with the provided pid, if found check for the file indicating +# expected crash and restart it. +# +sub check_expected_crash_and_restart($) +{ + my $ret_pid= shift; - # As mysqladmin is such a simple program, we trust it to terminate. - # I.e. we wait blocking, and wait wait for them all before we go on. - while (keys %mysql_admin_pids) + foreach my $mysqld (@{$::master}, @{$::slave}) { - foreach my $pid (keys %mysql_admin_pids) + if ( $mysqld->{'pid'} eq $ret_pid ) { - if ( waitpid($pid,0) > 0 ) + mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid"); + $mysqld->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/" . "$mysqld->{'type'}" . + "$mysqld->{'idx'}" . ".expect"; + if ( -f $expect_file ) { - delete $mysql_admin_pids{$pid}; + mtr_verbose("Crash was expected, file $expect_file exists"); + mysqld_start($mysqld, $mysqld->{'start_opts'}, + $mysqld->{'start_slave_master_info'}); + unlink($expect_file); } + + return; } } - # If we trusted "mysqladmin --shutdown_timeout= ..." we could just - # terminate now, but we don't (FIXME should be debugged). - # So we try again to ping and at least wait the same amount of time - # mysqladmin would for all to die. - - my $timeout= 20; # 20 seconds max - my $res= 1; # If we just fall through, we are done - # in the sense that the servers don't - # listen to their ports any longer - TIME: - while ( $timeout-- ) + foreach my $cluster (@{$::clusters}) { - foreach my $srv ( @to_kill_specs ) + if ( $cluster->{'pid'} eq $ret_pid ) { - $res= 1; # We are optimistic - if ( mtr_ping_mysqld_server($srv->{'port'}, $srv->{'sockfile'}) ) + mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid"); + $cluster->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/ndb_mgmd_" . "$cluster->{'type'}" . + ".expect"; + if ( -f $expect_file ) { - mtr_debug("Sleep 1 second waiting for processes to stop using port"); - sleep(1); # One second - $res= 0; - next TIME; + mtr_verbose("Crash was expected, file $expect_file exists"); + ndbmgmd_start($cluster); + unlink($expect_file); } + return; } - last; # If we got here, we are done - } - - $timeout or mtr_debug("At least one server is still listening to its port"); - sleep(5) if $::glob_win32; # FIXME next startup fails if no sleep + foreach my $ndbd (@{$cluster->{'ndbds'}}) + { + if ( $ndbd->{'pid'} eq $ret_pid ) + { + mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid"); + $ndbd->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/ndbd_" . "$cluster->{'type'}" . + "$ndbd->{'idx'}" . ".expect"; + if ( -f $expect_file ) + { + mtr_verbose("Crash was expected, file $expect_file exists"); + ndbd_start($cluster, $ndbd->{'idx'}, + $ndbd->{'start_extra_args'}); + unlink($expect_file); + } + return; + } + } + } + mtr_warning("check_expected_crash_and_restart couldn't find an entry for pid: $ret_pid"); - return $res; } ############################################################################## @@ -740,32 +955,18 @@ sub mtr_mysqladmin_shutdown { sub mtr_record_dead_children () { + my $process_died= 0; my $ret_pid; - # FIXME the man page says to wait for -1 to terminate, - # but on OS X we get '0' all the time... - while ( ($ret_pid= waitpid(-1,&WNOHANG)) > 0 ) + # Wait without blockinng to see if any processes had died + # -1 or 0 means there are no more procesess to wait for + while ( ($ret_pid= waitpid(-1,&WNOHANG)) != 0 and $ret_pid != -1) { - mtr_debug("waitpid() catched exit of child $ret_pid"); - foreach my $idx (0..1) - { - if ( $::master->[$idx]->{'pid'} eq $ret_pid ) - { - mtr_debug("child $ret_pid was master[$idx]"); - $::master->[$idx]->{'pid'}= 0; - } - } - - foreach my $idx (0..2) - { - if ( $::slave->[$idx]->{'pid'} eq $ret_pid ) - { - mtr_debug("child $ret_pid was slave[$idx]"); - $::slave->[$idx]->{'pid'}= 0; - last; - } - } + mtr_warning("mtr_record_dead_children: $ret_pid"); + mark_process_dead($ret_pid); + $process_died= 1; } + return $process_died; } sub start_reap_all { @@ -777,16 +978,24 @@ sub start_reap_all { # here. If a process terminated before setting $SIG{CHLD} (but after # any attempt to waitpid() it), it will still be a zombie. So we # have to handle any such process here. - while(waitpid(-1, &WNOHANG) > 0) { }; + my $pid; + while(($pid= waitpid(-1, &WNOHANG)) != 0 and $pid != -1) + { + mtr_warning("start_reap_all pid: $pid"); + mark_process_dead($pid); + }; } sub stop_reap_all { $SIG{CHLD}= 'DEFAULT'; } -sub mtr_ping_mysqld_server () { + +sub mtr_ping_port ($) { my $port= shift; + mtr_verbose("mtr_ping_port: $port"); + my $remote= "localhost"; my $iaddr= inet_aton($remote); if ( ! $iaddr ) @@ -799,13 +1008,18 @@ sub mtr_ping_mysqld_server () { { mtr_error("can't create socket: $!"); } + + mtr_debug("Pinging server (port: $port)..."); + if ( connect(SOCK, $paddr) ) { close(SOCK); # FIXME check error? + mtr_verbose("USED"); return 1; } else { + mtr_verbose("FREE"); return 0; } } @@ -822,30 +1036,36 @@ sub sleep_until_file_created ($$$) { my $pidfile= shift; my $timeout= shift; my $pid= shift; + my $sleeptime= 100; # Milliseconds + my $loops= ($timeout * 1000) / $sleeptime; - for ( my $loop= 1; $loop <= $timeout; $loop++ ) + for ( my $loop= 1; $loop <= $loops; $loop++ ) { if ( -r $pidfile ) { return $pid; } - # Check if it died after the fork() was successful - if ( waitpid($pid,&WNOHANG) == $pid ) + # Check if it died after the fork() was successful + if ( $pid != 0 && waitpid($pid,&WNOHANG) == $pid ) { + mtr_warning("Process $pid died"); return 0; } - mtr_debug("Sleep 1 second waiting for creation of $pidfile"); + mtr_debug("Sleep $sleeptime milliseconds waiting for $pidfile"); - if ( $loop % 60 == 0 ) + # Print extra message every 60 seconds + my $seconds= ($loop * $sleeptime) / 1000; + if ( $seconds > 1 and int($seconds) % 60 == 0 ) { - my $left= $timeout - $loop; - mtr_warning("Waited $loop seconds for $pidfile to be created, " . + my $left= $timeout - $seconds; + mtr_warning("Waited $seconds seconds for $pidfile to be created, " . "still waiting for $left seconds..."); } - sleep(1); + # Millisceond sleep emulated with select + select(undef, undef, undef, ($sleeptime/1000)); } return 0; @@ -855,18 +1075,18 @@ sub sleep_until_file_created ($$$) { sub mtr_kill_processes ($) { my $pids = shift; - foreach my $sig (15,9) + mtr_verbose("mtr_kill_processes " . join(" ", @$pids)); + + foreach my $pid (@$pids) { - my $retries= 20; # FIXME 20 seconds, this is silly! - kill($sig, @{$pids}); - while ( $retries-- and kill(0, @{$pids}) ) + foreach my $sig (15, 9) { - mtr_debug("Sleep 1 second waiting for processes to die"); - sleep(1) # Wait one second + last if mtr_im_kill_process([ $pid ], $sig, 10, 1); } } } + ############################################################################## # # When we exit, we kill off all children @@ -876,7 +1096,7 @@ sub mtr_kill_processes ($) { # FIXME something is wrong, we sometimes terminate with "Hangup" written # to tty, and no STDERR output telling us why. -# FIXME for some readon, setting HUP to 'IGNORE' will cause exit() to +# FIXME for some reason, setting HUP to 'IGNORE' will cause exit() to # write out "Hangup", and maybe loose some output. We insert a sleep... sub mtr_exit ($) { @@ -884,9 +1104,18 @@ sub mtr_exit ($) { # cluck("Called mtr_exit()"); mtr_timer_stop_all($::glob_timers); local $SIG{HUP} = 'IGNORE'; - kill('HUP', -$$); - sleep 2; + # ToDo: Signalling -$$ will only work if we are the process group + # leader (in fact on QNX it will signal our session group leader, + # which might be Do-compile or Pushbuild, causing tests to be + # aborted). So we only do it if we are the group leader. We might + # set ourselves as the group leader at startup (with + # POSIX::setpgrp(0,0)), but then care must be needed to always do + # proper child process cleanup. + kill('HUP', -$$) if !$::glob_win32_perl and $$ == getpgrp(); + exit($code); } +########################################################################### + 1; diff --git a/mysql-test/lib/mtr_report.pl b/mysql-test/lib/mtr_report.pl index 515988ee5c7..6e3796133f2 100644 --- a/mysql-test/lib/mtr_report.pl +++ b/mysql-test/lib/mtr_report.pl @@ -10,6 +10,7 @@ sub mtr_report_test_name($); sub mtr_report_test_passed($); sub mtr_report_test_failed($); sub mtr_report_test_skipped($); +sub mtr_report_test_not_skipped_though_disabled($); sub mtr_show_failed_diff ($); sub mtr_report_stats ($); @@ -21,6 +22,7 @@ sub mtr_warning (@); sub mtr_error (@); sub mtr_child_error (@); sub mtr_debug (@); +sub mtr_verbose (@); ############################################################################## @@ -36,6 +38,7 @@ sub mtr_show_failed_diff ($) { my $reject_file= "r/$tname.reject"; my $result_file= "r/$tname.result"; + my $log_file= "r/$tname.log"; my $eval_file= "r/$tname.eval"; if ( $::opt_suite ne "main" ) @@ -43,10 +46,11 @@ sub mtr_show_failed_diff ($) { $reject_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$reject_file"; $result_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$result_file"; $eval_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$eval_file"; + $log_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$log_file"; } if ( -f $eval_file ) - { + { $result_file= $eval_file; } elsif ( $::opt_result_ext and @@ -70,6 +74,12 @@ sub mtr_show_failed_diff ($) { print "http://www.mysql.com/doc/en/Reporting_mysqltest_bugs.html\n"; print "to find the reason to this problem and how to report this.\n\n"; } + + if ( -f $log_file ) + { + print "Result from queries before failure can be found in $log_file\n"; + # FIXME Maybe a tail -f -n 10 $log_file here + } } sub mtr_report_test_name ($) { @@ -88,7 +98,24 @@ sub mtr_report_test_skipped ($) { } else { - print "[ skipped ]\n"; + print "[ skipped ] $tinfo->{'comment'}\n"; + } +} + +sub mtr_report_tests_not_skipped_though_disabled ($) { + my $tests= shift; + + if ( $::opt_enable_disabled ) + { + my @disabled_tests= grep {$_->{'dont_skip_though_disabled'}} @$tests; + if ( @disabled_tests ) + { + print "\nTest(s) which will be run though they are marked as disabled:\n"; + foreach my $tinfo ( sort {$a->{'name'} cmp $b->{'name'}} @disabled_tests ) + { + printf " %-20s : %s\n", $tinfo->{'name'}, $tinfo->{'comment'}; + } + } } } @@ -99,7 +126,7 @@ sub mtr_report_test_passed ($) { if ( $::opt_timer and -f "$::opt_vardir/log/timer" ) { $timer= mtr_fromfile("$::opt_vardir/log/timer"); - $::glob_tot_real_time += $timer; + $::glob_tot_real_time += ($timer/1000); $timer= sprintf "%12s", $timer; } $tinfo->{'result'}= 'MTR_RES_PASSED'; @@ -114,6 +141,11 @@ sub mtr_report_test_failed ($) { { print "[ fail ] timeout\n"; } + elsif ( $tinfo->{'ndb_test'} and $::cluster->[0]->{'installed_ok'} eq "NO") + { + print "[ fail ] ndbcluster start failure\n"; + return; + } else { print "[ fail ]\n"; @@ -144,6 +176,8 @@ sub mtr_report_stats ($) { my $tot_passed= 0; my $tot_failed= 0; my $tot_tests= 0; + my $tot_restarts= 0; + my $found_problems= 0; # Some warnings are errors... foreach my $tinfo (@$tests) { @@ -161,6 +195,10 @@ sub mtr_report_stats ($) { $tot_tests++; $tot_failed++; } + if ( $tinfo->{'restarted'} ) + { + $tot_restarts++; + } } # ---------------------------------------------------------------------- @@ -183,41 +221,69 @@ sub mtr_report_stats ($) { "the documentation at\n", "http://www.mysql.com/doc/en/MySQL_test_suite.html\n"; } + print + "The servers were restarted $tot_restarts times\n"; + + if ( $::opt_timer ) + { + print + "Spent $::glob_tot_real_time seconds actually executing testcases\n" + } # ---------------------------------------------------------------------- + # If a debug run, there might be interesting information inside + # the "var/log/*.err" files. We save this info in "var/log/warnings" # ---------------------------------------------------------------------- if ( ! $::glob_use_running_server ) { + # Save and report if there was any fatal warnings/errors in err logs - # Report if there was any fatal warnings/errors in the log files - # - unlink("$::opt_vardir/log/warnings"); - unlink("$::opt_vardir/log/warnings.tmp"); - # Remove some non fatal warnings from the log files - -# FIXME what is going on ????? ;-) -# sed -e 's!Warning: Table:.* on delete!!g' -e 's!Warning: Setting lower_case_table_names=2!!g' -e 's!Warning: One can only use the --user.*root!!g' \ -# var/log/*.err \ -# | sed -e 's!Warning: Table:.* on rename!!g' \ -# > var/log/warnings.tmp; -# -# found_error=0; -# # Find errors -# for i in "^Warning:" "^Error:" "^==.* at 0x" -# do -# if ( $GREP "$i" var/log/warnings.tmp >> var/log/warnings ) -# { -# found_error=1 -# } -# done -# unlink("$::opt_vardir/log/warnings.tmp"); -# if ( $found_error= "1" ) -# { -# print "WARNING: Got errors/warnings while running tests. Please examine\n" -# print "$::opt_vardir/log/warnings for details.\n" -# } -# } + my $warnlog= "$::opt_vardir/log/warnings"; + + unless ( open(WARN, ">$warnlog") ) + { + mtr_warning("can't write to the file \"$warnlog\": $!"); + } + else + { + # We report different types of problems in order + foreach my $pattern ( "^Warning:", "^Error:", "^==.* at 0x", + "InnoDB: Warning", "missing DBUG_RETURN", + "mysqld: Warning", + "Attempting backtrace", "Assertion .* failed" ) + { + foreach my $errlog ( sort glob("$::opt_vardir/log/*.err") ) + { + unless ( open(ERR, $errlog) ) + { + mtr_warning("can't read $errlog"); + next; + } + while ( <ERR> ) + { + # Skip some non fatal warnings from the log files + if ( /Warning:\s+Table:.* on (delete|rename)/ or + /Warning:\s+Setting lower_case_table_names=2/ or + /Warning:\s+One can only use the --user.*root/ or + /InnoDB: Warning: we did not need to do crash recovery/) + { + next; # Skip these lines + } + if ( /$pattern/ ) + { + $found_problems= 1; + print WARN $_; + } + } + } + } + if ( $found_problems ) + { + mtr_warning("Got errors/warnings while running tests, please examine", + "\"$warnlog\" for details."); + } + } } print "\n"; @@ -235,6 +301,9 @@ sub mtr_report_stats ($) { } } print "\n"; + } + if ( $tot_failed != 0 || $found_problems) + { mtr_error("there where failing test cases"); } } @@ -298,5 +367,11 @@ sub mtr_debug (@) { print STDERR "####: ",join(" ", @_),"\n"; } } +sub mtr_verbose (@) { + if ( $::opt_verbose ) + { + print STDERR "> ",join(" ", @_),"\n"; + } +} 1; diff --git a/mysql-test/lib/mtr_stress.pl b/mysql-test/lib/mtr_stress.pl new file mode 100644 index 00000000000..a7d4b68b69d --- /dev/null +++ b/mysql-test/lib/mtr_stress.pl @@ -0,0 +1,178 @@ +# -*- cperl -*- + +# This is a library file used by the Perl version of mysql-test-run, +# and is part of the translation of the Bourne shell script with the +# same name. + +use strict; +use File::Spec; + +# These are not to be prefixed with "mtr_" + +sub run_stress_test (); + +############################################################################## +# +# Run tests in the stress mode +# +############################################################################## + +sub run_stress_test () +{ + + my $args; + my $stress_suitedir; + + mtr_report("Starting stress testing\n"); + + if ( ! $::glob_use_embedded_server ) + { + if ( ! mysqld_start($::master->[0],[],[]) ) + { + mtr_error("Can't start the mysqld server"); + } + } + + my $stress_basedir=File::Spec->catdir($::opt_vardir, "stress"); + + #Clean up stress dir + if ( -d $stress_basedir ) + { + rmtree($stress_basedir); + } + mkpath($stress_basedir); + + if ($::opt_stress_suite ne 'main' && $::opt_stress_suite ne 'default' ) + { + $stress_suitedir=File::Spec->catdir($::glob_mysql_test_dir, "suite", + $::opt_stress_suite); + } + else + { + $stress_suitedir=$::glob_mysql_test_dir; + } + + if ( -d $stress_suitedir ) + { + #$stress_suite_t_dir=File::Spec->catdir($stress_suitedir, "t"); + #$stress_suite_r_dir=File::Spec->catdir($stress_suitedir, "r"); + #FIXME: check dirs above for existence to ensure that test suite + # contains tests and results dirs + } + else + { + mtr_error("Specified test suite $::opt_stress_suite doesn't exist"); + } + + if ( @::opt_cases ) + { + $::opt_stress_test_file=File::Spec->catfile($stress_basedir, "stress_tests.txt"); + open(STRESS_FILE, ">$::opt_stress_test_file"); + print STRESS_FILE join("\n",@::opt_cases),"\n"; + close(STRESS_FILE); + } + elsif ( $::opt_stress_test_file ) + { + $::opt_stress_test_file=File::Spec->catfile($stress_suitedir, + $::opt_stress_test_file); + if ( ! -f $::opt_stress_test_file ) + { + mtr_error("Specified file $::opt_stress_test_file with list of tests does not exist\n", + "Please ensure that file exists and has proper permissions"); + } + } + else + { + $::opt_stress_test_file=File::Spec->catfile($stress_suitedir, + "stress_tests.txt"); + if ( ! -f $::opt_stress_test_file ) + { + mtr_error("Default file $::opt_stress_test_file with list of tests does not exist\n", + "Please use --stress-test-file option to specify custom one or you can\n", + "just specify name of test for testing as last argument in command line"); + + } + } + + if ( $::opt_stress_init_file ) + { + $::opt_stress_init_file=File::Spec->catfile($stress_suitedir, + $::opt_stress_init_file); + if ( ! -f $::opt_stress_init_file ) + { + mtr_error("Specified file $::opt_stress_init_file with list of tests does not exist\n", + "Please ensure that file exists and has proper permissions"); + } + } + else + { + $::opt_stress_init_file=File::Spec->catfile($stress_suitedir, + "stress_init.txt"); + if ( ! -f $::opt_stress_init_file ) + { + $::opt_stress_init_file=''; + } + } + + if ( $::opt_stress_mode ne 'random' && $::opt_stress_mode ne 'seq' ) + { + mtr_error("You specified wrong mode $::opt_stress_mode for stress test\n", + "Correct values are 'random' or 'seq'"); + } + + mtr_init_args(\$args); + + mtr_add_arg($args, "--server-socket=%s", $::master->[0]->{'path_mysock'}); + mtr_add_arg($args, "--server-user=%s", $::opt_user); + mtr_add_arg($args, "--server-database=%s", "test"); + mtr_add_arg($args, "--stress-suite-basedir=%s", $::glob_mysql_test_dir); + mtr_add_arg($args, "--suite=%s", $::opt_stress_suite); + mtr_add_arg($args, "--stress-tests-file=%s", $::opt_stress_test_file); + mtr_add_arg($args, "--stress-basedir=%s", $stress_basedir); + mtr_add_arg($args, "--server-logs-dir=%s", $stress_basedir); + mtr_add_arg($args, "--stress-mode=%s", $::opt_stress_mode); + mtr_add_arg($args, "--mysqltest=%s", $::exe_mysqltest); + mtr_add_arg($args, "--threads=%s", $::opt_stress_threads); + mtr_add_arg($args, "--verbose"); + mtr_add_arg($args, "--cleanup"); + mtr_add_arg($args, "--log-error-details"); + mtr_add_arg($args, "--abort-on-error"); + + if ( $::opt_stress_init_file ) + { + mtr_add_arg($args, "--stress-init-file=%", $::opt_stress_init_file); + } + + if ( !$::opt_stress_loop_count && !$::opt_stress_test_count && + !$::opt_stress_test_duration ) + { + #Limit stress testing with 20 loops in case when any limit parameter + #was specified + $::opt_stress_test_count=20; + } + + if ( $::opt_stress_loop_count ) + { + mtr_add_arg($args, "--loop-count=%s", $::opt_stress_loop_count); + } + + if ( $::opt_stress_test_count ) + { + mtr_add_arg($args, "--test-count=%s", $::opt_stress_test_count); + } + + if ( $::opt_stress_test_duration ) + { + mtr_add_arg($args, "--test-duration=%s", $::opt_stress_test_duration); + } + + #Run stress test + mtr_run("$::glob_mysql_test_dir/mysql-stress-test.pl", $args, "", "", "", ""); + + if ( ! $::glob_use_embedded_server ) + { + stop_masters(); + } +} + +1; |