From 15a5d6294a92f0e3ee1de100060c503088bcfa01 Mon Sep 17 00:00:00 2001 From: Dan Fandrich Date: Fri, 28 Apr 2023 20:49:28 -0700 Subject: runtests: turn singletest() into a state machine This allows it to run in a non-blocking manner. Ref: #10818 --- tests/runner.pm | 9 +- tests/runtests.pl | 338 ++++++++++++++++++++++++++++++++---------------------- 2 files changed, 206 insertions(+), 141 deletions(-) diff --git a/tests/runner.pm b/tests/runner.pm index c139ecc3e..1d48e979d 100644 --- a/tests/runner.pm +++ b/tests/runner.pm @@ -1137,12 +1137,13 @@ sub runnerar { ################################################################### # Returns nonzero if a response from an async call is ready +# argument is 0 for nonblocking, undef for blocking, anything else for timeout # Called by controller sub runnerar_ready { - my ($blocking) = @_; + my ($runnerid, $blocking) = @_; my $rin; vec($rin, fileno($controllerr), 1) = 1; - return select(my $rout=$rin, undef, my $eout=$rin, $blocking ? undef : 0); + return select(my $rout=$rin, undef, my $eout=$rin, $blocking); } ################################################################### @@ -1175,7 +1176,7 @@ sub ipcrecv { elsif($funcname eq "runner_shutdown") { runner_shutdown(@$argsarrayref); # Special case: no response - return; + return 1; } elsif($funcname eq "runner_stopservers") { @res = runner_stopservers(@$argsarrayref); @@ -1194,6 +1195,8 @@ sub ipcrecv { $buf = freeze \@res; syswrite($runnerw, (pack "L", length($buf)) . $buf); + + return 0; } ################################################################### diff --git a/tests/runtests.pl b/tests/runtests.pl index f82eedc4d..bd8f4ebb5 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -150,12 +150,22 @@ my %timetoolini; # timestamp for each test command run starting my %timetoolend; # timestamp for each test command run stopping my %timesrvrlog; # timestamp for each test server logs lock removal my %timevrfyend; # timestamp for each test result verification end -my $runnerid; # ID for runner async calls +my $globalabort; # flag signalling program abort + +# values for $singletest_state +use constant { + ST_INIT => 0, + ST_CLEARLOCKS => 1, + ST_INITED => 2, + ST_PREPROCESS => 3, + ST_RUN => 4, +}; +my $singletest_state = ST_INIT; # current state of singletest() + ####################################################################### # variables that command line options may set # - my $short; my $no_debuginfod; my $keepoutfiles; # keep stdout and stderr files after tests @@ -186,14 +196,7 @@ sub logmsg { sub catch_zap { my $signame = shift; logmsg "runtests.pl received SIG$signame, exiting\n"; - # TODO: make this set a flag that is checked in the main test loop - if($runnerid) { - runnerac_stopservers($runnerid); - runnerar(); # ignore the results - # Kill the runner entirely - runnerac_shutdown($runnerid); - } - die "Somebody sent me a SIG$signame"; + $globalabort = 1; } $SIG{INT} = \&catch_zap; $SIG{TERM} = \&catch_zap; @@ -1077,7 +1080,7 @@ sub singletest_count { ####################################################################### # Verify test succeeded sub singletest_check { - my ($testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind)=@_; + my ($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind)=@_; # Skip all the verification on torture tests if ($torture) { @@ -1621,116 +1624,146 @@ sub singletest_success { # Run a single specified test case # sub singletest { - my ($testnum, $count, $total)=@_; + my ($runnerid, $testnum, $count, $total)=@_; - my $logdir = getlogdir($testnum); + if($singletest_state == ST_INIT) { + my $logdir = getlogdir($testnum); - # first, remove all lingering log files - if(!cleardir($logdir) && $clearlocks) { - runnerac_clearlocks($runnerid, $logdir); + # first, remove all lingering log files + if(!cleardir($logdir) && $clearlocks) { + runnerac_clearlocks($runnerid, $logdir); + $singletest_state = ST_CLEARLOCKS; + } else { + $singletest_state = ST_INITED; + # Recursively call the state machine again because there is no + # event expected that would otherwise trigger a new call. + return singletest(@_); + } + + } elsif($singletest_state == ST_CLEARLOCKS) { my ($rid, $logs) = runnerar(); logmsg $logs; + my $logdir = getlogdir($testnum); cleardir($logdir); - } - - ################################################################### - # Restore environment variables that were modified in a previous run. - # Test definition may instruct to (un)set environment vars. - # This is done this early so that leftover variables don't affect - # starting servers or CI registration. - restore_test_env(1); + $singletest_state = ST_INITED; + # Recursively call the state machine again because there is no + # event expected that would otherwise trigger a new call. + return singletest(@_); + + } elsif($singletest_state == ST_INITED) { + ################################################################### + # Restore environment variables that were modified in a previous run. + # Test definition may instruct to (un)set environment vars. + # This is done this early so that leftover variables don't affect + # starting servers or CI registration. + restore_test_env(1); + + ################################################################### + # Load test file so CI registration can get the right data before the + # runner is called + loadtest("${TESTDIR}/test${testnum}"); + + ################################################################### + # Register the test case with the CI environment + citest_starttest($testnum); + + runnerac_test_preprocess($runnerid, $testnum); + $singletest_state = ST_PREPROCESS; + + } elsif($singletest_state == ST_PREPROCESS) { + my ($rid, $why, $error, $logs, $testtimings) = runnerar(); + logmsg $logs; + if($error == -2) { + if($postmortem) { + # Error indicates an actual problem starting the server, so + # display the server logs + displaylogs($testnum); + } + } + updatetesttimings($testnum, %$testtimings); + + ####################################################################### + # Print the test name and count tests + $error = singletest_count($testnum, $why); + if($error) { + # Submit the test case result with the CI environment + citest_finishtest($testnum, $error); + $singletest_state = ST_INIT; + return ($error, 0); + } + + ####################################################################### + # Execute this test number + my $cmdres; + my $CURLOUT; + my $tool; + my $usedvalgrind; + runnerac_test_run($runnerid, $testnum); + $singletest_state = ST_RUN; + + } elsif($singletest_state == ST_RUN) { + my ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar(); + logmsg $logs; + updatetesttimings($testnum, %$testtimings); + if($error == -1) { + # no further verification will occur + $timevrfyend{$testnum} = Time::HiRes::time(); + my $err = ignoreresultcode($testnum); + # Submit the test case result with the CI environment + citest_finishtest($testnum, $err); + $singletest_state = ST_INIT; + # return a test failure, either to be reported or to be ignored + return ($err, 0); + } + elsif($error == -2) { + # fill in the missing timings on error + timestampskippedevents($testnum); + # Submit the test case result with the CI environment + citest_finishtest($testnum, $error); + $singletest_state = ST_INIT; + return ($error, 0); + } + elsif($error > 0) { + # no further verification will occur + $timevrfyend{$testnum} = Time::HiRes::time(); + # Submit the test case result with the CI environment + citest_finishtest($testnum, $error); + $singletest_state = ST_INIT; + return ($error, 0); + } - ################################################################### - # Load test file so CI registration can get the right data before the - # runner is called - loadtest("${TESTDIR}/test${testnum}"); + ####################################################################### + # Verify that the test succeeded + $error = singletest_check($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind); + if($error == -1) { + my $err = ignoreresultcode($testnum); + # Submit the test case result with the CI environment + citest_finishtest($testnum, $err); + $singletest_state = ST_INIT; + # return a test failure, either to be reported or to be ignored + return ($err, 0); + } + elsif($error == -2) { + # torture test; there is no verification, so the run result holds the + # test success code + # Submit the test case result with the CI environment + citest_finishtest($testnum, $cmdres); + $singletest_state = ST_INIT; + return ($cmdres, 0); + } - ################################################################### - # Register the test case with the CI environment - citest_starttest($testnum); - runnerac_test_preprocess($runnerid, $testnum); - my ($rid, $why, $error, $logs, $testtimings) = runnerar(); - logmsg $logs; - if($error == -2) { - if($postmortem) { - # Error indicates an actual problem starting the server, so - # display the server logs - displaylogs($testnum); - } - } - updatetesttimings($testnum, %$testtimings); + ####################################################################### + # Report a successful test + singletest_success($testnum, $count, $total, ignoreresultcode($testnum)); - ####################################################################### - # Print the test name and count tests - $error = singletest_count($testnum, $why); - if($error) { - # Submit the test case result with the CI environment - citest_finishtest($testnum, $error); - return $error; - } - - ####################################################################### - # Execute this test number - my $cmdres; - my $CURLOUT; - my $tool; - my $usedvalgrind; - runnerac_test_run($runnerid, $testnum); - ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar(); - logmsg $logs; - updatetesttimings($testnum, %$testtimings); - if($error == -1) { - # no further verification will occur - $timevrfyend{$testnum} = Time::HiRes::time(); - my $err = ignoreresultcode($testnum); # Submit the test case result with the CI environment - citest_finishtest($testnum, $err); - # return a test failure, either to be reported or to be ignored - return $err; - } - elsif($error == -2) { - # fill in the missing timings on error - timestampskippedevents($testnum); - # Submit the test case result with the CI environment - citest_finishtest($testnum, $error); - return $error; - } - elsif($error > 0) { - # no further verification will occur - $timevrfyend{$testnum} = Time::HiRes::time(); - # Submit the test case result with the CI environment - citest_finishtest($testnum, $error); - return $error; - } + citest_finishtest($testnum, 0); + $singletest_state = ST_INIT; - ####################################################################### - # Verify that the test succeeded - $error = singletest_check($testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind); - if($error == -1) { - my $err = ignoreresultcode($testnum); - # Submit the test case result with the CI environment - citest_finishtest($testnum, $err); - # return a test failure, either to be reported or to be ignored - return $err; - } - elsif($error == -2) { - # torture test; there is no verification, so the run result holds the - # test success code - # Submit the test case result with the CI environment - citest_finishtest($testnum, $cmdres); - return $cmdres; + return (0, 0); # state machine is finished } - - - ####################################################################### - # Report a successful test - singletest_success($testnum, $count, $total, ignoreresultcode($testnum)); - - # Submit the test case result with the CI environment - citest_finishtest($testnum, 0); - - return 0; + return (0, 1); # state machine must be called again on event } ####################################################################### @@ -2522,48 +2555,77 @@ citest_starttestrun(); # Initialize the runner to prepare to run tests cleardir($LOGDIR); mkdir($LOGDIR, 0777); -$runnerid = runner_init($LOGDIR); +my $runnerid = runner_init($LOGDIR); ####################################################################### # The main test-loop # # run through each candidate test and execute it +nexttest: foreach my $testnum (@runtests) { $count++; - # execute one test case - my $error = singletest($testnum, $count, scalar(@runtests)); - - if($error < 0) { - # not a test we can run - next; - } - - $total++; # number of tests we've run - - if($error>0) { - if($error==2) { - # ignored test failures - $failedign .= "$testnum "; + # Loop over state machine waiting for singletest to complete + my $again; + while () { + # check the abort flag + if($globalabort) { + logmsg "Aborting tests\n"; + if($again) { + logmsg "Waiting for test to finish...\n"; + # Wait for the last request to complete and throw it away so + # that IPC calls & responses stay in sync + # TODO: send a signal to the runner to interrupt a long test + runnerar(); + } + last nexttest; } - else { - $failed.= "$testnum "; + + # execute one test case + my $error; + ($error, $again) = singletest($runnerid, $testnum, $count, scalar(@runtests)); + if($again) { + # Wait for asynchronous response + if(!runnerar_ready($runnerid, 0.05)) { + # TODO: If a response isn't ready, this is a chance to do + # something else first + } + next; # another iteration of the same singletest } - if($postmortem) { - # display all files in $LOGDIR/ in a nice way - displaylogs($testnum); + + # Test has completed + if($error < 0) { + # not a test we can run + next nexttest; } - if($error==2) { - $ign++; # ignored test result counter + + $total++; # number of tests we've run + + if($error>0) { + if($error==2) { + # ignored test failures + $failedign .= "$testnum "; + } + else { + $failed.= "$testnum "; + } + if($postmortem) { + # display all files in $LOGDIR/ in a nice way + displaylogs($testnum); + } + if($error==2) { + $ign++; # ignored test result counter + } + elsif(!$anyway) { + # a test failed, abort + logmsg "\n - abort tests\n"; + last; + } } - elsif(!$anyway) { - # a test failed, abort - logmsg "\n - abort tests\n"; - last; + elsif(!$error) { + $ok++; # successful test counter } - } - elsif(!$error) { - $ok++; # successful test counter + next nexttest; } # loop for next test -- cgit v1.2.1