summaryrefslogtreecommitdiff
path: root/run-tests.php
diff options
context:
space:
mode:
Diffstat (limited to 'run-tests.php')
-rwxr-xr-xrun-tests.php871
1 files changed, 490 insertions, 381 deletions
diff --git a/run-tests.php b/run-tests.php
index c5f1c02ad6..90418df584 100755
--- a/run-tests.php
+++ b/run-tests.php
@@ -25,10 +25,16 @@
/* $Id$ */
+/* Temporary variables while this file is being refactored. */
+/** @var ?JUnit */
+$junit = null;
+
+/* End temporary variables. */
+
/* Let there be no top-level code beyond this point:
* Only functions and classes, thanks!
*
- * Minimum required PHP version: 7.1.0
+ * Minimum required PHP version: 7.4.0
*/
function show_usage(): void
@@ -121,6 +127,10 @@ Options:
--color
--no-color Do/Don't colorize the result type in the test result.
+ --repeat [n]
+ Run the tests multiple times in the same process and check the
+ output of the last execution (CLI SAPI only).
+
HELP;
}
@@ -147,11 +157,15 @@ function main(): void
$repeat, $result_tests_file, $slow_min_ms, $start_time, $switch,
$temp_source, $temp_target, $test_cnt, $test_dirs,
$test_files, $test_idx, $test_list, $test_results, $testfile,
- $user_tests, $valgrind, $sum_results, $shuffle, $file_cache;
+ $user_tests, $valgrind, $sum_results, $shuffle, $file_cache, $num_repeats;
// Parallel testing
global $workers, $workerID;
global $context_line_count;
+ // Temporary for the duration of refactoring
+ /** @var JUnit */
+ global $junit;
+
define('IS_WINDOWS', substr(PHP_OS, 0, 3) == "WIN");
$workerID = 0;
@@ -230,65 +244,6 @@ function main(): void
$php_cgi = null;
$phpdbg = null;
- if (getenv('TEST_PHP_EXECUTABLE')) {
- $php = getenv('TEST_PHP_EXECUTABLE');
-
- if ($php == 'auto') {
- $php = TEST_PHP_SRCDIR . '/sapi/cli/php';
- putenv("TEST_PHP_EXECUTABLE=$php");
-
- if (!getenv('TEST_PHP_CGI_EXECUTABLE')) {
- $php_cgi = TEST_PHP_SRCDIR . '/sapi/cgi/php-cgi';
-
- if (file_exists($php_cgi)) {
- putenv("TEST_PHP_CGI_EXECUTABLE=$php_cgi");
- } else {
- $php_cgi = null;
- }
- }
- }
- $environment['TEST_PHP_EXECUTABLE'] = $php;
- }
-
- if (getenv('TEST_PHP_CGI_EXECUTABLE')) {
- $php_cgi = getenv('TEST_PHP_CGI_EXECUTABLE');
-
- if ($php_cgi == 'auto') {
- $php_cgi = TEST_PHP_SRCDIR . '/sapi/cgi/php-cgi';
- putenv("TEST_PHP_CGI_EXECUTABLE=$php_cgi");
- }
-
- $environment['TEST_PHP_CGI_EXECUTABLE'] = $php_cgi;
- }
-
- if (!getenv('TEST_PHPDBG_EXECUTABLE')) {
- if (IS_WINDOWS && file_exists(dirname($php) . "/phpdbg.exe")) {
- $phpdbg = realpath(dirname($php) . "/phpdbg.exe");
- } elseif (file_exists(dirname($php) . "/../../sapi/phpdbg/phpdbg")) {
- $phpdbg = realpath(dirname($php) . "/../../sapi/phpdbg/phpdbg");
- } elseif (file_exists("./sapi/phpdbg/phpdbg")) {
- $phpdbg = realpath("./sapi/phpdbg/phpdbg");
- } elseif (file_exists(dirname($php) . "/phpdbg")) {
- $phpdbg = realpath(dirname($php) . "/phpdbg");
- } else {
- $phpdbg = null;
- }
- if ($phpdbg) {
- putenv("TEST_PHPDBG_EXECUTABLE=$phpdbg");
- }
- }
-
- if (getenv('TEST_PHPDBG_EXECUTABLE')) {
- $phpdbg = getenv('TEST_PHPDBG_EXECUTABLE');
-
- if ($phpdbg == 'auto') {
- $phpdbg = TEST_PHP_SRCDIR . '/sapi/phpdbg/phpdbg';
- putenv("TEST_PHPDBG_EXECUTABLE=$phpdbg");
- }
-
- $environment['TEST_PHPDBG_EXECUTABLE'] = $phpdbg;
- }
-
if (getenv('TEST_PHP_LOG_FORMAT')) {
$log_format = strtoupper(getenv('TEST_PHP_LOG_FORMAT'));
} else {
@@ -302,7 +257,7 @@ function main(): void
$DETAILED = 0;
}
- junit_init();
+ $junit = new JUnit($environment, $workerID);
if (getenv('SHOW_ONLY_GROUPS')) {
$SHOW_ONLY_GROUPS = explode(",", getenv('SHOW_ONLY_GROUPS'));
@@ -395,7 +350,7 @@ function main(): void
if (function_exists('sapi_windows_vt100_support') && !sapi_windows_vt100_support(STDOUT, true)) {
$colorize = false;
}
- if (array_key_exists('NO_COLOR', $_ENV)) {
+ if (array_key_exists('NO_COLOR', $environment)) {
$colorize = false;
}
$selected_tests = false;
@@ -405,6 +360,7 @@ function main(): void
$shuffle = false;
$workers = null;
$context_line_count = 3;
+ $num_repeats = 1;
$cfgtypes = ['show', 'keep'];
$cfgfiles = ['skip', 'php', 'clean', 'out', 'diff', 'exp', 'mem'];
@@ -623,6 +579,10 @@ function main(): void
. ':print_suppressions=0';
}
break;
+ case '--repeat':
+ $num_repeats = (int) $argv[++$i];
+ $environment['SKIP_REPEAT'] = 1;
+ break;
//case 'w'
case '-':
// repeat check with full switch
@@ -683,14 +643,35 @@ function main(): void
return;
}
- // Default to PHP_BINARY as executable
- if (!isset($environment['TEST_PHP_EXECUTABLE'])) {
+ if (!$php) {
+ $php = getenv('TEST_PHP_EXECUTABLE');
+ }
+ if (!$php) {
$php = PHP_BINARY;
- putenv("TEST_PHP_EXECUTABLE=$php");
- $environment['TEST_PHP_EXECUTABLE'] = $php;
}
- if (strlen($conf_passed)) {
+ if (!$php_cgi) {
+ $php_cgi = getenv('TEST_PHP_CGI_EXECUTABLE');
+ }
+ if (!$php_cgi) {
+ $php_cgi = get_binary($php, 'php-cgi', 'sapi/cgi/php-cgi');
+ }
+
+ if (!$phpdbg) {
+ $phpdbg = getenv('TEST_PHPDBG_EXECUTABLE');
+ }
+ if (!$phpdbg) {
+ $phpdbg = get_binary($php, 'phpdbg', 'sapi/phpdbg/phpdbg');
+ }
+
+ putenv("TEST_PHP_EXECUTABLE=$php");
+ $environment['TEST_PHP_EXECUTABLE'] = $php;
+ putenv("TEST_PHP_CGI_EXECUTABLE=$php_cgi");
+ $environment['TEST_PHP_CGI_EXECUTABLE'] = $php_cgi;
+ putenv("TEST_PHPDBG_EXECUTABLE=$phpdbg");
+ $environment['TEST_PHPDBG_EXECUTABLE'] = $phpdbg;
+
+ if ($conf_passed !== null) {
if (IS_WINDOWS) {
$pass_options .= " -c " . escapeshellarg($conf_passed);
} else {
@@ -801,7 +782,7 @@ function main(): void
save_or_mail_results();
}
- junit_save_xml();
+ $junit->saveXML();
if (getenv('REPORT_EXIT_STATUS') !== '0' && getenv('REPORT_EXIT_STATUS') !== 'no' &&
($sum_results['FAILED'] || $sum_results['BORKED'] || $sum_results['LEAKED'])) {
exit(1);
@@ -890,7 +871,6 @@ More .INIs : " , (function_exists(\'php_ini_scanned_files\') ? str_replace("\n"
'tidy' => ['tidy.clean_output=0'],
'zlib' => ['zlib.output_compression=Off'],
'xdebug' => ['xdebug.mode=off'],
- 'mbstring' => ['mbstring.func_overload=0'],
];
foreach ($info_params_ex as $ext => $ini_overwrites_ex) {
@@ -1049,6 +1029,21 @@ function save_or_mail_results(): void
}
}
+function get_binary(string $php, string $sapi, string $sapi_path): ?string
+{
+ $dir = dirname($php);
+ if (IS_WINDOWS && file_exists("$dir/$sapi.exe")) {
+ return realpath("$dir/$sapi.exe");
+ }
+ if (file_exists("$dir/../../$sapi_path")) {
+ return realpath("$dir/../../$sapi_path");
+ }
+ if (file_exists("$dir/$sapi")) {
+ return realpath("$dir/$sapi");
+ }
+ return null;
+}
+
function find_files(string $dir, bool $is_ext_dir = false, bool $ignore = false): void
{
global $test_files, $exts_to_test, $ignored_by_ext, $exts_skipped;
@@ -1381,6 +1376,8 @@ function run_all_tests_parallel(array $test_files, array $env, $redir_tested): v
{
global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS, $valgrind;
+ global $junit;
+
// The PHP binary running run-tests.php, and run-tests.php itself
// This PHP executable is *not* necessarily the same as the tested version
$thisPHP = PHP_BINARY;
@@ -1463,7 +1460,7 @@ function run_all_tests_parallel(array $test_files, array $env, $redir_tested): v
[], // Inherit our stdin, stdout and stderr
$pipes,
null,
- $_ENV + [
+ $GLOBALS['environment'] + [
"TEST_PHP_WORKER" => $i,
"TEST_PHP_URI" => $sockUri,
],
@@ -1579,9 +1576,7 @@ escape:
}
}
}
- if (junit_enabled()) {
- junit_merge_results($message["junit"]);
- }
+ $junit->mergeResults($message["junit"]);
// no break
case "ready":
// Schedule sequential tests only once we are down to one worker.
@@ -1721,6 +1716,8 @@ function run_worker(): void
{
global $workerID, $workerSock;
+ global $junit;
+
$sockUri = getenv("TEST_PHP_URI");
$workerSock = stream_socket_client($sockUri, $_, $_, 5) or error("Couldn't connect to $sockUri");
@@ -1767,9 +1764,9 @@ function run_worker(): void
run_all_tests($command["test_files"], $command["env"], $command["redir_tested"]);
send_message($workerSock, [
"type" => "tests_finished",
- "junit" => junit_enabled() ? $GLOBALS['JUNIT'] : null,
+ "junit" => $junit->isEnabled() ? $junit : null,
]);
- junit_init();
+ $junit->clear();
break;
default:
send_message($workerSock, [
@@ -1805,6 +1802,15 @@ function show_file_block(string $file, string $block, ?string $section = null):
}
}
+function skip_test(string $tested, string $tested_file, string $shortname, string $reason) {
+ global $junit;
+
+ show_result('SKIP', $tested, $tested_file, "reason: $reason");
+ $junit->initSuite($junit->getSuiteName($shortname));
+ $junit->markTestAs('SKIP', $shortname, $tested, 0, $reason);
+ return 'SKIPPED';
+}
+
//
// Run an individual test case.
//
@@ -1821,8 +1827,19 @@ function run_test(string $php, $file, array $env): string
global $no_file_cache;
global $slow_min_ms;
global $preload, $file_cache;
+ global $num_repeats;
// Parallel testing
global $workerID;
+
+ // Temporary
+ /** @var JUnit */
+ global $junit;
+
+ static $skipCache;
+ if (!$skipCache) {
+ $skipCache = new SkipCache($cfg['keep']['skip']);
+ }
+
$temp_filenames = null;
$org_file = $file;
@@ -1914,6 +1931,10 @@ TEST $file
}
}
+ $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
+ $tested_file = $shortname;
+ $tested = trim($section_text['TEST']);
+
// the redirect section allows a set of tests to be reused outside of
// a given test dir
if ($bork_info === null) {
@@ -1931,6 +1952,10 @@ TEST $file
unset($section_text['FILEEOF']);
}
+ if ($num_repeats > 1 && isset($section_text['FILE_EXTERNAL'])) {
+ return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable');
+ }
+
foreach (['FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX'] as $prefix) {
$key = $prefix . '_EXTERNAL';
@@ -1954,9 +1979,6 @@ TEST $file
}
fclose($fp);
- $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
- $tested_file = $shortname;
-
if ($bork_info !== null) {
show_result("BORK", $bork_info, $tested_file);
$PHP_FAILED_TESTS['BORKED'][] = [
@@ -1967,7 +1989,7 @@ TEST $file
'info' => "$bork_info [$file]",
];
- junit_mark_test_as('BORK', $shortname, $tested_file, 0, $bork_info);
+ $junit->markTestAs('BORK', $shortname, $tested_file, 0, $bork_info);
return 'BORKED';
}
@@ -1986,30 +2008,16 @@ TEST $file
$cmdRedirect = '';
}
- $tested = trim($section_text['TEST']);
-
/* For GET/POST/PUT tests, check if cgi sapi is available and if it is, use it. */
if (array_key_exists('CGI', $section_text) || !empty($section_text['GET']) || !empty($section_text['POST']) || !empty($section_text['GZIP_POST']) || !empty($section_text['DEFLATE_POST']) || !empty($section_text['POST_RAW']) || !empty($section_text['PUT']) || !empty($section_text['COOKIE']) || !empty($section_text['EXPECTHEADERS'])) {
- if (isset($php_cgi)) {
- $php = $php_cgi . ' -C ';
- } elseif (IS_WINDOWS && file_exists(dirname($php) . "/php-cgi.exe")) {
- $php = realpath(dirname($php) . "/php-cgi.exe") . ' -C ';
- } else {
- if (file_exists(dirname($php) . "/../../sapi/cgi/php-cgi")) {
- $php = realpath(dirname($php) . "/../../sapi/cgi/php-cgi") . ' -C ';
- } elseif (file_exists("./sapi/cgi/php-cgi")) {
- $php = realpath("./sapi/cgi/php-cgi") . ' -C ';
- } elseif (file_exists(dirname($php) . "/php-cgi")) {
- $php = realpath(dirname($php) . "/php-cgi") . ' -C ';
- } else {
- show_result('SKIP', $tested, $tested_file, "reason: CGI not available");
-
- junit_init_suite(junit_get_suitename_for($shortname));
- junit_mark_test_as('SKIP', $shortname, $tested, 0, 'CGI not available');
- return 'SKIPPED';
- }
+ if (!$php_cgi) {
+ return skip_test($tested, $tested_file, $shortname, 'CGI not available');
}
+ $php = $php_cgi . ' -C ';
$uses_cgi = true;
+ if ($num_repeats > 1) {
+ return skip_test($tested, $tested_file, $shortname, 'CGI does not support --repeat');
+ }
}
/* For phpdbg tests, check if phpdbg sapi is available and if it is, use it. */
@@ -2026,11 +2034,22 @@ TEST $file
// be run straight away. For example, EXTENSIONS, SKIPIF, CLEAN.
$extra_options = '-rr';
} else {
- show_result('SKIP', $tested, $tested_file, "reason: phpdbg not available");
+ return skip_test($tested, $tested_file, $shortname, 'phpdbg not available');
+ }
+ if ($num_repeats > 1) {
+ return skip_test($tested, $tested_file, $shortname, 'phpdbg does not support --repeat');
+ }
+ }
- junit_init_suite(junit_get_suitename_for($shortname));
- junit_mark_test_as('SKIP', $shortname, $tested, 0, 'phpdbg not available');
- return 'SKIPPED';
+ if ($num_repeats > 1) {
+ if (array_key_exists('CLEAN', $section_text)) {
+ return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable');
+ }
+ if (array_key_exists('STDIN', $section_text)) {
+ return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable');
+ }
+ if (array_key_exists('CAPTURE_STDIO', $section_text)) {
+ return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable');
}
}
@@ -2144,9 +2163,8 @@ TEST $file
$ext_params = [];
settings2array($ini_overwrites, $ext_params);
$ext_params = settings2params($ext_params);
- $ext_dir = `$php $pass_options $extra_options $ext_params $no_file_cache -d display_errors=0 -r "echo ini_get('extension_dir');"`;
$extensions = preg_split("/[\n\r]+/", trim($section_text['EXTENSIONS']));
- $loaded = explode(",", `$php $pass_options $extra_options $ext_params $no_file_cache -d display_errors=0 -r "echo implode(',', get_loaded_extensions());"`);
+ [$ext_dir, $loaded] = $skipCache->getExtensions("$php $pass_options $extra_options $ext_params $no_file_cache");
$ext_prefix = IS_WINDOWS ? "php_" : "";
foreach ($extensions as $req_ext) {
if (!in_array($req_ext, $loaded)) {
@@ -2176,6 +2194,9 @@ TEST $file
// even though all the files are re-created.
$ini_settings['opcache.validate_timestamps'] = '0';
}
+ } else if ($num_repeats > 1) {
+ // Make sure warnings still show up on the second run.
+ $ini_settings['opcache.record_warnings'] = '1';
}
// Any special ini settings
@@ -2186,6 +2207,10 @@ TEST $file
$replacement = IS_WINDOWS ? '"' . PHP_BINARY . ' -r \"while ($in = fgets(STDIN)) echo $in;\" > $1"' : 'tee $1 >/dev/null';
$section_text['INI'] = preg_replace('/{MAIL:(\S+)}/', $replacement, $section_text['INI']);
settings2array(preg_split("/[\n\r]+/", $section_text['INI']), $ini_settings);
+
+ if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) {
+ return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable');
+ }
}
$ini_settings = settings2params($ini_settings);
@@ -2199,7 +2224,6 @@ TEST $file
if (array_key_exists('SKIPIF', $section_text)) {
if (trim($section_text['SKIPIF'])) {
show_file_block('skip', $section_text['SKIPIF']);
- save_text($test_skipif, $section_text['SKIPIF'], $temp_skipif);
$extra = !IS_WINDOWS ?
"unset REQUEST_METHOD; unset QUERY_STRING; unset PATH_TRANSLATED; unset SCRIPT_FILENAME; unset REQUEST_METHOD;" : "";
@@ -2208,15 +2232,24 @@ TEST $file
$env['ZEND_DONT_UNLOAD_MODULES'] = 1;
}
- junit_start_timer($shortname);
+ $junit->startTimer($shortname);
- $output = system_with_timeout("$extra $php $pass_options $extra_options -q $orig_ini_settings $no_file_cache -d display_errors=1 -d display_startup_errors=0 \"$test_skipif\"", $env);
- $output = trim($output);
+ $startTime = microtime(true);
+ $commandLine = "$extra $php $pass_options $extra_options -q $orig_ini_settings $no_file_cache -d display_errors=1 -d display_startup_errors=0";
+ $output = $skipCache->checkSkip($commandLine, $section_text['SKIPIF'], $test_skipif, $temp_skipif, $env);
- junit_finish_timer($shortname);
+ $time = microtime(true) - $startTime;
- if (!$cfg['keep']['skip']) {
- @unlink($test_skipif);
+ $junit->stopTimer($shortname);
+
+ if ($time > $slow_min_ms / 1000) {
+ $PHP_FAILED_TESTS['SLOW'][] = [
+ 'name' => $file,
+ 'test_name' => 'SKIPIF of ' . $tested . " [$tested_file]",
+ 'output' => '',
+ 'diff' => '',
+ 'info' => $time,
+ ];
}
if (!strncasecmp('skip', $output, 4)) {
@@ -2226,12 +2259,8 @@ TEST $file
show_result('SKIP', $tested, $tested_file, '', $temp_filenames);
}
- if (!$cfg['keep']['skip']) {
- @unlink($test_skipif);
- }
-
$message = !empty($m[1]) ? $m[1] : '';
- junit_mark_test_as('SKIP', $shortname, $tested, null, $message);
+ $junit->markTestAs('SKIP', $shortname, $tested, null, $message);
return 'SKIPPED';
}
@@ -2253,7 +2282,7 @@ TEST $file
'info' => "$output [$file]",
];
- junit_mark_test_as('BORK', $shortname, $tested, null, $output);
+ $junit->markTestAs('BORK', $shortname, $tested, null, $output);
return 'BORKED';
}
}
@@ -2264,7 +2293,7 @@ TEST $file
|| array_key_exists("DEFLATE_POST", $section_text))) {
$message = "ext/zlib required";
show_result('SKIP', $tested, $tested_file, "reason: $message", $temp_filenames);
- junit_mark_test_as('SKIP', $shortname, $tested, null, $message);
+ $junit->markTestAs('SKIP', $shortname, $tested, null, $message);
return 'SKIPPED';
}
@@ -2304,7 +2333,7 @@ TEST $file
// a redirected test never fails
$IN_REDIRECT = false;
- junit_mark_test_as('PASS', $shortname, $tested);
+ $junit->markTestAs('PASS', $shortname, $tested);
return 'REDIR';
} else {
$bork_info = "Redirect info must contain exactly one TEST string to be used as redirect directory.";
@@ -2334,7 +2363,7 @@ TEST $file
'info' => "$bork_info [$file]",
];
- junit_mark_test_as('BORK', $shortname, $tested, null, $bork_info);
+ $junit->markTestAs('BORK', $shortname, $tested, null, $bork_info);
return 'BORKED';
}
@@ -2405,7 +2434,7 @@ TEST $file
$env['REQUEST_METHOD'] = 'POST';
if (empty($request)) {
- junit_mark_test_as('BORK', $shortname, $tested, null, 'empty $request');
+ $junit->markTestAs('BORK', $shortname, $tested, null, 'empty $request');
return 'BORKED';
}
@@ -2436,7 +2465,7 @@ TEST $file
$env['REQUEST_METHOD'] = 'PUT';
if (empty($request)) {
- junit_mark_test_as('BORK', $shortname, $tested, null, 'empty $request');
+ $junit->markTestAs('BORK', $shortname, $tested, null, 'empty $request');
return 'BORKED';
}
@@ -2487,9 +2516,11 @@ TEST $file
$env['CONTENT_TYPE'] = '';
$env['CONTENT_LENGTH'] = '';
- $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args$cmdRedirect";
+ $repeat_option = $num_repeats > 1 ? "--repeat $num_repeats" : "";
+ $cmd = "$php $pass_options $repeat_option $ini_settings -f \"$test_file\" $args$cmdRedirect";
}
+ $orig_cmd = $cmd;
if ($valgrind) {
$env['USE_ZEND_ALLOC'] = '0';
$env['ZEND_DONT_UNLOAD_MODULES'] = 1;
@@ -2511,19 +2542,19 @@ COMMAND $cmd
";
}
- junit_start_timer($shortname);
+ $junit->startTimer($shortname);
$hrtime = hrtime();
$startTime = $hrtime[0] * 1000000000 + $hrtime[1];
$out = system_with_timeout($cmd, $env, $section_text['STDIN'] ?? null, $captureStdIn, $captureStdOut, $captureStdErr);
- junit_finish_timer($shortname);
+ $junit->stopTimer($shortname);
$hrtime = hrtime();
$time = $hrtime[0] * 1000000000 + $hrtime[1] - $startTime;
if ($time >= $slow_min_ms * 1000000) {
$PHP_FAILED_TESTS['SLOW'][] = [
'name' => $file,
- 'test_name' => (is_array($IN_REDIRECT) ? $IN_REDIRECT['via'] : '') . $tested . " [$tested_file]",
+ 'test_name' => $tested . " [$tested_file]",
'output' => '',
'diff' => '',
'info' => $time / 1000000000,
@@ -2560,6 +2591,25 @@ COMMAND $cmd
}
}
+ if ($num_repeats > 1) {
+ // In repeat mode, retain the output before the first execution,
+ // and of the last execution. Do this early, because the trimming below
+ // makes the newline handling complicated.
+ $separator1 = "Executing for the first time...\n";
+ $separator1_pos = strpos($out, $separator1);
+ if ($separator1_pos !== false) {
+ $separator2 = "Finished execution, repeating...\n";
+ $separator2_pos = strrpos($out, $separator2);
+ if ($separator2_pos !== false) {
+ $out = substr($out, 0, $separator1_pos)
+ . substr($out, $separator2_pos + strlen($separator2));
+ } else {
+ $out = substr($out, 0, $separator1_pos)
+ . substr($out, $separator1_pos + strlen($separator1));
+ }
+ }
+ }
+
// Does the output match what is expected?
$output = preg_replace("/\r\n/", "\n", trim($out));
@@ -2673,7 +2723,7 @@ COMMAND $cmd
if (preg_match("/^$wanted_re\$/s", $output)) {
$passed = true;
- if (!$cfg['keep']['php']) {
+ if (!$cfg['keep']['php'] && !$leaked) {
@unlink($test_file);
}
@unlink($tmp_post);
@@ -2687,7 +2737,7 @@ COMMAND $cmd
$info = " (warn: XLEAK section but test passes)";
} else {
show_result("PASS", $tested, $tested_file, '', $temp_filenames);
- junit_mark_test_as('PASS', $shortname, $tested);
+ $junit->markTestAs('PASS', $shortname, $tested);
return 'PASSED';
}
}
@@ -2715,7 +2765,7 @@ COMMAND $cmd
$info = " (warn: XLEAK section but test passes)";
} else {
show_result("PASS", $tested, $tested_file, '', $temp_filenames);
- junit_mark_test_as('PASS', $shortname, $tested);
+ $junit->markTestAs('PASS', $shortname, $tested);
return 'PASSED';
}
}
@@ -2778,6 +2828,20 @@ COMMAND $cmd
error("Cannot create test diff - $diff_filename");
}
+ // write .log
+ if (strpos($log_format, 'L') !== false && file_put_contents($log_filename, "
+---- EXPECTED OUTPUT
+$wanted
+---- ACTUAL OUTPUT
+$output
+---- FAILED
+") === false) {
+ error("Cannot create test log - $log_filename");
+ error_report($file, $log_filename, $tested);
+ }
+ }
+
+ if (!$passed || $leaked) {
// write .sh
if (strpos($log_format, 'S') !== false) {
$sh_script = <<<SH
@@ -2785,16 +2849,16 @@ COMMAND $cmd
case "$1" in
"gdb")
- gdb --args {$cmd}
+ gdb --args {$orig_cmd}
;;
"valgrind")
- USE_ZEND_ALLOC=0 valgrind $2 ${cmd}
+ USE_ZEND_ALLOC=0 valgrind $2 ${orig_cmd}
;;
"rr")
- rr record $2 ${cmd}
+ rr record $2 ${orig_cmd}
;;
*)
- {$cmd}
+ {$orig_cmd}
;;
esac
SH;
@@ -2803,18 +2867,6 @@ SH;
}
chmod($sh_filename, 0755);
}
-
- // write .log
- if (strpos($log_format, 'L') !== false && file_put_contents($log_filename, "
----- EXPECTED OUTPUT
-$wanted
----- ACTUAL OUTPUT
-$output
----- FAILED
-") === false) {
- error("Cannot create test log - $log_filename");
- error_report($file, $log_filename, $tested);
- }
}
if ($valgrind && $leaked && $cfg["show"]["mem"]) {
@@ -2835,7 +2887,7 @@ $output
$diff = empty($diff) ? '' : preg_replace('/\e/', '<esc>', $diff);
- junit_mark_test_as($restype, $shortname, $tested, null, $info, $diff);
+ $junit->markTestAs($restype, $shortname, $tested, null, $info, $diff);
return $restype[0] . 'ED';
}
@@ -3331,7 +3383,7 @@ function clear_show_test(): void
// Parallel testing
global $workerID;
- if (!$workerID) {
+ if (!$workerID && isset($line_length)) {
// Write over the last line to avoid random trailing chars on next echo
echo str_repeat(" ", $line_length), "\r";
}
@@ -3379,304 +3431,361 @@ function show_result(
}
-function junit_init(): void
+class JUnit
{
- // Check whether a junit log is wanted.
- global $workerID;
- $JUNIT = getenv('TEST_PHP_JUNIT');
- if (empty($JUNIT)) {
- $GLOBALS['JUNIT'] = false;
- return;
- }
- if ($workerID) {
- $fp = null;
- } elseif (!$fp = fopen($JUNIT, 'w')) {
- error("Failed to open $JUNIT for writing.");
- }
- $GLOBALS['JUNIT'] = [
- 'fp' => $fp,
- 'name' => 'PHP',
+ private bool $enabled = true;
+ private $fp = null;
+ private array $suites = [];
+ private array $rootSuite = self::EMPTY_SUITE + ['name' => 'php'];
+
+ private const EMPTY_SUITE = [
'test_total' => 0,
'test_pass' => 0,
'test_fail' => 0,
'test_error' => 0,
'test_skip' => 0,
'test_warn' => 0,
+ 'files' => [],
'execution_time' => 0,
- 'suites' => [],
- 'files' => []
];
-}
-function junit_save_xml(): void
-{
- global $JUNIT;
- if (!junit_enabled()) {
- return;
+ public function __construct(array $env, int $workerID)
+ {
+ // Check whether a junit log is wanted.
+ $fileName = $env['TEST_PHP_JUNIT'] ?? null;
+ if (empty($fileName)) {
+ $this->enabled = false;
+ return;
+ }
+ if (!$workerID && !$this->fp = fopen($fileName, 'w')) {
+ throw new Exception("Failed to open $fileName for writing.");
+ }
}
- $xml = '<' . '?' . 'xml version="1.0" encoding="UTF-8"' . '?' . '>' . PHP_EOL;
- $xml .= sprintf(
- '<testsuites name="%s" tests="%s" failures="%d" errors="%d" skip="%d" time="%s">' . PHP_EOL,
- $JUNIT['name'],
- $JUNIT['test_total'],
- $JUNIT['test_fail'],
- $JUNIT['test_error'],
- $JUNIT['test_skip'],
- $JUNIT['execution_time']
- );
- $xml .= junit_get_suite_xml();
- $xml .= '</testsuites>';
- fwrite($JUNIT['fp'], $xml);
-}
+ public function isEnabled(): bool
+ {
+ return $this->enabled;
+ }
-function junit_get_suite_xml(string $suite_name = ''): string
-{
- global $JUNIT;
-
- $result = "";
-
- foreach ($JUNIT['suites'] as $suite_name => $suite) {
- $result .= sprintf(
- '<testsuite name="%s" tests="%s" failures="%d" errors="%d" skip="%d" time="%s">' . PHP_EOL,
- $suite['name'],
- $suite['test_total'],
- $suite['test_fail'],
- $suite['test_error'],
- $suite['test_skip'],
- $suite['execution_time']
- );
+ public function clear(): void
+ {
+ $this->rootSuite = self::EMPTY_SUITE + ['name' => 'php'];
+ $this->suites = [];
+ }
- if (!empty($suite_name)) {
- foreach ($suite['files'] as $file) {
- $result .= $JUNIT['files'][$file]['xml'];
- }
+ public function saveXML(): void
+ {
+ if (!$this->enabled) {
+ return;
}
- $result .= '</testsuite>' . PHP_EOL;
+ $xml = '<' . '?' . 'xml version="1.0" encoding="UTF-8"' . '?' . '>' . PHP_EOL;
+ $xml .= sprintf(
+ '<testsuites name="%s" tests="%s" failures="%d" errors="%d" skip="%d" time="%s">' . PHP_EOL,
+ $this->rootSuite['name'],
+ $this->rootSuite['test_total'],
+ $this->rootSuite['test_fail'],
+ $this->rootSuite['test_error'],
+ $this->rootSuite['test_skip'],
+ $this->rootSuite['execution_time']
+ );
+ $xml .= $this->getSuitesXML();
+ $xml .= '</testsuites>';
+ fwrite($this->fp, $xml);
}
- return $result;
-}
+ private function getSuitesXML(string $suite_name = '')
+ {
+ // FIXME: $suite_name gets overwritten
+ $result = '';
+
+ foreach ($this->suites as $suite_name => $suite) {
+ $result .= sprintf(
+ '<testsuite name="%s" tests="%s" failures="%d" errors="%d" skip="%d" time="%s">' . PHP_EOL,
+ $suite['name'],
+ $suite['test_total'],
+ $suite['test_fail'],
+ $suite['test_error'],
+ $suite['test_skip'],
+ $suite['execution_time']
+ );
+
+ if (!empty($suite_name)) {
+ foreach ($suite['files'] as $file) {
+ $result .= $this->rootSuite['files'][$file]['xml'];
+ }
+ }
-function junit_enabled(): bool
-{
- global $JUNIT;
- return !empty($JUNIT);
-}
+ $result .= '</testsuite>' . PHP_EOL;
+ }
-/**
- * @param array|string $type
- */
-function junit_mark_test_as(
- $type,
- string $file_name,
- string $test_name,
- ?int $time = null,
- string $message = '',
- string $details = ''
-): void {
- global $JUNIT;
- if (!junit_enabled()) {
- return;
+ return $result;
}
- $suite = junit_get_suitename_for($file_name);
+ public function markTestAs(
+ $type,
+ string $file_name,
+ string $test_name,
+ ?int $time = null,
+ string $message = '',
+ string $details = ''
+ ): void {
+ if (!$this->enabled) {
+ return;
+ }
+
+ $suite = $this->getSuiteName($file_name);
- junit_suite_record($suite, 'test_total');
+ $this->record($suite, 'test_total');
- $time = $time ?? junit_get_timer($file_name);
- junit_suite_record($suite, 'execution_time', $time);
+ $time = $time ?? $this->getTimer($file_name);
+ $this->record($suite, 'execution_time', $time);
- $escaped_details = htmlspecialchars($details, ENT_QUOTES, 'UTF-8');
- $escaped_details = preg_replace_callback('/[\0-\x08\x0B\x0C\x0E-\x1F]/', function (array $c): string {
- return sprintf('[[0x%02x]]', ord($c[0]));
- }, $escaped_details);
- $escaped_message = htmlspecialchars($message, ENT_QUOTES, 'UTF-8');
+ $escaped_details = htmlspecialchars($details, ENT_QUOTES, 'UTF-8');
+ $escaped_details = preg_replace_callback('/[\0-\x08\x0B\x0C\x0E-\x1F]/', function ($c) {
+ return sprintf('[[0x%02x]]', ord($c[0]));
+ }, $escaped_details);
+ $escaped_message = htmlspecialchars($message, ENT_QUOTES, 'UTF-8');
- $escaped_test_name = htmlspecialchars($file_name . ' (' . $test_name . ')', ENT_QUOTES);
- $JUNIT['files'][$file_name]['xml'] = "<testcase name='$escaped_test_name' time='$time'>\n";
+ $escaped_test_name = htmlspecialchars($file_name . ' (' . $test_name . ')', ENT_QUOTES);
+ $this->rootSuite['files'][$file_name]['xml'] = "<testcase name='$escaped_test_name' time='$time'>\n";
- if (is_array($type)) {
- $output_type = $type[0] . 'ED';
- $temp = array_intersect(['XFAIL', 'XLEAK', 'FAIL', 'WARN'], $type);
- $type = reset($temp);
- } else {
- $output_type = $type . 'ED';
- }
-
- if ('PASS' == $type || 'XFAIL' == $type || 'XLEAK' == $type) {
- junit_suite_record($suite, 'test_pass');
- } elseif ('BORK' == $type) {
- junit_suite_record($suite, 'test_error');
- $JUNIT['files'][$file_name]['xml'] .= "<error type='$output_type' message='$escaped_message'/>\n";
- } elseif ('SKIP' == $type) {
- junit_suite_record($suite, 'test_skip');
- $JUNIT['files'][$file_name]['xml'] .= "<skipped>$escaped_message</skipped>\n";
- } elseif ('WARN' == $type) {
- junit_suite_record($suite, 'test_warn');
- $JUNIT['files'][$file_name]['xml'] .= "<warning>$escaped_message</warning>\n";
- } elseif ('FAIL' == $type) {
- junit_suite_record($suite, 'test_fail');
- $JUNIT['files'][$file_name]['xml'] .= "<failure type='$output_type' message='$escaped_message'>$escaped_details</failure>\n";
- } else {
- junit_suite_record($suite, 'test_error');
- $JUNIT['files'][$file_name]['xml'] .= "<error type='$output_type' message='$escaped_message'>$escaped_details</error>\n";
+ if (is_array($type)) {
+ $output_type = $type[0] . 'ED';
+ $temp = array_intersect(['XFAIL', 'XLEAK', 'FAIL', 'WARN'], $type);
+ $type = reset($temp);
+ } else {
+ $output_type = $type . 'ED';
+ }
+
+ if ('PASS' == $type || 'XFAIL' == $type || 'XLEAK' == $type) {
+ $this->record($suite, 'test_pass');
+ } elseif ('BORK' == $type) {
+ $this->record($suite, 'test_error');
+ $this->rootSuite['files'][$file_name]['xml'] .= "<error type='$output_type' message='$escaped_message'/>\n";
+ } elseif ('SKIP' == $type) {
+ $this->record($suite, 'test_skip');
+ $this->rootSuite['files'][$file_name]['xml'] .= "<skipped>$escaped_message</skipped>\n";
+ } elseif ('WARN' == $type) {
+ $this->record($suite, 'test_warn');
+ $this->rootSuite['files'][$file_name]['xml'] .= "<warning>$escaped_message</warning>\n";
+ } elseif ('FAIL' == $type) {
+ $this->record($suite, 'test_fail');
+ $this->rootSuite['files'][$file_name]['xml'] .= "<failure type='$output_type' message='$escaped_message'>$escaped_details</failure>\n";
+ } else {
+ $this->record($suite, 'test_error');
+ $this->rootSuite['files'][$file_name]['xml'] .= "<error type='$output_type' message='$escaped_message'>$escaped_details</error>\n";
+ }
+
+ $this->rootSuite['files'][$file_name]['xml'] .= "</testcase>\n";
}
- $JUNIT['files'][$file_name]['xml'] .= "</testcase>\n";
-}
+ private function record(string $suite, string $param, $value = 1): void
+ {
+ $this->rootSuite[$param] += $value;
+ $this->suites[$suite][$param] += $value;
+ }
-function junit_suite_record(string $suite, string $param, int $value = 1): void
-{
- global $JUNIT;
+ private function getTimer(string $file_name)
+ {
+ if (!$this->enabled) {
+ return 0;
+ }
- $JUNIT[$param] += $value;
- $JUNIT['suites'][$suite][$param] += $value;
-}
+ if (isset($this->rootSuite['files'][$file_name]['total'])) {
+ return number_format($this->rootSuite['files'][$file_name]['total'], 4);
+ }
-function junit_get_timer(string $file_name): int
-{
- global $JUNIT;
- if (!junit_enabled()) {
return 0;
}
- if (isset($JUNIT['files'][$file_name]['total'])) {
- return number_format($JUNIT['files'][$file_name]['total'], 4);
- }
+ public function startTimer(string $file_name): void
+ {
+ if (!$this->enabled) {
+ return;
+ }
- return 0;
-}
+ if (!isset($this->rootSuite['files'][$file_name]['start'])) {
+ $this->rootSuite['files'][$file_name]['start'] = microtime(true);
-function junit_start_timer(string $file_name): void
-{
- global $JUNIT;
- if (!junit_enabled()) {
- return;
+ $suite = $this->getSuiteName($file_name);
+ $this->initSuite($suite);
+ $this->suites[$suite]['files'][$file_name] = $file_name;
+ }
}
- if (!isset($JUNIT['files'][$file_name]['start'])) {
- $JUNIT['files'][$file_name]['start'] = microtime(true);
-
- $suite = junit_get_suitename_for($file_name);
- junit_init_suite($suite);
- $JUNIT['suites'][$suite]['files'][$file_name] = $file_name;
+ public function getSuiteName(string $file_name): string
+ {
+ return $this->pathToClassName(dirname($file_name));
}
-}
-function junit_get_suitename_for(string $file_name): string
-{
- return junit_path_to_classname(dirname($file_name));
-}
+ private function pathToClassName(string $file_name): string
+ {
+ if (!$this->enabled) {
+ return '';
+ }
-function junit_path_to_classname(string $file_name): string
-{
- global $JUNIT;
+ $ret = $this->rootSuite['name'];
+ $_tmp = [];
- if (!junit_enabled()) {
- return '';
+ // lookup whether we're in the PHP source checkout
+ $max = 5;
+ if (is_file($file_name)) {
+ $dir = dirname(realpath($file_name));
+ } else {
+ $dir = realpath($file_name);
+ }
+ do {
+ array_unshift($_tmp, basename($dir));
+ $chk = $dir . DIRECTORY_SEPARATOR . "main" . DIRECTORY_SEPARATOR . "php_version.h";
+ $dir = dirname($dir);
+ } while (!file_exists($chk) && --$max > 0);
+ if (file_exists($chk)) {
+ if ($max) {
+ array_shift($_tmp);
+ }
+ foreach ($_tmp as $p) {
+ $ret .= "." . preg_replace(",[^a-z0-9]+,i", ".", $p);
+ }
+ return $ret;
+ }
+
+ return $this->rootSuite['name'] . '.' . str_replace([DIRECTORY_SEPARATOR, '-'], '.', $file_name);
}
- $ret = $JUNIT['name'];
- $_tmp = [];
+ public function initSuite(string $suite_name): void
+ {
+ if (!$this->enabled) {
+ return;
+ }
+
+ if (!empty($this->suites[$suite_name])) {
+ return;
+ }
- // lookup whether we're in the PHP source checkout
- $max = 5;
- if (is_file($file_name)) {
- $dir = dirname(realpath($file_name));
- } else {
- $dir = realpath($file_name);
+ $this->suites[$suite_name] = self::EMPTY_SUITE + ['name' => $suite_name];
}
- do {
- array_unshift($_tmp, basename($dir));
- $chk = $dir . DIRECTORY_SEPARATOR . "main" . DIRECTORY_SEPARATOR . "php_version.h";
- $dir = dirname($dir);
- } while (!file_exists($chk) && --$max > 0);
- if (file_exists($chk)) {
- if ($max) {
- array_shift($_tmp);
+
+ public function stopTimer(string $file_name): void
+ {
+ if (!$this->enabled) {
+ return;
}
- foreach ($_tmp as $p) {
- $ret .= "." . preg_replace(",[^a-z0-9]+,i", ".", $p);
+
+ if (!isset($this->rootSuite['files'][$file_name]['start'])) {
+ throw new Exception("Timer for $file_name was not started!");
}
- return $ret;
- }
- return $JUNIT['name'] . '.' . str_replace([DIRECTORY_SEPARATOR, '-'], '.', $file_name);
-}
+ if (!isset($this->rootSuite['files'][$file_name]['total'])) {
+ $this->rootSuite['files'][$file_name]['total'] = 0;
+ }
-function junit_init_suite(string $suite_name): void
-{
- global $JUNIT;
- if (!junit_enabled()) {
- return;
+ $start = $this->rootSuite['files'][$file_name]['start'];
+ $this->rootSuite['files'][$file_name]['total'] += microtime(true) - $start;
+ unset($this->rootSuite['files'][$file_name]['start']);
}
- if (!empty($JUNIT['suites'][$suite_name])) {
- return;
+ public function mergeResults(?JUnit $other): void
+ {
+ if (!$this->enabled || !$other) {
+ return;
+ }
+
+ $this->mergeSuites($this->rootSuite, $other->rootSuite);
+ foreach ($other->suites as $name => $suite) {
+ if (!isset($this->suites[$name])) {
+ $this->suites[$name] = $suite;
+ continue;
+ }
+
+ $this->mergeSuites($this->suites[$name], $suite);
+ }
}
- $JUNIT['suites'][$suite_name] = [
- 'name' => $suite_name,
- 'test_total' => 0,
- 'test_pass' => 0,
- 'test_fail' => 0,
- 'test_error' => 0,
- 'test_skip' => 0,
- 'test_warn' => 0,
- 'files' => [],
- 'execution_time' => 0,
- ];
+ private function mergeSuites(array &$dest, array $source): void
+ {
+ $dest['test_total'] += $source['test_total'];
+ $dest['test_pass'] += $source['test_pass'];
+ $dest['test_fail'] += $source['test_fail'];
+ $dest['test_error'] += $source['test_error'];
+ $dest['test_skip'] += $source['test_skip'];
+ $dest['test_warn'] += $source['test_warn'];
+ $dest['execution_time'] += $source['execution_time'];
+ $dest['files'] += $source['files'];
+ }
}
-function junit_finish_timer(string $file_name): void
+class SkipCache
{
- global $JUNIT;
- if (!junit_enabled()) {
- return;
- }
+ private bool $keepFile;
- if (!isset($JUNIT['files'][$file_name]['start'])) {
- error("Timer for $file_name was not started!");
- }
+ private array $skips = [];
+ private array $extensions = [];
+
+ private int $hits = 0;
+ private int $misses = 0;
+ private int $extHits = 0;
+ private int $extMisses = 0;
- if (!isset($JUNIT['files'][$file_name]['total'])) {
- $JUNIT['files'][$file_name]['total'] = 0;
+ public function __construct(bool $keepFile)
+ {
+ $this->keepFile = $keepFile;
}
- $start = $JUNIT['files'][$file_name]['start'];
- $JUNIT['files'][$file_name]['total'] += microtime(true) - $start;
- unset($JUNIT['files'][$file_name]['start']);
-}
+ public function checkSkip(string $php, string $code, string $checkFile, string $tempFile, array $env): string
+ {
+ // Extension tests frequently use something like <?php require 'skipif.inc';
+ // for skip checks. This forces us to cache per directory to avoid pollution.
+ $dir = dirname($checkFile);
+ $key = "$php => $dir";
-function junit_merge_results(array $junit): void
-{
- global $JUNIT;
- $JUNIT['test_total'] += $junit['test_total'];
- $JUNIT['test_pass'] += $junit['test_pass'];
- $JUNIT['test_fail'] += $junit['test_fail'];
- $JUNIT['test_error'] += $junit['test_error'];
- $JUNIT['test_skip'] += $junit['test_skip'];
- $JUNIT['test_warn'] += $junit['test_warn'];
- $JUNIT['execution_time'] += $junit['execution_time'];
- $JUNIT['files'] += $junit['files'];
- foreach ($junit['suites'] as $name => $suite) {
- if (!isset($JUNIT['suites'][$name])) {
- $JUNIT['suites'][$name] = $suite;
- continue;
+ if (isset($this->skips[$key][$code])) {
+ $this->hits++;
+ if ($this->keepFile) {
+ save_text($checkFile, $code, $tempFile);
+ }
+ return $this->skips[$key][$code];
+ }
+
+ save_text($checkFile, $code, $tempFile);
+ $result = trim(system_with_timeout("$php \"$checkFile\"", $env));
+ $this->skips[$key][$code] = $result;
+ $this->misses++;
+
+ if (!$this->keepFile) {
+ @unlink($checkFile);
+ }
+
+ return $result;
+ }
+
+ public function getExtensions(string $php): array
+ {
+ if (isset($this->extensions[$php])) {
+ $this->extHits++;
+ return $this->extensions[$php];
}
- $SUITE =& $JUNIT['suites'][$name];
- $SUITE['test_total'] += $suite['test_total'];
- $SUITE['test_pass'] += $suite['test_pass'];
- $SUITE['test_fail'] += $suite['test_fail'];
- $SUITE['test_error'] += $suite['test_error'];
- $SUITE['test_skip'] += $suite['test_skip'];
- $SUITE['test_warn'] += $suite['test_warn'];
- $SUITE['execution_time'] += $suite['execution_time'];
- $SUITE['files'] += $suite['files'];
+ $extDir = `$php -d display_errors=0 -r "echo ini_get('extension_dir');"`;
+ $extensions = explode(",", `$php -d display_errors=0 -r "echo implode(',', get_loaded_extensions());"`);
+
+ $result = [$extDir, $extensions];
+ $this->extensions[$php] = $result;
+ $this->extMisses++;
+
+ return $result;
}
+
+// public function __destruct()
+// {
+// echo "Skips: {$this->hits} hits, {$this->misses} misses.\n";
+// echo "Extensions: {$this->extHits} hits, {$this->extMisses} misses.\n";
+// echo "Cache distribution:\n";
+//
+// foreach ($this->skips as $php => $cache) {
+// echo "$php: " . count($cache) . "\n";
+// }
+// }
}
class RuntestsValgrind
View Lorry Controller Status