diff options
Diffstat (limited to 'Tools/Scripts/run-gtk-tests')
| -rwxr-xr-x | Tools/Scripts/run-gtk-tests | 494 |
1 files changed, 494 insertions, 0 deletions
diff --git a/Tools/Scripts/run-gtk-tests b/Tools/Scripts/run-gtk-tests new file mode 100755 index 000000000..a3d3e6f28 --- /dev/null +++ b/Tools/Scripts/run-gtk-tests @@ -0,0 +1,494 @@ +#!/usr/bin/env python +# +# Copyright (C) 2011, 2012 Igalia S.L. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public License +# along with this library; see the file COPYING.LIB. If not, write to +# the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +import logging +import subprocess +import os +import sys +import optparse +import re +from signal import alarm, signal, SIGALRM, SIGKILL, SIGSEGV +from gi.repository import Gio, GLib + +top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) +sys.path.append(os.path.join(top_level_directory, "Tools", "jhbuild")) +sys.path.append(os.path.join(top_level_directory, "Tools", "gtk")) +import common +import jhbuildutils +from webkitpy.common.host import Host + +class SkippedTest: + ENTIRE_SUITE = None + + def __init__(self, test, test_case, reason, bug, build_type=None): + self.test = test + self.test_case = test_case + self.reason = reason + self.bug = bug + self.build_type = build_type + + def __str__(self): + skipped_test_str = "%s" % self.test + + if not(self.skip_entire_suite()): + skipped_test_str += " [%s]" % self.test_case + + skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug) + return skipped_test_str + + def skip_entire_suite(self): + return self.test_case == SkippedTest.ENTIRE_SUITE + + def skip_for_build_type(self, build_type): + if self.build_type is None: + return True; + + return self.build_type == build_type + +class TestTimeout(Exception): + pass + +class TestRunner: + TEST_DIRS = [ "WebKit2Gtk", "WebKit2", "JavaScriptCore", "WTF", "WebCore" ] + + SKIPPED = [ + SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689), + SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/usermedia-permission-requests", "Test times out", 158257), + SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/audio-usermedia-permission-request", "Test times out", 158257), + SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580), + SkippedTest("WebKit2Gtk/TestPrinting", "/webkit2/WebKitPrintOperation/custom-widget", "Test is flaky", 168196), + SkippedTest("WebKit2Gtk/TestWebViewEditor", "/webkit2/WebKitWebView/editable/editable", "Test hits an assertion in Debug builds", 151654, "Debug"), + SkippedTest("WebKit2Gtk/TestWebExtensions", "/webkit2/WebKitWebExtension/form-controls-associated-signal", "Test is flaky", 168194), + SkippedTest("WebKit2Gtk/TestWebExtensions", "/webkit2/WebKitWebView/install-missing-plugins-permission-request", "Test times out", 147822), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.ForceRepaint", "Test times out", 105532), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.ReloadPageAfterCrash", "Test flakily times out", 110129), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.DidAssociateFormControls", "Test times out", 120302), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.InjectedBundleFrameHitTest", "Test times out", 120303), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.TerminateTwice", "Test causes crash on the next test", 121970), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068), + SkippedTest("WebKit2/TestWebKit2", "WebKit2.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068), + ] + + SLOW = [ + "WTF_Lock.ContendedShortSection", + "WTF_Lock.ContendedLongSection", + "WTF_WordLock.ContendedShortSection", + "WTF_WordLock.ContendedLongSection", + "WebKit2Gtk/TestInspectorServer", + ] + + def __init__(self, options, tests=[]): + self._options = options + + self._build_type = "Debug" if self._options.debug else "Release" + common.set_build_types((self._build_type,)) + self._port = Host().port_factory.get("gtk") + self._driver = self._create_driver() + + self._programs_path = common.binary_build_path() + self._tests = self._get_tests(tests) + self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)] + self._disabled_tests = [] + + # These SPI daemons need to be active for the accessibility tests to work. + self._spi_registryd = None + self._spi_bus_launcher = None + + def _test_programs_base_dir(self): + return os.path.join(self._programs_path, "TestWebKitAPI") + + def _get_tests_from_dir(self, test_dir): + if not os.path.isdir(test_dir): + return [] + + tests = [] + for test_file in os.listdir(test_dir): + if not test_file.lower().startswith("test"): + continue + test_path = os.path.join(test_dir, test_file) + if os.path.isfile(test_path) and os.access(test_path, os.X_OK): + tests.append(test_path) + return tests + + def _get_tests(self, initial_tests): + tests = [] + for test in initial_tests: + if os.path.isdir(test): + tests.extend(self._get_tests_from_dir(test)) + else: + tests.append(test) + if tests: + return tests + + tests = [] + for test_dir in self.TEST_DIRS: + absolute_test_dir = os.path.join(self._test_programs_base_dir(), test_dir) + tests.extend(self._get_tests_from_dir(absolute_test_dir)) + return tests + + def _lookup_atspi2_binary(self, filename): + exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix') + if not exec_prefix: + return None + for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']: + filepath = os.path.join(exec_prefix, path, filename) + if os.path.isfile(filepath): + return filepath + + return None + + def _wait_for_accessibility_bus(self): + def timeout_accessibility_bus(): + self._accessibility_bus_found = False + sys.stderr.write("Timeout waiting for the accesibility bus.\n") + sys.stderr.flush() + loop.quit() + # Backup current environment, and temporally set the test one. + oldenv = dict(os.environ) + os.environ.clear() + os.environ.update(self._test_env) + # We spin a main loop until the bus name appears on DBus. + self._accessibility_bus_found = True + loop = GLib.MainLoop() + Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE, + lambda *args: loop.quit(), None) + GLib.timeout_add_seconds(5, timeout_accessibility_bus) + loop.run() + # Restore previous environment. + os.environ.clear() + os.environ.update(oldenv) + return self._accessibility_bus_found + + def _start_accessibility_daemons(self): + spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher') + spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd') + if not spi_bus_launcher_path or not spi_registryd_path: + return False + + try: + self._spi_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env) + except: + sys.stderr.write("Failed to launch the accessibility bus\n") + sys.stderr.flush() + return False + + # We need to wait until the SPI bus is launched before trying to start the SPI registry. + if not self._wait_for_accessibility_bus(): + sys.stderr.write("Failed checking the accessibility bus within D-Bus\n") + sys.stderr.flush() + return False + + try: + self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env) + except: + sys.stderr.write("Failed to launch the accessibility registry\n") + sys.stderr.flush() + return False + + return True + + def _create_driver(self, port_options=[]): + self._port._display_server = self._options.display_server + driver = self._port.create_driver(worker_number=0, no_timeout=True)._make_driver(pixel_tests=False) + if not driver.check_driver(self._port): + raise RuntimeError("Failed to check driver %s" %driver.__class__.__name__) + return driver + + def _setup_testing_environment(self): + self._test_env = self._driver._setup_environ_for_test() + self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2") + self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.library_build_path() + self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path + + # If we cannot start the accessibility daemons, we can just skip the accessibility tests. + if not self._start_accessibility_daemons(): + print "Could not start accessibility bus, so disabling TestWebKitAccessibility" + self._disabled_tests.append("WebKit2APITests/TestWebKitAccessibility") + return True + + def _tear_down_testing_environment(self): + if self._spi_registryd: + self._spi_registryd.terminate() + if self._spi_bus_launcher: + self._spi_bus_launcher.terminate() + if self._driver: + self._driver.stop() + + def _test_cases_to_skip(self, test_program): + if self._options.skipped_action != 'skip': + return [] + + test_cases = [] + for skipped in self._skipped_tests: + if test_program.endswith(skipped.test) and not skipped.skip_entire_suite(): + test_cases.append(skipped.test_case) + return test_cases + + def _should_run_test_program(self, test_program): + for disabled_test in self._disabled_tests: + if test_program.endswith(disabled_test): + return False + + if self._options.skipped_action != 'skip': + return True + + for skipped in self._skipped_tests: + if test_program.endswith(skipped.test) and skipped.skip_entire_suite(): + return False + return True + + def _get_child_pid_from_test_output(self, output): + if not output: + return -1 + match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output) + if not match: + return -1 + return int(match.group('child_pid')) + + def _kill_process(self, pid): + try: + os.kill(pid, SIGKILL) + except OSError: + # Process already died. + pass + + def _run_test_command(self, command, timeout=-1): + def alarm_handler(signum, frame): + raise TestTimeout + + child_pid = [-1] + def parse_line(line, child_pid = child_pid): + if child_pid[0] == -1: + child_pid[0] = self._get_child_pid_from_test_output(line) + + sys.stdout.write(line) + + def waitpid(pid): + while True: + try: + return os.waitpid(pid, 0) + except (OSError, IOError) as e: + if e.errno == errno.EINTR: + continue + raise + + def return_code_from_exit_status(status): + if os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + elif os.WIFEXITED(status): + return os.WEXITSTATUS(status) + else: + # Should never happen + raise RuntimeError("Unknown child exit status!") + + pid, fd = os.forkpty() + if pid == 0: + os.execvpe(command[0], command, self._test_env) + sys.exit(0) + + if timeout > 0: + signal(SIGALRM, alarm_handler) + alarm(timeout) + + try: + common.parse_output_lines(fd, parse_line) + if timeout > 0: + alarm(0) + except TestTimeout: + self._kill_process(pid) + if child_pid[0] > 0: + self._kill_process(child_pid[0]) + raise + + try: + dummy, status = waitpid(pid) + except OSError as e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + status = 0 + + return return_code_from_exit_status(status) + + def _run_test_glib(self, test_program): + tester_command = ['gtester', '-k'] + if self._options.verbose: + tester_command.append('--verbose') + for test_case in self._test_cases_to_skip(test_program): + tester_command.extend(['-s', test_case]) + tester_command.append(test_program) + # This timeout is supposed to be per test case, but in the case of GLib tests it affects all the tests cases of + # the same test program. Some test programs like TestLoaderClient, that have a lot of test cases, often time out + # in the bots because the timeout is not enough to run all the tests cases. So, we use a longer timeout for GLib + # tests (timeout * 2). + timeout = self._options.timeout * 2 + test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program)) + if test in TestRunner.SLOW: + timeout *= 5 + + return self._run_test_command(tester_command, timeout) + + def _get_tests_from_google_test_suite(self, test_program): + try: + output = subprocess.check_output([test_program, '--gtest_list_tests'], env=self._test_env) + except subprocess.CalledProcessError: + sys.stderr.write("ERROR: could not list available tests for binary %s.\n" % (test_program)) + sys.stderr.flush() + return 1 + + skipped_test_cases = self._test_cases_to_skip(test_program) + + tests = [] + prefix = None + for line in output.split('\n'): + if not line.startswith(' '): + prefix = line + continue + else: + test_name = prefix + line.strip() + if not test_name in skipped_test_cases: + tests.append(test_name) + return tests + + def _run_google_test(self, test_program, subtest): + test_command = [test_program, '--gtest_filter=%s' % (subtest)] + timeout = self._options.timeout + if subtest in TestRunner.SLOW: + timeout *= 5 + + status = self._run_test_command(test_command, timeout) + if status == -SIGSEGV: + sys.stdout.write("**CRASH** %s\n" % subtest) + sys.stdout.flush() + return status + + def _run_google_test_suite(self, test_program): + retcode = 0 + for subtest in self._get_tests_from_google_test_suite(test_program): + if self._run_google_test(test_program, subtest): + retcode = 1 + return retcode + + def _run_test(self, test_program): + basedir = os.path.basename(os.path.dirname(test_program)) + if basedir in ["WebKit2Gtk", "WebKitGtk"]: + return self._run_test_glib(test_program) + + if basedir in ["WebKit2", "JavaScriptCore", "WTF", "WebCore", "WebCoreGtk"]: + return self._run_google_test_suite(test_program) + + return 1 + + def run_tests(self): + if not self._tests: + sys.stderr.write("ERROR: tests not found in %s.\n" % (self._test_programs_base_dir())) + sys.stderr.flush() + return 1 + + if not self._setup_testing_environment(): + return 1 + + # Remove skipped tests now instead of when we find them, because + # some tests might be skipped while setting up the test environment. + self._tests = [test for test in self._tests if self._should_run_test_program(test)] + + crashed_tests = [] + failed_tests = [] + timed_out_tests = [] + try: + for test in self._tests: + exit_status_code = 0 + try: + exit_status_code = self._run_test(test) + except TestTimeout: + sys.stdout.write("TEST: %s: TIMEOUT\n" % test) + sys.stdout.flush() + timed_out_tests.append(test) + + if exit_status_code == -SIGSEGV: + sys.stdout.write("TEST: %s: CRASHED\n" % test) + sys.stdout.flush() + crashed_tests.append(test) + elif exit_status_code != 0: + failed_tests.append(test) + finally: + self._tear_down_testing_environment() + + if failed_tests: + names = [test.replace(self._test_programs_base_dir(), '', 1) for test in failed_tests] + sys.stdout.write("Tests failed (%d): %s\n" % (len(names), ", ".join(names))) + sys.stdout.flush() + + if crashed_tests: + names = [test.replace(self._test_programs_base_dir(), '', 1) for test in crashed_tests] + sys.stdout.write("Tests that crashed (%d): %s\n" % (len(names), ", ".join(names))) + sys.stdout.flush() + + if timed_out_tests: + names = [test.replace(self._test_programs_base_dir(), '', 1) for test in timed_out_tests] + sys.stdout.write("Tests that timed out (%d): %s\n" % (len(names), ", ".join(names))) + sys.stdout.flush() + + if self._skipped_tests and self._options.skipped_action == 'skip': + sys.stdout.write("Tests skipped (%d):\n%s\n" % + (len(self._skipped_tests), + "\n".join([str(skipped) for skipped in self._skipped_tests]))) + sys.stdout.flush() + + return len(failed_tests) + len(timed_out_tests) + +if __name__ == "__main__": + if not jhbuildutils.enter_jhbuild_environment_if_available("gtk"): + print "***" + print "*** Warning: jhbuild environment not present. Run update-webkitgtk-libs before build-webkit to ensure proper testing." + print "***" + + option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]') + option_parser.add_option('-r', '--release', + action='store_true', dest='release', + help='Run in Release') + option_parser.add_option('-d', '--debug', + action='store_true', dest='debug', + help='Run in Debug') + option_parser.add_option('-v', '--verbose', + action='store_true', dest='verbose', + help='Run gtester in verbose mode') + option_parser.add_option('--skipped', action='store', dest='skipped_action', + choices=['skip', 'ignore', 'only'], default='skip', + metavar='skip|ignore|only', + help='Specifies how to treat the skipped tests') + option_parser.add_option('-t', '--timeout', + action='store', type='int', dest='timeout', default=10, + help='Time in seconds until a test times out') + option_parser.add_option('--display-server', choices=['xvfb', 'xorg', 'weston', 'wayland'], default='xvfb', + help='"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. ' + '"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.'), + options, args = option_parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(message)s") + + runner = TestRunner(options, args) + sys.exit(runner.run_tests()) |