path: root/Tools/Scripts/webkitpy/test/main.py

# Copyright (C) 2010 Chris Jerdonek (cjerdonek@webkit.org)
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""unit testing code for webkitpy."""

import logging
import optparse
import os
import StringIO
import sys
import traceback
import unittest

# NOTE: We intentionally do not depend on anything else in webkitpy here to avoid breaking test-webkitpy.

_log = logging.getLogger(__name__)


class Tester(object):
    @staticmethod
    def clean_packages(dirs):
        """Delete all .pyc files under dirs that have no .py file."""
        for dir_to_clean in dirs:
            _log.debug("Cleaning orphaned *.pyc files from: %s" % dir_to_clean)
            for dir_path, dir_names, file_names in os.walk(dir_to_clean):
                for file_name in file_names:
                    if file_name.endswith(".pyc") and file_name[:-1] not in file_names:
                        file_path = os.path.join(dir_path, file_name)
                        _log.info("Deleting orphan *.pyc file: %s" % file_path)
                        os.remove(file_path)

    def __init__(self):
        self._verbosity = 1

    def parse_args(self, argv):
        parser = optparse.OptionParser(usage='usage: %prog [options] [args...]')
        parser.add_option('-a', '--all', action='store_true', default=False,
                          help='run all the tests'),
        parser.add_option('-c', '--coverage', action='store_true', default=False,
                          help='generate code coverage info (requires http://pypi.python.org/pypi/coverage)'),
        parser.add_option('-q', '--quiet', action='store_true', default=False,
                          help='run quietly (errors, warnings, and progress only)'),
        parser.add_option('-s', '--silent', action='store_true', default=False,
                          help='run silently (errors and warnings only)'),
        parser.add_option('-x', '--xml', action='store_true', default=False,
                          help='output xUnit-style XML output')
        parser.add_option('-v', '--verbose', action='count', default=0,
                          help='verbose output (specify once for individual test results, twice for debug messages)')
        parser.add_option('--skip-integrationtests', action='store_true', default=False,
                          help='do not run the integration tests')

        parser.epilog = ('[args...] is an optional list of modules, test_classes, or individual tests. '
                         'If no args are given, all the tests will be run.')

        self.progName = os.path.basename(argv[0])
        return parser.parse_args(argv[1:])

    def configure(self, options):
        self._options = options

        if options.silent:
            self._verbosity = 0
            self._configure_logging(logging.WARNING)
        elif options.quiet:
            self._verbosity = 1
            self._configure_logging(logging.WARNING)
        elif options.verbose == 0:
            self._verbosity = 1
            self._configure_logging(logging.INFO)
        elif options.verbose == 1:
            self._verbosity = 2
            self._configure_logging(logging.INFO)
        elif options.verbose == 2:
            self._verbosity = 2
            self._configure_logging(logging.DEBUG)

    def _configure_logging(self, log_level):
        """Configure the root logger.

        Configure the root logger not to log any messages from webkitpy --
        except for messages from the autoinstall module.  Also set the
        logging level as described below.
        """
        handler = logging.StreamHandler(sys.stderr)
        # We constrain the level on the handler rather than on the root
        # logger itself.  This is probably better because the handler is
        # configured and known only to this module, whereas the root logger
        # is an object shared (and potentially modified) by many modules.
        # Modifying the handler, then, is less intrusive and less likely to
        # interfere with modifications made by other modules (e.g. in unit
        # tests).
        handler.setLevel(log_level)
        formatter = logging.Formatter("%(message)s")
        handler.setFormatter(formatter)

        logger = logging.getLogger()
        logger.addHandler(handler)
        logger.setLevel(logging.NOTSET)

        # Filter out most webkitpy messages.
        #
        # Messages can be selectively re-enabled for this script by updating
        # this method accordingly.
        def filter(record):
            """Filter out autoinstall and non-third-party webkitpy messages."""
            # FIXME: Figure out a way not to use strings here, for example by
            #        using syntax like webkitpy.test.__name__.  We want to be
            #        sure not to import any non-Python 2.4 code, though, until
            #        after the version-checking code has executed.
            if (record.name.startswith("webkitpy.common.system.autoinstall") or
                record.name.startswith("webkitpy.test")):
                return True
            if record.name.startswith("webkitpy"):
                return False
            return True

        testing_filter = logging.Filter()
        testing_filter.filter = filter

        # Display a message so developers are not mystified as to why
        # logging does not work in the unit tests.
        _log.info("Suppressing most webkitpy logging while running unit tests.")
        handler.addFilter(testing_filter)

    def run(self, dirs, args):
        args = args or self._find_modules(dirs)
        return self._run_tests(dirs, args)

    def _find_modules(self, dirs):
        modules = []
        for dir_to_search in dirs:
            modules.extend(self._find_modules_under(dir_to_search, '_unittest.py'))
            if not self._options.skip_integrationtests:
                modules.extend(self._find_modules_under(dir_to_search, '_integrationtest.py'))
        modules.sort()

        for module in modules:
            _log.debug("Found: %s" % module)

        # FIXME: Figure out how to move this to test-webkitpy in order to to make this file more generic.
        if not self._options.all:
            slow_tests = ('webkitpy.common.checkout.scm.scm_unittest',)
            self._exclude(modules, slow_tests, 'are really, really slow', 31818)

            if sys.platform == 'win32':
                win32_blacklist = ('webkitpy.common.checkout',
                                   'webkitpy.common.config',
                                   'webkitpy.tool')
                self._exclude(modules, win32_blacklist, 'fail horribly on win32', 54526)

        return modules

    def _exclude(self, modules, module_prefixes, reason, bugid):
        _log.info('Skipping tests in the following modules or packages because they %s:' % reason)
        for prefix in module_prefixes:
            _log.info('    %s' % prefix)
            modules_to_exclude = filter(lambda m: m.startswith(prefix), modules)
            for m in modules_to_exclude:
                if len(modules_to_exclude) > 1:
                    _log.debug('        %s' % m)
                modules.remove(m)
        _log.info('    (https://bugs.webkit.org/show_bug.cgi?id=%d; use --all to include)' % bugid)
        _log.info('')

    def _find_modules_under(self, dir_to_search, suffix):

        def to_package(dir_path):
            return dir_path.replace(dir_to_search + os.sep, '').replace(os.sep, '.')

        def to_module(filename, package):
            return package + '.' + filename.replace('.py', '')

        modules = []
        for dir_path, _, filenames in os.walk(dir_to_search):
            package = to_package(dir_path)
            modules.extend(to_module(f, package) for f in filenames if f.endswith(suffix))
        return modules

    def _run_tests(self, dirs, args):
        if self._options.coverage:
            try:
                import webkitpy.thirdparty.autoinstalled.coverage as coverage
            except ImportError, e:
                _log.error("Failed to import 'coverage'; can't generate coverage numbers.")
                return False
            cov = coverage.coverage()
            cov.start()

        _log.debug("Loading the tests...")

        loader = unittest.defaultTestLoader
        suites = []
        for name in args:
            if self._is_module(dirs, name):
                # import modules explicitly before loading their tests because
                # loadTestsFromName() produces lousy error messages for bad modules.
                try:
                    __import__(name)
                except ImportError, e:
                    _log.fatal('Failed to import %s:' % name)
                    self._log_exception()
                    return False
            suites.append(loader.loadTestsFromName(name, None))

        test_suite = unittest.TestSuite(suites)
        if self._options.xml:
            from webkitpy.thirdparty.autoinstalled.xmlrunner import XMLTestRunner
            test_runner = XMLTestRunner(output='test-webkitpy-xml-reports')
        else:
            test_runner = unittest.TextTestRunner(verbosity=self._verbosity)

        _log.debug("Running the tests.")
        result = test_runner.run(test_suite)
        if self._options.coverage:
            cov.stop()
            cov.save()
            cov.report(show_missing=False)
        return result.wasSuccessful()

    def _is_module(self, dirs, name):
        relpath = name.replace('.', os.sep) + '.py'
        return any(os.path.exists(os.path.join(d, relpath)) for d in dirs)

    def _log_exception(self):
        s = StringIO.StringIO()
        traceback.print_exc(file=s)
        for l in s.buflist:
            _log.error('  ' + l.rstrip())