diff options
Diffstat (limited to 'test/testlib.py')
| -rw-r--r-- | test/testlib.py | 721 |
1 files changed, 0 insertions, 721 deletions
diff --git a/test/testlib.py b/test/testlib.py deleted file mode 100644 index 6c7bc3a..0000000 --- a/test/testlib.py +++ /dev/null @@ -1,721 +0,0 @@ -#!python -# Copyright (c) 2000-2008 ActiveState Software Inc. -# License: MIT License (http://www.opensource.org/licenses/mit-license.php) - -""" - test suite harness - - Usage: - - test --list [<tags>...] # list available tests modules - test [<tags>...] # run test modules - - Options: - -v, --verbose more verbose output - -q, --quiet don't print anything except if a test fails - -d, --debug log debug information - -h, --help print this text and exit - -l, --list Just list the available test modules. You can also - specify tags to play with module filtering. - -n, --no-default-tags Ignore default tags - -L <directive> Specify a logging level via - <logname>:<levelname> - For example: - codeintel.db:DEBUG - This option can be used multiple times. - - By default this will run all tests in all available "test_*" modules. - Tags can be specified to control which tests are run. For example: - - test python # run tests with the 'python' tag - test python cpln # run tests with both 'python' and 'cpln' tags - test -- -python # exclude tests with the 'python' tag - # (the '--' is necessary to end the option list) - - The full name and base name of a test module are implicit tags for that - module, e.g. module "test_xdebug.py" has tags "test_xdebug" and "xdebug". - A TestCase's class name (with and without "TestCase") is an implicit - tag for an test_* methods. A "test_foo" method also has "test_foo" - and "foo" implicit tags. - - Tags can be added explicitly added: - - to modules via a __tags__ global list; and - - to individual test_* methods via a "tags" attribute list (you can - use the testlib.tag() decorator for this). -""" -#TODO: -# - Document how tests are found (note the special "test_cases()" and -# "test_suite_class" hooks). -# - See the optparse "TODO" below. -# - Make the quiet option actually quiet. - -__version_info__ = (0, 6, 6) -__version__ = '.'.join(map(str, __version_info__)) - - -import os -from os.path import join, basename, dirname, abspath, splitext, \ - isfile, isdir, normpath, exists -import sys -import getopt -import glob -import time -import types -import tempfile -import unittest -from pprint import pprint -import imp -import optparse -import logging -import textwrap -import traceback - - - -#---- globals and exceptions - -log = logging.getLogger("test") - - - -#---- exports generally useful to test cases - -class TestError(Exception): - pass - -class TestSkipped(Exception): - """Raise this to indicate that a test is being skipped. - - ConsoleTestRunner knows to interpret these at NOT failures. - """ - pass - -class TestFailed(Exception): - pass - -def tag(*tags): - """Decorator to add tags to test_* functions. - - Example: - class MyTestCase(unittest.TestCase): - @testlib.tag("knownfailure") - def test_foo(self): - #... - """ - def decorate(f): - if not hasattr(f, "tags"): - f.tags = [] - f.tags += tags - return f - return decorate - - -#---- timedtest decorator -# Use this to assert that a test completes in a given amount of time. -# This is from http://www.artima.com/forums/flat.jsp?forum=122&thread=129497 -# Including here, becase it might be useful. -# NOTE: Untested and I suspect some breakage. - -TOLERANCE = 0.05 - -class DurationError(AssertionError): pass - -def timedtest(max_time, tolerance=TOLERANCE): - """ timedtest decorator - decorates the test method with a timer - when the time spent by the test exceeds - max_time in seconds, an Assertion error is thrown. - """ - def _timedtest(function): - def wrapper(*args, **kw): - start_time = time.time() - try: - function(*args, **kw) - finally: - total_time = time.time() - start_time - if total_time > max_time + tolerance: - raise DurationError(('Test was too long (%.2f s)' - % total_time)) - return wrapper - - return _timedtest - - - -#---- module api - -class Test(object): - def __init__(self, ns, testmod, testcase, testfn_name, - testsuite_class=None): - self.ns = ns - self.testmod = testmod - self.testcase = testcase - self.testfn_name = testfn_name - self.testsuite_class = testsuite_class - # Give each testcase some extra testlib attributes for useful - # introspection on TestCase instances later on. - self.testcase._testlib_shortname_ = self.shortname() - self.testcase._testlib_explicit_tags_ = self.explicit_tags() - self.testcase._testlib_implicit_tags_ = self.implicit_tags() - def __str__(self): - return self.shortname() - def __repr__(self): - return "<Test %s>" % self.shortname() - def shortname(self): - bits = [self._normname(self.testmod.__name__), - self._normname(self.testcase.__class__.__name__), - self._normname(self.testfn_name)] - if self.ns: - bits.insert(0, self.ns) - return '/'.join(bits) - def _flatten_tags(self, tags): - """Split tags with '/' in them into multiple tags. - - '/' is the reserved tag separator and allowing tags with - embedded '/' results in one being unable to select those via - filtering. As long as tag order is stable then presentation of - these subsplit tags should be fine. - """ - flattened = [] - for t in tags: - flattened += t.split('/') - return flattened - def explicit_tags(self): - tags = [] - if hasattr(self.testmod, "__tags__"): - tags += self.testmod.__tags__ - if hasattr(self.testcase, "__tags__"): - tags += self.testcase.__tags__ - testfn = getattr(self.testcase, self.testfn_name) - if hasattr(testfn, "tags"): - tags += testfn.tags - return self._flatten_tags(tags) - def implicit_tags(self): - tags = [ - self.testmod.__name__.lower(), - self._normname(self.testmod.__name__), - self.testcase.__class__.__name__.lower(), - self._normname(self.testcase.__class__.__name__), - self.testfn_name, - self._normname(self.testfn_name), - ] - if self.ns: - tags.insert(0, self.ns) - return self._flatten_tags(tags) - def tags(self): - return self.explicit_tags() + self.implicit_tags() - def doc(self): - testfn = getattr(self.testcase, self.testfn_name) - return testfn.__doc__ or "" - def _normname(self, name): - if name.startswith("test_"): - return name[5:].lower() - elif name.startswith("test"): - return name[4:].lower() - elif name.endswith("TestCase"): - return name[:-8].lower() - else: - return name - - -def testmod_paths_from_testdir(testdir): - """Generate test module paths in the given dir.""" - for path in glob.glob(join(testdir, "test_*.py")): - yield path - - for path in glob.glob(join(testdir, "test_*")): - if not isdir(path): continue - if not isfile(join(path, "__init__.py")): continue - yield path - -def testmods_from_testdir(testdir): - """Generate test modules in the given test dir. - - Modules are imported with 'testdir' first on sys.path. - """ - testdir = normpath(testdir) - for testmod_path in testmod_paths_from_testdir(testdir): - testmod_name = splitext(basename(testmod_path))[0] - log.debug("import test module '%s'", testmod_path) - try: - iinfo = imp.find_module(testmod_name, [dirname(testmod_path)]) - testabsdir = abspath(testdir) - sys.path.insert(0, testabsdir) - old_dir = os.getcwd() - os.chdir(testdir) - try: - testmod = imp.load_module(testmod_name, *iinfo) - finally: - os.chdir(old_dir) - sys.path.remove(testabsdir) - except TestSkipped: - _, ex, _ = sys.exc_info() - log.warn("'%s' module skipped: %s", testmod_name, ex) - except Exception: - _, ex, _ = sys.exc_info() - log.warn("could not import test module '%s': %s (skipping, " - "run with '-d' for full traceback)", - testmod_path, ex) - if log.isEnabledFor(logging.DEBUG): - traceback.print_exc() - else: - yield testmod - -def testcases_from_testmod(testmod): - """Gather tests from a 'test_*' module. - - Returns a list of TestCase-subclass instances. One instance for each - found test function. - - In general the normal unittest TestLoader.loadTests*() semantics are - used for loading tests with some differences: - - TestCase subclasses beginning with '_' are skipped (presumed to be - internal). - - If the module has a top-level "test_cases", it is called for a list of - TestCase subclasses from which to load tests (can be a generator). This - allows for run-time setup of test cases. - - If the module has a top-level "test_suite_class", it is used to group - all test cases from that module into an instance of that TestSuite - subclass. This allows for overriding of test running behaviour. - """ - class TestListLoader(unittest.TestLoader): - suiteClass = list - - loader = TestListLoader() - if hasattr(testmod, "test_cases"): - try: - for testcase_class in testmod.test_cases(): - if testcase_class.__name__.startswith("_"): - log.debug("skip private TestCase class '%s'", - testcase_class.__name__) - continue - for testcase in loader.loadTestsFromTestCase(testcase_class): - yield testcase - except Exception: - _, ex, _ = sys.exc_info() - testmod_path = testmod.__file__ - if testmod_path.endswith(".pyc"): - testmod_path = testmod_path[:-1] - log.warn("error running test_cases() in '%s': %s (skipping, " - "run with '-d' for full traceback)", - testmod_path, ex) - if log.isEnabledFor(logging.DEBUG): - traceback.print_exc() - else: - class_names_skipped = [] - for testcases in loader.loadTestsFromModule(testmod): - for testcase in testcases: - class_name = testcase.__class__.__name__ - if class_name in class_names_skipped: - pass - elif class_name.startswith("_"): - log.debug("skip private TestCase class '%s'", class_name) - class_names_skipped.append(class_name) - else: - yield testcase - - -def tests_from_manifest(testdir_from_ns): - """Return a list of `testlib.Test` instances for each test found in - the manifest. - - There will be a test for - (a) each "test*" function of - (b) each TestCase-subclass in - (c) each "test_*" Python module in - (d) each test dir in the manifest. - - If a "test_*" module has a top-level "test_suite_class", it will later - be used to group all test cases from that module into an instance of that - TestSuite subclass. This allows for overriding of test running behaviour. - """ - for ns, testdir in testdir_from_ns.items(): - for testmod in testmods_from_testdir(testdir): - if hasattr(testmod, "test_suite_class"): - testsuite_class = testmod.test_suite_class - if not issubclass(testsuite_class, unittest.TestSuite): - testmod_path = testmod.__file__ - if testmod_path.endswith(".pyc"): - testmod_path = testmod_path[:-1] - log.warn("'test_suite_class' of '%s' module is not a " - "subclass of 'unittest.TestSuite': ignoring", - testmod_path) - else: - testsuite_class = None - for testcase in testcases_from_testmod(testmod): - try: - yield Test(ns, testmod, testcase, - testcase._testMethodName, - testsuite_class) - except AttributeError: - # Python 2.4 and older: - yield Test(ns, testmod, testcase, - testcase._TestCase__testMethodName, - testsuite_class) - -def tests_from_manifest_and_tags(testdir_from_ns, tags): - include_tags = [tag.lower() for tag in tags if not tag.startswith('-')] - exclude_tags = [tag[1:].lower() for tag in tags if tag.startswith('-')] - - for test in tests_from_manifest(testdir_from_ns): - test_tags = [t.lower() for t in test.tags()] - - matching_exclude_tags = [t for t in exclude_tags if t in test_tags] - if matching_exclude_tags: - #log.debug("test '%s' matches exclude tag(s) '%s': skipping", - # test.shortname(), "', '".join(matching_exclude_tags)) - continue - - if not include_tags: - yield test - else: - for tag in include_tags: - if tag not in test_tags: - #log.debug("test '%s' does not match tag '%s': skipping", - # test.shortname(), tag) - break - else: - #log.debug("test '%s' matches tags: %s", test.shortname(), - # ' '.join(tags)) - yield test - -def test(testdir_from_ns, tags=[], setup_func=None): - log.debug("test(testdir_from_ns=%r, tags=%r, ...)", - testdir_from_ns, tags) - if setup_func is not None: - setup_func() - tests = list(tests_from_manifest_and_tags(testdir_from_ns, tags)) - if not tests: - return None - - # Groups test cases into a test suite class given by their test module's - # "test_suite_class" hook, if any. - suite = unittest.TestSuite() - suite_for_testmod = None - testmod = None - for test in tests: - if test.testmod != testmod: - if suite_for_testmod is not None: - suite.addTest(suite_for_testmod) - suite_for_testmod = (test.testsuite_class or unittest.TestSuite)() - testmod = test.testmod - suite_for_testmod.addTest(test.testcase) - if suite_for_testmod is not None: - suite.addTest(suite_for_testmod) - - runner = ConsoleTestRunner(sys.stdout) - result = runner.run(suite) - return result - -def list_tests(testdir_from_ns, tags): - # Say I have two test_* modules: - # test_python.py: - # __tags__ = ["guido"] - # class BasicTestCase(unittest.TestCase): - # def test_def(self): - # def test_class(self): - # class ComplexTestCase(unittest.TestCase): - # def test_foo(self): - # def test_bar(self): - # test_perl/__init__.py: - # __tags__ = ["larry", "wall"] - # class BasicTestCase(unittest.TestCase): - # def test_sub(self): - # def test_package(self): - # class EclecticTestCase(unittest.TestCase): - # def test_foo(self): - # def test_bar(self): - # The short-form list output for this should look like: - # python/basic/def [guido] - # python/basic/class [guido] - # python/complex/foo [guido] - # python/complex/bar [guido] - # perl/basic/sub [larry, wall] - # perl/basic/package [larry, wall] - # perl/eclectic/foo [larry, wall] - # perl/eclectic/bar [larry, wall] - log.debug("list_tests(testdir_from_ns=%r, tags=%r)", - testdir_from_ns, tags) - - tests = list(tests_from_manifest_and_tags(testdir_from_ns, tags)) - if not tests: - return - - WIDTH = 78 - if log.isEnabledFor(logging.INFO): # long-form - for i, t in enumerate(tests): - if i: - print() - testfile = t.testmod.__file__ - if testfile.endswith(".pyc"): - testfile = testfile[:-1] - print("%s:" % t.shortname()) - print(" from: %s#%s.%s" % (testfile, - t.testcase.__class__.__name__, t.testfn_name)) - wrapped = textwrap.fill(' '.join(t.tags()), WIDTH-10) - print(" tags: %s" % _indent(wrapped, 8, True)) - if t.doc(): - print(_indent(t.doc(), width=2)) - else: - for t in tests: - line = t.shortname() + ' ' - if t.explicit_tags(): - line += '[%s]' % ' '.join(t.explicit_tags()) - print(line) - - -#---- text test runner that can handle TestSkipped reasonably - -class ConsoleTestResult(unittest.TestResult): - """A test result class that can print formatted text results to a stream. - - Used by ConsoleTestRunner. - """ - separator1 = '=' * 70 - separator2 = '-' * 70 - - def __init__(self, stream): - unittest.TestResult.__init__(self) - self.skips = [] - self.stream = stream - - def getDescription(self, test): - if test._testlib_explicit_tags_: - return "%s [%s]" % (test._testlib_shortname_, - ', '.join(test._testlib_explicit_tags_)) - else: - return test._testlib_shortname_ - - def startTest(self, test): - unittest.TestResult.startTest(self, test) - self.stream.write(self.getDescription(test)) - self.stream.write(" ... ") - - def addSuccess(self, test): - unittest.TestResult.addSuccess(self, test) - self.stream.write("ok\n") - - def addSkip(self, test, err): - why = str(err[1]) - self.skips.append((test, why)) - self.stream.write("skipped (%s)\n" % why) - - def addError(self, test, err): - if isinstance(err[1], TestSkipped): - self.addSkip(test, err) - else: - unittest.TestResult.addError(self, test, err) - self.stream.write("ERROR\n") - - def addFailure(self, test, err): - unittest.TestResult.addFailure(self, test, err) - self.stream.write("FAIL\n") - - def printSummary(self): - self.stream.write('\n') - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - - def printErrorList(self, flavour, errors): - for test, err in errors: - self.stream.write(self.separator1 + '\n') - self.stream.write("%s: %s\n" - % (flavour, self.getDescription(test))) - self.stream.write(self.separator2 + '\n') - self.stream.write("%s\n" % err) - - -class ConsoleTestRunner(object): - """A test runner class that displays results on the console. - - It prints out the names of tests as they are run, errors as they - occur, and a summary of the results at the end of the test run. - - Differences with unittest.TextTestRunner: - - adds support for *skipped* tests (those that raise TestSkipped) - - no verbosity option (only have equiv of verbosity=2) - - test "short desc" is it 3-level tag name (e.g. 'foo/bar/baz' where - that identifies: 'test_foo.py::BarTestCase.test_baz'. - """ - def __init__(self, stream=sys.stderr): - self.stream = stream - - def run(self, test_or_suite, test_result_class=ConsoleTestResult): - """Run the given test case or test suite.""" - result = test_result_class(self.stream) - start_time = time.time() - test_or_suite.run(result) - time_taken = time.time() - start_time - - result.printSummary() - self.stream.write(result.separator2 + '\n') - self.stream.write("Ran %d test%s in %.3fs\n\n" - % (result.testsRun, result.testsRun != 1 and "s" or "", - time_taken)) - details = [] - num_skips = len(result.skips) - if num_skips: - details.append("%d skip%s" - % (num_skips, (num_skips != 1 and "s" or ""))) - if not result.wasSuccessful(): - num_failures = len(result.failures) - if num_failures: - details.append("%d failure%s" - % (num_failures, (num_failures != 1 and "s" or ""))) - num_errors = len(result.errors) - if num_errors: - details.append("%d error%s" - % (num_errors, (num_errors != 1 and "s" or ""))) - self.stream.write("FAILED (%s)\n" % ', '.join(details)) - elif details: - self.stream.write("OK (%s)\n" % ', '.join(details)) - else: - self.stream.write("OK\n") - return result - - - -#---- internal support stuff - -# Recipe: indent (0.2.1) -def _indent(s, width=4, skip_first_line=False): - """_indent(s, [width=4]) -> 's' indented by 'width' spaces - - The optional "skip_first_line" argument is a boolean (default False) - indicating if the first line should NOT be indented. - """ - lines = s.splitlines(1) - indentstr = ' '*width - if skip_first_line: - return indentstr.join(lines) - else: - return indentstr + indentstr.join(lines) - - - - - -#---- mainline - -#TODO: pass in add_help_option=False and add it ourself here. -## Optparse's handling of the doc passed in for -h|--help handling is -## abysmal. Hence we'll stick with getopt. -#def _parse_opts(args): -# """_parse_opts(args) -> (options, tags)""" -# usage = "usage: %prog [OPTIONS...] [TAGS...]" -# parser = optparse.OptionParser(prog="test", usage=usage, -# description=__doc__) -# parser.add_option("-v", "--verbose", dest="log_level", -# action="store_const", const=logging.DEBUG, -# help="more verbose output") -# parser.add_option("-q", "--quiet", dest="log_level", -# action="store_const", const=logging.WARNING, -# help="quieter output") -# parser.add_option("-l", "--list", dest="action", -# action="store_const", const="list", -# help="list available tests") -# parser.set_defaults(log_level=logging.INFO, action="test") -# opts, raw_tags = parser.parse_args() -# -# # Trim '.py' from user-supplied tags. They might have gotten there -# # via shell expansion. -# ... -# -# return opts, raw_tags - -def _parse_opts(args, default_tags): - """_parse_opts(args) -> (log_level, action, tags)""" - opts, raw_tags = getopt.getopt(args, "hvqdlL:n", - ["help", "verbose", "quiet", "debug", "list", "no-default-tags"]) - log_level = logging.WARN - action = "test" - no_default_tags = False - for opt, optarg in opts: - if opt in ("-h", "--help"): - action = "help" - elif opt in ("-v", "--verbose"): - log_level = logging.INFO - elif opt in ("-q", "--quiet"): - log_level = logging.ERROR - elif opt in ("-d", "--debug"): - log_level = logging.DEBUG - elif opt in ("-l", "--list"): - action = "list" - elif opt in ("-n", "--no-default-tags"): - no_default_tags = True - elif opt == "-L": - # Optarg is of the form '<logname>:<levelname>', e.g. - # "codeintel:DEBUG", "codeintel.db:INFO". - lname, llevelname = optarg.split(':', 1) - llevel = getattr(logging, llevelname) - logging.getLogger(lname).setLevel(llevel) - - # Clean up the given tags. - if no_default_tags: - tags = [] - else: - tags = default_tags - for raw_tag in raw_tags: - if splitext(raw_tag)[1] in (".py", ".pyc", ".pyo", ".pyw") \ - and exists(raw_tag): - # Trim '.py' from user-supplied tags if it looks to be from - # shell expansion. - tags.append(splitext(raw_tag)[0]) - elif '/' in raw_tag: - # Split one '/' to allow the shortname from the test listing - # to be used as a filter. - tags += raw_tag.split('/') - else: - tags.append(raw_tag) - - return log_level, action, tags - - -def harness(testdir_from_ns={None: os.curdir}, argv=sys.argv, - setup_func=None, default_tags=None): - """Convenience mainline for a test harness "test.py" script. - - "testdir_from_ns" (optional) is basically a set of directories in - which to look for test cases. It is a dict with: - <namespace>: <testdir> - where <namespace> is a (short) string that becomes part of the - included test names and an implicit tag for filtering those - tests. By default the current dir is use with an empty namespace: - {None: os.curdir} - "setup_func" (optional) is a callable that will be called once - before any tests are run to prepare for the test suite. It - is not called if no tests will be run. - "default_tags" (optional) - - Typically, if you have a number of test_*.py modules you can create - a test harness, "test.py", for them that looks like this: - - #!/usr/bin/env python - if __name__ == "__main__": - retval = testlib.harness() - sys.exit(retval) - """ - if not logging.root.handlers: - logging.basicConfig() - try: - log_level, action, tags = _parse_opts(argv[1:], default_tags or []) - except getopt.error: - _, ex, _ = sys.exc_info() - log.error(str(ex) + " (did you need a '--' before a '-TAG' argument?)") - return 1 - log.setLevel(log_level) - - if action == "help": - print(__doc__) - return 0 - if action == "list": - return list_tests(testdir_from_ns, tags) - elif action == "test": - result = test(testdir_from_ns, tags, setup_func=setup_func) - if result is None: - return None - return len(result.errors) + len(result.failures) - else: - raise TestError("unexpected action/mode: '%s'" % action) - - |