diff options
author | Stef Walter <stefw@gnome.org> | 2014-03-04 08:20:53 +0100 |
---|---|---|
committer | Stef Walter <stefw@gnome.org> | 2014-03-04 12:57:19 +0100 |
commit | b72048c920f50df85cb398f4309e68a575d8879e (patch) | |
tree | 78e89081991e80e68a05d76583d8e5d96ddb53fd /build | |
parent | ec89646b1a1594193d56dc18ffd0da15b211ff82 (diff) | |
download | libsecret-b72048c920f50df85cb398f4309e68a575d8879e.tar.gz |
Makefile.am: Use a single Makefile.am and parallel tests
Allow parallel building and testing by using a single Makefile.am
Implement parallel testing using TAP, with various drivers and
compilers living in the build/ directory.
Fix all sorts of issues that this caused, including builddir != srcdir,
leaks in tests and so on.
It would have been nice to break out all the above into separate
commits ... blush.
Diffstat (limited to 'build')
-rw-r--r-- | build/Makefile.am | 29 | ||||
-rwxr-xr-x | build/tap-compiler | 174 | ||||
-rwxr-xr-x | build/tap-driver | 287 | ||||
-rwxr-xr-x | build/tap-unittest | 84 | ||||
-rwxr-xr-x | build/test-driver | 127 |
5 files changed, 685 insertions, 16 deletions
diff --git a/build/Makefile.am b/build/Makefile.am index 73a8b1b..b73bf65 100644 --- a/build/Makefile.am +++ b/build/Makefile.am @@ -1,26 +1,23 @@ -include $(top_srcdir)/Makefile.decl - -VALGRIND_CONTRIB = \ - valgrind.h \ - memcheck.h \ - $(NULL) - SUPPRESSIONS = \ - $(srcdir)/gcrypt.supp \ - $(srcdir)/glib.supp \ - $(srcdir)/pthread.supp \ - $(srcdir)/unknown.supp \ + build/gcrypt.supp \ + build/glib.supp \ + build/pthread.supp \ + build/unknown.supp \ $(NULL) valgrind-suppressions: $(SUPPRESSIONS) - $(AM_V_GEN) cat $(SUPPRESSIONS) > $@ + $(AM_V_GEN) cat $^ > $@ -EXTRA_DIST = \ - valgrind \ +EXTRA_DIST += \ + build/valgrind \ + build/tap-compiler \ + build/tap-driver \ + build/tap-unittest \ + build/test-driver \ $(SUPPRESSIONS) -CLEANFILES = \ +CLEANFILES += \ valgrind-suppressions \ $(NULL) -all-local: valgrind-suppressions +nodist_noinst_DATA += valgrind-suppressions diff --git a/build/tap-compiler b/build/tap-compiler new file mode 100755 index 0000000..76b3171 --- /dev/null +++ b/build/tap-compiler @@ -0,0 +1,174 @@ +#!/usr/bin/python + +# Copyright (C) 2014 Red Hat, Inc. +# +# Cockpit is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# Cockpit is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Cockpit; If not, see <http://www.gnu.org/licenses/>. + +# +# This is a test output compiler which produces TAP from GTest output +# if GTest output is detected. +# +# Versions of glib later than 2.38.x output TAP natively when tests are +# run with the --tap option. However we can't depend on such a recent +# version of glib for our purposes. +# +# This implements the Test Anything Protocol (ie: TAP) +# https://metacpan.org/pod/release/PETDANCE/Test-Harness-2.64/lib/Test/Harness/TAP.pod +# + +import argparse +import os +import select +import subprocess +import sys + +class NullCompiler: + def __init__(self, command): + self.command = command + + def input(self, line): + sys.stdout.write(line) + + def process(self, proc): + while True: + line = proc.stdout.readline() + if not line: + break + self.input(line) + proc.wait() + return proc.returncode + + def run(self, proc, line=None): + if line: + self.input(line) + return self.process(proc) + + +class GTestCompiler(NullCompiler): + def __init__(self, filename): + NullCompiler.__init__(self, filename) + self.test_num = 0 + self.test_name = None + self.test_remaining = [] + + def input(self, line): + line = line.strip() + if line.startswith("GTest: "): + (cmd, unused, data) = line[7:].partition(": ") + cmd = cmd.strip() + data = data.strip() + if cmd == "run": + self.test_name = data + assert self.test_name in self.test_remaining, "%s %s" % (self.test_name, repr(self.test_remaining)) + self.test_remaining.remove(self.test_name) + self.test_num += 1 + elif cmd == "result": + if data == "OK": + print "ok %d %s" % (self.test_num, self.test_name) + if data == "FAIL": + print "not ok %d %s", (self.test_num, self.test_name) + self.test_name = None + elif cmd == "skipping": + print "ok %d # skip -- %s" % (self.test_num, self.test_name) + self.test_name = None + elif data: + print "# %s: %s" % (cmd, data) + else: + print "# %s" % cmd + elif line.startswith("(MSG: "): + print "# %s" % line[6:-1] + elif line: + print "# %s" % line + sys.stdout.flush() + + def run(self, proc, output=""): + # Complete retrieval of the list of tests + output += proc.stdout.read() + proc.wait() + if proc.returncode: + raise subprocess.CalledProcessError(proc.returncode, self.command) + self.test_remaining = [] + for line in output.split("\n"): + if line.startswith("/"): + self.test_remaining.append(line.strip()) + if not self.test_remaining: + print "Bail out! No tests found in GTest: %s" % self.command[0] + return 0 + + print "1..%d" % len(self.test_remaining) + + # First try to run all the tests in a batch + proc = subprocess.Popen(self.command + ["--verbose" ], close_fds=True, stdout=subprocess.PIPE) + result = self.process(proc) + if result == 0: + return 0 + + # Now pick up any stragglers due to failures + while True: + # Assume that the last test failed + if self.test_name: + print "not ok %d %s" % (self.test_num, self.test_name) + self.test_name = None + + # Run any tests which didn't get run + if not self.test_remaining: + break + + proc = subprocess.Popen(self.command + ["--verbose", "-p", self.test_remaining[0]], + close_fds=True, stdout=subprocess.PIPE) + result = self.process(proc) + + # The various exit codes and signals we continue for + if result not in [ 0, 1, -4, -5, -6, -7, -8, -11 ]: + break + + return result + +def main(argv): + parser = argparse.ArgumentParser(description='Automake TAP compiler') + parser.add_argument('--format', metavar='FORMAT', choices=[ "auto", "GTest", "TAP" ], + default="auto", help='The input format to compile') + parser.add_argument('--verbose', action='store_true', + default=True, help='Verbose mode (ignored)') + parser.add_argument('command', nargs='+', help="A test command to run") + args = parser.parse_args(argv[1:]) + + output = None + format = args.format + cmd = args.command + proc = None + + if format in ["auto", "GTest"]: + list_cmd = cmd + ["-l", "--verbose"] + proc = subprocess.Popen(list_cmd, close_fds=True, stdout=subprocess.PIPE) + output = proc.stdout.readline() + # Smell whether we're dealing with GTest list output from first line + if "random seed" in output or "GTest" in output or output.startswith("/"): + format = "GTest" + else: + format = "TAP" + else: + proc = subprocess.Popen(cmd, close_fds=True, stdout=subprocess.PIPE) + + if format == "GTest": + compiler = GTestCompiler(cmd) + elif format == "TAP": + compiler = NullCompiler(cmd) + else: + assert False, "not reached" + + return compiler.run(proc, output) + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build/tap-driver b/build/tap-driver new file mode 100755 index 0000000..1c5af40 --- /dev/null +++ b/build/tap-driver @@ -0,0 +1,287 @@ +#!/usr/bin/python + +# Copyright (C) 2013 Red Hat, Inc. +# +# Cockpit is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# Cockpit is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Cockpit; If not, see <http://www.gnu.org/licenses/>. + +# +# This is a TAP driver for automake +# +# In particular it leaves stderr untouched, and is cleaner than the +# one implemented in shell that is making the rounds. +# +# This implements the automake "Custom Test Driver" protocol: +# https://www.gnu.org/software/automake/manual/html_node/Custom-Test-Drivers.html +# +# This consumes the Test Anything Protocol (ie: TAP) +# https://metacpan.org/pod/release/PETDANCE/Test-Harness-2.64/lib/Test/Harness/TAP.pod +# + +import argparse +import os +import select +import subprocess +import sys + +class Driver: + def __init__(self, command, args): + self.argv = command + self.output = "" + self.test_name = args.test_name + self.log = open(args.log_file, "w") + self.trs = open(args.trs_file, "w") + self.color_tests = args.color_tests + self.expect_failure = args.expect_failure + self.reported = { } + self.test_plan = None + self.late_plan = False + self.errored = False + self.bail_out = False + + def report(self, code, num, *args): + CODES = { + "XPASS": '\x1b[0;31m', # red + "FAIL": '\x1b[0;31m', # red + "PASS": '\x1b[0;32m', # grn + "XFAIL": '\x1b[1;32m', # lgn + "SKIP": '\x1b[1;34m', # blu + "ERROR": '\x1b[0;35m', # mgn + } + + # Print out to console + if self.color_tests: + if code in CODES: + sys.stdout.write(CODES[code]) + sys.stdout.write(code) + if self.color_tests: + sys.stdout.write('\x1b[m') + sys.stdout.write(": ") + sys.stdout.write(self.test_name) + sys.stdout.write(" ") + if num: + sys.stdout.write(str(num)) + sys.stdout.write(" ") + for arg in args: + sys.stdout.write(str(arg)) + sys.stdout.write("\n") + sys.stdout.flush() + + # Book keeping + if code in CODES: + if num != None: + self.reported[num] = code + self.trs.write(":test-result: %s\n" % code) + if code == "ERROR": + self.errored = True + + def result_pass(self, num, description): + if self.expect_failure: + self.report("XPASS", num, description) + else: + self.report("PASS", num, description) + + def result_fail(self, num, description): + if self.expect_failure: + self.report("XFAIL", num, description) + else: + self.report("FAIL", num, description) + + def result_skip(self, num, description, ok): + if self.expect_failure: + self.report("XFAIL", num, description) + else: + self.report("SKIP", num, description) + + def report_error(self, problem): + self.report("ERROR", None, problem) + + def consume_test_line(self, ok, data): + # It's an error if the caller sends a test plan in the middle of tests + if self.late_plan: + self.report_error("Got tests after late TAP test plan") + self.late_plan = False + + # Parse out a number and then description + (num, unused, description) = data.partition(" ") + try: + num = int(num) + except ValueError: + self.report_error("Invalid test number: %s" % data) + return + description = description.lstrip() + + # Special case if description starts with this, then skip + if description.lower().startswith("# skip"): + self.result_skip(num, description, ok) + elif ok: + self.result_pass(num, description) + else: + self.result_fail(num, description) + + def consume_test_plan(self, first, last): + # Only one test plan is supported + if self.test_plan: + self.report_error("Get a second TAP test plan") + return + + try: + first = int(first) + last = int(last) + except ValueError: + self.report_error("Invalid test plan: %s..%s" % (first, last)) + return + + self.test_plan = (first, last) + self.late_plan = self.reported and True or False + + def consume_bail_out(self, line): + self.bail_out = True + self.report("SKIP", 0, line) + + def drain(self): + (ready, unused, self.output) = self.output.rpartition("\n") + for line in ready.split("\n"): + self.log.write(line) + self.log.write("\n") + + if line.startswith("ok "): + self.consume_test_line(True, line[3:]) + elif line.startswith("not ok "): + self.consume_test_line(False, line[7:]) + elif line and line[0].isdigit() and ".." in line: + (first, unused, last) = line.partition("..") + self.consume_test_plan(first, last) + elif line.lower().startswith("bail out!"): + self.consume_bail_out(line) + + def execute(self): + try: + proc = subprocess.Popen(self.argv, close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError, ex: + self.report_error("Couldn't run %s: %s" % (self.argv[0], str(ex))) + return + + outf = proc.stdout.fileno() + errf = proc.stderr.fileno() + rset = [outf, errf] + while len(rset) > 0: + ret = select.select(rset, [], [], 10) + if outf in ret[0]: + data = os.read(outf, 1024) + if data == "": + if self.output: + self.output += "\n" + rset.remove(outf) + else: + self.output += data + self.drain() + if errf in ret[0]: + data = os.read(errf, 1024) + if data == "": + rset.remove(errf) + self.log.write(data) + sys.stderr.write(data) + + proc.wait() + self.returncode = proc.returncode + + def run(self): + self.execute() + + failed = False + skipped = True + + # Basic collation of results + for (num, code) in self.reported.items(): + if code == "ERROR": + self.errored = True + elif code == "FAIL" or code == "XPASS": + failed = True + if code != "SKIP": + skipped = False + + # Check the plan + if not self.errored: + if not self.test_plan: + if not self.bail_out: + if self.returncode: + self.report_error("Test process failed: %d" % self.returncode) + else: + self.report_error("Didn't receive a TAP test plan") + else: + for i in range(self.test_plan[0], self.test_plan[1] + 1): + if i not in self.reported: + if self.bail_out: + self.report("SKIP", i, "- bailed out") + else: + self.report("ERROR", i, "- missing test") + skipped = False + self.errored = True + + if self.errored: + self.trs.write(":global-test-result: ERROR\n") + self.trs.write(":test-global-result: ERROR\n") + self.trs.write(":recheck: no\n") + elif failed: + self.trs.write(":global-test-result: FAIL\n") + self.trs.write(":test-global-result: FAIL\n") + self.trs.write(":recheck: yes\n") + elif skipped: + self.trs.write(":global-test-result: SKIP\n") + self.trs.write(":test-global-result: SKIP\n") + self.trs.write(":recheck: no\n") + if self.errored or failed: + self.trs.write(":copy-in-global-log: yes\n") + + # Process result code + return self.errored and 1 or 0 + +class YesNoAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + argparse.Action.__init__(self, option_strings, dest, **kwargs) + self.metavar = "[yes|no]" + def __call__(self, parser, namespace, values, option_string=None): + if not values or "yes" in values: + setattr(namespace, self.dest, True) + else: + setattr(namespace, self.dest, False) + +def main(argv): + parser = argparse.ArgumentParser(description='Automake TAP driver') + parser.add_argument('--test-name', metavar='NAME', + help='The name of the test') + parser.add_argument('--log-file', metavar='PATH.log', required=True, + help='The .log file the driver creates') + parser.add_argument('--trs-file', metavar='PATH.trs', required=True, + help='The .trs file the driver creates') + parser.add_argument('--color-tests', default=True, action=YesNoAction, + help='Whether the console output should be colorized or not') + parser.add_argument('--expect-failure', default=False, action=YesNoAction, + help="Whether the tested program is expected to fail") + parser.add_argument('--enable-hard-errors', default=False, action=YesNoAction, + help="Whether hard errors in the tested program are treated differently") + parser.add_argument('command', nargs='+', + help="A test command line to run") + args = parser.parse_args(argv[1:]) + + if not args.test_name: + args.test_name = os.path.basename(args.command[0]) + + driver = Driver(args.command, args) + return driver.run() + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build/tap-unittest b/build/tap-unittest new file mode 100755 index 0000000..915ec7d --- /dev/null +++ b/build/tap-unittest @@ -0,0 +1,84 @@ +#!/usr/bin/python + +# +# This is a TAP compiler for python unittest +# +# It hooks into python's standard unittest module, and produces TAP output. +# +# This produces the Test Anything Protocol (ie: TAP) +# https://metacpan.org/pod/release/PETDANCE/Test-Harness-2.64/lib/Test/Harness/TAP.pod +# +# Based on code from here: +# https://github.com/vit1251/unittest-tap-reporting +# + +import argparse +import imp +import os +import sys +import time +import traceback +import unittest + +def write_line(format, *args): + sys.stdout.write(format % args) + sys.stdout.write("\n") + sys.stdout.flush() + +class TAPTestResult(unittest.result.TestResult): + def __init__(self): + unittest.result.TestResult.__init__(self) + self.number = 0 + + def addSuccess(self, test): + self.number += 1 + write_line("ok %d %s", self.number, test.id()) + + def addSkip(self, test, reason): + self.number += 1 + write_line("not ok %d # skip %s", self.number, test.id()) + write_line("# %s", reason) + + def addError(self, test, exc): + (etype, evalue, etraceback) = exc + traceback.print_exception(etype, evalue, etraceback, file=sys.stderr) + self.number += 1 + write_line("not ok %d %s", self.number, test.id()) + + def addFailure(self, test, err): + (etype, evalue, etraceback) = exc + traceback.print_exception(etype, evalue, etraceback, limit=1, file=sys.stderr) + self.number += 1 + write_line("not ok %d %s", self.number, test.id()) + +class TAPTestRunner(object): + def __init__(self): + pass + + def run(self, test): + write_line("1..%d", test.countTestCases()) + result = TAPTestResult() + startTestRun = getattr(result, 'startTestRun', lambda : None) + startTestRun() + try: + test(result) + finally: + stopTestRun = getattr(result, 'stopTestRun', lambda : None) + stopTestRun() + return result + +def main(argv): + parser = argparse.ArgumentParser(description='Python unittest TAP driver') + parser.add_argument('module', nargs=1, + help="A unittest test module to run") + args = parser.parse_args(argv[1:]) + + (name, ext) = os.path.splitext(os.path.basename(args.module[0])) + module = imp.load_source(name, args.module[0]) + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(module) + runner = TAPTestRunner() + runner.run(tests) + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build/test-driver b/build/test-driver new file mode 100755 index 0000000..32bf39e --- /dev/null +++ b/build/test-driver @@ -0,0 +1,127 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2012-06-27.10; # UTC + +# Copyright (C) 2011-2013 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to <bug-automake@gnu.org> or send patches to +# <automake-patches@gnu.org>. + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <<END +Usage: + test-driver --test-name=NAME --log-file=PATH --trs-file=PATH + [--expect-failure={yes|no}] [--color-tests={yes|no}] + [--enable-hard-errors={yes|no}] [--] TEST-SCRIPT +The '--test-name', '--log-file' and '--trs-file' options are mandatory. +END +} + +# TODO: better error handling in option parsing (in particular, ensure +# TODO: $log_file, $trs_file and $test_name are defined). +test_name= # Used for reporting. +log_file= # Where to save the output of the test script. +trs_file= # Where to save the metadata of the test run. +expect_failure=no +color_tests=no +enable_hard_errors=yes +while test $# -gt 0; do + case $1 in + --help) print_usage; exit $?;; + --version) echo "test-driver $scriptversion"; exit $?;; + --test-name) test_name=$2; shift;; + --log-file) log_file=$2; shift;; + --trs-file) trs_file=$2; shift;; + --color-tests) color_tests=$2; shift;; + --expect-failure) expect_failure=$2; shift;; + --enable-hard-errors) enable_hard_errors=$2; shift;; + --) shift; break;; + -*) usage_error "invalid option: '$1'";; + esac + shift +done + +if test $color_tests = yes; then + # Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'. + red='[0;31m' # Red. + grn='[0;32m' # Green. + lgn='[1;32m' # Light green. + blu='[1;34m' # Blue. + mgn='[0;35m' # Magenta. + std='[m' # No color. +else + red= grn= lgn= blu= mgn= std= +fi + +do_exit='rm -f $log_file $trs_file; (exit $st); exit $st' +trap "st=129; $do_exit" 1 +trap "st=130; $do_exit" 2 +trap "st=141; $do_exit" 13 +trap "st=143; $do_exit" 15 + +# Test script is run here. +"$@" >$log_file 2>&1 +estatus=$? +if test $enable_hard_errors = no && test $estatus -eq 99; then + estatus=1 +fi + +case $estatus:$expect_failure in + 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; + 0:*) col=$grn res=PASS recheck=no gcopy=no;; + 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; + *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; + *:*) col=$red res=FAIL recheck=yes gcopy=yes;; +esac + +# Report outcome to console. +echo "${col}${res}${std}: $test_name" + +# Register the test result, and other relevant metadata. +echo ":test-result: $res" > $trs_file +echo ":global-test-result: $res" >> $trs_file +echo ":recheck: $recheck" >> $trs_file +echo ":copy-in-global-log: $gcopy" >> $trs_file + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: |