diff options
Diffstat (limited to 'valadate')
-rw-r--r-- | valadate/Makefile.am | 19 | ||||
-rw-r--r-- | valadate/assembly.vala | 73 | ||||
-rw-r--r-- | valadate/assemblyerror.vala | 27 | ||||
-rw-r--r-- | valadate/gnutestreportprinter.vala | 79 | ||||
-rw-r--r-- | valadate/taptestprinter.vala | 108 | ||||
-rw-r--r-- | valadate/taptestreportprinter.vala | 108 | ||||
-rw-r--r-- | valadate/test.vala | 42 | ||||
-rw-r--r-- | valadate/testadapter.vala | 107 | ||||
-rw-r--r-- | valadate/testassembly.vala | 93 | ||||
-rw-r--r-- | valadate/testcase.vala | 130 | ||||
-rw-r--r-- | valadate/testconfig.vala | 128 | ||||
-rw-r--r-- | valadate/testerror.vala | 27 | ||||
-rw-r--r-- | valadate/testgatherer.vala | 57 | ||||
-rw-r--r-- | valadate/testmodule.vala | 43 | ||||
-rw-r--r-- | valadate/testoptions.vala | 135 | ||||
-rw-r--r-- | valadate/testplan.vala | 283 | ||||
-rw-r--r-- | valadate/testreport.vala | 278 | ||||
-rw-r--r-- | valadate/testreportprinter.vala | 46 | ||||
-rw-r--r-- | valadate/testresult.vala | 220 | ||||
-rw-r--r-- | valadate/testrunner.vala | 194 | ||||
-rw-r--r-- | valadate/teststatus.vala | 30 | ||||
-rw-r--r-- | valadate/testsuite.vala | 72 | ||||
-rw-r--r-- | valadate/xmlfile.vala | 108 | ||||
-rw-r--r-- | valadate/xmltestreportprinter.vala | 69 |
24 files changed, 2035 insertions, 441 deletions
diff --git a/valadate/Makefile.am b/valadate/Makefile.am index 7d1cc4b87..2a73a4aef 100644 --- a/valadate/Makefile.am +++ b/valadate/Makefile.am @@ -11,6 +11,7 @@ AM_CPPFLAGS = \ $(GLIB_CFLAGS) \ $(GIO_CFLAGS) \ $(GMODULE_CFLAGS) \ + $(LIBXML_CFLAGS) \ -g \ $(NULL) @@ -21,15 +22,27 @@ lib_LTLIBRARIES = \ $(NULL) libvaladate_la_VALASOURCES = \ + assembly.vala \ + assemblyerror.vala \ + gnutestreportprinter.vala \ + taptestreportprinter.vala \ + testreportprinter.vala \ test.vala \ + testadapter.vala \ + testassembly.vala \ testcase.vala \ testconfig.vala \ - testexplorer.vala \ - testiterator.vala \ testmodule.vala \ + testoptions.vala \ + testgatherer.vala \ + testplan.vala \ testresult.vala \ + testreport.vala \ testrunner.vala \ + teststatus.vala \ testsuite.vala \ + xmlfile.vala \ + xmltestreportprinter.vala \ $(NULL) libvaladate_la_SOURCES = \ @@ -46,6 +59,7 @@ valadate.vapi valadate.vala.stamp: $(libvaladate_la_VALASOURCES) --vapidir $(top_srcdir)/vapi --pkg gobject-2.0 \ --pkg gio-2.0 \ --pkg gmodule-2.0 \ + --pkg libxml-2.0 \ --pkg config \ --pkg libvala@PACKAGE_SUFFIX@ \ -H valadate.h \ @@ -61,6 +75,7 @@ libvaladate_la_LIBADD = \ $(GLIB_LIBS) \ $(GIO_LIBS) \ $(GMODULE_LIBS) \ + $(LIBXML_LIBS) \ $(NULL) EXTRA_DIST = $(libvaladate_la_VALASOURCES) valadate.vapi valadate.vala.stamp diff --git a/valadate/assembly.vala b/valadate/assembly.vala new file mode 100644 index 000000000..8536460d5 --- /dev/null +++ b/valadate/assembly.vala @@ -0,0 +1,73 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public abstract class Valadate.Assembly { + + protected static SubprocessLauncher launcher; + + private static void init_launcher() { + if (launcher == null) { + launcher = new SubprocessLauncher( + GLib.SubprocessFlags.STDIN_PIPE | + GLib.SubprocessFlags.STDOUT_PIPE | + GLib.SubprocessFlags.STDERR_PIPE); + launcher.setenv("G_MESSAGES_DEBUG","all",true); + launcher.setenv("G_DEBUG","fatal-criticals fatal-warnings gc-friendly",true); + launcher.setenv("G_SLICE","always-malloc debug-blocks",true); + } + } + + public File binary {get;set;} + public InputStream stderr {get;set;} + public OutputStream stdin {get;set;} + public InputStream stdout {get;set;} + + protected Subprocess process; + + public Assembly(File binary) throws Error { + init_launcher(); + if(!binary.query_exists()) + throw new FileError.NOENT("The file %s does not exist", binary.get_path()); + if(!GLib.FileUtils.test(binary.get_path(), FileTest.IS_EXECUTABLE)) + throw new FileError.PERM("The file %s is not executable", binary.get_path()); + this.binary = binary; + } + + public abstract Assembly clone() throws Error; + + public virtual Assembly run(string? command = null, Cancellable? cancellable = null) throws Error { + string[] args; + Shell.parse_argv("%s %s".printf(binary.get_path(), command ?? ""), out args); + process = launcher.spawnv(args); + stdout = new DataInputStream (process.get_stdout_pipe()); + stderr = new DataInputStream (process.get_stderr_pipe()); + stdin = new DataOutputStream (process.get_stdin_pipe()); + process.wait_check(cancellable); + cancellable.set_error_if_cancelled(); + return this; + } + + public virtual void quit() { + if(process != null) + process.force_exit(); + } +} diff --git a/valadate/assemblyerror.vala b/valadate/assemblyerror.vala new file mode 100644 index 000000000..aa1b57e82 --- /dev/null +++ b/valadate/assemblyerror.vala @@ -0,0 +1,27 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public errordomain Valadate.AssemblyError { + NOT_FOUND, + LOAD, + METHOD +} diff --git a/valadate/gnutestreportprinter.vala b/valadate/gnutestreportprinter.vala new file mode 100644 index 000000000..87ce5e53b --- /dev/null +++ b/valadate/gnutestreportprinter.vala @@ -0,0 +1,79 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ +public class Valadate.GnuTestReportPrinter : TestReportPrinter { + + private const string TAP_VERSION = "13"; + + private List<TestCase> testcases = new List<TestCase>(); + + public GnuTestReportPrinter(TestConfig config) throws Error { + base(config); + } + + public override void print(TestReport report) { + + if(report.test is TestSuite) { + testcases = new List<TestCase>(); + } else if(report.test is TestCase) { + testcases.append(report.test as TestCase); + } else if(report.test is TestAdapter) { + var test = report.test as TestAdapter; + var idx = testcases.index(test.parent as TestCase); + int index = 1; + + if(idx > 0) + for(int i=0; i<idx;i++) + index += testcases.nth_data(i).count; + + for(int i=0;i<test.parent.count; i++) { + if(test.parent[i] == test) { + index += i; + } + } + + switch(report.test.status) { + case TestStatus.PASSED: + stdout.printf("PASS: %s %d - %s\n", test.parent.name, index, test.name); + break; + case TestStatus.SKIPPED: + stdout.printf("SKIP: %s %d - %s # SKIP %s \n", + test.parent.name, index, test.label, test.status_message ?? ""); + break; + case TestStatus.TODO: + var errs = report.xml.eval("//failure | //error"); + if(errs.size > 0) + stdout.printf("XFAIL: %s %d - %s # TODO %s \n", + test.parent.name, index, test.label, test.status_message ?? ""); + else + stdout.printf("PASS: %s %d - %s # TODO %s \n", + test.parent.name, index, test.label, test.status_message ?? ""); + break; + case TestStatus.FAILED: + case TestStatus.ERROR: + stdout.printf("FAIL: %s %d - %s\n", test.parent.name, index, test.label); + break; + default: + break; + } + } + } +} diff --git a/valadate/taptestprinter.vala b/valadate/taptestprinter.vala new file mode 100644 index 000000000..c1f9d7773 --- /dev/null +++ b/valadate/taptestprinter.vala @@ -0,0 +1,108 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TapTestReportPrinter : TestReportPrinter { + + private const string TAP_VERSION = "13"; + + private List<TestCase> testcases = new List<TestCase>(); + + public TapTestReportPrinter(TestConfig config) throws Error { + base(config); + if(!config.list_only) { + stdout.printf("TAP version %s\n", TAP_VERSION); + stdout.printf("# random seed: %s\n", config.seed); + } + } + + public override void print(TestReport report) { + + if(report.test is TestSuite && report.test.parent.name == "/") { + stdout.printf("1..%d\n", report.test.count); + + var props = report.xml.eval("//properties/property"); + stdout.puts("# Environment\n"); + foreach(Xml.Node* prop in props) { + stdout.printf("# %s : %s\n", + prop->get_prop("name"), prop->get_prop("value")); + } + + } else if(report.test is TestCase) { + testcases.append(report.test as TestCase); + stdout.printf("# Start of %s tests\n", report.test.label); + + } else if(report.test is TestAdapter) { + var test = report.test as TestAdapter; + var idx = testcases.index(test.parent as TestCase); + int index = 1; + bool lasttest = false; + + if(idx > 0) + for(int i=0; i<idx;i++) + index += testcases.nth_data(i).count; + + for(int i=0;i<test.parent.count; i++) { + if(test.parent[i] == test) { + index += i; + lasttest = (i == test.parent.count-1); + } + } + + switch(report.test.status) { + case TestStatus.PASSED: + stdout.printf("ok %d - %s\n", index, test.label); + break; + case TestStatus.SKIPPED: + stdout.printf("ok %d - %s # SKIP %s \n", index, test.label, test.status_message ?? "Skipping"); + break; + case TestStatus.TODO: + var errs = report.xml.eval("//failure | //error"); + if(errs.size > 0) + stdout.printf("not ok %d - %s # TODO %s \n", index, test.label, test.status_message ?? "Todo"); + else + stdout.printf("ok %d - %s # TODO %s \n", index, test.label, test.status_message ?? "Todo"); + break; + case TestStatus.FAILED: + case TestStatus.ERROR: + stdout.printf("not ok %d - %s\n", index, test.label); + break; + default: + stdout.printf("Bail out! %s\n", "There was an unexpected error"); + break; + } + stdout.puts(" ---\n"); + stdout.printf(" duration_ms: %.4f\n", test.time); + var messages = report.xml.eval("//failure | //error | //info"); + foreach(Xml.Node* mess in messages) { + stdout.printf(" message: >\n %s\n", mess->get_prop("message")); + stdout.printf(" severity: %s\n", mess->name); + } + stdout.puts(" ...\n"); + messages = report.xml.eval("//system-out | //system-err"); + foreach(Xml.Node* mess in messages) + stdout.printf("# %s\n", string.joinv("\n# ", mess->get_content().split("\n"))); + if(lasttest) + stdout.printf("# End of %s tests\n", test.parent.label); + stdout.flush(); + } + } +} diff --git a/valadate/taptestreportprinter.vala b/valadate/taptestreportprinter.vala new file mode 100644 index 000000000..c1f9d7773 --- /dev/null +++ b/valadate/taptestreportprinter.vala @@ -0,0 +1,108 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TapTestReportPrinter : TestReportPrinter { + + private const string TAP_VERSION = "13"; + + private List<TestCase> testcases = new List<TestCase>(); + + public TapTestReportPrinter(TestConfig config) throws Error { + base(config); + if(!config.list_only) { + stdout.printf("TAP version %s\n", TAP_VERSION); + stdout.printf("# random seed: %s\n", config.seed); + } + } + + public override void print(TestReport report) { + + if(report.test is TestSuite && report.test.parent.name == "/") { + stdout.printf("1..%d\n", report.test.count); + + var props = report.xml.eval("//properties/property"); + stdout.puts("# Environment\n"); + foreach(Xml.Node* prop in props) { + stdout.printf("# %s : %s\n", + prop->get_prop("name"), prop->get_prop("value")); + } + + } else if(report.test is TestCase) { + testcases.append(report.test as TestCase); + stdout.printf("# Start of %s tests\n", report.test.label); + + } else if(report.test is TestAdapter) { + var test = report.test as TestAdapter; + var idx = testcases.index(test.parent as TestCase); + int index = 1; + bool lasttest = false; + + if(idx > 0) + for(int i=0; i<idx;i++) + index += testcases.nth_data(i).count; + + for(int i=0;i<test.parent.count; i++) { + if(test.parent[i] == test) { + index += i; + lasttest = (i == test.parent.count-1); + } + } + + switch(report.test.status) { + case TestStatus.PASSED: + stdout.printf("ok %d - %s\n", index, test.label); + break; + case TestStatus.SKIPPED: + stdout.printf("ok %d - %s # SKIP %s \n", index, test.label, test.status_message ?? "Skipping"); + break; + case TestStatus.TODO: + var errs = report.xml.eval("//failure | //error"); + if(errs.size > 0) + stdout.printf("not ok %d - %s # TODO %s \n", index, test.label, test.status_message ?? "Todo"); + else + stdout.printf("ok %d - %s # TODO %s \n", index, test.label, test.status_message ?? "Todo"); + break; + case TestStatus.FAILED: + case TestStatus.ERROR: + stdout.printf("not ok %d - %s\n", index, test.label); + break; + default: + stdout.printf("Bail out! %s\n", "There was an unexpected error"); + break; + } + stdout.puts(" ---\n"); + stdout.printf(" duration_ms: %.4f\n", test.time); + var messages = report.xml.eval("//failure | //error | //info"); + foreach(Xml.Node* mess in messages) { + stdout.printf(" message: >\n %s\n", mess->get_prop("message")); + stdout.printf(" severity: %s\n", mess->name); + } + stdout.puts(" ...\n"); + messages = report.xml.eval("//system-out | //system-err"); + foreach(Xml.Node* mess in messages) + stdout.printf("# %s\n", string.joinv("\n# ", mess->get_content().split("\n"))); + if(lasttest) + stdout.printf("# End of %s tests\n", test.parent.label); + stdout.flush(); + } + } +} diff --git a/valadate/test.vala b/valadate/test.vala index 51c24ec17..0d853ebac 100644 --- a/valadate/test.vala +++ b/valadate/test.vala @@ -1,6 +1,6 @@ /* * Valadate - Unit testing library for GObject-based libraries. - * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -25,33 +25,41 @@ * It is the base interface for all runnable Tests. */ public interface Valadate.Test : Object { - + /** + * Runs the Tests and collects the results in a TestResult + * + * @param result the TestResult object used to store the results of the Test + */ + public abstract void run (TestResult result); /** * The name of the test */ public abstract string name { get; set; } - + /** + * The label of the test + */ + public abstract string label { get; set; } /** * Returns the number of tests that will be run by this test + * TestSuites should return the total number of tests that will + * be run. */ - public abstract int count { get; } - + public abstract int count {get;} /** - * Runs the Tests and collects the results in a TestResult - * - * @param result the TestResult object used to store the results of the Test + * This is used for the iterator and does not return the number of + * tests that will be run */ - public abstract void run (TestResult result); + public abstract int size {get;} + /** + * The #TestStatus of the test + */ + public abstract TestStatus status {get;set;default=TestStatus.NOT_RUN;} - public abstract Test get_test (int index); + public abstract double time {get;set;} - public virtual TestIterator iterator() { - return new TestIterator (this); - } + public abstract Test? parent {get;set;} - public virtual void set_up () { - } + public abstract Test get(int index); - public virtual void tear_down () { - } + public abstract void set(int index, Test test); } diff --git a/valadate/testadapter.vala b/valadate/testadapter.vala new file mode 100644 index 000000000..195d1a27a --- /dev/null +++ b/valadate/testadapter.vala @@ -0,0 +1,107 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +private class Valadate.TestAdapter : Object, Test { + + public string name {get;set;} + public string label {get;set;} + public double time {get;set;} + + public int timeout {get;set;} + + public TestStatus status {get;set;default=TestStatus.NOT_RUN;} + public string status_message {get;set;} + + public int count { + get { + return 1; + } + } + + public int size { + get { + return count; + } + } + + private TestCase.TestMethod test; + public Test? parent {get;set;} + + public new Test get(int index) { + return this; + } + + public TestAdapter(string name, int timeout) { + this.name = name; + this.timeout = timeout; + } + + public void add_test(owned TestPlan.TestMethod testmethod) { + this.test = () => { + testmethod(parent as TestCase); + }; + } + + public void add_async_test ( + TestPlan.AsyncTestMethod async_begin, + TestPlan.AsyncTestMethodResult async_finish) + { + var p = parent as TestCase; + this.test = () => { + AsyncResult? result = null; + var loop = new MainLoop(); + var thread = new Thread<void*>.try(name, () => { + async_begin(p, (o, r) => { result = r; loop.quit();}); + return null; + }); + Timeout.add(timeout, () => { + loop.quit(); + return false; + }, + Priority.HIGH); + loop.run(); + if(result == null) + throw new IOError.TIMED_OUT( + "The test timed out after %d milliseconds",timeout); + async_finish(p, result); + }; + } + + public void add_test_method(owned TestCase.TestMethod testmethod) { + this.test = (owned)testmethod; + } + + public void run(TestResult result) { + if(status == TestStatus.SKIPPED) + return; + var p = parent as TestCase; + result.add_test(this); + p.set_up(); + try { + test(); + } catch (Error e) { + result.add_failure(this, e.message); + } + p.tear_down(); + result.add_success(this); + } +} diff --git a/valadate/testassembly.vala b/valadate/testassembly.vala new file mode 100644 index 000000000..bc8e5aa9f --- /dev/null +++ b/valadate/testassembly.vala @@ -0,0 +1,93 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TestAssembly : TestModule { + + public File srcdir {get;set;} + public File builddir {get;set;} + + public TestOptions options {get;set;} + + public TestAssembly(string[] args) throws Error { + base(File.new_for_path(args[0])); + options = new TestOptions(args); + setup_dirs(); + } + + private TestAssembly.copy(TestAssembly other) throws Error { + base(other.binary); + options = other.options; + } + + private void setup_dirs() throws Error { + var buildstr = Environment.get_variable("G_TEST_BUILDDIR"); + + if(buildstr == null) { + builddir = binary.get_parent(); + if(builddir.get_basename() == ".libs") + builddir = builddir.get_parent(); + } else { + builddir = File.new_for_path(buildstr); + } + + var srcstr = Environment.get_variable("G_TEST_SRCDIR"); + + if(srcstr == null) { + // we're running outside the test harness + // check for buildir!=srcdir + // this currently on checks for autotools + if(!builddir.get_child("Makefile.in").query_exists()) { + // check for Makefile in builddir and extract VPATH + var makefile = builddir.get_child("Makefile"); + if(makefile.query_exists()) { + var reader = new DataInputStream(makefile.read()); + var line = reader.read_line(); + while(line!= null) { + if(line.has_prefix("VPATH = ")) { + srcstr = Path.build_path(Path.DIR_SEPARATOR_S, builddir.get_path(), line.split(" = ")[1]); + break; + } + line = reader.read_line(); + } + } + } + } + + if(srcstr == null) + srcdir = builddir; + else + srcdir = File.new_for_path(srcstr); + + var mesondir = srcdir.get_child(Path.get_basename(binary.get_path()) + "@exe"); + + if(mesondir.query_exists()) + srcdir = mesondir; + + Environment.set_variable("G_TEST_BUILDDIR", builddir.get_path(), true); + Environment.set_variable("G_TEST_SRCDIR", srcdir.get_path(), true); + + } + + public override Assembly clone() throws Error { + return new TestAssembly.copy(this); + } +} diff --git a/valadate/testcase.vala b/valadate/testcase.vala index 2a92377b5..fc2e52d9f 100644 --- a/valadate/testcase.vala +++ b/valadate/testcase.vala @@ -24,108 +24,106 @@ * Julien Peeters <contact@julienpeeters.fr> */ -public errordomain Valadate.TestError { - NOT_FOUND -} - -/** - * The TestMethod delegate represents a {@link Valadate.Test} method - * that can be added to a TestCase and run - */ -public delegate void Valadate.TestMethod (); - public abstract class Valadate.TestCase : Object, Test { - + /** + * The TestMethod delegate represents a {@link Valadate.Test} method + * that can be added to a TestCase and run + */ + public delegate void TestMethod () throws Error; /** * the name of the TestCase */ public string name { get; set; } - + /** + * the label of the TestCase + */ + public string label { get; set; } /** * Returns the number of {@link Valadate.Test}s that will be run by this TestCase */ public int count { get { int testcount = 0; - tests.foreach ((t) => { + _tests.foreach((t) => { + testcount += t.count; + }); + return testcount; + } + } + + public int size { + get { + int testcount = 0; + _tests.foreach((t) => { testcount += t.count; }); return testcount; } } - public string bug_base { get; set; } + public Test? parent {get;set;} - private List<Test> tests = new List<Test>(); + public TestStatus status {get;set;default=TestStatus.NOT_RUN;} + public string status_message {get;set;} + public double time {get;set;} - /** - * The public constructor takes an optional string parameter for the - * TestCase's name - */ - public TestCase (string? name = null) { - Object (name : name); - } + public string bug_base {get;set;} + + private List<Test> _tests = new List<Test>(); - construct { - if (name == null) - name = get_type ().name (); - } + private Test current_test; + private TestResult current_result; - public void add_test (string testname, owned TestMethod test) { - var adaptor = new TestAdaptor (testname, (owned) test, this); - tests.append (adaptor); + public new Test get(int index) { + return _tests.nth_data((uint)index); } - public Test get_test (int index) { - return tests.nth_data (index); + public new void set(int index, Test test) { + test.parent = this; + _tests.insert_before(_tests.nth(index), test); + var t = _tests.nth_data((uint)index++); + _tests.remove(t); } - public void bug (string reference) - requires (bug_base != null) - { - stdout.printf ("MSG Bug Reference: %s%s", bug_base, reference); - stdout.flush (); + public void add_test(Test test) { + test.parent = this; + _tests.append(test); } - public void skip (string message) { - stderr.printf ("SKIP %s", message); - stdout.flush (); + public void add_test_method(string testname, owned TestMethod test, int timeout, string? label = null) { + var adapter = new TestAdapter (testname, timeout); + adapter.add_test_method((owned)test); + adapter.label = label; + adapter.parent = this; + _tests.append(adapter); } - - public void fail (string? message = null) { - error ("FAIL %s", message ?? ""); + + public virtual void run(TestResult result) { + if(status != TestStatus.NOT_RUN) + return; + current_result = result; + _tests.foreach((t) => { + current_test = t; + t.run(result); + }); } - public virtual void run (TestResult result) { + public void bug(string reference) + requires(bug_base != null) + { + info("Bug Reference: %s%s",bug_base, reference); } -} -private class Valadate.TestAdaptor : Object, Test { + public void skip(string message) { + current_result.add_skip(current_test, message); - public string name { get; set; } - - public int count { - get { - return 1; - } } - private TestMethod test; - private unowned TestCase testcase; - - public TestAdaptor (string name, owned TestMethod test, TestCase testcase) { - this.test = (owned) test; - this.name = name; - this.testcase = testcase; + public void fail(string? message = null) { + current_result.add_failure(current_test, message); } - public Test get_test (int index) { - return this; - } + public virtual void set_up() {} - public void run (TestResult result) { - this.testcase.set_up (); - this.test (); - this.testcase.tear_down (); - } + public virtual void tear_down() {} } diff --git a/valadate/testconfig.vala b/valadate/testconfig.vala index 2a3252303..0c772a41a 100644 --- a/valadate/testconfig.vala +++ b/valadate/testconfig.vala @@ -20,130 +20,72 @@ * Chris Daley <chebizarro@gmail.com> */ -namespace Valadate { - public static string? get_current_test_path () { - return TestConfig._runtest; - } -} - public errordomain Valadate.TestConfigError { MODULE, - TESTPLAN + TESTPLAN, + METHOD, + TEST_PRINTER } public class Valadate.TestConfig : Object { - private static string _seed; - private static string testplan; - internal static string _runtest; - private static string format = "tap"; - private static bool list; - private static bool _keepgoing = true; - private static bool quiet; - private static bool timed; - private static bool verbose; - private static bool version; - private static bool vala_version; - - [CCode (array_length = false, array_null_terminated = true)] - private static string[] paths; - [CCode (array_length = false, array_null_terminated = true)] - private static string[] skip; - + public TestOptions options {get;construct set;} - public string seed { + public virtual string format { get { - return _seed; + return options.format; } } - public string runtest { + public virtual string seed { get { - return _runtest; + return options.seed; } } - public bool list_only { + public string? running_test { get { - return list; + return options.running_test; } } - public bool keep_going { + public bool in_subprocess { get { - return _keepgoing; + return options.running_test != null; } } - public TestSuite root {get;set;} - - public OptionContext opt_context; - - public const OptionEntry[] options = { - { "seed", 0, 0, OptionArg.STRING, ref _seed, "Start tests with random seed", "SEEDSTRING" }, - { "format", 'f', 0, OptionArg.STRING, ref format, "Output test results using format", "FORMAT" }, - { "list", 'l', 0, OptionArg.NONE, ref list, "List test cases available in a test executable", null }, - { "", 'k', 0, OptionArg.NONE, ref _keepgoing, "Skip failed tests and continue running", null }, - { "skip", 's', 0, OptionArg.STRING_ARRAY, ref skip, "Skip all tests matching", "TESTPATH..." }, - { "quiet", 'q', 0, OptionArg.NONE, ref quiet, "Run tests quietly", null }, - { "timed", 0, 0, OptionArg.NONE, ref timed, "Run timed tests", null }, - { "testplan", 0, 0, OptionArg.STRING, ref testplan, "Run the specified TestPlan", "FILE" }, - { "", 'r', 0, OptionArg.STRING, ref _runtest, null, null }, - { "verbose", 0, 0, OptionArg.NONE, ref verbose, "Run tests verbosely", null }, - { "version", 0, 0, OptionArg.NONE, ref version, "Display version number", null }, - { "vala-version", 0, 0, OptionArg.NONE, ref vala_version, "Display Vala version number", null }, - { "", 0, 0, OptionArg.STRING_ARRAY, ref paths, "Only start test cases matching", "TESTPATH..." }, - { null } - }; - - - public TestConfig() { - opt_context = new OptionContext ("- Valadate Testing Framework"); - opt_context.set_help_enabled (true); - opt_context.add_main_entries (options, null); + public virtual bool run_async { + get { + return options.run_async; + } } - public int parse(string[] args) { - var binary = args[0]; - GLib.Environment.set_prgname(binary); + public virtual bool list_only { + get { + return options.list; + } + } - try { - opt_context.parse (ref args); - } catch (OptionError e) { - stdout.printf ("%s\n", e.message); - stdout.printf ("Run '%s --help' to see a full list of available command line options.\n", args[0]); - return 1; + public virtual bool keep_going { + get { + return options.keepgoing; } + } - if (version) { - stdout.printf ("Valadate %s\n", "1.0"); - return 0; - } else if (vala_version) { - stdout.printf ("Vala %s\n", Config.PACKAGE_SUFFIX.substring (1)); - return 0; + public virtual int timeout { + get { + return options.timeout; } - - if(_seed == null) - _seed = "R02S%08x%08x%08x%08x".printf( - GLib.Random.next_int(), - GLib.Random.next_int(), - GLib.Random.next_int(), - GLib.Random.next_int()); - - root = new TestSuite("/"); - - try { - load(binary); - } catch (TestConfigError e) { - stdout.printf ("%s\n", e.message); - return 1; + } + + public virtual bool timed { + get { + return options.timed; } - - return -1; } - private void load(string binary) throws TestConfigError { - var testexplorer = new TestExplorer(binary, root); - testexplorer.load(); + public TestConfig(TestOptions options) { + Object(options : options); } } diff --git a/valadate/testerror.vala b/valadate/testerror.vala new file mode 100644 index 000000000..b173f71a1 --- /dev/null +++ b/valadate/testerror.vala @@ -0,0 +1,27 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright 2017 Chris Daley <bizarro@localhost.localdomain> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + * + */ + +public errordomain Valadate.TestError { + NOT_FOUND, + MODULE, + METHOD +} diff --git a/valadate/testgatherer.vala b/valadate/testgatherer.vala new file mode 100644 index 000000000..3d139346f --- /dev/null +++ b/valadate/testgatherer.vala @@ -0,0 +1,57 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TestGatherer : Vala.CodeVisitor { + + public HashTable<Type, Vala.Class> classes = + new HashTable<Type, Vala.Class>(direct_hash, direct_equal); + + private TestAssembly assembly; + + public TestGatherer(TestAssembly assembly) { + this.assembly = assembly; + } + + public override void visit_namespace(Vala.Namespace ns) { + ns.accept_children(this); + } + + public override void visit_class(Vala.Class cls) { + try { + var classtype = find_type(cls); + if (classtype.is_a(typeof(TestCase))) + classes.insert(classtype, cls); + } catch (Error e) { + warning(e.message); + } + cls.accept_children(this); + } + + private Type find_type(Vala.Class cls) throws Error { + var attr = new Vala.CCodeAttribute (cls); + unowned TestPlan.GetType node_get_type = + (TestPlan.GetType)assembly.get_method( + "%sget_type".printf(attr.lower_case_prefix)); + var ctype = node_get_type(); + return ctype; + } +} diff --git a/valadate/testmodule.vala b/valadate/testmodule.vala index eda6ac59e..c06ae6551 100644 --- a/valadate/testmodule.vala +++ b/valadate/testmodule.vala @@ -20,39 +20,36 @@ * Chris Daley <chebizarro@gmail.com> */ -public errordomain Valadate.TestModuleError { - NOT_FOUND, - LOAD, - METHOD -} - /** * Represents a loadable module containing {@link Valadate.Test}s */ -public class Valadate.TestModule : Object { - - private string lib_path; +public class Valadate.TestModule : Assembly { + private GLib.Module module; - - public TestModule (string libpath) { - lib_path = libpath; + + public TestModule(File binary) throws Error { + base(binary); } - public void load_module () throws TestModuleError { - if (!File.new_for_path (lib_path).query_exists ()) - throw new TestModuleError.NOT_FOUND ("Module: %s does not exist", lib_path); - - module = GLib.Module.open (lib_path, ModuleFlags.BIND_LOCAL); + private void load_module() throws AssemblyError { + module = GLib.Module.open (binary.get_path(), ModuleFlags.BIND_LAZY); if (module == null) - throw new TestModuleError.LOAD (GLib.Module.error ()); - module.make_resident (); + throw new AssemblyError.LOAD(GLib.Module.error()); + module.make_resident(); } - - internal void* get_method (string method_name) throws TestModuleError { + + public virtual void* get_method(string method_name) throws AssemblyError { + if(module == null) + load_module(); void* function; - if (module.symbol (method_name, out function)) + if(module.symbol (method_name, out function)) if (function != null) return function; - throw new TestModuleError.METHOD (GLib.Module.error ()); + throw new AssemblyError.METHOD(GLib.Module.error()); } + + public override Assembly clone() throws Error { + return new TestModule(binary); + } + } diff --git a/valadate/testoptions.vala b/valadate/testoptions.vala new file mode 100644 index 000000000..39e8abe03 --- /dev/null +++ b/valadate/testoptions.vala @@ -0,0 +1,135 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ +namespace Valadate { + + public class TestOptions { + + private static bool _async = true; + private static bool _tap; + private static string _format = "tap"; + private static bool _keepgoing = false; + private static bool _list; + private static bool _quiet; + private static string _runtest = null; + [CCode (array_length = false, array_null_terminated = true)] + private static string[] _skip; + private static int _timeout = 60000; + private static string _seed; + private static bool _timed = true; + private static string _testplan; + private static bool _verbose; + private static bool _version; + [CCode (array_length = false, array_null_terminated = true)] + private static string[] _paths; + + public const OptionEntry[] options = { + { "async", 'a', 0, OptionArg.NONE, ref _async, "Run tests asynchronously in a separate subprocess [Experimental]", null }, + { "format", 'f', 0, OptionArg.STRING, ref _format, "Output test results using format", "FORMAT" }, + { "", 'k', 0, OptionArg.NONE, ref _keepgoing, "Skip failed tests and continue running", null }, + { "list", 'l', 0, OptionArg.NONE, ref _list, "List test cases available in a test executable", null }, + { "quiet", 'q', 0, OptionArg.NONE, ref _quiet, "Run tests quietly", null }, + { "", 'r', 0, OptionArg.STRING, ref _runtest, null, null }, + { "skip", 's', 0, OptionArg.STRING_ARRAY, ref _skip, "Skip all tests matching", "TESTPATH..." }, + { "timeout", 't', 0, OptionArg.INT, ref _timeout, "Default timeout for tests", "MILLISECONDS" }, + { "seed", 0, 0, OptionArg.STRING, ref _seed, "Start tests with random seed", "SEEDSTRING" }, + { "timed", 0, 0, OptionArg.NONE, ref _timed, "Run timed tests", null }, { "tap", 0, 0, OptionArg.NONE, ref _tap, "Output test results using TAP format" }, + { "tap", 0, 0, OptionArg.NONE, ref _tap, "Output test results using TAP format" }, + { "testplan", 0, 0, OptionArg.STRING, ref _testplan, "Run the specified TestPlan", "FILE" }, + { "verbose", 0, 0, OptionArg.NONE, ref _verbose, "Run tests verbosely", null }, + { "version", 0, 0, OptionArg.NONE, ref _version, "Display version number", null }, + { "", 0, 0, OptionArg.STRING_ARRAY, ref _paths, "Only start test cases matching", "TESTPATH..." }, + { null } + }; + + public OptionContext opt_context; + + public static string? get_current_test_path() { + return _runtest; + } + + public string format { + get { + return _format; + } + } + + public string seed { + get { + return _seed; + } + } + + public string? running_test { + get { + return _runtest; + } + } + + public bool run_async { + get { + return _async; + } + } + + public bool list { + get { + return _list; + } + } + + public bool keepgoing { + get { + return _keepgoing; + } + } + + public int timeout { + get { + return _timeout; + } + } + + public bool timed { + get { + return _timed; + } + } + + public TestOptions(string[] args) throws OptionError { + _runtest = null; + + opt_context = new OptionContext ("- Valadate Testing Framework"); + opt_context.set_help_enabled (true); + opt_context.add_main_entries (options, null); + opt_context.parse (ref args); + + if(_seed == null) + _seed = "R02S%08x%08x%08x%08x".printf( + GLib.Random.next_int(), + GLib.Random.next_int(), + GLib.Random.next_int(), + GLib.Random.next_int()); + + } + + } +} diff --git a/valadate/testplan.vala b/valadate/testplan.vala new file mode 100644 index 000000000..fc8928e51 --- /dev/null +++ b/valadate/testplan.vala @@ -0,0 +1,283 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2016 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TestPlan : Vala.CodeVisitor { + + [CCode (has_target = false)] + public delegate void TestMethod(TestCase self) throws Error; + + public delegate void AsyncTestMethod(TestCase self, AsyncReadyCallback cb); + public delegate void AsyncTestMethodResult(TestCase self, AsyncResult res) throws Error; + + public delegate Type GetType(); + + public File plan {get;set;} + + public TestAssembly assembly {get;set;} + + public TestOptions options {get;set;} + + public TestConfig config {get;set;} + + public TestResult result {get;set;} + + public TestRunner runner {get;set;} + + public TestSuite root {get;protected set;} + + private Vala.CodeContext context; + private TestGatherer gatherer; + private delegate TestCase Constructor(); + private TestSuite testsuite; + private TestCase testcase; + + public TestPlan(TestAssembly assembly) throws Error { + + this.assembly = assembly; + options = assembly.options; + + var plan_name = Path.get_basename(assembly.binary.get_path()); + if(plan_name.has_prefix("lt-")) + plan_name = plan_name.substring(3); + + plan = assembly.srcdir.get_child(plan_name + ".vapi"); + if(!plan.query_exists()) { + plan = assembly.builddir.get_child(plan_name + ".vapi"); + if(!plan.query_exists()) { + throw new TestConfigError.TESTPLAN( + "Test Plan %s Not Found in %s or %s", plan_name, assembly.srcdir.get_path(), assembly.builddir.get_path()); + } + } + config = new TestConfig(options); + runner = new TestRunner(); + result = new TestResult(config); + testsuite = root = new TestSuite("/"); + setup_context(); + load_test_plan(); + } + + public int run() throws Error { + return runner.run_all(this); + } + + private void setup_context() { + context = new Vala.CodeContext (); + Vala.CodeContext.push (context); + context.report.enable_warnings = false; + context.report.set_verbose_errors (false); + context.verbose_mode = false; + } + + public void load_test_plan() throws Error { + context.add_source_file (new Vala.SourceFile ( + context, Vala.SourceFileType.PACKAGE, plan.get_path())); + var parser = new Vala.Parser (); + parser.parse (context); + gatherer = new TestGatherer(assembly); + context.accept(gatherer); + context.accept(this); + } + + public override void visit_namespace(Vala.Namespace ns) { + if (ns.name != null) { + var currpath = "/" + ns.get_full_name().replace(".","/"); + if(config.in_subprocess) + if(!options.running_test.has_prefix(currpath)) + return; + + if(currpath.last_index_of("/") == 0) + testsuite = root; + + var ts = new TestSuite(ns.name); + testsuite.add_test(ts); + testsuite = ts; + } + ns.accept_children(this); + } + + public override void visit_class(Vala.Class cls) { + + try { + if (is_subtype_of(cls, typeof(TestCase)) && !cls.is_abstract) { + unowned Constructor ctor = get_constructor(cls); + testcase = ctor(); + testcase.name = cls.name; + testcase.label = "/%s".printf(cls.get_full_name().replace(".","/")); + testsuite.add_test(testcase); + visit_testcase(cls); + + } else if (is_subtype_of(cls,typeof(TestSuite))) { + visit_testsuite(cls); + } + } catch (Error e) { + error(e.message); + } + cls.accept_children(this); + } + + private bool is_subtype_of(Vala.Class cls, Type type) { + var t = Type.from_name(cls.get_full_name().replace(".","")); + if(t.is_a(type)) + return true; + return false; + } + + private unowned Constructor get_constructor(Vala.Class cls) throws Error { + var attr = new Vala.CCodeAttribute (cls.default_construction_method); + return (Constructor)assembly.get_method(attr.name); + } + + public void visit_testcase(Vala.Class cls) { + + var t = Type.from_name(cls.get_full_name().replace(".","")); + var p = t.parent(); + if(p != typeof(TestCase)) { + var basecls = gatherer.classes.get(p); + if(basecls != null) + visit_testcase(basecls); + } + + foreach(var method in cls.get_methods()) { + + if(config.in_subprocess) + if (options.running_test != "%s/%s".printf( + testcase.label, method.name)) + continue; + + if(!is_test(method)) + continue; + + var added = false; + foreach(var test in testcase) + if(test.name == method.name) + added=true; + if(added) + continue; + + var adapter = new TestAdapter(method.name, config.timeout); + annotate_label(adapter); + annotate(adapter, method); + + if(config.in_subprocess && adapter.status != TestStatus.SKIPPED) { + var attr = new Vala.CCodeAttribute (method); + + if(method.coroutine) { + try { + unowned TestPlan.AsyncTestMethod beginmethod = + (TestPlan.AsyncTestMethod)assembly.get_method(attr.name); + unowned TestPlan.AsyncTestMethodResult testmethod = + (TestPlan.AsyncTestMethodResult)assembly.get_method(attr.finish_real_name); + adapter.add_async_test(beginmethod, testmethod); + } catch (Error e) { + var message = e.message; + adapter.add_test_method(()=> {debug(message);}); + } + } else { + try { + TestPlan.TestMethod testmethod = + (TestPlan.TestMethod)assembly.get_method(attr.name); + adapter.add_test((owned)testmethod); + } catch (Error e) { + var message = e.message; + adapter.add_test_method(()=> {debug(message);}); + } + } + } else { + adapter.add_test_method(()=> {assert_not_reached();}); + } + + adapter.label = "%s/%s".printf( + testcase.label, + adapter.label); + + testcase.add_test(adapter); + } + + } + + private void annotate_label(Test test) { + if(test.name.has_prefix("test_")) { + test.label = test.name.substring(5); + } else if(test.name.has_prefix("_test_")) { + test.label = test.name.substring(6); + test.status = TestStatus.SKIPPED; + } else if(test.name.has_prefix("todo_test_")) { + test.label = test.name.substring(10); + test.status = TestStatus.TODO; + } else { + test.label = test.name; + } + test.label = test.label.replace("_", " "); + } + + private void annotate(TestAdapter adapter, Vala.Method method) { + + foreach(var attr in method.attributes) { + if(attr.name == "Test") { + if(attr.has_argument("name")) + adapter.label = attr.get_string("name"); + if(attr.has_argument("skip")) { + adapter.status = TestStatus.SKIPPED; + adapter.status_message = attr.get_string("skip"); + } else if(attr.has_argument("todo")) { + adapter.status = TestStatus.SKIPPED; + adapter.status_message = attr.get_string("todo"); + } else if(attr.has_argument("timeout")) { + adapter.timeout = int.parse(attr.get_string("timeout")); + } + } + } + } + + private bool is_test(Vala.Method method) { + bool istest = false; + + if(method.is_virtual) + foreach(var test in testcase) + if(test.name == method.name) + return false; + + if (method.name.has_prefix("test_") || + method.name.has_prefix("_test_") || + method.name.has_prefix("todo_test_")) + istest = true; + + foreach(var attr in method.attributes) + if(attr.name == "Test") + istest = true; + + if(method.has_result) + istest = false; + + if(method.get_parameters().size > 0) + istest = false; + + return istest; + } + + public TestSuite visit_testsuite(Vala.Class testclass) throws Error { + unowned Constructor meth = get_constructor(testclass); + var testcase_test = meth() as TestSuite; + testcase_test.name = testclass.name; + return testcase_test; + } +} diff --git a/valadate/testreport.vala b/valadate/testreport.vala new file mode 100644 index 000000000..8e4dc734b --- /dev/null +++ b/valadate/testreport.vala @@ -0,0 +1,278 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ + +public class Valadate.TestReport { + + private const string XML_DECL ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; + private const string TESTSUITE_XML = + """<testsuite disabled="" errors="" failures="" hostname="" id="" """ + + """name="" package="" skipped="" tests="" time="" timestamp="" >"""+ + """<properties/></testsuite>"""; + private const string TESTCASE_XML = + """<testcase assertions="" classname="" name="" status="" time="" />"""; + private const string MESSAGE_XML = "<%s message=\"%s\" type=\"%s\">%s</%s>"; + private const string TESTCASE_START = + "<testcase assertions=\"\" classname=\"%s\" name=\"%s\" status=\"\" time=\"\">"; + private const string VDX_NS = "xmlns:vdx=\"https://www.valadate.org/vdx\""; + private const string TESTCASE_TAG = "testcase"; + private const string ROOT_TAG = "root"; + private const string SKIP_TAG = "skipped"; + private const string ERROR_TAG = "error"; + private const string FAILURE_TAG = "failure"; + private const string INFO_TAG = "info"; + private const string TIMER_TAG = "timer"; + private const string SYSTEM_OUT_TAG = "system-out"; + private const string SYSTEM_ERR_TAG = "system-err"; + + public Test test {get;set;} + public bool subprocess {get;set;} + + public XmlFile xml {get;set;} + + private static int64 start_time; + private static int64 end_time; + + private static Regex regex_err; + private const string regex_err_string = + """(\*{2}\n([A-Z]*):([\S]*) ([\S ]*)\n)"""; + + public TestReport(Test test, bool subprocess) throws Error { + this.test = test; + this.subprocess = subprocess; + + if(test.status == TestStatus.NOT_RUN) + test.status = TestStatus.RUNNING; + + if(subprocess) { + Log.set_default_handler (log_func); + GLib.set_printerr_handler (printerr_func); + regex_err = new Regex(regex_err_string); + } + + if(test is TestSuite || test is TestCase) + new_testsuite(); + else if (test is TestAdapter) + new_testcase(); + } + + private void new_testsuite() throws Error { + if(subprocess) + return; + + var decl = "%s<%s>%s</%s>".printf(XML_DECL, ROOT_TAG, TESTSUITE_XML, ROOT_TAG); + var doc = Xml.Parser.read_memory(decl, decl.length); + var root = doc->get_root_element()->children; + root->set_prop("tests", test.count.to_string()); + root->set_prop("name",test.label); + xml = new XmlFile.from_doc(doc); + + if(test.parent != null && test.parent.name != "/") + return; + + var props = root->children; + + foreach(var key in Environment.list_variables()) { + Xml.Node* node = new Xml.Node(null, "property"); + node->set_prop("name", key); + node->set_prop("value", Markup.escape_text(Environment.get_variable(key))); + props->add_child(node); + } + } + + private void new_testcase() throws Error { + if(subprocess) { + stderr.printf("%s<%s>",XML_DECL,ROOT_TAG); + stderr.printf(TESTCASE_START,test.parent.get_type().name(), test.label); + start_time = get_monotonic_time(); + } else { + var decl = "%s<%s>%s</%s>".printf(XML_DECL, ROOT_TAG, TESTCASE_XML, ROOT_TAG); + var doc = Xml.Parser.read_memory(decl, decl.length); + var root = doc->get_root_element()->children; + root->set_prop("classname",((TestAdapter)test).parent.name); + root->set_prop("status",test.status.to_string().substring(21)); + root->set_prop("name",test.label); + xml = new XmlFile.from_doc(doc); + } + } + + public void add_error(string message) { + if (test.status != TestStatus.SKIPPED && + test.status != TestStatus.TODO) + test.status = TestStatus.ERROR; + + add_message(ERROR_TAG, message); + + if(subprocess) { + emit_timer(); + stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG); + stderr.putc(0); + } + update_status(); + } + + public void add_failure(string message) { + if (test.status != TestStatus.SKIPPED && + test.status != TestStatus.TODO) + test.status = TestStatus.FAILED; + + add_message(FAILURE_TAG, message); + + if(subprocess) { + emit_timer(); + stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG); + stderr.putc(0); + } + update_status(); + } + + public void add_skip(string message) { + test.status = TestStatus.SKIPPED; + add_message(SKIP_TAG, message); + update_status(); + } + + public void add_success() { + if (test.status != TestStatus.SKIPPED && + test.status != TestStatus.TODO) + test.status = TestStatus.PASSED; + if(subprocess) { + emit_timer(); + stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG); + stderr.putc(0); + } + update_status(); + } + + private void add_message(string tag, string message) { + var escaped = Markup.escape_text(message); + if(subprocess) { + stderr.printf(MESSAGE_XML, tag, escaped, tag.up(), message, tag); + } else { + Xml.Node* child = new Xml.Node(null, tag); + child->set_content(escaped); + + string[] tags = {ERROR_TAG, FAILURE_TAG, INFO_TAG}; + + if(tag in tags) { + child->new_prop("message", escaped); + child->new_prop("type", tag.up()); + } + + Xml.Node* root = xml.eval("//testcase | //testsuite")[0]; + root->add_child(child); + } + } + /** + * Adds arbitrary text to the TestReport. In the xml output this + * text will be encapsulated in <system-out/> or <system-err/> tag + * + * @param text The text to be added to the {@link TestReport}. + * the text will be escaped before being added. + * @param tag The tag to use for adding the text + */ + public void add_text(string text, string tag) { + var markup = Markup.escape_text(text); + Xml.Node* child = new Xml.Node(null, tag); + child->set_content(markup); + + string[] tags = {ERROR_TAG, FAILURE_TAG, INFO_TAG}; + + if(tag in tags) { + child->new_prop("message", markup); + child->new_prop("type", tag.up()); + } + + Xml.Node* root = xml.eval("//testcase | //testsuite")[0]; + root->add_child(child); + } + + public void update_status() { + if(test is TestAdapter && !subprocess) { + Xml.Node* root = xml.eval("//testcase")[0]; + root->set_prop("status",test.status.to_string().substring(21)); + root->set_prop("time",test.time.to_string()); + } + } + + private static void emit_timer() { + end_time = get_monotonic_time(); + var ms = "%f".printf(((double)(end_time-start_time))/1000); + stderr.printf(MESSAGE_XML, TIMER_TAG, ms, TIMER_TAG, ms, TIMER_TAG); + } + + private static void printerr_func (string? text) { + if(text == null) + return; + MatchInfo info; + if(regex_err.match(text, 0, out info)) { + var escaped = Markup.escape_text(info.fetch(4)); + stderr.printf(MESSAGE_XML, ERROR_TAG, escaped, ERROR_TAG, text, ERROR_TAG); + emit_timer(); + stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG); + stderr.putc(0); + } + } + + private void log_func ( + string? log_domain, + LogLevelFlags log_levels, + string? message) { + + if (((log_levels & LogLevelFlags.LEVEL_INFO) != 0) || + ((log_levels & LogLevelFlags.LEVEL_MESSAGE) != 0) || + ((log_levels & LogLevelFlags.LEVEL_DEBUG) != 0)) { + add_message(INFO_TAG, message); + } else { + add_error(message); + } + } + + public void process_buffer(string buffer) throws Error { + + xml = new XmlFile.from_string(buffer); + + var bits = xml.eval("//testcase/text()"); + + if(bits.size != 0) { + Xml.Node* textnode = bits[0]; + add_message(SYSTEM_ERR_TAG, textnode->get_content()); + textnode->unlink(); + } + + var errs = xml.eval("//failure | //error"); + if (errs.size > 0 && + test.status != TestStatus.SKIPPED && + test.status != TestStatus.TODO) + test.status = TestStatus.FAILED; + + bits = xml.eval("//timer"); + Xml.Node* timer = bits[0]; + test.time = double.parse(timer->get_content()); + timer->unlink(); + + update_status(); + } + + public void add_stdout(string text) { + add_message(SYSTEM_OUT_TAG, text); + } +} diff --git a/valadate/testreportprinter.vala b/valadate/testreportprinter.vala new file mode 100644 index 000000000..5ddaaf099 --- /dev/null +++ b/valadate/testreportprinter.vala @@ -0,0 +1,46 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + * + */ + +public abstract class Valadate.TestReportPrinter { + + public static TestReportPrinter @new(TestConfig config) throws Error { + switch(config.format) { + case "tap" : + return new TapTestReportPrinter(config); + case "xml" : + return new XmlTestReportPrinter(config); + case "gnu" : + return new GnuTestReportPrinter(config); + default: + throw new TestConfigError.TEST_PRINTER("TestReportPrinter %s does not exist", config.format); + } + } + + public TestConfig config {get; set;} + + public TestReportPrinter(TestConfig config) throws Error { + this.config = config; + } + + public abstract void print(TestReport report); + +} diff --git a/valadate/testresult.vala b/valadate/testresult.vala index e95b70300..d04660d6b 100644 --- a/valadate/testresult.vala +++ b/valadate/testresult.vala @@ -20,165 +20,117 @@ * Chris Daley <chebizarro@gmail.com> */ -public class Valadate.TestResult : Object { - - private enum TestStatus { - NOT_RUN, - RUNNING, - PASSED, - SKIPPED, - ERROR, - FAILED - } - - private class TestReport { - - public signal void report (TestStatus status); - - public Test test { get; set; } +public class Valadate.TestResult { - public TestStatus status { get; set; } + public TestConfig config {get;set;} + public TestReportPrinter printer {get;set;} - public int index { get; set; } + private Queue<TestReport> reports = new Queue<TestReport>(); + private HashTable<Test, TestReport> tests = new HashTable<Test, TestReport>(direct_hash, direct_equal); - public string message { get; set; } + public TestResult(TestConfig config) throws Error { + this.config = config; + if(!config.in_subprocess) + printer = TestReportPrinter.new(config); + } - public TestReport (Test test, TestStatus status, int index, string? message = null) { - this.test = test; - this.status = status; - this.index = index; - this.message = message; + public bool report() { + if (reports.is_empty()) + return false; + + var rpt = reports.peek_head(); + + if (rpt.test.status == TestStatus.PASSED || + rpt.test.status == TestStatus.SKIPPED || + rpt.test.status == TestStatus.TODO || + rpt.test.status == TestStatus.FAILED || + rpt.test.status == TestStatus.ERROR) { + + printer.print(rpt); + reports.pop_head(); + report(); } + return true; } - private Queue<TestReport> reports = new Queue<TestReport> (); - private HashTable<Test, TestReport> tests = new HashTable<Test, TestReport> (direct_hash, direct_equal); - - private TestConfig config; - private MainLoop loop; - - public TestResult (TestConfig config) { - this.config = config; - } - - public void report () { + private void update_status(Test test) { + var rept = tests.get(test); + if(rept == null) + return; - if (reports.is_empty ()) { - loop.quit (); + var parent = test.parent; + if(parent == null) return; + + TestStatus status = TestStatus.PASSED; + foreach(var t in parent) { + if(t.status == TestStatus.RUNNING) + return; + else if(t.status == TestStatus.ERROR) + status = TestStatus.ERROR; + else if(t.status == TestStatus.FAILED) + status = TestStatus.FAILED; } - - var rpt = reports.peek_head (); - - if (rpt.status == TestStatus.PASSED || - rpt.status == TestStatus.SKIPPED || - rpt.status == TestStatus.FAILED || - rpt.status == TestStatus.ERROR) { - if (rpt.message != null) - stdout.puts (rpt.message); - stdout.flush (); - rpt.report (rpt.status); - reports.pop_head (); - report (); + parent.status = status; + update_status(parent); + } + + public void add_test(Test test) { + try { + reports.push_tail(new TestReport(test, config.in_subprocess)); + tests.insert(test, reports.peek_tail()); + } catch (Error e) { + error(e.message); } } - - public void add_error (Test test, string error) { - update_test (test, TestStatus.ERROR, "# %s\nnot ok %s %s\n".printf (error, "%d", test.name)); - } - public void add_failure (Test test, string failure) { - update_test (test, TestStatus.FAILED, "# %s\nnot ok %s %s\n".printf (failure, "%d", test.name)); + public void add_error(Test test, string error) { + tests.get(test).add_error(error); + update_status(test); } - public void add_success (Test test, string message) { - update_test (test, TestStatus.PASSED, "# %s\nok %s %s\n".printf (message, "%d", test.name)); + public void add_failure(Test test, string failure) { + tests.get(test).add_failure(failure); + update_status(test); } - public void add_skip (Test test, string reason, string message) { - update_test (test, TestStatus.SKIPPED, "# %s\nok %s %s # %s\n".printf (message, "%d", test.name, reason)); + public void add_success(Test test) { + tests.get(test).add_success(); + update_status(test); } - private void update_test (Test test, TestStatus status, string message) { - var rept = tests.get (test); - rept.status = status; - rept.message = message.printf (rept.index); + public void add_skip(Test test, string message) { + tests.get(test).add_skip(message); + update_status(test); } - /** - * Runs a the {@link Valadate.Test}s using the supplied - * {@link Valadate.TestRunner}. - * - * @param runner - */ - public void run (TestRunner runner) { - - var testcount = count_tests (config.root); - - if (!config.list_only && config.runtest == null) { - stdout.printf ("# random seed: %s\n", config.seed); - stdout.printf ("1..%d\n", testcount); - } + public void process_buffers(Test test, Assembly assembly) throws Error { + + var rept = tests.get(test); + if(rept == null) + return; - run_test (runner, config.root, ""); - - if (config.runtest == null) { - loop = new MainLoop (); - var time = new TimeoutSource (15); - time.set_callback (() => { - report (); - return true; - }); - time.attach (loop.get_context ()); - loop.run (); - } - } + var bis = new BufferedInputStream (assembly.stderr); + bis.fill(-1); + var xml = (string)bis.peek_buffer(); + if(xml.length < 8) + return; - private int count_tests (Test test) { - var testcount = 0; + rept.process_buffer(xml); - if (test is TestSuite) - foreach (var subtest in test) - testcount += count_tests (subtest); - else - testcount += test.count; + update_status(test); - return testcount; - } + uint8 outbuffer[4096] = {}; + assembly.stdout.read_all(outbuffer, null); + xml = ((string)outbuffer).strip(); + int i; + for(i=xml.length-1;i==0;i--) + if(xml.get_char(i).isgraph()) + break; - private int testno = 0; - - private void run_test (TestRunner runner, Test test, string path) { - foreach (var subtest in test) { - string testpath = "%s/%s".printf (path, subtest.name); - if (subtest is TestCase) { - if (config.runtest == null && !config.list_only) { - reports.push_tail (new TestReport (subtest, TestStatus.PASSED, -1, "# Start of %s tests\n".printf (testpath))); - run_test (runner, subtest, testpath); - reports.push_tail (new TestReport (subtest, TestStatus.PASSED, -1, "# End of %s tests\n".printf (testpath))); - } else { - run_test (runner, subtest, testpath); - } - } else if (subtest is TestSuite) { - run_test (runner, subtest, testpath); - if (config.runtest == null) { - var rpt = new TestReport (subtest, TestStatus.PASSED, -1); - rpt.report.connect ((s)=> ((TestSuite)subtest).tear_down ()); - reports.push_tail (rpt); - } - } else if (config.list_only) { - stdout.printf ("%s\n", testpath); - } else if (config.runtest != null) { - if (config.runtest == testpath) - runner.run_test (subtest, this); - } else { - testno++; - subtest.name = testpath; - var rept = new TestReport (subtest, TestStatus.RUNNING, testno); - reports.push_tail (rept); - tests.insert (subtest, rept); - runner.run.begin (subtest, this); - } - } + xml = xml.substring(0, i+1); + if(xml.length < 1 || xml == "\n") + return; + rept.add_stdout(xml); } } diff --git a/valadate/testrunner.vala b/valadate/testrunner.vala index 205966b82..b02953932 100644 --- a/valadate/testrunner.vala +++ b/valadate/testrunner.vala @@ -21,123 +21,145 @@ */ public class Valadate.TestRunner : Object { - - private uint _n_ongoing_tests = 0; - private Queue<DelegateWrapper> _pending_tests = new Queue<DelegateWrapper> (); - - /* Change this to change the cap on the number of concurrent operations. */ - private static uint _max_n_ongoing_tests = GLib.get_num_processors (); - private class DelegateWrapper { public SourceFunc cb; } - private SubprocessLauncher launcher = - new SubprocessLauncher (GLib.SubprocessFlags.STDOUT_PIPE | GLib.SubprocessFlags.STDERR_MERGE); - - private string binary; + private uint _n_ongoing_tests = 0; + private Queue<DelegateWrapper> _pending_tests = new Queue<DelegateWrapper> (); + private static uint _max_n_ongoing_tests = GLib.get_num_processors(); + private MainLoop loop; + private TestPlan plan; - public TestRunner (string binary) { - this.binary = binary; - this.launcher.setenv("G_MESSAGES_DEBUG","all", true); - this.launcher.setenv("G_DEBUG","fatal-criticals fatal-warnings gc-friendly", true); - this.launcher.setenv("G_SLICE","always-malloc debug-blocks", true); - GLib.set_printerr_handler (printerr_func_stack_trace); - Log.set_default_handler (log_func_stack_trace); + public void run(Test test, TestResult result) { + result.add_test(test); + test.run(result); + result.report(); } - private static void printerr_func_stack_trace (string? text) { - if (text == null || str_equal (text, "")) - return; - stderr.printf (text); - - /* Print a stack trace since we've hit some major issue */ - GLib.on_error_stack_trace ("libtool --mode=execute gdb"); + public int run_all(TestPlan plan) throws Error { + this.plan = plan; + + if (plan.config.list_only) { + list_tests(plan.root, ""); + return 0; + } else if (plan.root.count == 0) { + return 0; + } else if (!plan.config.in_subprocess) { + loop = new MainLoop(); + Timeout.add( + 10, + () => { + bool res = plan.result.report(); + if(!res) + loop.quit(); + return res; + + }, + Priority.HIGH_IDLE); + run_test_internal(plan.root, plan.result, ""); + loop.run(); + } else { + run_test_internal(plan.root, plan.result, ""); + } + return 0; } - - private void log_func_stack_trace ( - string? log_domain, - LogLevelFlags log_levels, - string message) { - Log.default_handler (log_domain, log_levels, message); - - /* Print a stack trace for any message at the warning level or above */ - if ((log_levels & ( - LogLevelFlags.LEVEL_WARNING | - LogLevelFlags.LEVEL_ERROR | - LogLevelFlags.LEVEL_CRITICAL)) != 0) { - GLib.on_error_stack_trace ("libtool --mode=execute gdb"); + + private void list_tests(Test test, string path) { + foreach(var subtest in test) { + string testpath = "%s/%s".printf(path, subtest.name); + if(subtest is TestAdapter) + stdout.printf("%s\n", testpath); + else + list_tests(subtest, testpath); } } - public void run_test (Test test, TestResult result) { - test.run (result); + public void run_test(Test test, TestResult result) { + test.run(result); } - public async void run (Test test, TestResult result) { - - string command = "%s -r %s".printf(binary, test.name); - string[] args; - string buffer = null; + private void run_test_internal(Test test, TestResult result, string path) throws Error { + + foreach(var subtest in test) { + + var testpath = "%s/%s".printf(path, subtest.name); + + if(subtest is TestCase) { + if(!plan.config.in_subprocess) + result.add_test(subtest); + run_test_internal(subtest, result, testpath); + } else if (subtest is TestSuite) { + result.add_test(subtest); + run_test_internal(subtest, result, testpath); + } else if (plan.config.in_subprocess) { + if(plan.config.running_test == testpath) + test.run(result); + } else if (subtest is TestAdapter) { + subtest.name = testpath; + result.add_test(subtest); + run_async.begin(subtest, result); + } + } + } + private async void run_async(Test test, TestResult result) throws Error + requires(plan.config.in_subprocess != true) { + var timeout = plan.config.timeout; + var testprog = plan.assembly.clone(); if (_n_ongoing_tests > _max_n_ongoing_tests) { var wrapper = new DelegateWrapper(); - wrapper.cb = run.callback; + wrapper.cb = run_async.callback; _pending_tests.push_tail((owned)wrapper); yield; } try { _n_ongoing_tests++; + var cancellable = new Cancellable (); + var tcase = test as TestAdapter; + if(timeout != tcase.timeout) + timeout = tcase.timeout; + + var time = new TimeoutSource (timeout); - Shell.parse_argv(command, out args); - var process = launcher.spawnv(args); - yield process.communicate_utf8_async(null, null, out buffer, null); - - if(process.wait_check()) - process_buffer(test, result, buffer); - + cancellable.cancelled.connect (() => { + testprog.quit(); + }); + + time.set_callback (() => { + if (tcase.status == TestStatus.RUNNING) + cancellable.cancel(); + return false; + }); + time.attach (loop.get_context()); + + testprog.run("-r %s".printf(test.name), cancellable); + result.add_success(test); + result.process_buffers(test, testprog); + } catch (IOError e) { + result.add_error(test, "The test timed out after %d milliseconds".printf(timeout)); + result.process_buffers(test, testprog); } catch (Error e) { - process_buffer(test, result, buffer, true); + result.add_error(test, e.message); + result.process_buffers(test, testprog); } finally { + result.report(); _n_ongoing_tests--; var wrapper = _pending_tests.pop_head (); if(wrapper != null) wrapper.cb(); } } - - public void process_buffer (Test test, TestResult result, string buffer, bool failed = false) { - string skip = null; - string[] message = {}; - - foreach(string line in buffer.split("\n")) - if (line.has_prefix("SKIP ")) - skip = line; - else - message += line; - - if (skip != null) - result.add_skip(test, skip, string.joinv("\n",message)); - else - if(failed) - result.add_failure(test, string.joinv("\n",message)); - else - result.add_success(test, string.joinv("\n",message)); - } - + public static int main (string[] args) { - var bin = args[0]; - var config = new TestConfig(); - int result = config.parse(args); - - if(result >= 0) - return result; - - var runner = new TestRunner(bin); - var testresult = new TestResult(config); - testresult.run(runner); - - return 0; + try { + var assembly = new TestAssembly(args); + var testplan = new TestPlan(assembly); + return testplan.run(); + } catch (Error e) { + error(e.message); + } } + } diff --git a/valadate/teststatus.vala b/valadate/teststatus.vala new file mode 100644 index 000000000..db9e4bf7d --- /dev/null +++ b/valadate/teststatus.vala @@ -0,0 +1,30 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Authors: + * Chris Daley <chebizarro@gmail.com> + */ +public enum Valadate.TestStatus { + NOT_RUN, + RUNNING, + PASSED, + SKIPPED, + TODO, + ERROR, + FAILED +} diff --git a/valadate/testsuite.vala b/valadate/testsuite.vala index 37762ade8..6b0239cb4 100644 --- a/valadate/testsuite.vala +++ b/valadate/testsuite.vala @@ -22,58 +22,90 @@ public class Valadate.TestSuite : Object, Test { + private List<Test> _tests = new List<Test>(); /** * the name of the TestSuite */ public string name { get; set; } - + /** + * the label of the TestSuite + */ + public string label { get; set; } + /** + * Iterator (not the actual number of Tests that will be run) + */ + public int size { + get { + return (int)_tests.length(); + } + } /** * Returns the number of {@link Valadate.Test}s that will be run by * this TestSuite */ public int count { get { - return (int) _tests.length (); + int testcount = 0; + _tests.foreach((t) => { + testcount += t.count; + }); + return testcount; } } - + public Test? parent {get;set;} /** * Returns a {@link GLib.List} of {@link Valadate.Test}s that will be * run by this TestSuite */ - public GLib.List<Test> tests { + public List<Test> tests { get { return _tests; } } - - private GLib.List<Test> _tests = new GLib.List<Test> (); - + public TestStatus status {get;set;default=TestStatus.NOT_RUN;} + public double time {get;set;} + public int skipped {get;set;} + public int errors {get;set;} + public int failures {get;set;} /** * The public constructor takes an optional string parameter for the * TestSuite's name */ - public TestSuite (string? name = null) { - Object (name : name); + public TestSuite(string? name = null) { + this.name = name ?? this.get_type().name(); + this.label = name; } - - construct { - if (name == null) - name = get_type ().name (); - } - /** * Adds a test to the suite. */ - public void add_test (Test test) { - _tests.append (test); + public void add_test(Test test) { + test.parent = this; + _tests.append(test); } - + /** + * Runs all of the tests in the Suite + */ public void run (TestResult result) { - _tests.foreach ((t) => { t.run (result); }); + + if(status != TestStatus.NOT_RUN) + return; + + _tests.foreach((t) => { + t.run(result); + }); } - public Test get_test (int index) { + public new Test get(int index) { return _tests.nth_data((uint)index); } + + public new void set(int index, Test test) { + test.parent = this; + _tests.insert_before(_tests.nth(index), test); + var t = _tests.nth_data((uint)index++); + _tests.remove(t); + } + + public virtual void set_up() {} + public virtual void tear_down() {} } diff --git a/valadate/xmlfile.vala b/valadate/xmlfile.vala new file mode 100644 index 000000000..ca76e178b --- /dev/null +++ b/valadate/xmlfile.vala @@ -0,0 +1,108 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright 2016 Chris Daley <chebizarro@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + */ + +public errordomain Valadate.XmlFileError { + ERROR +} + +public class Valadate.XmlSearchResults { + + private Xml.XPath.Object* result; + + public int size { + get { + if(result == null || result->type != Xml.XPath.ObjectType.NODESET || result->nodesetval == null) + return 0; + return result->nodesetval->length(); + } + } + + public void* get(int i) + requires(size > 0) + requires(i < size) + requires(i >= 0) + { + return result->nodesetval->item (i); + } + + + internal XmlSearchResults(Xml.XPath.Object* result) { + this.result = result; + } + + ~XmlSearchResults() { + if(result != null) delete result; + } + +} + +public class Valadate.XmlFile { + + private Xml.Doc* document; + private Xml.XPath.Context context; + private bool owns_doc = false; + + public XmlFile(File path) throws Error { + this.from_doc(Xml.Parser.parse_file(path.get_path())); + owns_doc = true; + } + + internal XmlFile.from_doc(Xml.Doc* xmldoc) throws Error { + document = xmldoc; + owns_doc = true; + + if (document == null) + throw new XmlFileError.ERROR( + "There was an error parsing the Xml.Doc"); + + set_context(); + } + + public XmlFile.from_string(string xml) throws Error { + document = Xml.Parser.read_memory(xml, xml.length, null, null, + Xml.ParserOption.RECOVER | Xml.ParserOption.NOERROR | + Xml.ParserOption.NOWARNING | Xml.ParserOption.NOBLANKS); + owns_doc = true; + + if (document == null) + throw new XmlFileError.ERROR( + "There was an error parsing the string %s", xml); + set_context(); + } + + private void set_context() { + context = new Xml.XPath.Context (document); + } + + ~XmlFile() { + if (owns_doc) + delete document; + } + + public void register_ns(string prefix, string ns) { + context.register_ns(prefix, ns); + } + + public XmlSearchResults eval(string expression) { + return new XmlSearchResults(context.eval_expression (expression)); + } + +} diff --git a/valadate/xmltestreportprinter.vala b/valadate/xmltestreportprinter.vala new file mode 100644 index 000000000..3ae7c3afb --- /dev/null +++ b/valadate/xmltestreportprinter.vala @@ -0,0 +1,69 @@ +/* + * Valadate - Unit testing library for GObject-based libraries. + * Copyright (C) 2017 Chris Daley <chebizarro@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + * + */ + +public class Valadate.XmlTestReportPrinter : TestReportPrinter { + + private const string XML_DECL ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; + private const string TESTSUITES_XML = + """<testsuites disabled="" errors="" failures="" name="" """ + + """tests="" time=""></testsuites>"""; + + public XmlFile xml {get;set;} + + private Xml.Node* testsuite; + private Xml.Node* oldtestsuite; + private int testcount = -1; + private int casecount = -1; + + public XmlTestReportPrinter(TestConfig config) throws Error { + base(config); + this.config = config; + xml = new XmlFile.from_string(XML_DECL + TESTSUITES_XML); + } + + public override void print(TestReport report) { + Xml.Node* root = xml.eval("//testsuites")[0]; + Xml.Node* node = report.xml.eval("//testsuite | //testcase")[0]; + + if(report.test is TestSuite) { + if(testsuite == null) { + testcount = report.test.count; + testsuite = root->add_child(node->copy_list()); + } else { + oldtestsuite = testsuite; + testsuite = testsuite->add_child(node->copy_list()); + } + } else if (report.test is TestCase) { + oldtestsuite = testsuite; + testsuite = testsuite->add_child(node->copy_list()); + casecount = report.test.count; + } else if(report.test is TestAdapter) { + testsuite->add_child(node->copy_list()); + testcount--; + casecount--; + if(casecount == 0) + testsuite = oldtestsuite; + } + if(testcount == 0) + root->doc->dump_format(stdout); + } +} |