Enables output in the JUnit XML format.

Adds basic support for optionally outputting in the XML format
commonly used by JUnit compatible tools.

This format is supported by default by many tools, including
the Jenkins build system. It also is more detailed and
captures more information than the more simplistic TAP
format.

Signed-off-by: Jon A. Cruz <jonc@osg.samsung.com>
Reviewed-by: Pekka Paalanen <pekka.paalanen@collabora.co.uk>

3 files changed, 525 insertions, 0 deletions

diff --git a/tools/zunitc/src/zuc_junit_reporter.c b/tools/zunitc/src/zuc_junit_reporter.c
new file mode 100644
index 00000000..5c30eaab
--- /dev/null
+++ b/tools/zunitc/src/zuc_junit_reporter.c
@@ -0,0 +1,470 @@
+/*
+ * Copyright © 2015 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+
+#include "zuc_junit_reporter.h"
+
+#if ENABLE_JUNIT_XML
+
+#include <fcntl.h>
+#include <libxml/parser.h>
+#include <memory.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "zuc_event_listener.h"
+#include "zuc_types.h"
+
+#include "shared/zalloc.h"
+
+/**
+ * Hardcoded output name.
+ * @todo follow-up with refactoring to avoid filename hardcoding.
+ * Will allow for better testing in parallel etc. in general.
+ */
+#define XML_FNAME "test_detail.xml"
+
+#define ISO_8601_FORMAT "%Y-%m-%dT%H:%M:%SZ"
+
+/**
+ * Internal data.
+ */
+struct junit_data
+{
+	int fd;
+	time_t begin;
+};
+
+#define MAX_64BIT_STRLEN 20
+
+static void
+set_attribute(xmlNodePtr node, const char *name, int value)
+{
+	xmlChar scratch[MAX_64BIT_STRLEN + 1] = {};
+	xmlStrPrintf(scratch, sizeof(scratch), BAD_CAST "%d", value);
+	xmlSetProp(node, BAD_CAST name, scratch);
+}
+
+/**
+ * Output the given event.
+ *
+ * @param parent the parent node to add new content to.
+ * @param event the event to write out.
+ */
+static void
+emit_event(xmlNodePtr parent, struct zuc_event *event)
+{
+	char *msg = NULL;
+
+	switch (event->op) {
+	case ZUC_OP_TRUE:
+		if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+			     "  Actual: false\n"
+			     "Expected: true\n", event->file, event->line,
+			     event->expr1) < 0) {
+			msg = NULL;
+		}
+		break;
+	case ZUC_OP_FALSE:
+		if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+			     "  Actual: true\n"
+			     "Expected: false\n", event->file, event->line,
+			     event->expr1) < 0) {
+			msg = NULL;
+		}
+		break;
+	case ZUC_OP_NULL:
+		if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+			     "  Actual: %p\n"
+			     "Expected: %p\n", event->file, event->line,
+			     event->expr1, (void *)event->val1, NULL) < 0) {
+			msg = NULL;
+		}
+		break;
+	case ZUC_OP_NOT_NULL:
+		if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+			     "  Actual: %p\n"
+			     "Expected: not %p\n", event->file, event->line,
+			     event->expr1, (void *)event->val1, NULL) < 0) {
+			msg = NULL;
+		}
+		break;
+	case ZUC_OP_EQ:
+		if (event->valtype == ZUC_VAL_CSTR) {
+			if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+				     "  Actual: %s\n"
+				     "Expected: %s\n"
+				     "Which is: %s\n",
+				     event->file, event->line, event->expr2,
+				     (char *)event->val2, event->expr1,
+				     (char *)event->val1) < 0) {
+				msg = NULL;
+			}
+		} else {
+			if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
+				     "  Actual: %ld\n"
+				     "Expected: %s\n"
+				     "Which is: %ld\n",
+				     event->file, event->line, event->expr2,
+				     event->val2, event->expr1,
+				     event->val1) < 0) {
+				msg = NULL;
+			}
+		}
+		break;
+	case ZUC_OP_NE:
+		if (event->valtype == ZUC_VAL_CSTR) {
+			if (asprintf(&msg, "%s:%d: error: "
+				     "Expected: (%s) %s (%s),"
+				     " actual: %s == %s\n",
+				     event->file, event->line,
+				     event->expr1, zuc_get_opstr(event->op),
+				     event->expr2, (char *)event->val1,
+				     (char *)event->val2) < 0) {
+				msg = NULL;
+			}
+		} else {
+			if (asprintf(&msg, "%s:%d: error: "
+				     "Expected: (%s) %s (%s),"
+				     " actual: %ld vs %ld\n",
+				     event->file, event->line,
+				     event->expr1, zuc_get_opstr(event->op),
+				     event->expr2, event->val1,
+				     event->val2) < 0) {
+				msg = NULL;
+			}
+		}
+		break;
+	case ZUC_OP_TERMINATE:
+	{
+		char const *level = (event->val1 == 0) ? "error"
+			: (event->val1 == 1) ? "warning"
+			: "note";
+		if (asprintf(&msg, "%s:%d: %s: %s\n",
+			     event->file, event->line, level,
+			     event->expr1) < 0) {
+			msg = NULL;
+		}
+		break;
+	}
+	case ZUC_OP_TRACEPOINT:
+		if (asprintf(&msg, "%s:%d: note: %s\n",
+			     event->file, event->line, event->expr1) < 0) {
+			msg = NULL;
+		}
+		break;
+	default:
+		if (asprintf(&msg, "%s:%d: error: "
+			     "Expected: (%s) %s (%s), actual: %ld vs %ld\n",
+			     event->file, event->line,
+			     event->expr1, zuc_get_opstr(event->op),
+			     event->expr2, event->val1, event->val2) < 0) {
+			msg = NULL;
+		}
+	}
+
+	if ((event->op == ZUC_OP_TERMINATE) && (event->val1 > 1)) {
+		xmlNewChild(parent, NULL, BAD_CAST "skipped", NULL);
+	} else {
+		xmlNodePtr node = xmlNewChild(parent, NULL,
+					      BAD_CAST "failure", NULL);
+
+		if (msg) {
+			xmlSetProp(node, BAD_CAST "message", BAD_CAST msg);
+		}
+		xmlSetProp(node, BAD_CAST "type", BAD_CAST "");
+		if (msg) {
+			xmlNodePtr cdata = xmlNewCDataBlock(node->doc,
+							    BAD_CAST msg,
+							    strlen(msg));
+			xmlAddChild(node, cdata);
+		}
+	}
+
+	free(msg);
+}
+
+/**
+ * Formats a time in milliseconds to the normal JUnit elapsed form, or
+ * NULL if there is a problem.
+ * The caller should release this with free()
+ *
+ * @return the formatted time string upon success, NULL otherwise.
+ */
+static char *
+as_duration(long ms) {
+	char *str = NULL;
+
+	if (asprintf(&str, "%1.3f", ms / 1000.0) < 0) {
+		str = NULL;
+	} else {
+		/*
+		 * Special case to match behavior of standard JUnit output
+		 * writers. Asumption is certain readers might have
+		 * limitations, etc. so it is best to keep 100% identical
+		 * output.
+		 */
+		if (!strcmp("0.000", str)) {
+			free(str);
+			str = strdup("0");
+		}
+	}
+	return str;
+}
+
+/**
+ * Returns the status string for the tests (run/notrun).
+ *
+ * @param test the test to check status of.
+ * @return the status string.
+ */
+static char const *
+get_test_status(struct zuc_test *test)
+{
+	if (test->disabled || test->skipped)
+		return "notrun";
+	else
+		return "run";
+}
+
+/**
+ * Output the given test.
+ *
+ * @param parent the parent node to add new content to.
+ * @param test the test to write out.
+ */
+static void
+emit_test(xmlNodePtr parent, struct zuc_test *test)
+{
+	char *time_str = as_duration(test->elapsed);
+	xmlNodePtr node = xmlNewChild(parent, NULL, BAD_CAST "testcase", NULL);
+
+	xmlSetProp(node, BAD_CAST "name", BAD_CAST test->name);
+	xmlSetProp(node, BAD_CAST "status", BAD_CAST get_test_status(test));
+
+	if (time_str) {
+		xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str);
+
+		free(time_str);
+		time_str = NULL;
+	}
+
+	xmlSetProp(node, BAD_CAST "classname", BAD_CAST test->test_case->name);
+
+	if ((test->failed || test->fatal || test->skipped) && test->events) {
+		struct zuc_event *evt;
+		for (evt = test->events; evt; evt = evt->next)
+			emit_event(node, evt);
+	}
+}
+
+/**
+ * Output the given test case.
+ *
+ * @param parent the parent node to add new content to.
+ * @param test_case the test case to write out.
+ */
+static void
+emit_case(xmlNodePtr parent, struct zuc_case *test_case)
+{
+	int i;
+	int skipped = 0;
+	int disabled = 0;
+	int failures = 0;
+	xmlNodePtr node = NULL;
+	char *time_str = as_duration(test_case->elapsed);
+
+	for (i = 0; i < test_case->test_count; ++i) {
+		if (test_case->tests[i]->disabled )
+			disabled++;
+		if (test_case->tests[i]->skipped )
+			skipped++;
+		if (test_case->tests[i]->failed
+		    || test_case->tests[i]->fatal )
+			failures++;
+	}
+
+	node = xmlNewChild(parent, NULL, BAD_CAST "testsuite", NULL);
+	xmlSetProp(node, BAD_CAST "name", BAD_CAST test_case->name);
+
+	set_attribute(node, "tests", test_case->test_count);
+	set_attribute(node, "failures", failures);
+	set_attribute(node, "disabled", disabled);
+	set_attribute(node, "skipped", skipped);
+
+	if (time_str) {
+		xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str);
+		free(time_str);
+		time_str = NULL;
+	}
+
+	for (i = 0; i < test_case->test_count; ++i)
+		emit_test(node, test_case->tests[i]);
+}
+
+/**
+ * Formats a time in milliseconds to the full ISO-8601 date/time string
+ * format, or NULL if there is a problem.
+ * The caller should release this with free()
+ *
+ * @return the formatted time string upon success, NULL otherwise.
+ */
+static char *
+as_iso_8601(time_t const *t)
+{
+	char *result = NULL;
+	char buf[32] = {};
+	struct tm when;
+
+	if (gmtime_r(t, &when) != NULL)
+		if (strftime(buf, sizeof(buf), ISO_8601_FORMAT, &when))
+			result = strdup(buf);
+
+	return result;
+}
+
+
+static void
+run_started(void *data, int live_case_count, int live_test_count,
+	    int disabled_count)
+{
+	struct junit_data *jdata = data;
+
+	jdata->begin = time(NULL);
+	jdata->fd = open(XML_FNAME, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC,
+			 S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
+}
+
+static void
+run_ended(void *data, int case_count, struct zuc_case **cases,
+	  int live_case_count, int live_test_count, int total_passed,
+	  int total_failed, int total_disabled, long total_elapsed)
+{
+	int i;
+	long time = 0;
+	char *time_str = NULL;
+	char *timestamp = NULL;
+	xmlNodePtr root = NULL;
+	xmlDocPtr doc = NULL;
+	xmlChar *xmlchars = NULL;
+	int xmlsize = 0;
+	struct junit_data *jdata = data;
+
+	for (i = 0; i < case_count; ++i)
+		time += cases[i]->elapsed;
+
+	time_str = as_duration(time);
+	timestamp = as_iso_8601(&jdata->begin);
+
+	/* here would be where to add errors? */
+
+	doc = xmlNewDoc(BAD_CAST "1.0");
+	root = xmlNewNode(NULL, BAD_CAST "testsuites");
+	xmlDocSetRootElement(doc, root);
+
+	set_attribute(root, "tests", live_test_count);
+	set_attribute(root, "failures", total_failed);
+	set_attribute(root, "disabled", total_disabled);
+
+	if (timestamp) {
+		xmlSetProp(root, BAD_CAST "timestamp", BAD_CAST timestamp);
+		free(timestamp);
+		timestamp = NULL;
+	}
+
+	if (time_str) {
+		xmlSetProp(root, BAD_CAST "time", BAD_CAST time_str);
+		free(time_str);
+		time_str = NULL;
+	}
+
+	xmlSetProp(root, BAD_CAST "name", BAD_CAST "AllTests");
+
+	for (i = 0; i < case_count; ++i) {
+		emit_case(root, cases[i]);
+	}
+
+	xmlDocDumpFormatMemoryEnc(doc, &xmlchars, &xmlsize, "UTF-8", 1);
+	dprintf(jdata->fd, "%s", (char *) xmlchars);
+	xmlFree(xmlchars);
+	xmlchars = NULL;
+	xmlFreeDoc(doc);
+
+	if ((jdata->fd != fileno(stdout))
+	    && (jdata->fd != fileno(stderr))
+	    && (jdata->fd != -1)) {
+		close(jdata->fd);
+		jdata->fd = -1;
+	}
+}
+
+static void
+destroy(void *data)
+{
+	xmlCleanupParser();
+
+	free(data);
+}
+
+struct zuc_event_listener *
+zuc_junit_reporter_create(void)
+{
+	struct zuc_event_listener *listener =
+		zalloc(sizeof(struct zuc_event_listener));
+
+	struct junit_data *data = zalloc(sizeof(struct junit_data));
+	data->fd = -1;
+
+	listener->data = data;
+	listener->destroy = destroy;
+	listener->run_started = run_started;
+	listener->run_ended = run_ended;
+
+	return listener;
+}
+
+#else /* ENABLE_JUNIT_XML */
+
+#include "shared/zalloc.h"
+#include "zuc_event_listener.h"
+
+/*
+ * Simple stub version if junit output (including libxml2 support) has
+ * been disabled.
+ * Will return NULL to cause failures as calling this when the #define
+ * has not been enabled is an invalid scenario.
+ */
+
+struct zuc_event_listener *
+zuc_junit_reporter_create(void)
+{
+	return NULL;
+}
+
+#endif /* ENABLE_JUNIT_XML */
diff --git a/tools/zunitc/src/zuc_junit_reporter.h b/tools/zunitc/src/zuc_junit_reporter.h
new file mode 100644
index 00000000..66f3c7b3
--- /dev/null
+++ b/tools/zunitc/src/zuc_junit_reporter.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2015 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef ZUC_JUNIT_REPORTER_H
+#define ZUC_JUNIT_REPORTER_H
+
+struct zuc_event_listener;
+
+/**
+ * Creates an instance of a reporter that will write data in the JUnit
+ * XML format.
+ */
+struct zuc_event_listener *
+zuc_junit_reporter_create(void);
+
+#endif /* ZUC_JUNIT_REPORTER_H */
diff --git a/tools/zunitc/src/zunitc_impl.c b/tools/zunitc/src/zunitc_impl.c
index b8ab0b2c..6f591f08 100644
--- a/tools/zunitc/src/zunitc_impl.c
+++ b/tools/zunitc/src/zunitc_impl.c
@@ -44,6 +44,7 @@
 #include "zuc_collector.h"
 #include "zuc_context.h"
 #include "zuc_event_listener.h"
+#include "zuc_junit_reporter.h"
 
 #include "shared/config-parser.h"
 #include "shared/helpers.h"
@@ -152,6 +153,12 @@ zuc_set_break_on_failure(bool break_on_failure)
 	g_ctx.break_on_failure = break_on_failure;
 }
 
+void
+zuc_set_output_junit(bool enable)
+{
+	g_ctx.output_junit = enable;
+}
+
 const char *
 zuc_get_program_name(void)
 {
@@ -523,6 +530,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
 	int opt_repeat = 0;
 	int opt_random = 0;
 	int opt_break_on_failure = 0;
+	int opt_junit = 0;
 	char *opt_filter = NULL;
 
 	char *help_param = NULL;
@@ -535,6 +543,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
 		{ WESTON_OPTION_INTEGER, "zuc-random", 0, &opt_random },
 		{ WESTON_OPTION_BOOLEAN, "zuc-break-on-failure", 0,
 		  &opt_break_on_failure },
+#if ENABLE_JUNIT_XML
+		{ WESTON_OPTION_BOOLEAN, "zuc-output-xml", 0, &opt_junit },
+#endif
 		{ WESTON_OPTION_STRING, "zuc-filter", 0, &opt_filter },
 	};
 
@@ -623,6 +634,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
 		       "  --zuc-filter=FILTER\n"
 		       "  --zuc-list-tests\n"
 		       "  --zuc-nofork\n"
+#if ENABLE_JUNIT_XML
+		       "  --zuc-output-xml\n"
+#endif
 		       "  --zuc-random=N            [0|1|<seed number>]\n"
 		       "  --zuc-repeat=N\n"
 		       "  --help\n",
@@ -638,6 +652,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
 		zuc_set_random(opt_random);
 		zuc_set_spawn(!opt_nofork);
 		zuc_set_break_on_failure(opt_break_on_failure);
+		zuc_set_output_junit(opt_junit);
 		rc = EXIT_SUCCESS;
 	}
 
@@ -1301,6 +1316,8 @@ zucimpl_run_tests(void)
 	if (g_ctx.listeners == NULL) {
 		zuc_add_event_listener(zuc_collector_create(&(g_ctx.fds[1])));
 		zuc_add_event_listener(zuc_base_logger_create());
+		if (g_ctx.output_junit)
+			zuc_add_event_listener(zuc_junit_reporter_create());
 	}
 
 	if (g_ctx.case_count < 1) {