Tests: Add libusbx stress test

See https://github.com/tobygray/libusbx/tree/testing as well
as http://libusbx.1081486.n5.nabble.com/Libusbx-devel-Crashes-tt433.html#a438

4 files changed, 526 insertions, 0 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
new file mode 100644
index 0000000..e5d6d75
--- /dev/null
+++ b/tests/Makefile.am
@@ -0,0 +1,6 @@
+AM_CPPFLAGS = -I$(top_srcdir)/libusb
+LDADD = ../libusb/libusb-1.0.la
+
+noinst_PROGRAMS = stress
+
+stress_SOURCES = stress.c testlib.c
diff --git a/tests/libusbx_testlib.h b/tests/libusbx_testlib.h
new file mode 100644
index 0000000..cb9ac9c
--- /dev/null
+++ b/tests/libusbx_testlib.h
@@ -0,0 +1,106 @@
+/*
+ * libusbx test library helper functions
+ * Copyright © 2012 Toby Gray <toby.gray@realvnc.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef LIBUSBX_TESTLIB_H
+#define LIBUSBX_TESTLIB_H
+
+#include <stdio.h>
+
+#if !defined(bool)
+#define bool int
+#endif
+#if !defined(true)
+#define true (1 == 1)
+#endif
+#if !defined(false)
+#define false (!true)
+#endif
+
+/** Values returned from a test function to indicate test result */
+typedef enum {
+	/** Indicates that the test ran successfully. */
+	TEST_STATUS_SUCCESS,
+	/** Indicates that the test failed one or more test. */
+	TEST_STATUS_FAILURE,
+	/** Indicates that an unexpected error occurred. */
+	TEST_STATUS_ERROR,
+	/** Indicates that the test can't be run. For example this may be
+	* due to no suitable device being connected to perform the tests.*/
+	TEST_STATUS_SKIP
+} libusbx_testlib_result;
+
+/**
+ * Context for test library functions
+ */
+typedef struct {
+	char ** test_names;
+	int test_count;
+	bool list_tests;
+	bool verbose;
+	int output_fd;
+	FILE* output_file;
+	int null_fd;
+} libusbx_testlib_ctx;
+
+/**
+ * Logs some test information or state
+ */
+void libusbx_testlib_logf(libusbx_testlib_ctx * ctx, 
+                          const char* fmt, ...);
+
+/**
+ * Function pointer for a libusbx test function.
+ *
+ * Should return TEST_STATUS_SUCCESS on success or another TEST_STATUS value.
+ */
+typedef libusbx_testlib_result
+(*libusbx_testlib_test_function)(libusbx_testlib_ctx * ctx);
+
+/**
+ * Structure holding a test description.
+ */
+typedef struct {
+	/** Human readable name of the test. */
+	const char * name;
+	/** The test library will call this function to run the test. */
+	libusbx_testlib_test_function function;
+} libusbx_testlib_test;
+
+/**
+ * Value to use at the end of a test array to indicate the last
+ * element.
+ */
+#define LIBUSBX_NULL_TEST {NULL, NULL}
+
+/**
+ * Runs the tests provided.
+ *
+ * Before running any tests argc and argv will be processed
+ * to determine the mode of operation.
+ *
+ * \param argc The argc from main
+ * \param argv The argv from main
+ * \param tests A NULL_TEST terminated array of tests
+ * \return 0 on success, non-zero on failure
+ */
+int libusbx_testlib_run_tests(int argc,
+                              char ** argv,
+                              const libusbx_testlib_test * tests);
+
+#endif //LIBUSBX_TESTLIB_H
diff --git a/tests/stress.c b/tests/stress.c
new file mode 100644
index 0000000..c4a92fa
--- /dev/null
+++ b/tests/stress.c
@@ -0,0 +1,161 @@
+/*
+ * libusbx stress test program to perform simple stress tests
+ * Copyright © 2012 Toby Gray <toby.gray@realvnc.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <memory.h>
+
+#include "libusb.h"
+
+#include "libusbx_testlib.h"
+
+/** Test that creates and destroys a single concurrent context
+ * 10000 times. */
+static libusbx_testlib_result test_init_and_exit(libusbx_testlib_ctx * tctx)
+{
+	libusb_context * ctx = NULL;
+	int i;
+	for (i = 0; i < 10000; ++i) {
+		int r = libusb_init(&ctx);
+		if (r != LIBUSB_SUCCESS) {
+			libusbx_testlib_logf(tctx,
+				"Failed to init libusb on iteration %d: %d",
+				i, r);
+			return TEST_STATUS_FAILURE;
+		}
+		libusb_exit(ctx);
+		ctx = NULL;
+	}
+
+	return TEST_STATUS_SUCCESS;
+}
+
+/** Tests that devices can be listed 1000 times. */
+static libusbx_testlib_result test_get_device_list(libusbx_testlib_ctx * tctx)
+{
+	libusb_context * ctx = NULL;
+	int r, i;
+	r = libusb_init(&ctx);
+	if (r != LIBUSB_SUCCESS) {
+		libusbx_testlib_logf(tctx, "Failed to init libusb: %d", r);
+		return TEST_STATUS_FAILURE;
+	}
+	for (i = 0; i < 1000; ++i) {
+		libusb_device ** device_list;
+		ssize_t list_size = libusb_get_device_list(ctx, &device_list);
+		if (list_size < 0 || device_list == NULL) {
+			libusbx_testlib_logf(tctx,
+				"Failed to get device list on iteration %d: %d (%p)",
+				i, -list_size, device_list);
+			return TEST_STATUS_FAILURE;
+		}
+		libusb_free_device_list(device_list, 1);
+	}
+	libusb_exit(ctx);
+	return TEST_STATUS_SUCCESS;
+}
+
+/** Tests that 100 concurrent device lists can be open at a time. */
+static libusbx_testlib_result test_many_device_lists(libusbx_testlib_ctx * tctx)
+{
+#define LIST_COUNT 100
+	libusb_context * ctx = NULL;
+	libusb_device ** device_lists[LIST_COUNT];
+	int r, i;
+	memset(device_lists, 0, sizeof(device_lists));
+
+	r = libusb_init(&ctx);
+	if (r != LIBUSB_SUCCESS) {
+		libusbx_testlib_logf(tctx, "Failed to init libusb: %d", r);
+		return TEST_STATUS_FAILURE;
+	}
+
+	/* Create the 100 device lists. */
+	for (i = 0; i < LIST_COUNT; ++i) {
+		ssize_t list_size = libusb_get_device_list(ctx, &(device_lists[i]));
+		if (list_size < 0 || device_lists[i] == NULL) {
+			libusbx_testlib_logf(tctx,
+				"Failed to get device list on iteration %d: %d (%p)",
+				i, -list_size, device_lists[i]);
+			return TEST_STATUS_FAILURE;
+		}
+	}
+
+	/* Destroy the 100 device lists. */
+	for (i = 0; i < LIST_COUNT; ++i) {
+		if (device_lists[i]) {
+			libusb_free_device_list(device_lists[i], 1);
+			device_lists[i] = NULL;
+		}
+	}
+
+	libusb_exit(ctx);
+	return TEST_STATUS_SUCCESS;
+#undef LIST_COUNT
+}
+
+/** Tests that the default context (used for various things including
+ * logging) works correctly when the first context created in a
+ * process is destroyed. */
+static libusbx_testlib_result test_default_context_change(libusbx_testlib_ctx * tctx)
+{
+	libusb_context * ctx = NULL;
+	int r, i;
+
+	for (i = 0; i < 100; ++i) {
+		/* First create a new context */
+		r = libusb_init(&ctx);
+		if (r != LIBUSB_SUCCESS) {
+			libusbx_testlib_logf(tctx, "Failed to init libusb: %d", r);
+			return TEST_STATUS_FAILURE;
+		}
+
+		/* Enable debug output, to be sure to use the context */
+		libusb_set_debug(NULL, LIBUSB_LOG_LEVEL_DEBUG);
+		libusb_set_debug(ctx, LIBUSB_LOG_LEVEL_DEBUG);
+
+		/* Now create a reference to the default context */
+		r = libusb_init(NULL);
+		if (r != LIBUSB_SUCCESS) {
+			libusbx_testlib_logf(tctx, "Failed to init libusb: %d", r);
+			return TEST_STATUS_FAILURE;
+		}
+
+		/* Destroy the first context */
+		libusb_exit(ctx);
+		/* Destroy the default context */
+		libusb_exit(NULL);
+	}
+
+	return TEST_STATUS_SUCCESS;
+}
+
+/* Fill in the list of tests. */
+static const libusbx_testlib_test tests[] = {
+	{"init_and_exit", &test_init_and_exit},
+	{"get_device_list", &test_get_device_list},
+	{"many_device_lists", &test_many_device_lists},
+	{"default_context_change", &test_default_context_change},
+	LIBUSBX_NULL_TEST
+};
+
+int main (int argc, char ** argv)
+{
+	return libusbx_testlib_run_tests(argc, argv, tests);
+}
diff --git a/tests/testlib.c b/tests/testlib.c
new file mode 100644
index 0000000..9e45d31
--- /dev/null
+++ b/tests/testlib.c
@@ -0,0 +1,253 @@
+/*
+ * libusbx test library helper functions
+ * Copyright © 2012 Toby Gray <toby.gray@realvnc.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libusbx_testlib.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifdef _WIN32
+#include <io.h>
+#define dup _dup
+#define dup2 _dup2
+#define open _open
+#define close _close
+#define fdopen _fdopen
+#define NULL_PATH "nul"
+#define STDOUT_FILENO 1
+#define STDERR_FILENO 2
+#else
+#include <unistd.h>
+#define NULL_PATH "/dev/null"
+#endif
+#define INVALID_FD -1
+
+/**
+ * Converts a test result code into a human readable string.
+ */
+static const char* test_result_to_str(libusbx_testlib_result result)
+{
+	switch (result) {
+	case TEST_STATUS_SUCCESS:
+		return "Success";
+	case TEST_STATUS_FAILURE:
+		return "Failure";
+	case TEST_STATUS_ERROR:
+		return "Error";
+	case TEST_STATUS_SKIP:
+		return "Skip";
+	default:
+		return "Unknown";
+	}
+}
+
+static void print_usage(int argc, char ** argv)
+{
+	printf("Usage: %s [-l] [-v] [<test_name> ...]\n",
+		argc > 0 ? argv[0] : "test_*");
+	printf("   -l   List available tests\n");
+	printf("   -v   Don't redirect STDERR/STDOUT during tests\n");
+}
+
+static void cleanup_test_output(libusbx_testlib_ctx * ctx)
+{
+	if (ctx->output_file != NULL) {
+		fclose(ctx->output_file);
+		ctx->output_file = NULL;
+	}
+	if (ctx->output_fd != INVALID_FD) {
+		close(ctx->output_fd);
+		ctx->output_fd = INVALID_FD;
+	}
+	if (ctx->null_fd != INVALID_FD) {
+		close(ctx->null_fd);
+		ctx->null_fd = INVALID_FD;
+	}
+}
+
+/**
+ * Setup test output handles
+ * \return zero on success, non-zero on failure
+ */
+static int setup_test_output(libusbx_testlib_ctx * ctx)
+{
+	/* Keep a copy of STDOUT for test output */
+	ctx->output_fd = dup(STDOUT_FILENO);
+	if (ctx->output_fd < 0) {
+		ctx->output_fd = INVALID_FD;
+		printf("Failed to duplicate output handle: %d\n", errno);
+		return 1;
+	}
+	ctx->output_file = fdopen(ctx->output_fd, "w");
+	if (!ctx->output_file) {
+		cleanup_test_output(ctx);
+		printf("Failed to open FILE for output handle: %d\n", errno);
+		return 1;
+	}
+	/* Stop output to stdout and stderr from being displayed if using non-verbose output */
+	if (!ctx->verbose) {
+		/* Redirect STDOUT_FILENO and STDERR_FILENO to /dev/null or "nul"*/
+		ctx->null_fd = open(NULL_PATH, O_WRONLY);
+		if (ctx->null_fd < 0) {
+			ctx->null_fd = INVALID_FD;
+			cleanup_test_output(ctx);
+			printf("Failed to open null handle: %d\n", errno);
+			return 1;
+		}
+		if ((dup2(ctx->null_fd, STDOUT_FILENO) < 0) ||
+			(dup2(ctx->null_fd, STDERR_FILENO) < 0)) {
+				cleanup_test_output(ctx);
+				return 1;
+		}
+	}
+	return 0;
+}
+
+void libusbx_testlib_logf(libusbx_testlib_ctx * ctx,
+	const char* fmt, ...)
+{
+	va_list va;
+	if (!ctx->output_file)
+		return;
+	va_start(va, fmt);
+	vfprintf(ctx->output_file, fmt, va);
+	va_end(va);
+	fprintf(ctx->output_file, "\n");
+	fflush(ctx->output_file);
+}
+
+int libusbx_testlib_run_tests(int argc,
+	char ** argv,
+	const libusbx_testlib_test * tests)
+{
+	int run_count = 0;
+	int idx = 0;
+	int pass_count = 0;
+	int fail_count = 0;
+	int error_count = 0;
+	int skip_count = 0;
+	int r, j;
+	size_t arglen;
+	libusbx_testlib_result test_result;
+	libusbx_testlib_ctx ctx;
+
+	/* Setup default mode of operation */
+	ctx.test_names = NULL;
+	ctx.test_count = 0;
+	ctx.list_tests = false;
+	ctx.verbose = false;
+	ctx.output_fd = INVALID_FD;
+	ctx.output_file = NULL;
+	ctx.null_fd = INVALID_FD;
+
+	/* Parse command line options */
+	if (argc >= 2) {
+		for (j = 1; j < argc; j++) {
+			arglen = strlen(argv[j]);
+			if ( ((argv[j][0] == '-') || (argv[j][0] == '/')) &&
+				arglen >=2 ) {
+					switch (argv[j][1]) {
+					case 'l':
+						ctx.list_tests = true;
+						break;
+					case 'v':
+						ctx.verbose = true;
+						break;
+					default:
+						printf("Unknown option: '%s'\n", argv[j]);
+						print_usage(argc, argv);
+						return 1;
+					}
+			} else {
+				/* End of command line options, remaining must be list of tests to run */
+				ctx.test_names = argv + j;
+				ctx.test_count = argc - j;
+				break;
+			}
+		}
+	}
+
+	/* Validate command line options */
+	if (ctx.test_names && ctx.list_tests) {
+		printf("List of tests requested but test list provided\n");
+		print_usage(argc, argv);
+		return 1;
+	}
+
+	/* Setup test log output */
+	r = setup_test_output(&ctx);
+	if (r != 0)
+		return r;  
+
+	/* Act on any options not related to running tests */
+	if (ctx.list_tests) {
+		while (tests[idx].function != NULL) {
+			libusbx_testlib_logf(&ctx, tests[idx].name);
+			++idx;
+		}
+		cleanup_test_output(&ctx);
+		return 0;
+	}
+
+	/* Run any requested tests */
+	while (tests[idx].function != NULL) {
+		const libusbx_testlib_test * test = &tests[idx];
+		++idx;
+		if (ctx.test_count > 0) {
+			/* Filtering tests to run, check if this is one of them */
+			int i;
+			for (i = 0; i < ctx.test_count; ++i) {
+				if (strcmp(ctx.test_names[i], test->name) == 0)
+					/* Matches a requested test name */
+					break;
+			}
+			if (i >= ctx.test_count) {
+				/* Failed to find a test match, so do the next loop iteration */
+				continue;
+			}
+		}
+		libusbx_testlib_logf(&ctx,
+			"Starting test run: %s...", test->name);
+		test_result = test->function(&ctx);
+		libusbx_testlib_logf(&ctx,
+			"%s (%d)",
+			test_result_to_str(test_result), test_result);
+		switch (test_result) {
+		case TEST_STATUS_SUCCESS: pass_count++; break;
+		case TEST_STATUS_FAILURE: fail_count++; break;
+		case TEST_STATUS_ERROR: error_count++; break;
+		case TEST_STATUS_SKIP: skip_count++; break;
+		}
+		++run_count;
+	}
+	libusbx_testlib_logf(&ctx, "---");
+	libusbx_testlib_logf(&ctx, "Ran %d tests", run_count);
+	libusbx_testlib_logf(&ctx, "Passed %d tests", pass_count);
+	libusbx_testlib_logf(&ctx, "Failed %d tests", fail_count);
+	libusbx_testlib_logf(&ctx, "Error in %d tests", error_count);
+	libusbx_testlib_logf(&ctx, "Skipped %d tests", skip_count);
+
+	cleanup_test_output(&ctx);
+	return pass_count != run_count;
+}