diff options
| author | Pekka Paalanen <pekka.paalanen@collabora.com> | 2021-03-03 14:21:01 +0200 |
|---|---|---|
| committer | Pekka Paalanen <pekka.paalanen@collabora.com> | 2021-03-12 16:14:28 +0200 |
| commit | ee38ed80d8ae00a5b7e6c80b8022804df52e9265 (patch) | |
| tree | 71bb074741e1bf76d483f561249ec979d16517ff | |
| parent | cde58fd20a6e0a3f83ad3dafde2a0e89994c6e29 (diff) | |
| download | weston-ee38ed80d8ae00a5b7e6c80b8022804df52e9265.tar.gz | |
tests: add build option to force skips as failures
This will be useful in CI, where we do not want to see any skips. If
something starts to skip, that's a mistake somewhere, and want to catch
it.
Signed-off-by: Pekka Paalanen <pekka.paalanen@collabora.com>
| -rw-r--r-- | meson_options.txt | 6 | ||||
| -rw-r--r-- | tests/meson.build | 1 | ||||
| -rw-r--r-- | tests/weston-test-runner.c | 25 |
3 files changed, 31 insertions, 1 deletions
diff --git a/meson_options.txt b/meson_options.txt index 239bd2da..ea225a90 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -211,6 +211,12 @@ option( description: 'Tests: output JUnit XML results' ) option( + 'test-skip-is-failure', + type: 'boolean', + value: false, + description: 'Tests: consider skip to be a failure' +) +option( 'test-gl-renderer', type: 'boolean', value: true, diff --git a/tests/meson.build b/tests/meson.build index 90d776f5..212bba0b 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -276,6 +276,7 @@ test_config_h.set_quoted('WESTON_TEST_REFERENCE_PATH', meson.current_source_dir( test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap) test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data')) test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path()) +test_config_h.set10('WESTON_TEST_SKIP_IS_FAILURE', get_option('test-skip-is-failure')) configure_file(output: 'test-config.h', configuration: test_config_h) foreach t : tests diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c index d8becc31..0ab7bc1b 100644 --- a/tests/weston-test-runner.c +++ b/tests/weston-test-runner.c @@ -35,6 +35,7 @@ #include <signal.h> #include <getopt.h> +#include "test-config.h" #include "weston-test-runner.h" #include "weston-testsuite-data.h" #include "shared/string-helpers.h" @@ -243,7 +244,11 @@ result_to_str(enum test_result_code ret) [RESULT_FAIL] = "fail", [RESULT_HARD_ERROR] = "hard error", [RESULT_OK] = "ok", +#if WESTON_TEST_SKIP_IS_FAILURE + [RESULT_SKIP] = "skip error", +#else [RESULT_SKIP] = "skip", +#endif }; assert(ret >= 0 && ret < ARRAY_LENGTH(names)); @@ -284,6 +289,9 @@ run_case(struct wet_testsuite_data *suite_data, case RESULT_SKIP: suite_data->skipped++; skip = " # SKIP"; +#if WESTON_TEST_SKIP_IS_FAILURE + fail = "not "; +#endif break; } @@ -327,10 +335,16 @@ skip_case(struct wet_testsuite_data *suite_data, const void *test_data, int iteration) { + const char *skip_error = ""; int iteration_nr = iteration + 1; +#if WESTON_TEST_SKIP_IS_FAILURE + skip_error = "not "; +#endif + suite_data->counter++; - printf("ok %d %s %s/%d # SKIP fixture\n", suite_data->counter, + printf("%sok %d %s %s/%d # SKIP fixture\n", + skip_error, suite_data->counter, suite_data->fixture_name, t->name, iteration_nr); } @@ -477,6 +491,11 @@ weston_test_harness_destroy(struct weston_test_harness *harness) static enum test_result_code counts_to_result(const struct wet_testsuite_data *data) { +#if WESTON_TEST_SKIP_IS_FAILURE + if (data->skipped > 0) + return RESULT_FAIL; +#endif + /* RESULT_SKIP is reserved for fixture setup itself skipping everything */ if (data->total == data->passed + data->skipped) return RESULT_OK; @@ -644,7 +663,11 @@ main(int argc, char *argv[]) if (ret == RESULT_SKIP) { tap_skip_fixture(&harness->data); +#if WESTON_TEST_SKIP_IS_FAILURE + ret = RESULT_FAIL; +#else continue; +#endif } if (ret != RESULT_OK && result != RESULT_HARD_ERROR) |