Normally skip tests that are not expected to succeed

If a test is not expected to succeed, then running it could be considered
to be a waste of resources, particularly if the failure might manifest
as an indefinite hang (see !11), or if the test is likely to dump core
and trigger "expensive" crash-reporting mechanisms like systemd-coredump,
corekeeper, abrt or apport.

Skip the tests that are expected to fail. They can still be requested via
an environment variable, which can be set after fixing a bug to check which
tests are now passing.

Signed-off-by: Simon McVittie <smcv@debian.org>

4 files changed, 27 insertions, 12 deletions

diff --git a/test-fixtures/test-utils.c b/test-fixtures/test-utils.c
index 59e3fd8c..fcb0d4c0 100644
--- a/test-fixtures/test-utils.c
+++ b/test-fixtures/test-utils.c
@@ -125,7 +125,7 @@ is_boolean_env_set (const char *variable)
   return ret;
 }
 
-void
+CoglBool
 test_utils_init (TestFlags requirement_flags,
                  TestFlags known_failure_flags)
 {
@@ -204,6 +204,8 @@ test_utils_init (TestFlags requirement_flags,
     g_print ("WARNING: Missing required feature[s] for this test\n");
   else if (known_failure)
     g_print ("WARNING: Test is known to fail\n");
+
+  return (!missing_requirement && !known_failure);
 }
 
 void
diff --git a/test-fixtures/test-utils.h b/test-fixtures/test-utils.h
index 9c3ced9b..e59163c8 100644
--- a/test-fixtures/test-utils.h
+++ b/test-fixtures/test-utils.h
@@ -69,7 +69,7 @@ typedef enum {
 extern CoglContext *test_ctx;
 extern CoglFramebuffer *test_fb;
 
-void
+CoglBool
 test_utils_init (TestFlags requirement_flags,
                  TestFlags known_failure_flags);
 
diff --git a/tests/conform/test-conform-main.c b/tests/conform/test-conform-main.c
index 9b6573d9..d84d755b 100644
--- a/tests/conform/test-conform-main.c
+++ b/tests/conform/test-conform-main.c
@@ -15,10 +15,17 @@
     extern void FUNC (void);                                            \
     if (strcmp (#FUNC, argv[1]) == 0)                                   \
       {                                                                 \
-        test_utils_init (REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS);        \
-        FUNC ();                                                        \
-        test_utils_fini ();                                             \
-        exit (0);                                                       \
+        if (test_utils_init (REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS)     \
+            || g_getenv ("COGL_TEST_TRY_EVERYTHING") != NULL)           \
+          {                                                             \
+            FUNC ();                                                    \
+            test_utils_fini ();                                         \
+            exit (0);                                                   \
+          }                                                             \
+        else                                                            \
+          {                                                             \
+            exit (1);                                                   \
+          }                                                             \
       }                                                                 \
   } G_STMT_END
 
diff --git a/tests/unit/test-unit-main.c b/tests/unit/test-unit-main.c
index b1f78645..6065b083 100644
--- a/tests/unit/test-unit-main.c
+++ b/tests/unit/test-unit-main.c
@@ -36,10 +36,16 @@ main (int argc, char **argv)
       return 1;
     }
 
-  test_utils_init (unit_test->requirement_flags,
-                   unit_test->known_failure_flags);
-  unit_test->run ();
-  test_utils_fini ();
-
-  return 0;
+  if (test_utils_init (unit_test->requirement_flags,
+                       unit_test->known_failure_flags)
+      || g_getenv ("COGL_TEST_TRY_EVERYTHING") != NULL)
+    {
+      unit_test->run ();
+      test_utils_fini ();
+      return 0;
+    }
+  else
+    {
+      return 1;
+    }
 }