patch 8.1.0529: flaky test sometimes fails in different waysv8.1.0529

Problem:    Flaky test sometimes fails in different ways.
Solution:   When the second run gives a different error, try running the test
            again, up to five times.

1 files changed, 35 insertions, 16 deletions

diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index cacfa7558..2c2249848 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -314,28 +314,47 @@ endif
 for s:test in sort(s:tests)
   " Silence, please!
   set belloff=all
+  let prev_error = ''
+  let total_errors = []
+  let run_nr = 1
 
   call RunTheTest(s:test)
 
+  " Repeat a flaky test.  Give up when:
+  " - it fails again with the same message
+  " - it fails five times (with a different mesage)
   if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
-    call add(s:messages, 'Found errors in ' . s:test . ':')
-    call extend(s:messages, v:errors)
-    call add(s:messages, 'Flaky test failed, running it again')
-    let first_run = v:errors
+    while 1
+      call add(s:messages, 'Found errors in ' . s:test . ':')
+      call extend(s:messages, v:errors)
 
-    " Flakiness is often caused by the system being very busy.  Sleep a couple
-    " of seconds to have a higher chance of succeeding the second time.
-    sleep 2
+      call add(total_errors, 'Run ' . run_nr . ':')
+      call extend(total_errors, v:errors)
 
-    let v:errors = []
-    call RunTheTest(s:test)
-    if len(v:errors) > 0
-      let second_run = v:errors
-      let v:errors = ['First run:']
-      call extend(v:errors, first_run)
-      call add(v:errors, 'Second run:')
-      call extend(v:errors, second_run)
-    endif
+      if run_nr == 5 || prev_error == v:errors[0]
+        call add(total_errors, 'Flaky test failed too often, giving up')
+        let v:errors = total_errors
+        break
+      endif
+
+      call add(s:messages, 'Flaky test failed, running it again')
+
+      " Flakiness is often caused by the system being very busy.  Sleep a
+      " couple of seconds to have a higher chance of succeeding the second
+      " time.
+      sleep 2
+
+      let prev_error = v:errors[0]
+      let v:errors = []
+      let run_nr += 1
+
+      call RunTheTest(s:test)
+
+      if len(v:errors) == 0
+        " Test passed on rerun.
+        break
+      endif
+    endwhile
   endif
 
   call AfterTheTest()