Bug #57840 MTR: parallel execution breaks with smart ordering of test cases

There were actually more problems in this area:
  Slaves (if any) were unconditionally restarted, this appears unnecessary.
  Sort criteria were suboptimal, included the test name.
Added logic to "reserve" a sequence of tests with same config for one thread
Got rid of sort_criteria hash, put it into the test case itself
Adds little sanity check that expected worker picks up test
Fixed some tests that may fail if starting on running server
Some of these fail only if *same* test is repeated.
Finally, special sorting of tests that do --force-restart

14 files changed, 62 insertions, 30 deletions

diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm
index 7d25a72212a..7961fa6fa7d 100644
--- a/mysql-test/lib/mtr_cases.pm
+++ b/mysql-test/lib/mtr_cases.pm
@@ -170,8 +170,6 @@ sub collect_test_cases ($$$) {
   if ( $opt_reorder && !$quick_collect)
   {
     # Reorder the test cases in an order that will make them faster to run
-    my %sort_criteria;
-
     # Make a mapping of test name to a string that represents how that test
     # should be sorted among the other tests.  Put the most important criterion
     # first, then a sub-criterion, then sub-sub-criterion, etc.
@@ -183,24 +181,31 @@ sub collect_test_cases ($$$) {
       # Append the criteria for sorting, in order of importance.
       #
       push(@criteria, "ndb=" . ($tinfo->{'ndb_test'} ? "A" : "B"));
+      push(@criteria, $tinfo->{template_path});
       # Group test with equal options together.
       # Ending with "~" makes empty sort later than filled
       my $opts= $tinfo->{'master_opt'} ? $tinfo->{'master_opt'} : [];
       push(@criteria, join("!", sort @{$opts}) . "~");
+      # Add slave opts if any
+      if ($tinfo->{'slave_opt'})
+      {
+	push(@criteria, join("!", sort @{$tinfo->{'slave_opt'}}));
+      }
+      # This sorts tests with force-restart *before* identical tests
+      push(@criteria, $tinfo->{force_restart} ? "force-restart" : "no-restart");
 
-      $sort_criteria{$tinfo->{name}} = join(" ", @criteria);
+      $tinfo->{criteria}= join(" ", @criteria);
     }
 
-    @$cases = sort {
-      $sort_criteria{$a->{'name'}} . $a->{'name'} cmp
-	$sort_criteria{$b->{'name'}} . $b->{'name'}; } @$cases;
+    @$cases = sort {$a->{criteria} cmp $b->{criteria}; } @$cases;
 
     # For debugging the sort-order
     # foreach my $tinfo (@$cases)
     # {
-    #   print("$sort_criteria{$tinfo->{'name'}} -> \t$tinfo->{'name'}\n");
+    #   my $tname= $tinfo->{name} . ' ' . $tinfo->{combination};
+    #   my $crit= $tinfo->{criteria};
+    #   print("$tname\n\t$crit\n");
     # }
-
   }
 
   if (defined $print_testcases){
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 651dabcd408..b4cd53cff09 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -662,22 +662,40 @@ sub run_test_server ($$$) {
 	    next;
 	  }
 
-	  # Prefer same configuration, or just use next if --noreorder
-	  if (!$opt_reorder or (defined $result and
-	      $result->{template_path} eq $t->{template_path}))
-	  {
-	    #mtr_report("Test uses same config => good match");
-	    # Test uses same config => good match
-	    $next= splice(@$tests, $i, 1);
-	    last;
-	  }
-
 	  # Second best choice is the first that does not fulfill
 	  # any of the above conditions
 	  if (!defined $second_best){
 	    #mtr_report("Setting second_best to $i");
 	    $second_best= $i;
 	  }
+
+	  # Smart allocation of next test within this thread.
+
+	  if ($opt_reorder and $opt_parallel > 1 and defined $result)
+	  {
+	    my $wid= $result->{worker};
+	    # Reserved for other thread, try next
+	    next if (defined $t->{reserved} and $t->{reserved} != $wid);
+	    if (! defined $t->{reserved})
+	    {
+	      # Force-restart not relevant when comparing *next* test
+	      $t->{criteria} =~ s/force-restart$/no-restart/;
+	      my $criteria= $t->{criteria};
+	      # Reserve similar tests for this worker, but not too many
+	      my $maxres= (@$tests - $i) / $opt_parallel + 1;
+	      for (my $j= $i+1; $j <= $i + $maxres; $j++)
+	      {
+		my $tt= $tests->[$j];
+		last unless defined $tt;
+		last if $tt->{criteria} ne $criteria;
+		$tt->{reserved}= $wid;
+	      }
+	    }
+	  }
+
+	  # At this point we have found next suitable test
+	  $next= splice(@$tests, $i, 1);
+	  last;
 	}
 
 	# Use second best choice if no other test has been found
@@ -686,10 +704,12 @@ sub run_test_server ($$$) {
 	  mtr_error("Internal error, second best too large($second_best)")
 	    if $second_best >  $#$tests;
 	  $next= splice(@$tests, $second_best, 1);
+	  delete $next->{reserved};
 	}
 
 	if ($next) {
-	  #$next->print_test();
+	  # We don't need this any more
+	  delete $next->{criteria};
 	  $next->write_test($sock, 'TESTCASE');
 	  $running{$next->key()}= $next;
 	  $num_ndb_tests++ if ($next->{ndb_test});
@@ -772,6 +792,11 @@ sub run_worker ($) {
       delete($test->{'comment'});
       delete($test->{'logfile'});
 
+      # A sanity check. Should this happen often we need to look at it.
+      if (defined $test->{reserved} && $test->{reserved} != $thread_num) {
+	my $tres= $test->{reserved};
+	mtr_warning("Test reserved for w$tres picked up by w$thread_num");
+      }
       $test->{worker} = $thread_num if $opt_parallel > 1;
 
       run_testcase($test);
@@ -4575,17 +4600,6 @@ sub server_need_restart {
     }
   }
 
-  # Temporary re-enable the "always restart slave" hack
-  # this should be removed asap, but will require that each rpl
-  # testcase cleanup better after itself - ie. stop and reset
-  # replication
-  # Use the "#!use-slave-opt" marker to detect that this is a "slave"
-  # server
-  if ( $server->option("#!use-slave-opt") ){
-    mtr_verbose_restart($server, "Always restart slave(s)");
-    return 1;
-  }
-
   my $is_mysqld= grep ($server eq $_, mysqlds());
   if ($is_mysqld)
   {
diff --git a/mysql-test/suite/binlog/t/binlog_index-master.opt b/mysql-test/suite/binlog/t/binlog_index-master.opt
new file mode 100644
index 00000000000..cef79bc8585
--- /dev/null
+++ b/mysql-test/suite/binlog/t/binlog_index-master.opt
@@ -0,0 +1 @@
+--force-restart
diff --git a/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt b/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt
index ad2c6a647b5..377e2114039 100644
--- a/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt
+++ b/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt
@@ -1 +1,2 @@
 -O max_binlog_size=4096
+--force-restart
diff --git a/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt b/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt
index e2cfcb299cf..2a1187d3fe4 100644
--- a/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt
+++ b/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt
@@ -1 +1,2 @@
 --binlog-do-db=b42829
+--force-restart
diff --git a/mysql-test/suite/rpl/r/rpl_ignore_table.result b/mysql-test/suite/rpl/r/rpl_ignore_table.result
index e77be425270..6b845ddcac9 100644
--- a/mysql-test/suite/rpl/r/rpl_ignore_table.result
+++ b/mysql-test/suite/rpl/r/rpl_ignore_table.result
@@ -141,3 +141,4 @@ HEX(word)
 SELECT * FROM tmptbl504451f4258$1;
 ERROR 42S02: Table 'test.tmptbl504451f4258$1' doesn't exist
 DROP TABLE t5;
+call mtr.force_restart();
diff --git a/mysql-test/suite/rpl/t/rpl_cross_version-master.opt b/mysql-test/suite/rpl/t/rpl_cross_version-master.opt
index 0ea05290c11..2b2d357b124 100644
--- a/mysql-test/suite/rpl/t/rpl_cross_version-master.opt
+++ b/mysql-test/suite/rpl/t/rpl_cross_version-master.opt
@@ -1 +1,2 @@
 --replicate-same-server-id --relay-log=slave-relay-bin --secure-file-priv=$MYSQL_TMP_DIR
+--force-restart
diff --git a/mysql-test/suite/rpl/t/rpl_current_user-master.opt b/mysql-test/suite/rpl/t/rpl_current_user-master.opt
new file mode 100644
index 00000000000..cef79bc8585
--- /dev/null
+++ b/mysql-test/suite/rpl/t/rpl_current_user-master.opt
@@ -0,0 +1 @@
+--force-restart
diff --git a/mysql-test/suite/rpl/t/rpl_ignore_table.test b/mysql-test/suite/rpl/t/rpl_ignore_table.test
index 66f96e8f4e8..b5666ad6e91 100644
--- a/mysql-test/suite/rpl/t/rpl_ignore_table.test
+++ b/mysql-test/suite/rpl/t/rpl_ignore_table.test
@@ -174,3 +174,5 @@ SELECT * FROM tmptbl504451f4258$1;
 connection master;
 DROP TABLE t5;
 sync_slave_with_master;
+
+call mtr.force_restart();
diff --git a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh
index 066f72926af..e5bb3e61d11 100644
--- a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh
+++ b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh
@@ -1 +1,2 @@
+rm -f $MYSQLTEST_VARDIR/std_data_master_link
 ln -s $MYSQLTEST_VARDIR/std_data $MYSQLTEST_VARDIR/std_data_master_link
diff --git a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh
index 218209a2542..7a0c0bb382a 100644
--- a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh
+++ b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh
@@ -1 +1,2 @@
+rm -f $MYSQLTEST_VARDIR/std_data_slave_link
 ln -s $MYSQLTEST_VARDIR/std_data $MYSQLTEST_VARDIR/std_data_slave_link
diff --git a/mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt b/mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt
new file mode 100644
index 00000000000..cef79bc8585
--- /dev/null
+++ b/mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt
@@ -0,0 +1 @@
+--force-restart
diff --git a/mysql-test/t/key_cache-master.opt b/mysql-test/t/key_cache-master.opt
index 66e19c18a8a..6398e3e0a26 100644
--- a/mysql-test/t/key_cache-master.opt
+++ b/mysql-test/t/key_cache-master.opt
@@ -1 +1,2 @@
 --key_buffer_size=2M --small.key_buffer_size=256K --small.key_buffer_size=128K
+--force-restart
diff --git a/mysql-test/t/mysqlbinlog-master.opt b/mysql-test/t/mysqlbinlog-master.opt
index ac1a87c73b3..a9f4a6010d8 100644
--- a/mysql-test/t/mysqlbinlog-master.opt
+++ b/mysql-test/t/mysqlbinlog-master.opt
@@ -1 +1,2 @@
 --max-binlog-size=4096
+--force-restart