Fix for BUG#20716: SHOW INSTANCES statement causes races in IM tests.

  
Fix for the bug in mysql-test-run.pl which prevents other tests succeed
after IM-test failure.
  
The idea of the fix of BUG#20716 is to:
  1. Check each SHOW INSTANCES statement, add necessary "sleep" instruction before;
  2. Move all environment checkings into the one file and include it everywhere.


mysql-test/mysql-test-run.pl:
  Fix bug in mysql-test-run.pl -- kill leftovers if some
  guarded mysqld-instance is still alive after IM shutdown.
mysql-test/r/im_daemon_life_cycle.result:
  Updated result file.
mysql-test/r/im_life_cycle.result:
  Updated result file.
mysql-test/r/im_options_set.result:
  Updated result file.
mysql-test/r/im_options_unset.result:
  Updated result file.
mysql-test/r/im_utils.result:
  Updated result file.
mysql-test/t/im_daemon_life_cycle.imtest:
  Include im_check_env.inc for the checking of environment.
mysql-test/t/im_life_cycle.imtest:
  Include im_check_env.inc for the checking of environment.
mysql-test/t/im_options_set.imtest:
  Include im_check_env.inc for the checking of environment.
mysql-test/t/im_options_unset.imtest:
  Include im_check_env.inc for the checking of environment.
mysql-test/t/im_utils.imtest:
  Include im_check_env.inc for the checking of environment.
mysql-test/include/im_check_env.inc:
  A new file to be included in each IM-test.
  The statements in the file ensure that starting
  conditions (environment) are as expected.

12 files changed, 98 insertions, 149 deletions

diff --git a/mysql-test/include/im_check_env.inc b/mysql-test/include/im_check_env.inc
new file mode 100644
index 00000000000..169edbac6b3
--- /dev/null
+++ b/mysql-test/include/im_check_env.inc
@@ -0,0 +1,27 @@
+# This file is intended to be used in each IM-test. It contains stamements,
+# that ensure that starting conditions (environment) for the IM-test are as
+# expected.
+
+# Wait for mysqld1 (guarded instance) to start.
+
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
+
+# Check the running instances.
+
+--connect (mysql1_con,localhost,root,,mysql,$IM_MYSQLD1_PORT,$IM_MYSQLD1_SOCK)
+
+--connection mysql1_con
+
+SHOW VARIABLES LIKE 'server_id';
+
+--connection default
+
+# Let IM detect that mysqld1 is online. This delay should be longer than
+# monitoring interval.
+
+--sleep 2
+
+# Check that IM understands that mysqld1 is online, while mysqld2 is
+# offline.
+
+SHOW INSTANCES;
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 14bd9b198a2..5a7609a55fb 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -2898,12 +2898,16 @@ sub im_stop($) {
 
   while (1)
   {
+    # Check that IM-main died.
+
     if (kill (0, $instance_manager->{'pid'}))
     {
       mtr_debug("IM-main is still alive.");
       last;
     }
 
+    # Check that IM-angel died.
+
     if (defined $instance_manager->{'angel_pid'} &&
         kill (0, $instance_manager->{'angel_pid'}))
     {
@@ -2911,21 +2915,39 @@ sub im_stop($) {
       last;
     }
 
+    # Check that all guarded mysqld-instances died.
+
+    my $guarded_mysqlds_dead= 1;
+
     foreach my $pid (@mysqld_pids)
     {
       if (kill (0, $pid))
       {
         mtr_debug("Guarded mysqld ($pid) is still alive.");
+        $guarded_mysqlds_dead= 0;
         last;
       }
     }
 
+    last unless $guarded_mysqlds_dead;
+
+    # Ok, all necessary processes are dead.
+
     $clean_shutdown= 1;
     last;
   }
 
   # Kill leftovers (the order is important).
 
+  if ($clean_shutdown)
+  {
+    mtr_debug("IM-shutdown was clean -- all processed died.");
+  }
+  else
+  {
+    mtr_debug("IM failed to shutdown gracefully. We have to clean the mess...");
+  }
+
   unless ($clean_shutdown)
   {
 
diff --git a/mysql-test/r/im_daemon_life_cycle.result b/mysql-test/r/im_daemon_life_cycle.result
index ea27fcb6db1..4f7dd77a88f 100644
--- a/mysql-test/r/im_daemon_life_cycle.result
+++ b/mysql-test/r/im_daemon_life_cycle.result
@@ -1,4 +1,7 @@
 Success: the process has been started.
+SHOW VARIABLES LIKE 'server_id';
+Variable_name	Value
+server_id	1
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
diff --git a/mysql-test/r/im_life_cycle.result b/mysql-test/r/im_life_cycle.result
index a9f78aea7d3..8394ec491e5 100644
--- a/mysql-test/r/im_life_cycle.result
+++ b/mysql-test/r/im_life_cycle.result
@@ -1,8 +1,7 @@
-
---------------------------------------------------------------------
--- 1.1.1.
---------------------------------------------------------------------
 Success: the process has been started.
+SHOW VARIABLES LIKE 'server_id';
+Variable_name	Value
+server_id	1
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
@@ -40,10 +39,6 @@ ERROR HY000: Bad instance name. Check that the instance with such a name exists
 --------------------------------------------------------------------
 -- 1.1.6.
 --------------------------------------------------------------------
-SHOW INSTANCES;
-instance_name	status
-mysqld1	online
-mysqld2	offline
 Killing the process...
 Sleeping...
 Success: the process was restarted.
diff --git a/mysql-test/r/im_options_set.result b/mysql-test/r/im_options_set.result
index 5e6c740624e..c3035079b39 100644
--- a/mysql-test/r/im_options_set.result
+++ b/mysql-test/r/im_options_set.result
@@ -1,8 +1,11 @@
-server_id           = 1
-server_id           = 2
+Success: the process has been started.
 SHOW VARIABLES LIKE 'server_id';
 Variable_name	Value
 server_id	1
+SHOW INSTANCES;
+instance_name	status
+mysqld1	online
+mysqld2	offline
 SET mysqld1.server_id = 11;
 server_id =11
 server_id           = 2
diff --git a/mysql-test/r/im_options_unset.result b/mysql-test/r/im_options_unset.result
index bf54025edb7..ba468c78a5b 100644
--- a/mysql-test/r/im_options_unset.result
+++ b/mysql-test/r/im_options_unset.result
@@ -1,8 +1,11 @@
-server_id           = 1
-server_id           = 2
+Success: the process has been started.
 SHOW VARIABLES LIKE 'server_id';
 Variable_name	Value
 server_id	1
+SHOW INSTANCES;
+instance_name	status
+mysqld1	online
+mysqld2	offline
 UNSET mysqld1.server_id;
 server_id           = 2
 SHOW VARIABLES LIKE 'server_id';
diff --git a/mysql-test/r/im_utils.result b/mysql-test/r/im_utils.result
index e6a5e007ed4..be696921812 100644
--- a/mysql-test/r/im_utils.result
+++ b/mysql-test/r/im_utils.result
@@ -1,4 +1,7 @@
 Success: the process has been started.
+SHOW VARIABLES LIKE 'server_id';
+Variable_name	Value
+server_id	1
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
diff --git a/mysql-test/t/im_daemon_life_cycle.imtest b/mysql-test/t/im_daemon_life_cycle.imtest
index 3afc36935f8..fe2345a9987 100644
--- a/mysql-test/t/im_daemon_life_cycle.imtest
+++ b/mysql-test/t/im_daemon_life_cycle.imtest
@@ -7,21 +7,7 @@
 ###########################################################################
 
 --source include/im_check_os.inc
-
-###########################################################################
-
-# Wait for mysqld1 (guarded instance) to start.
-
---exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
-
-# Let IM detect that mysqld1 is online. This delay should be longer than
-# monitoring interval.
-
---sleep 3
-
-# Check that start conditions are as expected.
-
-SHOW INSTANCES;
+--source include/im_check_env.inc
 
 ###########################################################################
 
diff --git a/mysql-test/t/im_life_cycle.imtest b/mysql-test/t/im_life_cycle.imtest
index 2cbe53a7b28..35258396415 100644
--- a/mysql-test/t/im_life_cycle.imtest
+++ b/mysql-test/t/im_life_cycle.imtest
@@ -7,33 +7,7 @@
 ###########################################################################
 
 --source include/im_check_os.inc
-
-###########################################################################
-#
-# 1.1.1. Check that Instance Manager is able:
-#  - to read definitions of two mysqld-instances;
-#  - to start the first instance;
-#  - to understand 'nonguarded' option and keep the second instance down;
-#
-###########################################################################
-
---echo
---echo --------------------------------------------------------------------
---echo -- 1.1.1.
---echo --------------------------------------------------------------------
-
-# Wait for mysqld1 (guarded instance) to start.
-
---exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
-
-# Let IM detect that mysqld1 is online. This delay should be longer than
-# monitoring interval.
-
---sleep 3
-
-# Check that start conditions are as expected.
-
-SHOW INSTANCES;
+--source include/im_check_env.inc
 
 ###########################################################################
 #
@@ -54,9 +28,10 @@ START INSTANCE mysqld2;
 # FIXME: START INSTANCE should be synchronous.
 --exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 started
 
-# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
-# synchronous. Even waiting for mysqld to start by looking at its pid file is
-# not enough, because IM may not detect that mysqld has started.
+# FIXME: Result of SHOW INSTANCES here is not deterministic unless START
+# INSTANCE is synchronous. Even waiting for mysqld to start by looking at
+# its pid file is not enough, because it is unknown when IM detects that
+# mysqld has started.
 # SHOW INSTANCES;
 
 --connect (mysql_con,localhost,root,,mysql,$IM_MYSQLD2_PORT,$IM_MYSQLD2_SOCK)
@@ -86,9 +61,10 @@ STOP INSTANCE mysqld2;
 # FIXME: STOP INSTANCE should be synchronous.
 --exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 stopped
 
-# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
-# synchronous. Even waiting for mysqld to start by looking at its pid file is
-# not enough, because IM may not detect that mysqld has started.
+# FIXME: Result of SHOW INSTANCES here is not deterministic unless START
+# INSTANCE is synchronous. Even waiting for mysqld to start by looking at
+# its pid file is not enough, because it is unknown when IM detects that
+# mysqld has started.
 # SHOW INSTANCES;
 
 ###########################################################################
@@ -114,8 +90,8 @@ START INSTANCE mysqld1;
 
 ###########################################################################
 #
-# 1.1.5. Check that Instance Manager reports correct errors for 'STOP INSTANCE'
-# command:
+# 1.1.5. Check that Instance Manager reports correct errors for
+#        'STOP INSTANCE' command:
 #   - if the client tries to start unregistered instance;
 #   - if the client tries to start already stopped instance;
 #   - if the client submits invalid arguments;
@@ -146,12 +122,10 @@ STOP INSTANCE mysqld3;
 --echo -- 1.1.6.
 --echo --------------------------------------------------------------------
 
-SHOW INSTANCES;
-
 --exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD1_PATH_PID restarted 30
 
-# Give some time to IM to detect that mysqld was restarted. It should be longer
-# than monitoring interval.
+# Give some time to IM to detect that mysqld was restarted. It should be
+# longer than monitoring interval.
 
 --sleep 3
 
@@ -172,16 +146,18 @@ START INSTANCE mysqld2;
 # FIXME: START INSTANCE should be synchronous.
 --exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 started
 
-# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
-# synchronous. Even waiting for mysqld to start by looking at its pid file is
-# not enough, because IM may not detect that mysqld has started.
+# FIXME: Result of SHOW INSTANCES here is not deterministic unless START
+# INSTANCE is synchronous. Even waiting for mysqld to start by looking at
+# its pid file is not enough, because it is unknown when IM detects that
+# mysqld has started.
 # SHOW INSTANCES;
 
 --exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD2_PATH_PID killed 10
 
-# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
-# synchronous. Even waiting for mysqld to start by looking at its pid file is
-# not enough, because IM may not detect that mysqld has started.
+# FIXME: Result of SHOW INSTANCES here is not deterministic unless START
+# INSTANCE is synchronous. Even waiting for mysqld to start by looking at
+# its pid file is not enough, because it is unknown when IM detects that
+# mysqld has started.
 # SHOW INSTANCES;
 
 ###########################################################################
diff --git a/mysql-test/t/im_options_set.imtest b/mysql-test/t/im_options_set.imtest
index a9b64861f99..76e209b6a66 100644
--- a/mysql-test/t/im_options_set.imtest
+++ b/mysql-test/t/im_options_set.imtest
@@ -39,32 +39,7 @@
 ###########################################################################
 
 --source include/im_check_os.inc
-
-###########################################################################
-#
-# 0. Check starting conditions.
-#
-###########################################################################
-
-# - check the configuration file;
-
---exec grep server_id $MYSQLTEST_VARDIR/im.cnf ;
-
-# - check the running instances.
-
---connect (mysql1_con,localhost,root,,mysql,$IM_MYSQLD1_PORT,$IM_MYSQLD1_SOCK)
-
---connection mysql1_con
-
-SHOW VARIABLES LIKE 'server_id';
-
---connection default
-
-# - check the internal cache.
-#   TODO: we should check only server_id option here.
-
-# SHOW INSTANCE OPTIONS mysqld1;
-# SHOW INSTANCE OPTIONS mysqld2;
+--source include/im_check_env.inc
 
 ###########################################################################
 #
diff --git a/mysql-test/t/im_options_unset.imtest b/mysql-test/t/im_options_unset.imtest
index 40629805d45..06f59e79ffe 100644
--- a/mysql-test/t/im_options_unset.imtest
+++ b/mysql-test/t/im_options_unset.imtest
@@ -46,32 +46,7 @@
 ###########################################################################
 
 --source include/im_check_os.inc
-
-###########################################################################
-#
-# 0. Check starting conditions.
-#
-###########################################################################
-
-# - check the configuration file;
-
---exec grep server_id $MYSQLTEST_VARDIR/im.cnf ;
-
-# - check the running instances.
-
---connect (mysql1_con,localhost,root,,mysql,$IM_MYSQLD1_PORT,$IM_MYSQLD1_SOCK)
-
---connection mysql1_con
-
-SHOW VARIABLES LIKE 'server_id';
-
---connection default
-
-# - check the internal cache.
-#   TODO: we should check only server_id option here.
-
-# SHOW INSTANCE OPTIONS mysqld1;
-# SHOW INSTANCE OPTIONS mysqld2;
+--source include/im_check_env.inc
 
 ###########################################################################
 #
diff --git a/mysql-test/t/im_utils.imtest b/mysql-test/t/im_utils.imtest
index 47902eeba52..4c05b342af5 100644
--- a/mysql-test/t/im_utils.imtest
+++ b/mysql-test/t/im_utils.imtest
@@ -7,36 +7,17 @@
 ###########################################################################
 
 --source include/im_check_os.inc
+--source include/im_check_env.inc
 
 ###########################################################################
 
 #
-# Check starting conditions. This test case assumes that:
-#  - two mysqld-instances are registered;
-#  - the first instance is online;
-#  - the second instance is offline;
+# Check 'SHOW INSTANCE OPTIONS' command.
 #
-
-# Wait for mysqld1 (guarded instance) to start.
-
---exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
-
-# Let IM detect that mysqld1 is online. This delay should be longer than
-# monitoring interval.
-
---sleep 3
-
-# Check that start conditions are as expected.
-
-SHOW INSTANCES;
-
-#
-# Check 'SHOW INSTANCE OPTIONS' command:
-#  - check that options of both offline and online instances are accessible;
-#  - since configuration of an mysqld-instance contains directories, we should
-#    completely ignore the second column (values) in order to make the test
-#    case produce the same results on different installations;
-#    TODO: ignore values of only directory-specific options.
+# Since configuration of an mysqld-instance contains directories, we should
+# completely ignore the second column (values) in order to make the test
+# case produce the same results on different installations;
+# TODO: ignore values of only directory-specific options.
 #
 
 --replace_column 2 VALUE