Fix of test suite in scope of fixing BUG#18023: IM: instance

can be started several times; monitor interval must be > 2sec


mysql-test/r/im_daemon_life_cycle.result:
  Updated result file.
mysql-test/r/im_life_cycle.result:
  Updated result file.
mysql-test/r/im_utils.result:
  Updated result file.
mysql-test/t/im_daemon_life_cycle-im.opt:
  Set monitoring interval to 1 second in order to:
    - be consistent with 5.1;
    - speed up tests;
mysql-test/t/im_daemon_life_cycle.imtest:
  1. Use wait_for_start.sh script to minimize chance of race condition.
  2. Polishing.
mysql-test/t/im_life_cycle.imtest:
  1. Use wait_for_start.sh and wait_for_stop.sh scripts to
     minimize chance of race condition;
  2. Remove some statements, because there is no way now to
     stabilize their output.
  3. Polishing;
mysql-test/t/im_utils.imtest:
  1. Use wait_for_start.sh script to minimize chance of race condition.
  2. Polishing.
mysql-test/t/kill_n_check.sh:
  1. Make timeout configurable by command-line argument;
  2. Change algorithm of waiting for process to restart to be
     more robust.
mysql-test/t/im_life_cycle-im.opt:
  Set monitoring interval to 1 second in order to:
    - be consistent with 5.1;
    - speed up tests;
mysql-test/t/im_utils-im.opt:
  Set monitoring interval to 1 second in order to:
    - be consistent with 5.1;
    - speed up tests;
mysql-test/t/wait_for_process.sh:
  A new helper script, intended to be used instead of dummy "sleep"
  when waiting for some process to start or stop.

11 files changed, 319 insertions, 111 deletions

diff --git a/mysql-test/r/im_daemon_life_cycle.result b/mysql-test/r/im_daemon_life_cycle.result
index d0a76b450fe..ea27fcb6db1 100644
--- a/mysql-test/r/im_daemon_life_cycle.result
+++ b/mysql-test/r/im_daemon_life_cycle.result
@@ -1,3 +1,4 @@
+Success: the process has been started.
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
diff --git a/mysql-test/r/im_life_cycle.result b/mysql-test/r/im_life_cycle.result
index 36277f6aa28..a9f78aea7d3 100644
--- a/mysql-test/r/im_life_cycle.result
+++ b/mysql-test/r/im_life_cycle.result
@@ -1,44 +1,45 @@
+
+--------------------------------------------------------------------
+-- 1.1.1.
+--------------------------------------------------------------------
+Success: the process has been started.
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
 mysqld2	offline
-SHOW INSTANCE STATUS mysqld1;
-instance_name	status	version
-mysqld1	online	VERSION
-SHOW INSTANCE STATUS mysqld2;
-instance_name	status	version
-mysqld2	offline	VERSION
+
+--------------------------------------------------------------------
+-- 1.1.2.
+--------------------------------------------------------------------
 START INSTANCE mysqld2;
-SHOW INSTANCES;
-instance_name	status
-mysqld1	online
-mysqld2	online
-SHOW INSTANCE STATUS mysqld1;
-instance_name	status	version
-mysqld1	online	VERSION
-SHOW INSTANCE STATUS mysqld2;
-instance_name	status	version
-mysqld2	online	VERSION
+Success: the process has been started.
 SHOW VARIABLES LIKE 'port';
 Variable_name	Value
-port	IM_MYSQLD1_PORT
+port	IM_MYSQLD2_PORT
+
+--------------------------------------------------------------------
+-- 1.1.3.
+--------------------------------------------------------------------
 STOP INSTANCE mysqld2;
-SHOW INSTANCES;
-instance_name	status
-mysqld1	online
-mysqld2	offline
-SHOW INSTANCE STATUS mysqld1;
-instance_name	status	version
-mysqld1	online	VERSION
-SHOW INSTANCE STATUS mysqld2;
-instance_name	status	version
-mysqld2	offline	VERSION
+Success: the process has been stopped.
+
+--------------------------------------------------------------------
+-- 1.1.4.
+--------------------------------------------------------------------
 START INSTANCE mysqld3;
 ERROR HY000: Bad instance name. Check that the instance with such a name exists
 START INSTANCE mysqld1;
 ERROR HY000: The instance is already started
+
+--------------------------------------------------------------------
+-- 1.1.5.
+--------------------------------------------------------------------
 STOP INSTANCE mysqld3;
 ERROR HY000: Bad instance name. Check that the instance with such a name exists
+
+--------------------------------------------------------------------
+-- 1.1.6.
+--------------------------------------------------------------------
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
@@ -50,20 +51,25 @@ SHOW INSTANCES;
 instance_name	status
 mysqld1	online
 mysqld2	offline
+
+--------------------------------------------------------------------
+-- 1.1.7.
+--------------------------------------------------------------------
 START INSTANCE mysqld2;
-SHOW INSTANCES;
-instance_name	status
-mysqld1	online
-mysqld2	online
+Success: the process has been started.
 Killing the process...
 Sleeping...
 Success: the process was killed.
-SHOW INSTANCES;
-instance_name	status
-mysqld1	online
-mysqld2	offline
+
+--------------------------------------------------------------------
+-- 1.1.8.
+--------------------------------------------------------------------
 SHOW INSTANCE STATUS;
 ERROR 42000: You have an error in your command syntax. Check the manual that corresponds to your MySQL Instance Manager version for the right syntax to use
+
+--------------------------------------------------------------------
+-- BUG#12813
+--------------------------------------------------------------------
 START INSTANCE mysqld1,mysqld2,mysqld3;
 ERROR 42000: You have an error in your command syntax. Check the manual that corresponds to your MySQL Instance Manager version for the right syntax to use
 STOP INSTANCE mysqld1,mysqld2,mysqld3;
diff --git a/mysql-test/r/im_utils.result b/mysql-test/r/im_utils.result
index fbfaeaebcac..e6a5e007ed4 100644
--- a/mysql-test/r/im_utils.result
+++ b/mysql-test/r/im_utils.result
@@ -1,3 +1,4 @@
+Success: the process has been started.
 SHOW INSTANCES;
 instance_name	status
 mysqld1	online
@@ -42,7 +43,9 @@ skip-innodb	VALUE
 skip-bdb	VALUE
 skip-ndbcluster	VALUE
 START INSTANCE mysqld2;
+Success: the process has been started.
 STOP INSTANCE mysqld2;
+Success: the process has been stopped.
 SHOW mysqld1 LOG FILES;
 Logfile	Path	File size
 ERROR LOG	PATH	FILE_SIZE
diff --git a/mysql-test/t/im_daemon_life_cycle-im.opt b/mysql-test/t/im_daemon_life_cycle-im.opt
index 21c01191e4c..3a45c7a41f7 100644
--- a/mysql-test/t/im_daemon_life_cycle-im.opt
+++ b/mysql-test/t/im_daemon_life_cycle-im.opt
@@ -1,2 +1,3 @@
 --run-as-service
 --log=$MYSQLTEST_VARDIR/log/im.log
+--monitoring-interval=1
diff --git a/mysql-test/t/im_daemon_life_cycle.imtest b/mysql-test/t/im_daemon_life_cycle.imtest
index 87388d7c1e6..3afc36935f8 100644
--- a/mysql-test/t/im_daemon_life_cycle.imtest
+++ b/mysql-test/t/im_daemon_life_cycle.imtest
@@ -10,6 +10,22 @@
 
 ###########################################################################
 
+# Wait for mysqld1 (guarded instance) to start.
+
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
+
+# Let IM detect that mysqld1 is online. This delay should be longer than
+# monitoring interval.
+
+--sleep 3
+
+# Check that start conditions are as expected.
+
 SHOW INSTANCES;
 
---exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_PATH_PID restarted
+###########################################################################
+
+# Kill the IM main process and check that the IM Angel will restart the main
+# process.
+
+--exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_PATH_PID restarted 30
diff --git a/mysql-test/t/im_life_cycle-im.opt b/mysql-test/t/im_life_cycle-im.opt
new file mode 100644
index 00000000000..34b74ce0c95
--- /dev/null
+++ b/mysql-test/t/im_life_cycle-im.opt
@@ -0,0 +1 @@
+--monitoring-interval=1
diff --git a/mysql-test/t/im_life_cycle.imtest b/mysql-test/t/im_life_cycle.imtest
index 445ae7f72fa..2cbe53a7b28 100644
--- a/mysql-test/t/im_life_cycle.imtest
+++ b/mysql-test/t/im_life_cycle.imtest
@@ -17,11 +17,23 @@
 #
 ###########################################################################
 
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.1.
+--echo --------------------------------------------------------------------
+
+# Wait for mysqld1 (guarded instance) to start.
+
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
+
+# Let IM detect that mysqld1 is online. This delay should be longer than
+# monitoring interval.
+
+--sleep 3
+
+# Check that start conditions are as expected.
+
 SHOW INSTANCES;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld1;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld2;
 
 ###########################################################################
 #
@@ -33,20 +45,24 @@ SHOW INSTANCE STATUS mysqld2;
 #
 ###########################################################################
 
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.2.
+--echo --------------------------------------------------------------------
+
 START INSTANCE mysqld2;
-# FIXME
---sleep 3
+# FIXME: START INSTANCE should be synchronous.
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 started
 
-SHOW INSTANCES;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld1;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld2;
+# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
+# synchronous. Even waiting for mysqld to start by looking at its pid file is
+# not enough, because IM may not detect that mysqld has started.
+# SHOW INSTANCES;
 
---connect (mysql_con,localhost,root,,mysql,$IM_MYSQLD1_PORT,$IM_MYSQLD1_SOCK)
+--connect (mysql_con,localhost,root,,mysql,$IM_MYSQLD2_PORT,$IM_MYSQLD2_SOCK)
 --connection mysql_con
 
---replace_result $IM_MYSQLD1_PORT IM_MYSQLD1_PORT
+--replace_result $IM_MYSQLD2_PORT IM_MYSQLD2_PORT
 SHOW VARIABLES LIKE 'port';
 
 --connection default
@@ -61,15 +77,19 @@ SHOW VARIABLES LIKE 'port';
 #
 ###########################################################################
 
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.3.
+--echo --------------------------------------------------------------------
+
 STOP INSTANCE mysqld2;
-# FIXME
---sleep 3
+# FIXME: STOP INSTANCE should be synchronous.
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 stopped
 
-SHOW INSTANCES;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld1;
---replace_column 3 VERSION
-SHOW INSTANCE STATUS mysqld2;
+# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
+# synchronous. Even waiting for mysqld to start by looking at its pid file is
+# not enough, because IM may not detect that mysqld has started.
+# SHOW INSTANCES;
 
 ###########################################################################
 #
@@ -81,16 +101,17 @@ SHOW INSTANCE STATUS mysqld2;
 #
 ###########################################################################
 
---error 3000
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.4.
+--echo --------------------------------------------------------------------
+
+--error 3000 # ER_BAD_INSTANCE_NAME
 START INSTANCE mysqld3;
 
---error 3002
+--error 3002 # ER_INSTANCE_ALREADY_STARTED
 START INSTANCE mysqld1;
 
-# FIXME TODO
-# BUG#12813: START/STOP INSTANCE commands accept a list as argument
-# START INSTANCE mysqld1, mysqld2;
-
 ###########################################################################
 #
 # 1.1.5. Check that Instance Manager reports correct errors for 'STOP INSTANCE'
@@ -101,27 +122,40 @@ START INSTANCE mysqld1;
 #
 ###########################################################################
 
---error 3000
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.5.
+--echo --------------------------------------------------------------------
+
+--error 3000 # ER_BAD_INSTANCE_NAME
 STOP INSTANCE mysqld3;
 
 # TODO: IM should be fixed.
 # BUG#12673: Instance Manager allows to stop the instance many times
-# --error 3002
+# --error 3002 # ER_INSTANCE_ALREADY_STARTED
 # STOP INSTANCE mysqld2;
 
-# FIXME TODO
-# BUG#12813: START/STOP INSTANCE commands accept a list as argument
-# STOP INSTANCE mysqld1, mysqld2;
-
 ###########################################################################
 #
 # 1.1.6. Check that Instance Manager is able to restart guarded instances.
 #
 ###########################################################################
 
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.6.
+--echo --------------------------------------------------------------------
+
 SHOW INSTANCES;
 
---exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD1_PATH_PID restarted
+--exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD1_PATH_PID restarted 30
+
+# Give some time to IM to detect that mysqld was restarted. It should be longer
+# than monitoring interval.
+
+--sleep 3
+
+SHOW INSTANCES;
 
 ###########################################################################
 #
@@ -129,17 +163,26 @@ SHOW INSTANCES;
 #
 ###########################################################################
 
-SHOW INSTANCES;
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.7.
+--echo --------------------------------------------------------------------
 
 START INSTANCE mysqld2;
-# FIXME
---sleep 3
+# FIXME: START INSTANCE should be synchronous.
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 started
 
-SHOW INSTANCES;
+# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
+# synchronous. Even waiting for mysqld to start by looking at its pid file is
+# not enough, because IM may not detect that mysqld has started.
+# SHOW INSTANCES;
 
---exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD2_PATH_PID killed
+--exec $MYSQL_TEST_DIR/t/kill_n_check.sh $IM_MYSQLD2_PATH_PID killed 10
 
-SHOW INSTANCES;
+# FIXME: SHOW INSTANCES is not deterministic unless START INSTANCE is
+# synchronous. Even waiting for mysqld to start by looking at its pid file is
+# not enough, because IM may not detect that mysqld has started.
+# SHOW INSTANCES;
 
 ###########################################################################
 #
@@ -147,7 +190,13 @@ SHOW INSTANCES;
 # incomplete SHOW INSTANCE STATUS command.
 #
 ###########################################################################
---error 1149
+
+--echo
+--echo --------------------------------------------------------------------
+--echo -- 1.1.8.
+--echo --------------------------------------------------------------------
+
+--error ER_SYNTAX_ERROR
 SHOW INSTANCE STATUS;
 
 #
@@ -159,8 +208,13 @@ SHOW INSTANCE STATUS;
 #                                 a list as argument.
 #
 
---error 1149
+--echo
+--echo --------------------------------------------------------------------
+--echo -- BUG#12813
+--echo --------------------------------------------------------------------
+
+--error ER_SYNTAX_ERROR
 START INSTANCE mysqld1,mysqld2,mysqld3;
 
---error 1149
+--error ER_SYNTAX_ERROR
 STOP INSTANCE mysqld1,mysqld2,mysqld3;
diff --git a/mysql-test/t/im_utils-im.opt b/mysql-test/t/im_utils-im.opt
new file mode 100644
index 00000000000..34b74ce0c95
--- /dev/null
+++ b/mysql-test/t/im_utils-im.opt
@@ -0,0 +1 @@
+--monitoring-interval=1
diff --git a/mysql-test/t/im_utils.imtest b/mysql-test/t/im_utils.imtest
index dc6fb93c4ff..47902eeba52 100644
--- a/mysql-test/t/im_utils.imtest
+++ b/mysql-test/t/im_utils.imtest
@@ -17,6 +17,17 @@
 #  - the second instance is offline;
 #
 
+# Wait for mysqld1 (guarded instance) to start.
+
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD1_PATH_PID 30 started
+
+# Let IM detect that mysqld1 is online. This delay should be longer than
+# monitoring interval.
+
+--sleep 3
+
+# Check that start conditions are as expected.
+
 SHOW INSTANCES;
 
 #
@@ -40,11 +51,10 @@ SHOW INSTANCE OPTIONS mysqld2;
 #
 
 START INSTANCE mysqld2;
-
-# FIXME
--- sleep 3
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 started
 
 STOP INSTANCE mysqld2;
+--exec $MYSQL_TEST_DIR/t/wait_for_process.sh $IM_MYSQLD2_PATH_PID 30 stopped
 
 #
 # Check 'SHOW LOG FILES' command:
diff --git a/mysql-test/t/kill_n_check.sh b/mysql-test/t/kill_n_check.sh
index e722b3a180d..64cc869d1ec 100755
--- a/mysql-test/t/kill_n_check.sh
+++ b/mysql-test/t/kill_n_check.sh
@@ -1,66 +1,115 @@
 #!/bin/sh
 
-if [ $# -ne 2 ]; then
-  echo "Usage: kill_n_check.sh <pid file path> killed|restarted"
+###########################################################################
+
+# NOTE: this script returns 0 (success) even in case of failure. This is
+# because this script is executed under mysql-test-run[.pl] and it's better to
+# examine particular problem in log file, than just having said that the test
+# case has failed.
+
+###########################################################################
+
+check_restart()
+{
+  if [ ! -r "$pid_path" ]; then
+    user_msg='the process was killed'
+    return 1
+  fi
+
+  new_pid=`cat "$pid_path" 2>/dev/null`
+
+  if [ $? -eq 0 -a "$original_pid" = "$new_pid" ]; then
+    user_msg='the process was not restarted'
+    return 1
+  fi
+
+  user_msg='the process was restarted'
+  return 0
+}
+
+###########################################################################
+
+if [ $# -ne 3 ]; then
+  echo "Usage: kill_n_check.sh <pid file path> killed|restarted <timeout>"
   exit 0
 fi
 
 pid_path="$1"
 expected_result="$2"
+total_timeout="$3"
 
-if [ -z "$pid_path" -o ! -r "$pid_path" ]; then
-  echo "Error: invalid PID path ($pid_path) or PID file does not exist."
+if [ "$expected_result" != 'killed' -a \
+     "$expected_result" != 'restarted' ]; then
+  echo "Error: invalid second argument ('killed' or 'restarted' expected)."
   exit 0
 fi
 
-if [ "$expected_result" != "killed" -a \
-     "$expected_result" != "restarted" ]; then
-  echo "Error: expected result must be either 'killed' or 'restarted'."
+if [ -z "$pid_path" ]; then
+  echo "Error: invalid PID path ($pid_path)."
   exit 0
 fi
 
-# echo "PID path: '$pid_path'"
+if [ $expected_result = 'killed' -a ! -r "$pid_path" ]; then
+  echo "Error: PID file ($pid_path) does not exist."
+  exit 0
+fi
 
-original_pid=`cat "$pid_path"`
+if [ -z "$total_timeout" ]; then
+  echo "Error: timeout is not specified."
+  exit 0
+fi
 
-# echo "Original PID: $original_pid"
+###########################################################################
+
+original_pid=`cat "$pid_path"`
 
 echo "Killing the process..."
 
 kill -9 $original_pid
 
+###########################################################################
+
 echo "Sleeping..."
 
-sleep 3
+if [ "$expected_result" = "restarted" ]; then
 
-new_pid=""
+  # Wait for the process to restart.
 
-[ -r "$pid_path" ] && new_pid=`cat "$pid_path"`
+  cur_attempt=1
 
-# echo "New PID: $new_pid"
+  while true; do
 
-if [ "$expected_result" = "restarted" ]; then
+    if check_restart; then
+      echo "Success: $user_msg."
+      exit 0
+    fi
 
-  if [ -z "$new_pid" ]; then
-    echo "Error: the process was killed."
-    exit 0
-  fi
+    [ $cur_attempt -ge $total_timeout ] && break
 
-  if [ "$original_pid" -eq "$new_pid" ]; then
-    echo "Error: the process was not restarted."
-    exit 0
-  fi
-  
-  echo "Success: the process was restarted."
+    sleep 1
+
+    cur_attempt=`expr $cur_attempt + 1`
+
+  done
+
+  echo "Error: $user_msg."
   exit 0
-  
-else # $expected_result = killed
-    
+
+else # $expected_result == killed
+
+  # Here we have to sleep for some long time to ensure that the process will
+  # not be restarted.
+
+  sleep $total_timeout
+
+  new_pid=`cat "$pid_path" 2>/dev/null`
+
   if [ "$new_pid" -a "$new_pid" -ne "$original_pid" ]; then
     echo "Error: the process was restarted."
-    exit 0
+  else
+    echo "Success: the process was killed."
   fi
 
-  echo "Success: the process was killed."
   exit 0
+
 fi
diff --git a/mysql-test/t/wait_for_process.sh b/mysql-test/t/wait_for_process.sh
new file mode 100755
index 00000000000..df0f4a17e3a
--- /dev/null
+++ b/mysql-test/t/wait_for_process.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+
+###########################################################################
+
+pid_path="$1"
+total_attempts="$2"
+event="$3"
+
+case "$3" in
+  started)
+    check_fn='check_started';
+    ;;
+
+  stopped)
+    check_fn='check_stopped';
+    ;;
+
+  *)
+    echo "Error: invalid third argument ('started' or 'stopped' expected)."
+    exit 0
+esac
+
+###########################################################################
+
+check_started()
+{
+  [ ! -r "$pid_path" ] && return 1
+
+  new_pid=`cat "$pid_path" 2>/dev/null`
+
+  [ $? -eq 0 -a "$original_pid" = "$new_pid" ] && return 1
+
+  return 0
+}
+
+###########################################################################
+
+check_stopped()
+{
+  [ -r "$pid_path" ] && return 1
+
+  return 0
+}
+
+###########################################################################
+
+cur_attempt=1
+
+while true; do
+
+  if ( eval $check_fn ); then
+    echo "Success: the process has been $event."
+    exit 0
+  fi
+
+  [ $cur_attempt -ge $total_attempts ] && break
+
+  sleep 1
+
+  cur_attempt=`expr $cur_attempt + 1`
+
+done
+
+echo "Error: the process has not been $event in $total_attempts secs."
+exit 0
+