Fix for galera_3nodes.galera_gtid_2_cluster

Temporary disable WSREP while executing RESET MASTER. In situation when 2 nodes are both master/slave first stop slave on both and than reset master.
Enforce stricter causality check with wsrep_sync_wait.

4 files changed, 129 insertions, 6 deletions

diff --git a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
index 2707fb69f4d..7aac1662edd 100644
--- a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
+++ b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
@@ -13,8 +13,7 @@ gtid_ignore_duplicates
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
 # enforce read-committed characteristics across the cluster
-# wsrep-causal-reads=ON
-# wsrep-sync-wait=15
+wsrep-sync-wait=15
 
 [mysqld.1]
 #galera_port=@OPT.port
diff --git a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result
index 35ca84119e7..afb94d493c4 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result
@@ -1,27 +1,40 @@
+connection node_2;
+connection node_1;
+connection node_1;
 cluster 1 node 1
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connection node_2;
 cluster 1 node 2
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_3;
 cluster 1 node 3
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connection node_4;
 cluster 2 node 1
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connect node_5, 127.0.0.1, root, , test, $NODE_MYPORT_5;
+connection node_5;
 cluster 2 node 2
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connect node_6, 127.0.0.1, root, , test, $NODE_MYPORT_6;
+connection node_6;
 cluster 2 node 3
 SHOW STATUS LIKE 'wsrep_cluster_size';
 Variable_name	Value
 wsrep_cluster_size	3
+connection node_1;
 change master to master_host='127.0.0.1', master_user='root', master_port=NODE_MYPORT_4, master_use_gtid=current_pos, ignore_server_ids=(12,13);;
 start slave;
 include/wait_for_slave_to_start.inc
@@ -31,6 +44,7 @@ select @@gtid_binlog_state;
 select @@gtid_slave_pos;
 @@gtid_slave_pos
 
+connection node_4;
 change master to master_host='127.0.0.1', master_user='root', master_port=NODE_MYPORT_1, master_use_gtid=current_pos, ignore_server_ids=(22,23);;
 start slave;
 include/wait_for_slave_to_start.inc
@@ -41,15 +55,19 @@ select @@gtid_slave_pos;
 @@gtid_slave_pos
 
 cluster 1 node 1
+connection node_1;
 create table t1 (cluster_domain_id int ,node_server_id int, seq_no int);
 insert into t1 values (1, 11, 2);
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 2 node 1
+connection node_4;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2
@@ -62,9 +80,12 @@ cluster_domain_id	node_server_id	seq_no
 1	11	2
 2	21	1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 1 node 2
+connection node_2;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,2-21-1
@@ -73,9 +94,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,1-12-3,2-21-1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 1 node 3
+connection node_3;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,1-12-3,2-21-1
@@ -84,9 +108,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 2 node 2
+connection node_5;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1
@@ -95,9 +122,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2
 #wait for sync  cluster 2 and 1
+connection node_4;
 include/save_master_gtid.inc
+connection node_1;
 include/sync_with_master_gtid.inc
 cluster 2 node 3
+connection node_6;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2
@@ -106,28 +136,53 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2,2-23-3
 #wait for sync  cluster 2 and 1
+connection node_4;
 include/save_master_gtid.inc
+connection node_1;
 include/sync_with_master_gtid.inc
 cluster 1 node 1
+connection node_1;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2,2-23-3
 drop table t1;
 stop slave;
 reset slave;
+cluster 2 node 1
+connection node_4;
+stop slave;
+reset slave;
+cluster 1 node 1
+connection node_1;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 cluster 2 node 1
-stop slave;
-reset slave;
+connection node_4;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
+connection node_2;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_3;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_5;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_6;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_1;
 change master to master_host='127.0.0.1', master_user='root', master_port=NODE_MYPORT_6, master_use_gtid=current_pos, ignore_server_ids=(12,13);;
 start slave;
 include/wait_for_slave_to_start.inc
@@ -137,6 +192,7 @@ select @@gtid_binlog_state;
 select @@gtid_slave_pos;
 @@gtid_slave_pos
 
+connection node_4;
 change master to master_host='127.0.0.1', master_user='root', master_port=NODE_MYPORT_3, master_use_gtid=current_pos, ignore_server_ids=(22,23);;
 start slave;
 include/wait_for_slave_to_start.inc
@@ -147,15 +203,19 @@ select @@gtid_slave_pos;
 @@gtid_slave_pos
 
 cluster 1 node 1
+connection node_1;
 create table t1 (cluster_domain_id int ,node_server_id int, seq_no int);
 insert into t1 values (1, 11, 2);
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 2 node 1
+connection node_4;
 insert into t1 values (2, 21, 1);
 select @@gtid_binlog_state;
 @@gtid_binlog_state
@@ -165,9 +225,12 @@ cluster_domain_id	node_server_id	seq_no
 1	11	2
 2	21	1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 1 node 2
+connection node_2;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,2-21-1
@@ -176,9 +239,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,1-12-3,2-21-1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 1 node 3
+connection node_3;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-11-2,1-12-3,2-21-1
@@ -187,9 +253,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1
 #wait for sync  cluster 1 and 2
+connection node_1;
 include/save_master_gtid.inc
+connection node_4;
 include/sync_with_master_gtid.inc
 cluster 2 node 2
+connection node_5;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1
@@ -198,9 +267,12 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2
 #wait for sync  cluster 2 and 1
+connection node_4;
 include/save_master_gtid.inc
+connection node_1;
 include/sync_with_master_gtid.inc
 cluster 2 node 3
+connection node_6;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2
@@ -209,23 +281,43 @@ select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2,2-23-3
 #wait for sync  cluster 2 and 1
+connection node_4;
 include/save_master_gtid.inc
+connection node_1;
 include/sync_with_master_gtid.inc
 cluster 1 node 1
+connection node_1;
 select @@gtid_binlog_state;
 @@gtid_binlog_state
 1-12-3,1-11-2,1-13-4,2-21-1,2-22-2,2-23-3
 drop table t1;
 stop slave;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 cluster 2 node 1
+connection node_4;
 stop slave;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
+connection node_2;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_3;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_5;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
+connection node_6;
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test
index c679db1305d..925600ffaa8 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test
@@ -138,27 +138,45 @@ select @@gtid_binlog_state;
 drop table t1;
 stop slave;
 reset slave;
+
+--echo cluster 2 node 1
+--connection node_4
+stop slave;
+reset slave;
+
+--echo cluster 1 node 1
+--connection node_1
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 --sleep 2
 
 --echo cluster 2 node 1
 --connection node_4
-stop slave;
-reset slave;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 
 --connection node_2
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_3
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_5
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_6
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 #--enable_parsing
 #
 # This test creates 2x 3 nodes galera cluster.
@@ -272,21 +290,33 @@ select @@gtid_binlog_state;
 drop table t1;
 stop slave;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 
 --echo cluster 2 node 1
 --connection node_4
 stop slave;
 change master to master_use_gtid=no, ignore_server_ids=();
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 set global GTID_SLAVE_POS="";
 
 --connection node_2
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_3
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_5
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
 --connection node_6
+set global wsrep_on=OFF;
 reset master;
+set global wsrep_on=ON;
diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc
index fc08399bb88..59f6a45a52f 100644
--- a/sql/sql_repl.cc
+++ b/sql/sql_repl.cc
@@ -3369,6 +3369,7 @@ static my_bool kill_callback(THD *thd, kill_callback_arg *arg)
       thd->variables.server_id == arg->slave_server_id)
   {
     arg->thd= thd;
+    if (WSREP(thd)) mysql_mutex_lock(&thd->LOCK_thd_data);
     mysql_mutex_lock(&thd->LOCK_thd_kill);    // Lock from delete
     return 1;
   }
@@ -3390,6 +3391,7 @@ void kill_zombie_dump_threads(uint32 slave_server_id)
     */
     arg.thd->awake_no_mutex(KILL_SLAVE_SAME_ID);
     mysql_mutex_unlock(&arg.thd->LOCK_thd_kill);
+    if (WSREP(arg.thd)) mysql_mutex_unlock(&arg.thd->LOCK_thd_data);
   }
 }