Merge pull request #1142 from codership/10.2-MDEV-15740

MDEV-15740 Fixes to Galera transaction recovery

8 files changed, 245 insertions, 1 deletions

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index 9a8adba7a03..4dcfdec84ce 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -44,4 +44,3 @@ galera.MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C]
 galera.galera_sst_xtrabackup-v2 : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir]
 galera.galera_sst_xtrabackup-v2_data_dir : MDEV-17848 Galera test failure on galera_sst_xtrabackup-v2[_data_dir]
 query_cache : MDEV-18137: Galera test failure on query_cache
-galera_gcache_recover_manytrx : MDEV-15740
diff --git a/mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result b/mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result
index 868b39bfbd6..604b24a8fab 100644
--- a/mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result
+++ b/mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result
@@ -70,28 +70,55 @@ WHILE 1 DO
 INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
 END WHILE;
 END|
+connect node_1_insert_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_multi, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_transaction, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_update_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_1k, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_1m, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1_insert_10m, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1_insert_simple;
 CALL insert_simple();;
+connection node_1_insert_multi;
 CALL insert_multi();;
+connection node_1_insert_transaction;
 CALL insert_transaction ();;
+connection node_1_update_simple;
 CALL update_simple ();;
+connection node_1_insert_1k;
 CALL insert_1k ();;
+connection node_1_insert_1m;
 CALL insert_1m ();;
+connection node_1_insert_10m;
 CALL insert_10m ();;
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 Killing server ...
+connection node_1;
 Killing server ...
+connection node_1_insert_simple;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_multi;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_transaction;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_update_simple;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_1k;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_1m;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1_insert_10m;
 ERROR HY000: Lost connection to MySQL server during query
+connection node_1;
 Performing --wsrep-recover ...
 Using --wsrep-start-position when starting mysqld ...
+connection node_2;
 Performing --wsrep-recover ...
 Using --wsrep-start-position when starting mysqld ...
+connection node_1;
 include/diff_servers.inc [servers=1 2]
+connection node_1;
 DROP TABLE t1;
 DROP TABLE ten;
 DROP PROCEDURE insert_simple;
@@ -100,8 +127,10 @@ DROP PROCEDURE insert_transaction;
 DROP PROCEDURE update_simple;
 DROP PROCEDURE insert_1k;
 DROP PROCEDURE insert_1m;
+connection node_1;
 CALL mtr.add_suppression("conflict state 7 after post commit");
 CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
 include/assert_grep.inc [async IST sender starting to serve]
+connection node_2;
 CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
 include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]
diff --git a/mysql-test/suite/wsrep/r/wsrep-recover-v25,binlogon.rdiff b/mysql-test/suite/wsrep/r/wsrep-recover-v25,binlogon.rdiff
new file mode 100644
index 00000000000..5bf6502d0d8
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/wsrep-recover-v25,binlogon.rdiff
@@ -0,0 +1,17 @@
+--- r/wsrep-recover-v25.result	2019-01-27 15:38:58.819204748 +0200
++++ r/wsrep-recover-v25.reject	2019-01-27 15:39:49.967358994 +0200
+@@ -18,11 +18,10 @@
+ connection default;
+ SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+ # Kill the server
+-Expect seqno 3
+-3
+-Expect 5 7
++Expect seqno 2
++2
++Expect 5
+ SELECT * FROM t1;
+ f1
+ 5
+-7
+ DROP TABLE t1;
diff --git a/mysql-test/suite/wsrep/r/wsrep-recover-v25.result b/mysql-test/suite/wsrep/r/wsrep-recover-v25.result
new file mode 100644
index 00000000000..6d146f67bdf
--- /dev/null
+++ b/mysql-test/suite/wsrep/r/wsrep-recover-v25.result
@@ -0,0 +1,28 @@
+# Kill the server
+Expect seqno 0
+0
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+# Kill the server
+Expect seqno 1
+1
+INSERT INTO t1 VALUES (5);
+# Kill the server
+Expect seqno 2
+2
+SELECT VARIABLE_VALUE `expect 2` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+expect 2
+2
+connect con1, localhost, root;
+SET DEBUG_SYNC = "ha_commit_trans_after_prepare SIGNAL after_prepare_reached WAIT_FOR continue";
+INSERT INTO t1 VALUES (7);
+connection default;
+SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+# Kill the server
+Expect seqno 3
+3
+Expect 5 7
+SELECT * FROM t1;
+f1
+5
+7
+DROP TABLE t1;
diff --git a/mysql-test/suite/wsrep/t/wsrep-recover-step.inc b/mysql-test/suite/wsrep/t/wsrep-recover-step.inc
new file mode 100644
index 00000000000..22669438fe0
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/wsrep-recover-step.inc
@@ -0,0 +1,41 @@
+#
+# Macro to run wsrep recovery step. This is adapted from
+# suite/galera/include/galera_wsrep_recover.inc, with additional
+# option to pass binlog argument to recovery command. The macro
+# returns recovered position split in uuid and seqno parts.
+#
+# Arguments:
+#
+# wsrep_recover_binlog_opt - Binlog options to recovery command
+#
+# Return:
+#
+# wsrep_recover_start_position_uuid - UUID corresponding to recovered position
+# wsrep_recover_start_position_seqno - seqno corresponding to recovered position
+#
+
+--exec $MYSQLD --defaults-group-suffix=.1 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --log-error=$MYSQL_TMP_DIR/galera_wsrep_recover.log --innodb --wsrep-recover $wsrep_recover_binlog_opt --core-file > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+
+--perl
+	use strict;
+	my $wsrep_start_position = `grep 'WSREP: Recovered position:' $ENV{MYSQL_TMP_DIR}/galera_wsrep_recover.log | sed 's/.*WSREP\:\ Recovered\ position://' | sed 's/^[ \t]*//'`;
+	chomp($wsrep_start_position);
+        die if $wsrep_start_position eq '';
+	open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/galera_wsrep_start_position.inc") or die;
+	my ($uuid, $seqno) = split /:/, $wsrep_start_position;
+	print FILE "--let \$wsrep_recover_start_position_uuid = $uuid\n";
+	print FILE "--let \$wsrep_recover_start_position_seqno = $seqno\n";
+	close FILE;
+EOF
+
+--source $MYSQL_TMP_DIR/galera_wsrep_start_position.inc
+
+if ($wsrep_recover_start_position_uuid == '') {
+   --die "Could not obtain start_position_uuid."
+}
+
+if ($wsrep_recover_start_position_seqno == '') {
+   --die "Could not obtain start_position_seqno."
+}
+
+--remove_file $MYSQL_TMP_DIR/galera_wsrep_start_position.inc
diff --git a/mysql-test/suite/wsrep/t/wsrep-recover-v25.cnf b/mysql-test/suite/wsrep/t/wsrep-recover-v25.cnf
new file mode 100644
index 00000000000..b1c96d2614d
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/wsrep-recover-v25.cnf
@@ -0,0 +1,7 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
diff --git a/mysql-test/suite/wsrep/t/wsrep-recover-v25.combinations b/mysql-test/suite/wsrep/t/wsrep-recover-v25.combinations
new file mode 100644
index 00000000000..1ce3b45aa1a
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/wsrep-recover-v25.combinations
@@ -0,0 +1,4 @@
+[binlogon]
+log-bin
+
+[binlogoff]
diff --git a/mysql-test/suite/wsrep/t/wsrep-recover-v25.test b/mysql-test/suite/wsrep/t/wsrep-recover-v25.test
new file mode 100644
index 00000000000..cfd77fbdef4
--- /dev/null
+++ b/mysql-test/suite/wsrep/t/wsrep-recover-v25.test
@@ -0,0 +1,119 @@
+#
+# Verify that the wsrep XID gets updated in InnoDB rollback segment
+# properly and can be recovered with --wsrep-recover
+#
+# The test runs the following scenarios:
+#
+# 1) The server is started but no SQL is run
+# 2) DDL is executed
+# 3) INSERT is executed
+# 4) Two INSERTs are executed so that the first one in order will be
+#    blocked after certification and the second one before entering
+#    commit order critical section.
+# 5) Two DMLs are executed so that the prepare step is run out of order.
+#    Both transactions are blocked before commit order critical section.
+#
+# After each scenario server is killed and the recovered position
+# is validated.
+#
+
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+--source include/have_wsrep_provider.inc
+--source include/have_debug_sync.inc
+
+#
+# Binlog option for recovery run. This must be set in the test because
+# combinations file causes log-bin option to be set from command line,
+# not via my.cnf.
+#
+--let $log_bin = `SELECT @@log_bin`
+if ($log_bin) {
+--let $wsrep_recover_binlog_opt = --log-bin
+}
+
+#
+# Scenario 1
+# The expected recovered seqno is 1 corresponding to initial cluster
+# configuration change.
+#
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 0
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Senario 2
+# The expected recovered seqno is 1 corresponding to CREATE TABLE
+#
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 1
+--echo $wsrep_recover_start_position_seqno
+
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+#
+# Scenario 3
+# The expected recovered seqno is 2 corresponding to CREATE TABLE and INSERT.
+#
+# The expected wsrep_last_committed after the server is restarted is 2.
+#
+
+INSERT INTO t1 VALUES (5);
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+--echo Expect seqno 2
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+SELECT VARIABLE_VALUE `expect 2` FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed';
+
+#
+# Scenario 4
+#
+# The INSERT gets prepared but not committed.
+#
+# If binlog is off, the expected outcome is that the INSERT gets committed
+# since it is already committed in the cluster. If binlog is on, the INSERT
+# should be rolled back during recovery phase since it has not yet
+# been logged into binlog.
+#
+--connect con1, localhost, root
+SET DEBUG_SYNC = "ha_commit_trans_after_prepare SIGNAL after_prepare_reached WAIT_FOR continue";
+--send INSERT INTO t1 VALUES (7)
+
+--connection default
+SET DEBUG_SYNC = "now WAIT_FOR after_prepare_reached";
+--source include/kill_mysqld.inc
+--source wsrep-recover-step.inc
+if ($log_bin) {
+  --echo Expect seqno 2
+}
+if (!$log_bin) {
+  --echo Expect seqno 3
+}
+--echo $wsrep_recover_start_position_seqno
+--let $restart_parameters = --wsrep-start-position=$wsrep_recover_start_position_uuid:$wsrep_recover_start_position_seqno
+--source include/start_mysqld.inc
+--source include/wait_wsrep_ready.inc
+
+if ($log_bin) {
+  --echo Expect 5
+}
+if (!$log_bin) {
+   --echo Expect 5 7
+}
+SELECT * FROM t1;
+
+DROP TABLE t1;