diff options
24 files changed, 1632 insertions, 46 deletions
diff --git a/mysql-test/suite/binlog/r/binlog_xa_multi_binlog_crash_recovery.result b/mysql-test/suite/binlog/r/binlog_xa_multi_binlog_crash_recovery.result new file mode 100644 index 00000000000..60c5d50820e --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_xa_multi_binlog_crash_recovery.result @@ -0,0 +1,39 @@ +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +connect con1,localhost,root,,; +XA START 'xa1'; +INSERT INTO t1 SET a=1; +SET DEBUG_SYNC= "simulate_hang_after_binlog_prepare SIGNAL con1_ready WAIT_FOR con1_go"; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +XA END 'xa1'; +XA PREPARE 'xa1';; +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; +FLUSH LOGS; +FLUSH LOGS; +FLUSH LOGS; +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +master-bin.000003 # +master-bin.000004 # +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Format_desc # # SERVER_VERSION, BINLOG_VERSION +master-bin.000004 # Gtid_list # # [#-#-#] +master-bin.000004 # Binlog_checkpoint # # master-bin.000001 +SET DEBUG_SYNC= "now SIGNAL con1_go"; +connection con1; +ERROR HY000: Lost connection to MySQL server during query +connection default; +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa1 +XA COMMIT 'xa1'; +SELECT * FROM t1; +a b +1 NULL +connection default; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/t/binlog_xa_multi_binlog_crash_recovery.test b/mysql-test/suite/binlog/t/binlog_xa_multi_binlog_crash_recovery.test new file mode 100644 index 00000000000..d0852de2fcc --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_xa_multi_binlog_crash_recovery.test @@ -0,0 +1,86 @@ +# ==== Purpose ==== +# +# Test verifies that XA crash recovery works fine across multiple binary logs. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate an explicit XA transaction. Using debug simulation hold the +# execution of XA PREPARE statement after the XA PREPARE is written to +# the binary log. With this the prepare will not be done in engine. +# 1 - By executing FLUSH LOGS generate multiple binary logs. +# 2 - Now make the server to disappear at this point. +# 3 - Restart the server. During recovery the XA PREPARE from the binary +# log will be read. It is cross checked with engine. Since it is not +# present in engine it will be executed once again. +# 4 - When server is up execute XA RECOVER to check that the XA is +# prepared in engine as well. +# 5 - XA COMMIT the transaction and check the validity of the data. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA +# + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_log_bin.inc + +RESET MASTER; + +CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); + +connect(con1,localhost,root,,); +XA START 'xa1'; +INSERT INTO t1 SET a=1; +SET DEBUG_SYNC= "simulate_hang_after_binlog_prepare SIGNAL con1_ready WAIT_FOR con1_go"; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +XA END 'xa1'; +--send XA PREPARE 'xa1'; + +connection default; +SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; +FLUSH LOGS; +FLUSH LOGS; +FLUSH LOGS; + +--source include/show_binary_logs.inc +--let $binlog_file= master-bin.000004 +--let $binlog_start= 4 +--source include/show_binlog_events.inc + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +--error 0,2013 +SET DEBUG_SYNC= "now SIGNAL con1_go"; +--source include/wait_until_disconnected.inc + +--connection con1 +--error 2013 +--reap +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +XA RECOVER; +XA COMMIT 'xa1'; + +SELECT * FROM t1; + +# Clean up. +connection default; +DROP TABLE t1; + diff --git a/mysql-test/suite/rpl/r/rpl_xa_commit_crash_safe.result b/mysql-test/suite/rpl/r/rpl_xa_commit_crash_safe.result new file mode 100644 index 00000000000..ce7d257690a --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_commit_crash_safe.result @@ -0,0 +1,50 @@ +include/master-slave.inc +[connection master] +connect master2,localhost,root,,; +connection master; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +connection slave; +include/stop_slave.inc +connection master1; +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +XA COMMIT 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +connection master1; +connection master; +connection default; +connection server_1; +connection master; +connection slave; +include/start_slave.inc +connection master; +SELECT * FROM t; +f +20 +40 +XA RECOVER; +formatID gtrid_length bqual_length data +XA COMMIT 'xa2'; +ERROR XAE04: XAER_NOTA: Unknown XID +SELECT * FROM t; +f +20 +40 +connection slave; +SELECT * FROM t; +f +20 +40 +connection master; +DROP TABLE t; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_xa_event_apply_failure.result b/mysql-test/suite/rpl/r/rpl_xa_event_apply_failure.result new file mode 100644 index 00000000000..dc9930b77b1 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_event_apply_failure.result @@ -0,0 +1,60 @@ +include/master-slave.inc +[connection master] +connect master2,localhost,root,,; +connection master; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CALL mtr.add_suppression("Failed to execute binlog query event"); +CALL mtr.add_suppression("Recovery: Error .Out of memory.."); +CALL mtr.add_suppression("Crash recovery failed."); +CALL mtr.add_suppression("Can.t init tc log"); +CALL mtr.add_suppression("Aborting"); +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +connection slave; +include/stop_slave.inc +connection master1; +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +XA COMMIT 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +connection master1; +connection master; +connection default; +connection default; +connection master; +*** must be no 'xa2' commit seen, as it's still prepared: +SELECT * FROM t; +f +20 +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa2 +SET GLOBAL DEBUG_DBUG=""; +SET SQL_LOG_BIN=0; +XA COMMIT 'xa2'; +SET SQL_LOG_BIN=1; +connection server_1; +connection master; +connection slave; +include/start_slave.inc +connection master; +SELECT * FROM t; +f +20 +40 +connection slave; +SELECT * FROM t; +f +20 +40 +connection master; +DROP TABLE t; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_xa_prepare_commit_prepare.result b/mysql-test/suite/rpl/r/rpl_xa_prepare_commit_prepare.result new file mode 100644 index 00000000000..9ba24716639 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_prepare_commit_prepare.result @@ -0,0 +1,48 @@ +include/master-slave.inc +[connection master] +connect master2,localhost,root,,; +connection master; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +connection slave; +include/stop_slave.inc +connection master1; +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +XA PREPARE 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +connection master1; +connection master; +connection default; +connection server_1; +connection master; +connection slave; +include/start_slave.inc +connection master; +SELECT * FROM t; +f +20 +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa2 +XA COMMIT 'xa2'; +SELECT * FROM t; +f +20 +40 +connection slave; +SELECT * FROM t; +f +20 +40 +connection master; +DROP TABLE t; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_xa_prepare_crash_safe.result b/mysql-test/suite/rpl/r/rpl_xa_prepare_crash_safe.result new file mode 100644 index 00000000000..99baf59a3c1 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_prepare_crash_safe.result @@ -0,0 +1,62 @@ +include/master-slave.inc +[connection master] +connect master2,localhost,root,,; +connection master; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CALL mtr.add_suppression("Found 2 prepared XA transactions"); +CALL mtr.add_suppression("Found 3 prepared XA transactions"); +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +connection slave; +include/stop_slave.inc +connection master1; +use test; +xa start 'xa2'; +insert into t values (30); +xa end 'xa2'; +SET DEBUG_SYNC="simulate_hang_after_binlog_prepare SIGNAL reached WAIT_FOR go"; +xa prepare 'xa2'; +connection master2; +XA START 'xa3'; +INSERT INTO t VALUES (40); +XA END 'xa3'; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +XA PREPARE 'xa3'; +ERROR HY000: Lost connection to MySQL server during query +connection master1; +ERROR HY000: Lost connection to MySQL server during query +connection master; +connection default; +connection server_1; +connection master; +connection slave; +include/start_slave.inc +connection master; +SELECT * FROM t; +f +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa3 +1 3 0 xa1 +1 3 0 xa2 +XA COMMIT 'xa1'; +XA COMMIT 'xa2'; +XA COMMIT 'xa3'; +SELECT * FROM t; +f +20 +30 +40 +connection slave; +SELECT * FROM t; +f +20 +30 +40 +connection master; +DROP TABLE t; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_xa_rollback_commit_crash_safe.result b/mysql-test/suite/rpl/r/rpl_xa_rollback_commit_crash_safe.result new file mode 100644 index 00000000000..dc26c224ea9 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_rollback_commit_crash_safe.result @@ -0,0 +1,47 @@ +include/master-slave.inc +[connection master] +connect master2,localhost,root,,; +connection master; +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +connection slave; +include/stop_slave.inc +connection master1; +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +XA ROLLBACK 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +connection master1; +connection master; +connection default; +connection server_1; +connection master; +connection slave; +include/start_slave.inc +connection master; +SELECT * FROM t; +f +20 +XA RECOVER; +formatID gtrid_length bqual_length data +XA ROLLBACK 'xa2'; +ERROR XAE04: XAER_NOTA: Unknown XID +SELECT * FROM t; +f +20 +connection slave; +SELECT * FROM t; +f +20 +connection master; +DROP TABLE t; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_xa_commit_crash_safe.test b/mysql-test/suite/rpl/t/rpl_xa_commit_crash_safe.test new file mode 100644 index 00000000000..e972e3f09de --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_commit_crash_safe.test @@ -0,0 +1,98 @@ +# ==== Purpose ==== +# +# Test verifies that XA COMMIT statements are crash safe. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate 2 explicit XA transactions. 'xa1' and 'xa2'. +# 'xa1' will be prepared and committed. +# 1 - For 'xa2' let the XA COMMIT be done in binary log and crash the +# server so that it is not committed in engine. +# 2 - Restart the server. The recovery code should successfully recover +# 'xa2'. The COMMIT should be executed during recovery. +# 3 - Check the data in table. Both rows should be present in table. +# 4 - Trying to commit 'xa2' should report unknown 'XA' error as COMMIT is +# already complete during recovery. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA + + +--source include/have_innodb.inc +--source include/master-slave.inc +--source include/have_debug.inc + +connect (master2,localhost,root,,); +--connection master +CALL mtr.add_suppression("Found 1 prepared XA transactions"); + +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +--sync_slave_with_master +--source include/stop_slave.inc + +--connection master1 +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +--error 2013 # CR_SERVER_LOST +XA COMMIT 'xa2'; +--source include/wait_until_disconnected.inc + +--connection master1 +--source include/wait_until_disconnected.inc + +--connection master +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# rpl_end.inc needs to use the connection server_1 +connection server_1; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection slave +--source include/start_slave.inc +--sync_with_master + +--connection master +SELECT * FROM t; +XA RECOVER; +--error 1397 # ER_XAER_NOTA +XA COMMIT 'xa2'; +SELECT * FROM t; +--sync_slave_with_master + +SELECT * FROM t; + +--connection master +DROP TABLE t; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_xa_event_apply_failure.test b/mysql-test/suite/rpl/t/rpl_xa_event_apply_failure.test new file mode 100644 index 00000000000..14cebbd9b13 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_event_apply_failure.test @@ -0,0 +1,119 @@ +# ==== Purpose ==== +# +# Test verifies that if for some reason an event cannot be applied during +# recovery, appropriate error is reported. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate 2 explicit XA transactions. 'xa1' and 'xa2'. +# 'xa1' will be prepared and committed. +# 1 - For 'xa2' let the XA COMMIT be done in binary log and crash the +# server so that it is not committed in engine. +# 2 - Restart the server. Using debug simulation point make XA COMMIT 'xa2' +# execution to fail. The server will resume anyway +# to leave the error in the errlog (see "Recovery: Error.."). +# 3 - Work around the simulated failure with Commit once again +# from a connection that turns OFF binlogging. +# Slave must catch up with the master. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA + + +--source include/have_innodb.inc +--source include/master-slave.inc +--source include/have_debug.inc + +connect (master2,localhost,root,,); +--connection master +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CALL mtr.add_suppression("Failed to execute binlog query event"); +CALL mtr.add_suppression("Recovery: Error .Out of memory.."); +CALL mtr.add_suppression("Crash recovery failed."); +CALL mtr.add_suppression("Can.t init tc log"); +CALL mtr.add_suppression("Aborting"); + +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +--sync_slave_with_master +--source include/stop_slave.inc + +--connection master1 +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +--error 2013 # CR_SERVER_LOST +XA COMMIT 'xa2'; +--source include/wait_until_disconnected.inc + +--connection master1 +--source include/wait_until_disconnected.inc + +--connection master +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart: --debug-dbug=d,trans_xa_commit_fail +EOF + +connection default; +--source include/wait_until_disconnected.inc + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--enable_reconnect +--echo *** must be no 'xa2' commit seen, as it's still prepared: +SELECT * FROM t; +XA RECOVER; + +# Commit it manually now to work around the extra binlog record +# by turning binlogging OFF by the connection. + +SET GLOBAL DEBUG_DBUG=""; +SET SQL_LOG_BIN=0; +--error 0 +XA COMMIT 'xa2'; +SET SQL_LOG_BIN=1; + + +# rpl_end.inc needs to use the connection server_1 +connection server_1; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--source include/wait_until_connected_again.inc + +--connection slave +--source include/start_slave.inc +--sync_with_master + +--connection master +SELECT * FROM t; + +--sync_slave_with_master +SELECT * FROM t; + +--connection master +DROP TABLE t; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_xa_prepare_commit_prepare.test b/mysql-test/suite/rpl/t/rpl_xa_prepare_commit_prepare.test new file mode 100644 index 00000000000..7b987c7f29b --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_prepare_commit_prepare.test @@ -0,0 +1,95 @@ +# ==== Purpose ==== +# +# Test verifies that XA PREPARE transactions are crash safe. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate 2 explicit XA transactions. 'xa1' and 'xa2'. +# 'xa1' will be prepared and committed. +# 1 - For 'xa2' let the XA PREPARE be done in binary log and crash the +# server so that it is not prepared in engine. +# 2 - Restart the server. The recovery code should successfully recover +# 'xa2'. +# 3 - When server is up, execute XA RECOVER and verify that 'xa2' is +# present. +# 4 - Commit the XA transaction and verify its correctness. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA + +--source include/have_innodb.inc +--source include/master-slave.inc +--source include/have_debug.inc + +connect (master2,localhost,root,,); +--connection master +CALL mtr.add_suppression("Found 1 prepared XA transactions"); + +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +--sync_slave_with_master +--source include/stop_slave.inc + +--connection master1 +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +--error 2013 # CR_SERVER_LOST +XA PREPARE 'xa2'; +--source include/wait_until_disconnected.inc + +--connection master1 +--source include/wait_until_disconnected.inc + +--connection master +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# rpl_end.inc needs to use the connection server_1 +connection server_1; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection slave +--source include/start_slave.inc +--sync_with_master + +--connection master +SELECT * FROM t; +XA RECOVER; +XA COMMIT 'xa2'; +SELECT * FROM t; +--sync_slave_with_master + +SELECT * FROM t; + +--connection master +DROP TABLE t; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_xa_prepare_crash_safe.test b/mysql-test/suite/rpl/t/rpl_xa_prepare_crash_safe.test new file mode 100644 index 00000000000..9d2c5cce528 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_prepare_crash_safe.test @@ -0,0 +1,117 @@ +# ==== Purpose ==== +# +# Test verifies that XA PREPARE transactions are crash safe. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate 3 explicit XA transactions. 'xa1', 'xa2' and 'xa3'. +# Using debug simulation hold the execution of second XA PREPARE +# statement after the XA PREPARE is written to the binary log. +# With this the prepare will not be done in engine. +# 1 - For 'xa3' allow the PREPARE statement to be written to binary log and +# simulate server crash. +# 2 - Restart the server. The recovery code should successfully recover +# 'xa2' and 'xa3'. +# 3 - When server is up, execute XA RECOVER and verify that 'xa2' and 'xa3' +# are present along with 'xa1'. +# 4 - Commit all the XA transactions and verify their correctness. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA + + +--source include/have_innodb.inc +--source include/master-slave.inc +--source include/have_debug.inc + +connect (master2,localhost,root,,); +--connection master +CALL mtr.add_suppression("Found 1 prepared XA transactions"); +CALL mtr.add_suppression("Found 2 prepared XA transactions"); +CALL mtr.add_suppression("Found 3 prepared XA transactions"); + +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +--sync_slave_with_master +--source include/stop_slave.inc + +--connection master1 +use test; +xa start 'xa2'; +insert into t values (30); +xa end 'xa2'; +SET DEBUG_SYNC="simulate_hang_after_binlog_prepare SIGNAL reached WAIT_FOR go"; +send xa prepare 'xa2'; + +--connection master2 +let $wait_condition= + SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST + WHERE STATE like "debug sync point: simulate_hang_after_binlog_prepare%"; +--source include/wait_condition.inc + +XA START 'xa3'; +INSERT INTO t VALUES (40); +XA END 'xa3'; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +--error 2013 # CR_SERVER_LOST +XA PREPARE 'xa3'; +--source include/wait_until_disconnected.inc + +--connection master1 +--error 2013 +--reap +--source include/wait_until_disconnected.inc + +--connection master +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# rpl_end.inc needs to use the connection server_1 +connection server_1; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--enable_reconnect +--source include/wait_until_connected_again.inc + + +--connection slave +--source include/start_slave.inc +--sync_with_master + +--connection master +SELECT * FROM t; +XA RECOVER; +XA COMMIT 'xa1'; +XA COMMIT 'xa2'; +XA COMMIT 'xa3'; +SELECT * FROM t; +--sync_slave_with_master + +SELECT * FROM t; + +--connection master +DROP TABLE t; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_xa_rollback_commit_crash_safe.test b/mysql-test/suite/rpl/t/rpl_xa_rollback_commit_crash_safe.test new file mode 100644 index 00000000000..1d19f96116e --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_rollback_commit_crash_safe.test @@ -0,0 +1,97 @@ +# ==== Purpose ==== +# +# Test verifies that XA COMMIT statements are crash safe. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Generate 2 explicit XA transactions. 'xa1' and 'xa2'. +# 'xa1' will be prepared and committed. +# 1 - For 'xa2' let the XA ROLLBACK be done in binary log and crash the +# server so that it is not committed in engine. +# 2 - Restart the server. The recovery code should successfully recover +# 'xa2'. The ROLLBACK should be executed during recovery. +# 3 - Check the data in table. Only one row should be present in table. +# 4 - Trying to rollback 'xa2' should report unknown 'XA' error as rollback +# is already complete during recovery. +# +# ==== References ==== +# +# MDEV-21469: Implement crash-safe logging of the user XA + +--source include/have_innodb.inc +--source include/master-slave.inc +--source include/have_debug.inc + +connect (master2,localhost,root,,); +--connection master +CALL mtr.add_suppression("Found 1 prepared XA transactions"); + +CREATE TABLE t ( f INT ) ENGINE=INNODB; +XA START 'xa1'; +INSERT INTO t VALUES (20); +XA END 'xa1'; +XA PREPARE 'xa1'; +XA COMMIT 'xa1'; +--sync_slave_with_master +--source include/stop_slave.inc + +--connection master1 +XA START 'xa2'; +INSERT INTO t VALUES (40); +XA END 'xa2'; +XA PREPARE 'xa2'; + +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +wait +EOF + +SET GLOBAL DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +--error 2013 # CR_SERVER_LOST +XA ROLLBACK 'xa2'; +--source include/wait_until_disconnected.inc + +--connection master1 +--source include/wait_until_disconnected.inc + +--connection master +--source include/wait_until_disconnected.inc + +# +# Server restart +# +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +restart +EOF + +connection default; +--enable_reconnect +--source include/wait_until_connected_again.inc + +# rpl_end.inc needs to use the connection server_1 +connection server_1; +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection master +--enable_reconnect +--source include/wait_until_connected_again.inc + +--connection slave +--source include/start_slave.inc +--sync_with_master + +--connection master +SELECT * FROM t; +XA RECOVER; +--error 1397 # ER_XAER_NOTA +XA ROLLBACK 'xa2'; +SELECT * FROM t; +--sync_slave_with_master + +SELECT * FROM t; + +--connection master +DROP TABLE t; +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/sql/handler.cc b/sql/handler.cc index ec14e6cbf95..b981294f0d6 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1345,6 +1345,9 @@ int ha_prepare(THD *thd) handlerton *ht= ha_info->ht(); if (ht->prepare) { + DBUG_EXECUTE_IF("simulate_crash_after_first_engine_prepare", + if (!ha_info->next()) DBUG_SUICIDE();); + if (unlikely(prepare_or_error(ht, thd, all))) { ha_rollback_trans(thd, all); @@ -1375,22 +1378,22 @@ int ha_prepare(THD *thd) } /* - Like ha_check_and_coalesce_trx_read_only to return counted number of - read-write transaction participants limited to two, but works in the 'all' - context. - Also returns the last found rw ha_info through the 2nd argument. + Returns counted number of + read-write recoverable transaction participants optionally limited to two. + Also optionally returns the last found rw ha_info through the 2nd argument. */ -uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info) +uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info, bool count_through) { unsigned rw_ha_count= 0; for (auto ha_info= thd->transaction.all.ha_list; ha_info; ha_info= ha_info->next()) { - if (ha_info->is_trx_read_write()) + if (ha_info->is_trx_read_write() && ha_info->ht()->recover) { - *ptr_ha_info= ha_info; - if (++rw_ha_count > 1) + if (ptr_ha_info) + *ptr_ha_info= ha_info; + if (++rw_ha_count > 1 && !count_through) break; } } @@ -1403,7 +1406,7 @@ uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info) A helper function to evaluate if two-phase commit is mandatory. As a side effect, propagates the read-only/read-write flags of the statement transaction to its enclosing normal transaction. - + If we have at least two engines with read-write changes we must run a two-phase commit. Otherwise we can run several independent commits as the only transactional engine has read-write changes @@ -1883,6 +1886,10 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) { int err; handlerton *ht= ha_info->ht(); + + DBUG_EXECUTE_IF("simulate_crash_after_first_engine_commit_or_rollback", + if (!ha_info->next()) DBUG_SUICIDE();); + if ((err= ht->commit(ht, thd, all))) { my_error(ER_ERROR_DURING_COMMIT, MYF(0), err); @@ -1994,6 +2001,10 @@ int ha_rollback_trans(THD *thd, bool all) { int err; handlerton *ht= ha_info->ht(); + + DBUG_EXECUTE_IF("simulate_crash_after_first_engine_commit_or_rollback", + if (!ha_info->next()) DBUG_SUICIDE();); + if ((err= ht->rollback(ht, thd, all))) { // cannot happen my_error(ER_ERROR_DURING_ROLLBACK, MYF(0), err); @@ -2233,7 +2244,9 @@ struct xarecover_st int len, found_foreign_xids, found_my_xids; XID *list; HASH *commit_list; + HASH *xa_prepared_list; // prepared user xa list bool dry_run; + uint recover_htons; // number of recoverable htons for XA recovery }; static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin, @@ -2245,6 +2258,7 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin, if (hton->recover) { + info->recover_htons++; while ((got= hton->recover(hton, info->list, info->len)) > 0 ) { sql_print_information("Found %d prepared transaction(s) in %s", @@ -2286,7 +2300,21 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin, _db_doprnt_("ignore xid %s", xid_to_str(buf, info->list[i])); }); xid_cache_insert(info->list + i); + XID *foreign_xid= info->list + i; info->found_foreign_xids++; + + /* + For each foreign xid prepraed in engine, check if it is present in + xa_prepared_list of binlog. + */ + if (info->xa_prepared_list) + { + struct xa_recovery_member *member= NULL; + if ((member= (xa_recovery_member *) + my_hash_search(info->xa_prepared_list, foreign_xid->key(), + foreign_xid->key_length()))) + member->in_engine_prepare++; + } continue; } if (IF_WSREP(!(wsrep_emulate_bin_log && @@ -2333,14 +2361,23 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin, return FALSE; } -int ha_recover(HASH *commit_list) +/* + The function accepts two xid record hashes of regular and user XA resp. + The regular transactions recovery is decided right here. + The user XA recovery will proceed. For that the function + update the states of the user xa xid records and also returns + the number of recoverable htons. +*/ +int ha_recover(HASH *commit_list, HASH *xa_prepared_list, uint *ptr_count) { struct xarecover_st info; DBUG_ENTER("ha_recover"); info.found_foreign_xids= info.found_my_xids= 0; info.commit_list= commit_list; + info.xa_prepared_list= xa_prepared_list; info.dry_run= (info.commit_list==0 && tc_heuristic_recover==0); info.list= NULL; + info.recover_htons= 0; /* commit_list and tc_heuristic_recover cannot be set both */ DBUG_ASSERT(info.commit_list==0 || tc_heuristic_recover==0); @@ -2367,12 +2404,14 @@ int ha_recover(HASH *commit_list) DBUG_RETURN(1); } - plugin_foreach(NULL, xarecover_handlerton, + plugin_foreach(NULL, xarecover_handlerton, MYSQL_STORAGE_ENGINE_PLUGIN, &info); + if (ptr_count) + *ptr_count= info.recover_htons; my_free(info.list); if (info.found_foreign_xids) - sql_print_warning("Found %d prepared XA transactions", + sql_print_warning("Found %d prepared XA transactions", info.found_foreign_xids); if (info.dry_run && info.found_my_xids) { @@ -2385,7 +2424,7 @@ int ha_recover(HASH *commit_list) info.found_my_xids, opt_tc_log_file); DBUG_RETURN(1); } - if (info.commit_list) + if (info.commit_list && !info.found_foreign_xids) sql_print_information("Crash recovery finished."); DBUG_RETURN(0); } diff --git a/sql/handler.h b/sql/handler.h index 92e4001a395..9acdd47bfc2 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -530,6 +530,8 @@ enum legacy_db_type DB_TYPE_FIRST_DYNAMIC=45, DB_TYPE_DEFAULT=127 // Must be last }; + +enum xa_binlog_state {XA_PREPARE=0, XA_COMPLETE}; /* Better name for DB_TYPE_UNKNOWN. Should be used for engines that do not have a hard-coded type value here. @@ -816,7 +818,6 @@ struct st_system_tablename const char *tablename; }; - typedef ulonglong my_xid; // this line is the same as in log_event.h #define MYSQL_XID_PREFIX "MySQLXid" #define MYSQL_XID_PREFIX_LEN 8 // must be a multiple of 8 @@ -904,6 +905,16 @@ struct xid_t { }; typedef struct xid_t XID; +/* + Struct to describe the user XA state for recovery. +*/ +struct xa_recovery_member +{ + XID xid; + enum xa_binlog_state state; // xid's binlog status - prepared or completed + uint in_engine_prepare; // number of engines that have xid prepared +}; + /* for recover() handlerton call */ #define MIN_XID_LIST_SIZE 128 #define MAX_XID_LIST_SIZE (1024*128) @@ -5176,7 +5187,7 @@ int ha_commit_one_phase(THD *thd, bool all); int ha_commit_trans(THD *thd, bool all); int ha_rollback_trans(THD *thd, bool all); int ha_prepare(THD *thd); -int ha_recover(HASH *commit_list); +int ha_recover(HASH *commit_list, HASH *xa_recover_list, uint *xa_recover_htons); /* transactions: these functions never call handlerton functions directly */ int ha_enable_transaction(THD *thd, bool on); @@ -5302,5 +5313,5 @@ void print_keydup_error(TABLE *table, KEY *key, myf errflag); int del_global_index_stat(THD *thd, TABLE* table, KEY* key_info); int del_global_table_stat(THD *thd, const LEX_CSTRING *db, const LEX_CSTRING *table); -uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info); +uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info, bool count_through); #endif /* HANDLER_INCLUDED */ diff --git a/sql/log.cc b/sql/log.cc index 908586260d1..c69b8518cf4 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -37,6 +37,7 @@ #include "log_event.h" // Query_log_event #include "rpl_filter.h" #include "rpl_rli.h" +#include "rpl_mi.h" #include "sql_audit.h" #include "mysqld.h" @@ -2223,6 +2224,12 @@ static int binlog_commit(handlerton *hton, THD *thd, bool all) if (!all) cache_mngr->trx_cache.set_prev_position(MY_OFF_T_UNDEF); + DBUG_EXECUTE_IF("simulate_crash_after_binlog_commit_or_rollback", + DBUG_SUICIDE();); + DEBUG_SYNC(thd, "simulate_hang_after_binlog_prepare"); + DBUG_EXECUTE_IF("simulate_crash_after_binlog_prepare", + DBUG_SUICIDE();); + THD_STAGE_INFO(thd, org_stage); DBUG_RETURN(error); } @@ -2326,6 +2333,9 @@ static int binlog_rollback(handlerton *hton, THD *thd, bool all) cache_mngr->trx_cache.set_prev_position(MY_OFF_T_UNDEF); thd->reset_binlog_for_next_statement(); + DBUG_EXECUTE_IF("simulate_crash_after_binlog_commit_or_rollback", + DBUG_SUICIDE();); + DBUG_RETURN(error); } @@ -3420,6 +3430,8 @@ MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period) index_file_name[0] = 0; bzero((char*) &index_file, sizeof(index_file)); bzero((char*) &purge_index_file, sizeof(purge_index_file)); + /* non-zero is a marker to conduct xa recovery and related cleanup */ + xa_recover_list.records= xa_recover_htons= 0; } void MYSQL_BIN_LOG::stop_background_thread() @@ -3481,6 +3493,11 @@ void MYSQL_BIN_LOG::cleanup() mysql_cond_destroy(&COND_xid_list); mysql_cond_destroy(&COND_binlog_background_thread); mysql_cond_destroy(&COND_binlog_background_thread_end); + if (!is_relay_log && xa_recover_list.records) + { + free_root(&mem_root, MYF(0)); + my_hash_free(&xa_recover_list); + } } /* @@ -8156,7 +8173,7 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) /* Now we have in queue the list of transactions to be committed in order. */ } - + DBUG_ASSERT(is_open()); if (likely(is_open())) // Should always be true { @@ -9847,7 +9864,7 @@ int TC_LOG_MMAP::recover() goto err2; // OOM } - if (ha_recover(&xids)) + if (ha_recover(&xids, 0, NULL)) goto err2; my_hash_free(&xids); @@ -9888,7 +9905,7 @@ int TC_LOG::using_heuristic_recover() return 0; sql_print_information("Heuristic crash recovery mode"); - if (ha_recover(0)) + if (ha_recover(0, 0, NULL)) sql_print_error("Heuristic crash recovery failed"); sql_print_information("Please restart mysqld without --tc-heuristic-recover"); return 1; @@ -10156,7 +10173,7 @@ int TC_LOG_BINLOG::unlog_xa_prepare(THD *thd, bool all) if (!cache_mngr->need_unlog) { Ha_trx_info *ha_info; - uint rw_count= ha_count_rw_all(thd, &ha_info); + uint rw_count= ha_count_rw_all(thd, &ha_info, false); bool rc= false; if (rw_count > 0) @@ -10369,14 +10386,108 @@ start_binlog_background_thread() return 0; } +#ifdef HAVE_REPLICATION +/** + Auxiliary function for TC_LOG::recover(). + @returns a successfully created and inserted @c xa_recovery_member + into hash @c hash_arg, + or NULL. +*/ +static xa_recovery_member* +xa_member_insert(HASH *hash_arg, xid_t *xid_arg, xa_binlog_state state_arg, + MEM_ROOT *ptr_mem_root) +{ + xa_recovery_member *member= (xa_recovery_member*) + alloc_root(ptr_mem_root, sizeof(xa_recovery_member)); + if (!member) + return NULL; + + member->xid.set(xid_arg); + member->state= state_arg; + member->in_engine_prepare= 0; + return my_hash_insert(hash_arg, (uchar*) member) ? NULL : member; +} + +/* Inserts or updates an existing hash member with a proper state */ +static bool xa_member_replace(HASH *hash_arg, xid_t *xid_arg, bool is_prepare, + MEM_ROOT *ptr_mem_root) +{ + if(is_prepare) + { + if (!(xa_member_insert(hash_arg, xid_arg, XA_PREPARE, ptr_mem_root))) + return true; + } + else + { + /* + Search if XID is already present in recovery_list. If found + and the state is 'XA_PREPRAED' mark it as XA_COMPLETE. + Effectively, there won't be XA-prepare event group replay. + */ + xa_recovery_member* member; + if ((member= (xa_recovery_member *) + my_hash_search(hash_arg, xid_arg->key(), xid_arg->key_length()))) + { + if (member->state == XA_PREPARE) + member->state= XA_COMPLETE; + } + else // We found only XA COMMIT during recovery insert to list + { + if (!(member= xa_member_insert(hash_arg, + xid_arg, XA_COMPLETE, ptr_mem_root))) + return true; + } + } + return false; +} +#endif + +extern "C" uchar *xid_get_var_key(xid_t *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length= entry->key_length(); + return (uchar*) entry->key(); +} +/** + Performs recovery based on transaction coordinator log for 2pc. At the + time of crash, if the binary log was in active state, then recovery for + "implicit" 'xid's and explicit 'XA' transactions is initiated, + otherwise merely the gtid binlog state is updated. + For 'xid' and 'XA' based recovery the following steps are performed. + + Identify the active binlog checkpoint file. + Scan the binary log from the beginning. + From GTID_LIST and GTID_EVENTs reconstruct the gtid binlog state. + Prepare a list of 'xid's for recovery. + Prepare a list of explicit 'XA' transactions for recovery. + Recover the 'xid' transactions. + The explicit 'XA' transaction recovery is initiated once all the server + components are initialized. Please check 'execute_xa_for_recovery()'. + + Called from @c MYSQL_BIN_LOG::do_binlog_recovery() + + @param linfo Store here the found log file name and position to + the NEXT log file name in the index file. + + @param last_log_name Name of the last active binary log at the time of + crash. + + @param first_log Pointer to IO_CACHE of active binary log + @param fdle Format_description_log_event of active binary log + @param do_xa Is 2pc recovery needed for 'xid's and explicit XA + transactions. + @return indicates success or failure of recovery. + @retval 0 success + @retval 1 failure + +*/ int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, IO_CACHE *first_log, Format_description_log_event *fdle, bool do_xa) { Log_event *ev= NULL; HASH xids; - MEM_ROOT mem_root; char binlog_checkpoint_name[FN_REFLEN]; bool binlog_checkpoint_found; bool first_round; @@ -10389,9 +10500,14 @@ int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, bool last_gtid_valid= false; #endif - if (! fdle->is_valid() || - (do_xa && my_hash_init(key_memory_binlog_recover_exec, &xids, &my_charset_bin, TC_LOG_PAGE_SIZE/3, 0, - sizeof(my_xid), 0, 0, MYF(0)))) + binlog_checkpoint_name[0]= 0; + if (!fdle->is_valid() || + (do_xa && + (my_hash_init(key_memory_binlog_recover_exec, &xids, &my_charset_bin, + TC_LOG_PAGE_SIZE/3, 0, sizeof(my_xid), 0, 0, MYF(0)) || + my_hash_init(key_memory_binlog_recover_exec, &xa_recover_list, + &my_charset_bin, TC_LOG_PAGE_SIZE/3, 0, 0, + (my_hash_get_key) xid_get_var_key, 0, MYF(0))))) goto err1; if (do_xa) @@ -10465,21 +10581,29 @@ int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, #ifdef HAVE_REPLICATION case GTID_EVENT: - if (first_round) { Gtid_log_event *gev= (Gtid_log_event *)ev; - - /* Update the binlog state with any GTID logged after Gtid_list. */ - last_gtid.domain_id= gev->domain_id; - last_gtid.server_id= gev->server_id; - last_gtid.seq_no= gev->seq_no; - last_gtid_standalone= - ((gev->flags2 & Gtid_log_event::FL_STANDALONE) ? true : false); - last_gtid_valid= true; + if (first_round) + { + /* Update the binlog state with any GTID logged after Gtid_list. */ + last_gtid.domain_id= gev->domain_id; + last_gtid.server_id= gev->server_id; + last_gtid.seq_no= gev->seq_no; + last_gtid_standalone= + ((gev->flags2 & Gtid_log_event::FL_STANDALONE) ? true : false); + last_gtid_valid= true; + } + if (do_xa && + (gev->flags2 & + (Gtid_log_event::FL_PREPARED_XA | + Gtid_log_event::FL_COMPLETED_XA)) && + xa_member_replace(&xa_recover_list, &gev->xid, + gev->flags2 & Gtid_log_event::FL_PREPARED_XA, + &mem_root)) + goto err2; + break; } - break; #endif - case START_ENCRYPTION_EVENT: { if (fdle->start_decryption((Start_encryption_log_event*) ev)) @@ -10569,10 +10693,22 @@ int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, if (do_xa) { - if (ha_recover(&xids)) + if (ha_recover(&xids, &xa_recover_list, &xa_recover_htons)) goto err2; - free_root(&mem_root, MYF(0)); + DBUG_ASSERT(!xa_recover_list.records || + (binlog_checkpoint_found && binlog_checkpoint_name[0] != 0)); + + if (!xa_recover_list.records) + { + free_root(&mem_root, MYF(0)); + my_hash_free(&xa_recover_list); + } + else + { + xa_binlog_checkpoint_name= strmake_root(&mem_root, binlog_checkpoint_name, + strlen(binlog_checkpoint_name)); + } my_hash_free(&xids); } return 0; @@ -10588,6 +10724,7 @@ err2: { free_root(&mem_root, MYF(0)); my_hash_free(&xids); + my_hash_free(&xa_recover_list); } err1: sql_print_error("Crash recovery failed. Either correct the problem " @@ -10597,6 +10734,219 @@ err1: return 1; } +void MYSQL_BIN_LOG::execute_xa_for_recovery() +{ + if (xa_recover_list.records) + (void) recover_explicit_xa_prepare(); + free_root(&mem_root, MYF(0)); + my_hash_free(&xa_recover_list); +}; + +/** + Performs recovery of user XA transactions. + 'xa_recover_list' contains the list of XA transactions to be recovered. + with possible replaying replication event from the binary log. + + @return indicates success or failure of recovery. + @retval false success + @retval true failure + +*/ +bool MYSQL_BIN_LOG::recover_explicit_xa_prepare() +{ +#ifndef HAVE_REPLICATION + /* Can't be supported without replication applier built in. */ + return false; +#else + bool err= true; + int error=0; + Relay_log_info *rli= NULL; + rpl_group_info *rgi; + THD *thd= new THD(0); /* Needed by start_slave_threads */ + thd->thread_stack= (char*) &thd; + thd->store_globals(); + thd->security_ctx->skip_grants(); + IO_CACHE log; + const char *errmsg; + File file; + bool enable_apply_event= false; + Log_event *ev = 0; + LOG_INFO linfo; + int recover_xa_count= xa_recover_list.records; + xa_recovery_member *member= NULL; + + if (!(rli= thd->rli_fake= new Relay_log_info(FALSE, "Recovery"))) + { + my_error(ER_OUTOFMEMORY, MYF(ME_FATAL), 1); + goto err2; + } + rli->sql_driver_thd= thd; + static LEX_CSTRING connection_name= { STRING_WITH_LEN("Recovery") }; + rli->mi= new Master_info(&connection_name, false); + if (!(rgi= thd->rgi_fake)) + rgi= thd->rgi_fake= new rpl_group_info(rli); + rgi->thd= thd; + thd->system_thread_info.rpl_sql_info= + new rpl_sql_thread_info(rli->mi->rpl_filter); + + if (rli && !rli->relay_log.description_event_for_exec) + { + rli->relay_log.description_event_for_exec= + new Format_description_log_event(4); + } + if (find_log_pos(&linfo, xa_binlog_checkpoint_name, 1)) + { + sql_print_error("Binlog file '%s' not found in binlog index, needed " + "for recovery. Aborting.", xa_binlog_checkpoint_name); + goto err2; + } + + tmp_disable_binlog(thd); + thd->variables.pseudo_slave_mode= TRUE; + for (;;) + { + if ((file= open_binlog(&log, linfo.log_file_name, &errmsg)) < 0) + { + sql_print_error("%s", errmsg); + goto err1; + } + while (recover_xa_count > 0 && + (ev= Log_event::read_log_event(&log, + rli->relay_log.description_event_for_exec, + opt_master_verify_checksum))) + { + if (!ev->is_valid()) + { + sql_print_error("Found invalid binlog query event %s" + " at %s:%llu; error %d %s", ev->get_type_str(), + linfo.log_file_name, + (ev->log_pos - ev->data_written)); + goto err1; + } + enum Log_event_type typ= ev->get_type_code(); + ev->thd= thd; + + if (typ == FORMAT_DESCRIPTION_EVENT) + enable_apply_event= true; + + if (typ == GTID_EVENT) + { + Gtid_log_event *gev= (Gtid_log_event *)ev; + if (gev->flags2 & + (Gtid_log_event::FL_PREPARED_XA | Gtid_log_event::FL_COMPLETED_XA)) + { + if ((member= + (xa_recovery_member*) my_hash_search(&xa_recover_list, + gev->xid.key(), + gev->xid.key_length()))) + { + /* + When XA PREPARE group of events (as flagged so) check + its actual binlog state which may be COMPLETED. If the + state is also PREPARED then analyze through + in_engine_prepare whether the transaction needs replay. + */ + if (gev->flags2 & Gtid_log_event::FL_PREPARED_XA) + { + if (member->state == XA_PREPARE) + { + // XA prepared is not present in (some) engine then apply it + if (member->in_engine_prepare == 0) + enable_apply_event= true; + else if (gev->flags2 & Gtid_log_event::FL_MULTI_ENGINE_XA && + xa_recover_htons > member->in_engine_prepare) + { + enable_apply_event= true; + // partially engine-prepared XA is first cleaned out prior replay + thd->lex->sql_command= SQLCOM_XA_ROLLBACK; + ha_commit_or_rollback_by_xid(&gev->xid, 0); + } + else + --recover_xa_count; + } + } + else if (gev->flags2 & Gtid_log_event::FL_COMPLETED_XA) + { + if (member->state == XA_COMPLETE && + member->in_engine_prepare > 0) + enable_apply_event= true; + else + --recover_xa_count; + } + } + } + } + + if (enable_apply_event) + { + if ((err= ev->apply_event(rgi))) + { + sql_print_error("Failed to execute binlog query event of type: %s," + " at %s:%llu; error %d %s", ev->get_type_str(), + linfo.log_file_name, + (ev->log_pos - ev->data_written), + thd->get_stmt_da()->sql_errno(), + thd->get_stmt_da()->message()); + delete ev; + goto err1; + } + else if (typ == FORMAT_DESCRIPTION_EVENT) + enable_apply_event=false; + else if (thd->lex->sql_command == SQLCOM_XA_PREPARE || + thd->lex->sql_command == SQLCOM_XA_COMMIT || + thd->lex->sql_command == SQLCOM_XA_ROLLBACK) + { + --recover_xa_count; + enable_apply_event=false; + + sql_print_information("Binlog event %s at %s:%llu" + " successfully applied", + typ == XA_PREPARE_LOG_EVENT ? + static_cast<XA_prepare_log_event *>(ev)->get_query() : + static_cast<Query_log_event *>(ev)->query, + linfo.log_file_name, (ev->log_pos - ev->data_written)); + } + } + if (typ != FORMAT_DESCRIPTION_EVENT) + delete ev; + } + end_io_cache(&log); + mysql_file_close(file, MYF(MY_WME)); + file= -1; + if (unlikely((error= find_next_log(&linfo, 1)))) + { + if (error != LOG_INFO_EOF) + sql_print_error("find_log_pos() failed (error: %d)", error); + else + break; + } + } +err1: + reenable_binlog(thd); + /* + There should be no more XA transactions to recover upon successful + completion. + */ + if (recover_xa_count > 0) + goto err2; + sql_print_information("Crash recovery finished."); + err= false; +err2: + if (file >= 0) + { + end_io_cache(&log); + mysql_file_close(file, MYF(MY_WME)); + } + thd->variables.pseudo_slave_mode= FALSE; + delete rli->mi; + delete thd->system_thread_info.rpl_sql_info; + rgi->slave_close_thread_tables(thd); + thd->reset_globals(); + delete thd; + + return err; +#endif /* !HAVE_REPLICATION */ +} int MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery) diff --git a/sql/log.h b/sql/log.h index 063513fe908..a795cf511ae 100644 --- a/sql/log.h +++ b/sql/log.h @@ -63,6 +63,7 @@ class TC_LOG virtual int unlog(ulong cookie, my_xid xid)=0; virtual int unlog_xa_prepare(THD *thd, bool all)= 0; virtual void commit_checkpoint_notify(void *cookie)= 0; + virtual void execute_xa_for_recovery() {}; protected: /* @@ -710,6 +711,8 @@ public: void commit_checkpoint_notify(void *cookie); int recover(LOG_INFO *linfo, const char *last_log_name, IO_CACHE *first_log, Format_description_log_event *fdle, bool do_xa); + bool recover_explicit_xa_prepare(); + int do_binlog_recovery(const char *opt_name, bool do_xa_recovery); #if !defined(MYSQL_CLIENT) @@ -934,7 +937,7 @@ public: mysql_mutex_t* get_binlog_end_pos_lock() { return &LOCK_binlog_end_pos; } int wait_for_update_binlog_end_pos(THD* thd, struct timespec * timeout); - + void execute_xa_for_recovery(); /* Binlog position of end of the binlog. Access to this is protected by LOCK_binlog_end_pos @@ -947,6 +950,10 @@ public: */ my_off_t binlog_end_pos; char binlog_end_pos_file[FN_REFLEN]; + MEM_ROOT mem_root; + char *xa_binlog_checkpoint_name; // binlog file to start off xa recovery + HASH xa_recover_list; // user xids with their binlog/engine status + uint xa_recover_htons; // number of detected recoverable hton:s }; class Log_event_handler diff --git a/sql/log_event.h b/sql/log_event.h index 639cbfbe7aa..e1880339e85 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -3249,12 +3249,7 @@ public: #ifdef MYSQL_SERVER bool write(); -#endif - -private: -#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) - char query[sizeof("XA COMMIT ONE PHASE") + 1 + ser_buf_size]; - int do_commit(); +#ifdef HAVE_REPLICATION const char* get_query() { sprintf(query, @@ -3262,6 +3257,13 @@ private: m_xid.serialize()); return query; } +#endif /* HAVE_REPLICATION */ +#endif /* MYSQL_SERVER */ + +private: +#if defined(MYSQL_SERVER) && defined(HAVE_REPLICATION) + char query[sizeof("XA COMMIT ONE PHASE") + 1 + ser_buf_size]; + int do_commit(); #endif }; @@ -3614,6 +3616,12 @@ public: static const uchar FL_PREPARED_XA= 64; /* FL_"COMMITTED or ROLLED-BACK"_XA is set for XA transaction. */ static const uchar FL_COMPLETED_XA= 128; + /* + To mark the fact of multiple transactional engine participants + in the prepared XA. The FL_COMPLETED_XA bit is reused by XA_PREPARE_LOG_EVENT, + oth the XA completion events do not need such marking. + */ + static const uchar FL_MULTI_ENGINE_XA= 128; #ifdef MYSQL_SERVER Gtid_log_event(THD *thd_arg, uint64 seq_no, uint32 domain_id, bool standalone, diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index b6c626e7735..51d15bb58f4 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -3263,6 +3263,8 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg, flags2|= thd->lex->sql_command == SQLCOM_XA_PREPARE ? FL_PREPARED_XA : FL_COMPLETED_XA; + flags2|= (thd->lex->sql_command == SQLCOM_XA_PREPARE && + ha_count_rw_all(thd, NULL, true) > 2) ? FL_MULTI_ENGINE_XA : 0; xid.set(xid_state.get_xid()); } } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 2ed732329d2..53ba297c11b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5075,7 +5075,7 @@ static int init_server_components() unireg_abort(1); } - if (ha_recover(0)) + if (ha_recover(0, 0, NULL)) { unireg_abort(1); } @@ -5499,7 +5499,7 @@ int mysqld_main(int argc, char **argv) initialize_information_schema_acl(); execute_ddl_log_recovery(); - + tc_log->execute_xa_for_recovery(); /* Change EVENTS_ORIGINAL to EVENTS_OFF (the default value) as there is no point in using ORIGINAL during startup diff --git a/sql/xa.cc b/sql/xa.cc index 69e9fd70af6..f74c96c8d89 100644 --- a/sql/xa.cc +++ b/sql/xa.cc @@ -568,6 +568,11 @@ bool trans_xa_commit(THD *thd) DBUG_ENTER("trans_xa_commit"); + DBUG_EXECUTE_IF("trans_xa_commit_fail", + { my_error(ER_OUT_OF_RESOURCES, MYF(0)); + DBUG_RETURN(TRUE); }); + + if (!xid_state.is_explicit_XA() || !xid_state.xid_cache_element->xid.eq(thd->lex->xid)) { diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_xa_recover.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_xa_recover.result new file mode 100644 index 00000000000..6561d0e1ce3 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_xa_recover.result @@ -0,0 +1,132 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +connection master; +CALL mtr.add_suppression("Found . prepared XA transactions"); +CALL mtr.add_suppression("Failed to execute binlog query event"); +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=rocksdb; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=innodb; +INSERT INTO t1 SET a = 1; +INSERT INTO t2 SET a = 1; +XA START 'xa1'; +INSERT INTO t1 SET a=1 + 1; +INSERT INTO t2 SET a=1 + 1; +XA END 'xa1'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_binlog_prepare"; +XA PREPARE 'xa1'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** xa1 in the list" +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa1 +XA ROLLBACK 'xa1'; +SELECT MAX(a) - 1 as zero FROM t1; +zero +0 +SELECT MAX(a) - 1 as zero FROM t2; +zero +0 +XA START 'xa1'; +INSERT INTO t1 SET a=1 + 2; +INSERT INTO t2 SET a=1 + 2; +XA END 'xa1'; +XA PREPARE 'xa1'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +XA ROLLBACK 'xa1'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** empty list (rolled back)" +XA RECOVER; +formatID gtrid_length bqual_length data +SELECT MAX(a) - 1 as zero FROM t1; +zero +0 +SELECT MAX(a) - 1 as zero FROM t2; +zero +0 +XA START 'xa1'; +INSERT INTO t1 SET a=1 + 3; +INSERT INTO t2 SET a=1 + 3; +XA END 'xa1'; +XA PREPARE 'xa1'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_binlog_commit_or_rollback"; +XA COMMIT 'xa1'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** empty list (committed away)" +XA RECOVER; +formatID gtrid_length bqual_length data +SELECT MAX(a) - 1 as three FROM t1; +three +3 +SELECT MAX(a) - 1 as three FROM t2; +three +3 +XA START 'xa2'; +INSERT INTO t1 SET a=4 + 1; +INSERT INTO t2 SET a=4 + 1; +XA END 'xa2'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_first_engine_prepare"; +XA PREPARE 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** xa2 in the list" +XA RECOVER; +formatID gtrid_length bqual_length data +1 3 0 xa2 +XA ROLLBACK 'xa2'; +SELECT MAX(a) - 4 as zero FROM t1; +zero +0 +SELECT MAX(a) - 4 as zero FROM t2; +zero +0 +XA START 'xa2'; +INSERT INTO t1 SET a=4 + 2; +INSERT INTO t2 SET a=4 + 2; +XA END 'xa2'; +XA PREPARE 'xa2'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_first_engine_commit_or_rollback"; +XA ROLLBACK 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** empty list (rolled back)" +XA RECOVER; +formatID gtrid_length bqual_length data +SELECT MAX(a) - 4 as zero FROM t1; +zero +0 +SELECT MAX(a) - 4 as zero FROM t2; +zero +0 +XA START 'xa2'; +INSERT INTO t1 SET a=4 + 3; +INSERT INTO t2 SET a=4 + 3; +XA END 'xa2'; +XA PREPARE 'xa2'; +SET SESSION DEBUG_DBUG="d,simulate_crash_after_first_engine_commit_or_rollback"; +XA COMMIT 'xa2'; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +"*** empty list (committed away)" +XA RECOVER; +formatID gtrid_length bqual_length data +SELECT MAX(a) - 4 as three FROM t1; +three +3 +SELECT MAX(a) - 4 as three FROM t2; +three +3 +connection slave; +include/start_slave.inc +connection master; +connection slave; +include/diff_tables.inc [master:t1, slave:t1] +include/diff_tables.inc [master:t2, slave:t2] +connection master; +SET SESSION DEBUG_DBUG=""; +drop table t1,t2; +connection slave; +include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover-master.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover-master.opt new file mode 100644 index 00000000000..0fbeabe503f --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover-master.opt @@ -0,0 +1 @@ +--log_bin --rocksdb_flush_log_at_trx_commit=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.inc new file mode 100644 index 00000000000..312b8b6a19e --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.inc @@ -0,0 +1,67 @@ +# callee of rpl_rocksdb_xa_recover.test +# requires t1,t2 as defined in the caller. + +--let $at=_prepare +--let $finally_expected=$xa in the list +--let $init_val=`SELECT MAX(a) from t1 as t_1` +--eval XA START '$xa' +--eval INSERT INTO t1 SET a=$init_val + 1 +--eval INSERT INTO t2 SET a=$init_val + 1 +--eval XA END '$xa' +--eval SET SESSION DEBUG_DBUG="d,simulate_crash_$when$at" + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--error 2013 # CR_SERVER_LOST +--eval XA PREPARE '$xa' +--source include/wait_until_disconnected.inc +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc +--echo "*** $finally_expected" +XA RECOVER; +--eval XA ROLLBACK '$xa' +--eval SELECT MAX(a) - $init_val as zero FROM t1 +--eval SELECT MAX(a) - $init_val as zero FROM t2 + + +--let $at=_commit_or_rollback +--let $finally_expected=empty list (rolled back) +--eval XA START '$xa' +--eval INSERT INTO t1 SET a=$init_val + 2 +--eval INSERT INTO t2 SET a=$init_val + 2 +--eval XA END '$xa' +--eval XA PREPARE '$xa' + +--eval SET SESSION DEBUG_DBUG="d,simulate_crash_$when$at" +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--error 2013 # CR_SERVER_LOST +--eval XA ROLLBACK '$xa' +--source include/wait_until_disconnected.inc +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +--echo "*** $finally_expected" +XA RECOVER; +--eval SELECT MAX(a) - $init_val as zero FROM t1 +--eval SELECT MAX(a) - $init_val as zero FROM t2 + + +--let $at=_commit_or_rollback +--let $finally_expected=empty list (committed away) +--eval XA START '$xa' +--eval INSERT INTO t1 SET a=$init_val + 3 +--eval INSERT INTO t2 SET a=$init_val + 3 +--eval XA END '$xa' +--eval XA PREPARE '$xa' + +--eval SET SESSION DEBUG_DBUG="d,simulate_crash_$when$at" +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--error 2013 # CR_SERVER_LOST +--eval XA COMMIT '$xa' +--source include/wait_until_disconnected.inc +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +--echo "*** $finally_expected" +XA RECOVER; +--eval SELECT MAX(a) - $init_val as three FROM t1 +--eval SELECT MAX(a) - $init_val as three FROM t2 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.test new file mode 100644 index 00000000000..c7cefbda4bf --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_xa_recover.test @@ -0,0 +1,46 @@ +# MDEV-742, MDEV-21469 XA replication, and xa crash-safe. +# Tests prove xa state is recovered to a prepared or completed state +# upon post-crash recovery, incl a multi-engine case. + +--source include/have_rocksdb.inc +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--connection slave +--source include/stop_slave.inc + +--connection master +CALL mtr.add_suppression("Found . prepared XA transactions"); +CALL mtr.add_suppression("Failed to execute binlog query event"); +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=rocksdb; +CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=innodb; +INSERT INTO t1 SET a = 1; +INSERT INTO t2 SET a = 1; + +--let $xa=xa1 +--let $when=after_binlog +--source rpl_rocksdb_xa_recover.inc + +--let $xa=xa2 +--let $when=after_first_engine +--let $finally_expected=$xa in the list +--source rpl_rocksdb_xa_recover.inc + +--connection slave +--source include/start_slave.inc + +--connection master +--sync_slave_with_master +let $diff_tables= master:t1, slave:t1; +source include/diff_tables.inc; +let $diff_tables= master:t2, slave:t2; +source include/diff_tables.inc; + +--connection master +SET SESSION DEBUG_DBUG=""; +drop table t1,t2; +--sync_slave_with_master + +--source include/rpl_end.inc |