MDEV-12352 InnoDB shutdown should not be blocked by a large transaction rollback

row_undo_step(), trx_rollback_active(): Abort the rollback of a
recovered ordinary transaction if fast shutdown has been initiated.

trx_rollback_resurrected(): Convert an aborted-rollback transaction
into a fake XA PREPARE transaction, so that fast shutdown can proceed.

8 files changed, 83 insertions, 14 deletions

diff --git a/mysql-test/suite/innodb/r/recovery_shutdown.result b/mysql-test/suite/innodb/r/recovery_shutdown.result
index 028a0bd6239..861461dd072 100644
--- a/mysql-test/suite/innodb/r/recovery_shutdown.result
+++ b/mysql-test/suite/innodb/r/recovery_shutdown.result
@@ -45,6 +45,11 @@ BEGIN;
 INSERT INTO t1 (a) SELECT NULL FROM t;
 UPDATE t1 SET a=a+100, b=a;
 DELETE FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 CREATE TABLE u(a SERIAL) ENGINE=INNODB;
 # Kill and restart
diff --git a/mysql-test/suite/innodb/t/recovery_shutdown.test b/mysql-test/suite/innodb/t/recovery_shutdown.test
index ea38bd19a9f..28b80cd3818 100644
--- a/mysql-test/suite/innodb/t/recovery_shutdown.test
+++ b/mysql-test/suite/innodb/t/recovery_shutdown.test
@@ -31,6 +31,12 @@ eval DELETE FROM t$c;
 dec $c;
 }
 
+INSERT INTO t1(a) SELECT NULL FROM t;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+
 --connection default
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 CREATE TABLE u(a SERIAL) ENGINE=INNODB;
diff --git a/storage/innobase/include/trx0roll.h b/storage/innobase/include/trx0roll.h
index 98a667b2ec1..e9e3f3a77c8 100644
--- a/storage/innobase/include/trx0roll.h
+++ b/storage/innobase/include/trx0roll.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -32,7 +33,8 @@ Created 3/26/1996 Heikki Tuuri
 #include "mtr0mtr.h"
 #include "trx0sys.h"
 
-extern bool	trx_rollback_or_clean_is_active;
+extern bool		trx_rollback_or_clean_is_active;
+extern const trx_t*	trx_roll_crash_recv_trx;
 
 /*******************************************************************//**
 Determines if this transaction is rolling back an incomplete transaction
diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc
index 149dc671930..95b88f6ce42 100644
--- a/storage/innobase/row/row0undo.cc
+++ b/storage/innobase/row/row0undo.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -44,6 +45,7 @@ Created 1/8/1997 Heikki Tuuri
 #include "row0upd.h"
 #include "row0mysql.h"
 #include "srv0srv.h"
+#include "srv0start.h"
 
 /* How to undo row operations?
 (1) For an insert, we have stored a prefix of the clustered index record
@@ -348,6 +350,14 @@ row_undo_step(
 
 	ut_ad(que_node_get_type(node) == QUE_NODE_UNDO);
 
+	if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx)
+	    && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE
+	    && !srv_undo_sources && srv_fast_shutdown) {
+		/* Shutdown has been initiated. */
+		trx->error_state = DB_INTERRUPTED;
+		return(NULL);
+	}
+
 	err = row_undo(node, thr);
 
 	trx->error_state = err;
diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc
index dbb902258c6..a0bc85e0433 100644
--- a/storage/innobase/trx/trx0roll.cc
+++ b/storage/innobase/trx/trx0roll.cc
@@ -55,7 +55,7 @@ rollback */
 bool			trx_rollback_or_clean_is_active;
 
 /** In crash recovery, the current trx to be rolled back; NULL otherwise */
-static const trx_t*	trx_roll_crash_recv_trx	= NULL;
+const trx_t*		trx_roll_crash_recv_trx;
 
 /** In crash recovery we set this to the undo n:o of the current trx to be
 rolled back. Then we can print how many % the rollback has progressed. */
@@ -605,6 +605,14 @@ trx_rollback_active(
 
 	que_run_threads(roll_node->undo_thr);
 
+	if (trx->error_state != DB_SUCCESS) {
+		ut_ad(trx->error_state == DB_INTERRUPTED);
+		ut_ad(!srv_undo_sources);
+		ut_ad(srv_fast_shutdown);
+		ut_ad(!dictionary_locked);
+		goto func_exit;
+	}
+
 	trx_rollback_finish(thr_get_trx(roll_node->undo_thr));
 
 	/* Free the memory reserved by the undo graph */
@@ -649,13 +657,14 @@ trx_rollback_active(
 		}
 	}
 
+	ib_logf(IB_LOG_LEVEL_INFO,
+		"Rollback of trx with id " TRX_ID_FMT " completed", trx->id);
+
+func_exit:
 	if (dictionary_locked) {
 		row_mysql_unlock_data_dictionary(trx);
 	}
 
-	ib_logf(IB_LOG_LEVEL_INFO,
-		"Rollback of trx with id " TRX_ID_FMT " completed", trx->id);
-
 	mem_heap_free(heap);
 
 	trx_roll_crash_recv_trx	= NULL;
@@ -700,8 +709,8 @@ func_exit:
 		trx_free_for_background(trx);
 		return(TRUE);
 	case TRX_STATE_ACTIVE:
-		if (srv_shutdown_state != SRV_SHUTDOWN_NONE
-		    && srv_fast_shutdown) {
+		if (!srv_undo_sources && srv_fast_shutdown) {
+fake_prepared:
 			trx->state = TRX_STATE_PREPARED;
 			trx_sys->n_prepared_trx++;
 			trx_sys->n_prepared_recovered_trx++;
@@ -713,6 +722,14 @@ func_exit:
 		if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 			mutex_exit(&trx_sys->mutex);
 			trx_rollback_active(trx);
+			if (trx->error_state != DB_SUCCESS) {
+				ut_ad(trx->error_state == DB_INTERRUPTED);
+				ut_ad(!srv_undo_sources);
+				ut_ad(srv_fast_shutdown);
+				mutex_enter(&trx_sys->mutex);
+				trx_mutex_enter(trx);
+				goto fake_prepared;
+			}
 			trx_free_for_background(trx);
 			return(TRUE);
 		}
diff --git a/storage/xtradb/include/trx0roll.h b/storage/xtradb/include/trx0roll.h
index b2e9d8a077f..399b29610ff 100644
--- a/storage/xtradb/include/trx0roll.h
+++ b/storage/xtradb/include/trx0roll.h
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -33,7 +34,8 @@ Created 3/26/1996 Heikki Tuuri
 #include "mtr0mtr.h"
 #include "trx0sys.h"
 
-extern bool	trx_rollback_or_clean_is_active;
+extern bool		trx_rollback_or_clean_is_active;
+extern const trx_t*	trx_roll_crash_recv_trx;
 
 /*******************************************************************//**
 Determines if this transaction is rolling back an incomplete transaction
diff --git a/storage/xtradb/row/row0undo.cc b/storage/xtradb/row/row0undo.cc
index 149dc671930..95b88f6ce42 100644
--- a/storage/xtradb/row/row0undo.cc
+++ b/storage/xtradb/row/row0undo.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -44,6 +45,7 @@ Created 1/8/1997 Heikki Tuuri
 #include "row0upd.h"
 #include "row0mysql.h"
 #include "srv0srv.h"
+#include "srv0start.h"
 
 /* How to undo row operations?
 (1) For an insert, we have stored a prefix of the clustered index record
@@ -348,6 +350,14 @@ row_undo_step(
 
 	ut_ad(que_node_get_type(node) == QUE_NODE_UNDO);
 
+	if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx)
+	    && trx_get_dict_operation(trx) == TRX_DICT_OP_NONE
+	    && !srv_undo_sources && srv_fast_shutdown) {
+		/* Shutdown has been initiated. */
+		trx->error_state = DB_INTERRUPTED;
+		return(NULL);
+	}
+
 	err = row_undo(node, thr);
 
 	trx->error_state = err;
diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc
index dbb902258c6..a0bc85e0433 100644
--- a/storage/xtradb/trx/trx0roll.cc
+++ b/storage/xtradb/trx/trx0roll.cc
@@ -55,7 +55,7 @@ rollback */
 bool			trx_rollback_or_clean_is_active;
 
 /** In crash recovery, the current trx to be rolled back; NULL otherwise */
-static const trx_t*	trx_roll_crash_recv_trx	= NULL;
+const trx_t*		trx_roll_crash_recv_trx;
 
 /** In crash recovery we set this to the undo n:o of the current trx to be
 rolled back. Then we can print how many % the rollback has progressed. */
@@ -605,6 +605,14 @@ trx_rollback_active(
 
 	que_run_threads(roll_node->undo_thr);
 
+	if (trx->error_state != DB_SUCCESS) {
+		ut_ad(trx->error_state == DB_INTERRUPTED);
+		ut_ad(!srv_undo_sources);
+		ut_ad(srv_fast_shutdown);
+		ut_ad(!dictionary_locked);
+		goto func_exit;
+	}
+
 	trx_rollback_finish(thr_get_trx(roll_node->undo_thr));
 
 	/* Free the memory reserved by the undo graph */
@@ -649,13 +657,14 @@ trx_rollback_active(
 		}
 	}
 
+	ib_logf(IB_LOG_LEVEL_INFO,
+		"Rollback of trx with id " TRX_ID_FMT " completed", trx->id);
+
+func_exit:
 	if (dictionary_locked) {
 		row_mysql_unlock_data_dictionary(trx);
 	}
 
-	ib_logf(IB_LOG_LEVEL_INFO,
-		"Rollback of trx with id " TRX_ID_FMT " completed", trx->id);
-
 	mem_heap_free(heap);
 
 	trx_roll_crash_recv_trx	= NULL;
@@ -700,8 +709,8 @@ func_exit:
 		trx_free_for_background(trx);
 		return(TRUE);
 	case TRX_STATE_ACTIVE:
-		if (srv_shutdown_state != SRV_SHUTDOWN_NONE
-		    && srv_fast_shutdown) {
+		if (!srv_undo_sources && srv_fast_shutdown) {
+fake_prepared:
 			trx->state = TRX_STATE_PREPARED;
 			trx_sys->n_prepared_trx++;
 			trx_sys->n_prepared_recovered_trx++;
@@ -713,6 +722,14 @@ func_exit:
 		if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 			mutex_exit(&trx_sys->mutex);
 			trx_rollback_active(trx);
+			if (trx->error_state != DB_SUCCESS) {
+				ut_ad(trx->error_state == DB_INTERRUPTED);
+				ut_ad(!srv_undo_sources);
+				ut_ad(srv_fast_shutdown);
+				mutex_enter(&trx_sys->mutex);
+				trx_mutex_enter(trx);
+				goto fake_prepared;
+			}
 			trx_free_for_background(trx);
 			return(TRUE);
 		}