MDEV-24612: innodb hangs if it's initialization is broken before encryption threads are startedbb-10.5-MDEV-24612

Do not init encryption threads if shutdown is in progress.

4 files changed, 51 insertions, 0 deletions

diff --git a/mysql-test/suite/innodb/r/encryption_threads_shutdown.result b/mysql-test/suite/innodb/r/encryption_threads_shutdown.result
new file mode 100644
index 00000000000..3f0dcd90426
--- /dev/null
+++ b/mysql-test/suite/innodb/r/encryption_threads_shutdown.result
@@ -0,0 +1,12 @@
+call mtr.add_suppression("Creating system tablespace with existing redo log file is not recommended.");
+call mtr.add_suppression("InnoDB: Database creation was aborted");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
+CREATE TABLE t(a INT) ENGINE=InnoDB;
+# Kill the server
+# restart
+SELECT * FROM t;
+ERROR 42000: Unknown storage engine 'InnoDB'
+# Kill the server
+# restart
+DROP TABLE t;
diff --git a/mysql-test/suite/innodb/t/encryption_threads_shutdown.opt b/mysql-test/suite/innodb/t/encryption_threads_shutdown.opt
new file mode 100644
index 00000000000..e5a02a1a1c9
--- /dev/null
+++ b/mysql-test/suite/innodb/t/encryption_threads_shutdown.opt
@@ -0,0 +1,6 @@
+--innodb-encryption-rotate-key-age=2
+--innodb-encryption-threads=4
+--innodb-tablespaces-encryption
+--plugin-load-add=$FILE_KEY_MANAGEMENT_SO
+--loose-file-key-management
+--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt
diff --git a/mysql-test/suite/innodb/t/encryption_threads_shutdown.test b/mysql-test/suite/innodb/t/encryption_threads_shutdown.test
new file mode 100644
index 00000000000..c3ed2b29c52
--- /dev/null
+++ b/mysql-test/suite/innodb/t/encryption_threads_shutdown.test
@@ -0,0 +1,31 @@
+# This test is for MDEV-24612 fix
+--source include/have_innodb.inc
+
+call mtr.add_suppression("Creating system tablespace with existing redo log file is not recommended.");
+call mtr.add_suppression("InnoDB: Database creation was aborted");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
+
+--let $MYSQLD_DATADIR=`SELECT @@datadir`
+--let $DATA= $MYSQLD_DATADIR/ibdata1
+--let $DATACOPY=$MYSQLTEST_VARDIR/tmp/ibdata1
+CREATE TABLE t(a INT) ENGINE=InnoDB;
+--source include/kill_mysqld.inc
+# Move the file to cause srv_init_abort_low() call from srv_start()
+--move_file $DATA $DATACOPY
+
+# If the bug is not fixed, the server will hang here. Note that the test is
+# unstable because the condition
+# "srv_n_fil_crypt_threads_started != srv_n_fil_crypt_threads" can be true
+# in fil_crypt_set_thread_cnt() functions if encryption threads have not yet
+# been finished. To make the test stable we need to add debug code which would
+# check that all encryption threads were finished before the "while" loop
+# with the above condition. And in this test we can't detect if the  debug code
+# itself caused the hang or the code we are testing did it.
+--source include/start_mysqld.inc
+--error ER_UNKNOWN_STORAGE_ENGINE
+SELECT * FROM t;
+--source include/kill_mysqld.inc
+--move_file $DATACOPY $DATA
+--source include/start_mysqld.inc
+DROP TABLE t;
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index acc196a910d..26c73e7d0eb 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -2216,6 +2216,8 @@ fil_crypt_set_thread_cnt(
 	const uint	new_cnt)
 {
 	if (!fil_crypt_threads_inited) {
+		if (srv_shutdown_state != SRV_SHUTDOWN_NONE)
+			return;
 		fil_crypt_threads_init();
 	}