MDEV-12198 innodb_defragment=1 crashes server on OPTIMIZE TABLE when FULLTEXT index exists

ha_innobase::defragment_table(): Skip corrupted indexes and
FULLTEXT INDEX. In InnoDB, FULLTEXT INDEX is implemented with
auxiliary tables. We will not defragment them on OPTIMIZE TABLE.

8 files changed, 77 insertions, 20 deletions

diff --git a/mysql-test/suite/innodb/r/innodb_defragment.result b/mysql-test/suite/innodb/r/innodb_defragment.result
index 5f3fd523946..5d53fde3549 100644
--- a/mysql-test/suite/innodb/r/innodb_defragment.result
+++ b/mysql-test/suite/innodb/r/innodb_defragment.result
@@ -1,4 +1,3 @@
-DROP TABLE if exists t1;
 set global innodb_defragment_stats_accuracy = 80;
 CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB;
 optimize table t1;
diff --git a/mysql-test/suite/innodb/r/innodb_defragment_small.result b/mysql-test/suite/innodb/r/innodb_defragment_small.result
new file mode 100644
index 00000000000..3f594f79e70
--- /dev/null
+++ b/mysql-test/suite/innodb/r/innodb_defragment_small.result
@@ -0,0 +1,24 @@
+SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment;
+SET GLOBAL innodb_defragment = 1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+INSERT INTO t1 VALUES (100000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (200000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (300000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (400000, REPEAT('A', 256));
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+DROP TABLE t1;
+#
+# MDEV-12198 innodb_defragment=1 crashes server on
+# OPTIMIZE TABLE when FULLTEXT index exists
+#
+CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+DROP TABLE t1;
+SET GLOBAL innodb_defragment = @innodb_defragment_orig;
diff --git a/mysql-test/suite/innodb/t/innodb_defragment-master.opt b/mysql-test/suite/innodb/t/innodb_defragment-master.opt
deleted file mode 100644
index 6fc7f343b24..00000000000
--- a/mysql-test/suite/innodb/t/innodb_defragment-master.opt
+++ /dev/null
@@ -1,2 +0,0 @@
---innodb_file_per_table
---innodb-defragment=1
\ No newline at end of file
diff --git a/mysql-test/suite/innodb/t/innodb_defragment.opt b/mysql-test/suite/innodb/t/innodb_defragment.opt
index 6426bac41a0..aea3d480c24 100644
--- a/mysql-test/suite/innodb/t/innodb_defragment.opt
+++ b/mysql-test/suite/innodb/t/innodb_defragment.opt
@@ -1,4 +1,5 @@
 --loose-innodb-buffer-pool-stats
 --loose-innodb-buffer-page
 --loose-innodb-buffer-page-lru
+--innodb-file-per-table
 --innodb-defragment=1
\ No newline at end of file
diff --git a/mysql-test/suite/innodb/t/innodb_defragment.test b/mysql-test/suite/innodb/t/innodb_defragment.test
index 22b72a4aa6b..d9f5f56316e 100644
--- a/mysql-test/suite/innodb/t/innodb_defragment.test
+++ b/mysql-test/suite/innodb/t/innodb_defragment.test
@@ -1,17 +1,7 @@
 --source include/have_innodb.inc
 --source include/big_test.inc
---source include/not_valgrind.inc
 --source include/not_embedded.inc
 
---disable_warnings
-DROP TABLE if exists t1;
---enable_warnings
-
---disable_query_log
-let $innodb_defragment_n_pages_orig=`select @@innodb_defragment_n_pages`;
-let $innodb_defragment_stats_accuracy_orig=`select @@innodb_defragment_stats_accuracy`;
---enable_query_log
-
 set global innodb_defragment_stats_accuracy = 80;
 
 # Create table.
@@ -47,12 +37,14 @@ delimiter ;//
 # Populate table.
 let $i = $data_size;
 --disable_query_log
+BEGIN;
 while ($i)
 {
   eval
     INSERT INTO t1 VALUES ($data_size + 1 - $i, REPEAT('A', 256));
   dec $i;
 }
+COMMIT;
 --enable_query_log
 
 select count(stat_value) from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name  in ('n_pages_freed');
@@ -161,10 +153,3 @@ select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like
 
 DROP PROCEDURE defragment;
 DROP TABLE t1;
-
-# reset system
---disable_query_log
-EVAL SET GLOBAL innodb_defragment_n_pages = $innodb_defragment_n_pages_orig;
-EVAL SET GLOBAL innodb_defragment_stats_accuracy = $innodb_defragment_stats_accuracy_orig;
---enable_query_log
-
diff --git a/mysql-test/suite/innodb/t/innodb_defragment_small.test b/mysql-test/suite/innodb/t/innodb_defragment_small.test
new file mode 100644
index 00000000000..454333d6986
--- /dev/null
+++ b/mysql-test/suite/innodb/t/innodb_defragment_small.test
@@ -0,0 +1,28 @@
+--source include/have_innodb.inc
+
+SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment;
+SET GLOBAL innodb_defragment = 1;
+
+# Small tests copied from innodb.innodb_defragment
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB;
+OPTIMIZE TABLE t1;
+
+INSERT INTO t1 VALUES (100000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (200000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (300000, REPEAT('A', 256));
+INSERT INTO t1 VALUES (400000, REPEAT('A', 256));
+
+OPTIMIZE TABLE t1;
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-12198 innodb_defragment=1 crashes server on
+--echo # OPTIMIZE TABLE when FULLTEXT index exists
+--echo #
+
+CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB;
+
+OPTIMIZE TABLE t1;
+DROP TABLE t1;
+
+SET GLOBAL innodb_defragment = @innodb_defragment_orig;
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index e447af9bdad..f198a18656c 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -12599,6 +12599,17 @@ ha_innobase::defragment_table(
 	for (index = dict_table_get_first_index(table); index;
 	     index = dict_table_get_next_index(index)) {
 
+		if (dict_index_is_corrupted(index)) {
+			continue;
+		}
+
+		if (index->page == FIL_NULL) {
+			/* Do not defragment auxiliary tables related
+			to FULLTEXT INDEX. */
+			ut_ad(index->type & DICT_FTS);
+			continue;
+		}
+
 		if (one_index && strcasecmp(index_name, index->name) != 0) {
 			continue;
 		}
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index 3130dc980d2..d2cd1951bd7 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -13272,6 +13272,17 @@ ha_innobase::defragment_table(
 	for (index = dict_table_get_first_index(table); index;
 	     index = dict_table_get_next_index(index)) {
 
+		if (dict_index_is_corrupted(index)) {
+			continue;
+		}
+
+		if (index->page == FIL_NULL) {
+			/* Do not defragment auxiliary tables related
+			to FULLTEXT INDEX. */
+			ut_ad(index->type & DICT_FTS);
+			continue;
+		}
+
 		if (one_index && strcasecmp(index_name, index->name) != 0) {
 			continue;
 		}