MDEV-25951 MariaDB crash after ALTER TABLE convert to utf8mb4bb-10.4-MDEV-25951-instant-utf

Bug happens when partially indexed CHAR or VARCHAR field in converted from
utf8mb3 to utf8mb4.

Fixing by relaxing assertions. For some time dict_index_t and dict_table_t
are becoming not synchronized. Namely, dict_index_t has a new prefix_len which
is a multiple of a user-provided length and charset->mbmaxlen. But
the table still have and old mbmaxlen and assertion fails. This happens only
during utf8mb3 -> utf8mb4 conversions and the magic number 4 comes from
utf8mb_4_.

At the end of ALTER TABLE (innobase_rename_or_enlarge_columns_cache())
dict_index_t and dict_table_t became synchronized
again and will stay so at all times. For, example, they will be synchronized
on table load and newly added assertion proves that.

6 files changed, 76 insertions, 3 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter_charset.result b/mysql-test/suite/innodb/r/instant_alter_charset.result
index 8b1171191fa..43b4b2453e8 100644
--- a/mysql-test/suite/innodb/r/instant_alter_charset.result
+++ b/mysql-test/suite/innodb/r/instant_alter_charset.result
@@ -2043,3 +2043,36 @@ constraint a foreign key (id) references t1  (id)
 alter table t1 change id id2 int;
 drop table t2;
 drop table t1;
+#
+# MDEV-25951 MariaDB crash after ALTER TABLE convert to utf8mb4
+#
+CREATE TABLE t1 (id INT PRIMARY KEY, a VARCHAR(32), KEY (a(7))) ENGINE=INNODB DEFAULT CHARSET=UTF8;
+INSERT INTO t1 VALUES (1, 'a1'), (2, 'a1');
+ALTER TABLE t1
+CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci,
+ADD UNIQUE INDEX test_key (a);
+ERROR 23000: Duplicate entry 'a1' for key 'test_key'
+ALTER TABLE t1 CONVERT TO CHARACTER SET UTF8MB4 COLLATE UTF8MB4_UNICODE_520_CI;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SELECT * FROM t1;
+id	a
+1	a1
+2	a1
+DROP TABLE t1;
+CREATE TABLE t1 (id INT PRIMARY KEY, a CHAR(32), KEY (a(7))) ENGINE=INNODB DEFAULT CHARSET=UTF8;
+INSERT INTO t1 VALUES (1, 'a1'), (2, 'a1');
+ALTER TABLE t1
+CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci,
+ADD UNIQUE INDEX test_key (a);
+ERROR 23000: Duplicate entry 'a1' for key 'test_key'
+ALTER TABLE t1 CONVERT TO CHARACTER SET UTF8MB4 COLLATE UTF8MB4_UNICODE_520_CI;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SELECT * FROM t1;
+id	a
+1	a1
+2	a1
+DROP TABLE t1;
diff --git a/mysql-test/suite/innodb/t/instant_alter_charset.test b/mysql-test/suite/innodb/t/instant_alter_charset.test
index 1d444b88a7f..d64e30402ed 100644
--- a/mysql-test/suite/innodb/t/instant_alter_charset.test
+++ b/mysql-test/suite/innodb/t/instant_alter_charset.test
@@ -857,3 +857,34 @@ create table t2 (input_id int primary key, id int not null,
 alter table t1 change id id2 int;
 drop table t2;
 drop table t1;
+
+
+--echo #
+--echo # MDEV-25951 MariaDB crash after ALTER TABLE convert to utf8mb4
+--echo #
+
+CREATE TABLE t1 (id INT PRIMARY KEY, a VARCHAR(32), KEY (a(7))) ENGINE=INNODB DEFAULT CHARSET=UTF8;
+INSERT INTO t1 VALUES (1, 'a1'), (2, 'a1');
+
+--error ER_DUP_ENTRY
+ALTER TABLE t1
+  CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci,
+  ADD UNIQUE INDEX test_key (a);
+
+ALTER TABLE t1 CONVERT TO CHARACTER SET UTF8MB4 COLLATE UTF8MB4_UNICODE_520_CI;
+CHECK TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (id INT PRIMARY KEY, a CHAR(32), KEY (a(7))) ENGINE=INNODB DEFAULT CHARSET=UTF8;
+INSERT INTO t1 VALUES (1, 'a1'), (2, 'a1');
+
+--error ER_DUP_ENTRY
+ALTER TABLE t1
+  CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci,
+  ADD UNIQUE INDEX test_key (a);
+
+ALTER TABLE t1 CONVERT TO CHARACTER SET UTF8MB4 COLLATE UTF8MB4_UNICODE_520_CI;
+CHECK TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;
diff --git a/storage/innobase/data/data0type.cc b/storage/innobase/data/data0type.cc
index 7de4cc026d1..b1952bcc2a4 100644
--- a/storage/innobase/data/data0type.cc
+++ b/storage/innobase/data/data0type.cc
@@ -61,10 +61,10 @@ dtype_get_at_most_n_mbchars(
 					length is being determined */
 {
 	ut_a(len_is_stored(data_len));
-	ut_ad(!mbmaxlen || !(prefix_len % mbmaxlen));
+	ut_ad(!mbmaxlen || !(prefix_len % mbmaxlen) || !(prefix_len % 4));
 
 	if (mbminlen != mbmaxlen) {
-		ut_a(!(prefix_len % mbmaxlen));
+		ut_a(!(prefix_len % mbmaxlen) || !(prefix_len % 4));
 		return(innobase_get_at_most_n_mbchars(
 			dtype_get_charset_coll(prtype),
 			prefix_len, data_len, str));
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index f22e106b6b8..8357732e73b 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -1916,7 +1916,8 @@ dict_index_add_to_cache(
 			/* Set the max_prefix value based on the
 			prefix_len. */
 			ut_ad(field->col->is_binary()
-			      || field->prefix_len % field->col->mbmaxlen == 0);
+			      || field->prefix_len % field->col->mbmaxlen == 0
+			      || field->prefix_len % 4 == 0);
 			field->col->max_prefix = field->prefix_len;
 		}
 		ut_ad(field->col->ord_part == 1);
diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc
index 0c6810dccc3..623403db3c8 100644
--- a/storage/innobase/dict/dict0load.cc
+++ b/storage/innobase/dict/dict0load.cc
@@ -2576,6 +2576,12 @@ corrupted:
 			    != DB_SUCCESS) {
 				goto func_exit;
 			}
+
+			for (uint i = 0; i < index->n_fields; i++) {
+				dict_field_t &f = index->fields[i];
+				ut_ad(f.col->mbmaxlen == 0
+				      || f.prefix_len % f.col->mbmaxlen == 0);
+			}
 		}
 next_rec:
 		btr_pcur_move_to_next_user_rec(&pcur, &mtr);
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index ac066be7b42..4316a3a4671 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -11351,6 +11351,8 @@ create_index(
 			prefix_len = 0;
 		}
 
+		ut_ad(prefix_len % field->charset()->mbmaxlen == 0);
+
 		field_lengths[i] = key_part->length;
 
 		if (!key_part->field->stored_in_db()) {