MDEV-20726 InnoDB: Assertion failure in file data0type.cc line 67bb-10.4-MDEV-20726-alter-utf8

Do not rebuild index when it's key part converted from utf8mb3 to utf8mb4
but key part stays the same.

dict_index_add_to_cache(): assert that prefix_len is divided by mbmaxlen

ha_innobase::compare_key_parts(): compare key part lenght in symbols instead
of bytes.

4 files changed, 35 insertions, 9 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter_charset.result b/mysql-test/suite/innodb/r/instant_alter_charset.result
index f8c21978327..cbb49819bfe 100644
--- a/mysql-test/suite/innodb/r/instant_alter_charset.result
+++ b/mysql-test/suite/innodb/r/instant_alter_charset.result
@@ -1918,3 +1918,14 @@ check table t;
 Table	Op	Msg_type	Msg_text
 test.t	check	status	OK
 drop table t;
+#
+# MDEV-20726: InnoDB: Assertion failure in file data0type.cc line 67
+#
+CREATE TABLE t (
+id int(10) unsigned NOT NULL PRIMARY KEY,
+a text CHARSET utf8mb3,
+KEY a_idx(a(1))
+) ENGINE=InnoDB;
+INSERT INTO t VALUES (1, 'something in the air');
+ALTER TABLE t MODIFY a text CHARSET utf8mb4;
+DROP TABLE t;
diff --git a/mysql-test/suite/innodb/t/instant_alter_charset.test b/mysql-test/suite/innodb/t/instant_alter_charset.test
index 82ae1040f57..270db2d6462 100644
--- a/mysql-test/suite/innodb/t/instant_alter_charset.test
+++ b/mysql-test/suite/innodb/t/instant_alter_charset.test
@@ -715,3 +715,18 @@ alter table t modify c varchar(10) collate latin1_general_cs, algorithm=instant;
 check table t;
 
 drop table t;
+
+--echo #
+--echo # MDEV-20726: InnoDB: Assertion failure in file data0type.cc line 67
+--echo #
+
+CREATE TABLE t (
+  id int(10) unsigned NOT NULL PRIMARY KEY,
+  a text CHARSET utf8mb3,
+  KEY a_idx(a(1))
+) ENGINE=InnoDB;
+
+INSERT INTO t VALUES (1, 'something in the air');
+ALTER TABLE t MODIFY a text CHARSET utf8mb4;
+
+DROP TABLE t;
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index af6e45e3ad3..b1ea5788222 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -1839,6 +1839,7 @@ dict_index_add_to_cache(
 			   > field->col->max_prefix) {
 			/* Set the max_prefix value based on the
 			prefix_len. */
+			ut_ad(field->prefix_len % field->col->mbmaxlen == 0);
 			field->col->max_prefix = field->prefix_len;
 		}
 		ut_ad(field->col->ord_part == 1);
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 7fe707ac1e2..ba02f71058e 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -21143,26 +21143,25 @@ Compare_keys ha_innobase::compare_key_parts(
     const KEY_PART_INFO &old_part, const KEY_PART_INFO &new_part) const
 {
   const bool is_equal= old_field.is_equal(new_field);
+  const CHARSET_INFO *old_cs= old_field.charset();
+  const CHARSET_INFO *new_cs= new_field.charset;
 
   if (!is_equal)
   {
     if (!old_field.can_be_converted_by_engine(new_field))
       return Compare_keys::NotEqual;
 
-    if (!Charset(old_field.charset())
-             .eq_collation_specific_names(new_field.charset))
-    {
+    if (!Charset(old_cs).eq_collation_specific_names(new_cs))
       return Compare_keys::NotEqual;
-    }
   }
 
-  if (old_part.length != new_part.length)
+  if (old_part.length / old_cs->mbmaxlen != new_part.length / new_cs->mbmaxlen)
   {
-    if (old_part.length != old_field.field_length ||
-        old_part.length >= new_part.length || is_equal)
-    {
+    if (old_part.length != old_field.field_length)
+      return Compare_keys::NotEqual;
+
+    if (old_part.length >= new_part.length)
       return Compare_keys::NotEqual;
-    }
 
     return Compare_keys::EqualButKeyPartLength;
   }