MDEV-23600 SIGFPE in row_search_with_covering_prefixbb-10.2-MDEV-23600

The InnoDB index fields store bytes, not characters.
Remove some unnecessary conversions from characters to bytes.

TODO: Check the validity of the changed results,
and check that all tests were correctly imported from
percona/percona-server@1eee538087ffcf121c37f844b447ba5480faf081.

3 files changed, 24 insertions, 17 deletions

diff --git a/mysql-test/r/fast_prefix_index_fetch_innodb.result b/mysql-test/r/fast_prefix_index_fetch_innodb.result
index c6d96389b08..433f826bd34 100644
--- a/mysql-test/r/fast_prefix_index_fetch_innodb.result
+++ b/mysql-test/r/fast_prefix_index_fetch_innodb.result
@@ -346,10 +346,10 @@ f1
 🐱🌑
 select @cluster_lookups;
 @cluster_lookups
-2
+1
 select @cluster_lookups_avoided;
 @cluster_lookups_avoided
-0
+1
 # Eligible - record length is shorter than prefix length
 SELECT f1 FROM t1 FORCE INDEX (`f1`) WHERE f1 like '🌑%';
 f1
@@ -366,10 +366,10 @@ f1
 🌒
 select @cluster_lookups;
 @cluster_lookups
-1
+0
 select @cluster_lookups_avoided;
 @cluster_lookups_avoided
-1
+2
 DROP TABLE t1;
 CREATE TABLE t1(
 col1 INT,
@@ -398,4 +398,9 @@ select @cluster_lookups_avoided;
 @cluster_lookups_avoided
 0
 DROP TABLE t1;
+CREATE TABLE t(c POINT UNIQUE) ENGINE=InnoDB;
+INSERT t SET c=POINT(1,1);
+SELECT * FROM t WHERE c > (SELECT MAX(c) FROM t);
+c
+DROP TABLE t;
 set global innodb_prefix_index_cluster_optimization = OFF;
diff --git a/mysql-test/t/fast_prefix_index_fetch_innodb.test b/mysql-test/t/fast_prefix_index_fetch_innodb.test
index c3b3440d82d..a3daf5a2cf0 100644
--- a/mysql-test/t/fast_prefix_index_fetch_innodb.test
+++ b/mysql-test/t/fast_prefix_index_fetch_innodb.test
@@ -665,4 +665,10 @@ select @cluster_lookups;
 select @cluster_lookups_avoided;
 
 DROP TABLE t1;
+
+CREATE TABLE t(c POINT UNIQUE) ENGINE=InnoDB;
+INSERT t SET c=POINT(1,1);
+SELECT * FROM t WHERE c > (SELECT MAX(c) FROM t);
+DROP TABLE t;
+
 set global innodb_prefix_index_cluster_optimization = OFF;
diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc
index 9340d5060d9..897b0991933 100644
--- a/storage/innobase/row/row0sel.cc
+++ b/storage/innobase/row/row0sel.cc
@@ -2815,6 +2815,9 @@ row_sel_field_store_in_mysql_format_func(
 	ut_ad(len != UNIV_SQL_NULL);
 	MEM_CHECK_DEFINED(data, len);
 	MEM_CHECK_ADDRESSABLE(dest, templ->mysql_col_len);
+	for (ulint i = templ->mysql_col_len; i--; ) {
+		dest[i] = static_cast<byte>(ut_rnd_gen());
+	}
 #ifdef HAVE_valgrind_or_MSAN
 	MEM_UNDEFINED(dest, templ->mysql_col_len);
 #endif /* HAVE_valgrind_or_MSAN */
@@ -4184,6 +4187,7 @@ bool row_search_with_covering_prefix(
 	for (ulint i = 0; i < prebuilt->n_template; i++) {
 		mysql_row_templ_t* templ = prebuilt->mysql_template + i;
 		ulint j = templ->rec_prefix_field_no;
+		ut_ad(!templ->mbminlen == !templ->mbmaxlen);
 
 		/** Condition (1) : is the field in the index. */
 		if (j == ULINT_UNDEFINED) {
@@ -4199,27 +4203,19 @@ bool row_search_with_covering_prefix(
 
 		ulint rec_size = rec_offs_nth_size(offsets, j);
 		const dict_field_t* field = dict_index_get_nth_field(index, j);
-		ulint max_chars = field->prefix_len / templ->mbmaxlen;
 
-		ut_a(field->prefix_len > 0);
+		ut_ad(field->prefix_len);
 
-		if (rec_size < max_chars) {
-			/* Record in bytes shorter than the index
-			prefix length in char. */
-			continue;
-		}
-
-		if (rec_size * templ->mbminlen >= field->prefix_len) {
+		if (rec_size >= field->prefix_len) {
 			/* Shortest representation string by the
 			byte length of the record is longer than the
 			maximum possible index prefix. */
 			return false;
 		}
 
-		size_t num_chars = rec_field_len_in_chars(
-			field->col, j, rec, offsets);
-
-		if (num_chars >= max_chars) {
+		if (templ->mbminlen != templ->mbmaxlen
+		    && rec_field_len_in_chars(field->col, j, rec, offsets)
+		    >= field->prefix_len / templ->mbmaxlen) {
 			/* No of chars to store the record exceeds
 			the index prefix character length. */
 			return false;