MDEV-22728: SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect on optimized builds10.1-mdev22715

For low sort_buffer_size, in the cost calculation of using the Unique object the elements in the tree were evaluated to 0, make sure to have atleast 1 element in the Unique tree.

Also for the function Unique::get allocate memory for atleast MERGEBUFF2+1 keys.

1 files changed, 27 insertions, 0 deletions

diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test
index 31ca1c253e4..07306b59c35 100644
--- a/mysql-test/t/index_merge_innodb.test
+++ b/mysql-test/t/index_merge_innodb.test
@@ -198,4 +198,31 @@ SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE  (key1 between 10 and 819
 drop table t1;
 set optimizer_switch=@tmp_optimizer_switch;
 
+--echo #
+--echo # MDEV-22728: SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect
+--echo # on optimized builds
+--echo #
+
+SET @save_sort_buffer_size=@@sort_buffer_size;
+SET @save_innodb_file_format= @@innodb_file_format;
+SET @save_innodb_large_prefix= @@innodb_large_prefix;
+SET sort_buffer_size=2048;
+SET GLOBAL innodb_file_format = BARRACUDA;
+SET GLOBAL innodb_large_prefix = ON;
+
+CREATE TABLE t1 (
+  a VARCHAR(1024) CHARACTER SET UTF8 PRIMARY KEY,
+  b INT,
+  c INT,
+  INDEX (b)
+) ENGINE=InnoDB CHARSET utf8 ROW_FORMAT= DYNAMIC;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_100;
+EXPLAIN SELECT * FROM t1 WHERE a='1' OR b < 5;
+SELECT * FROM t1 WHERE a='1' OR b < 5;
+DROP TABLE t1;
+
+SET GLOBAL innodb_file_format = @save_innodb_file_format;
+SET GLOBAL innodb_large_prefix = @save_innodb_large_prefix;
+SET sort_buffer_size= @save_sort_buffer_size;
+
 disconnect disable_purge;