MDEV-5817 MySQL BUG#11825482: Broken key length calculation for btree index

just as in 5.6 fix - copy the correct null-handling code from MyISAM

2 files changed, 69 insertions, 0 deletions

diff --git a/mysql-test/suite/heap/heap.result b/mysql-test/suite/heap/heap.result
index 0142cfa66b0..1e7656a3a44 100644
--- a/mysql-test/suite/heap/heap.result
+++ b/mysql-test/suite/heap/heap.result
@@ -758,6 +758,40 @@ SELECT * from t1;
 id	color	ts
 7	GREEN	2
 DROP TABLE t1;
+#
+# BUG#11825482: Broken key length calculation for btree index
+#
+CREATE TABLE h1 (f1 VARCHAR(1), f2 INT NOT NULL,
+UNIQUE KEY h1i (f1,f2) USING BTREE ) ENGINE=HEAP;
+INSERT INTO h1 VALUES(NULL,0),(NULL,1);
+SELECT 'wrong' as 'result' FROM dual WHERE ('h', 0) NOT IN (SELECT * FROM h1);
+result
+CREATE TABLE t1 (  
+pk int NOT NULL,  
+col_int_nokey INT,  
+col_varchar_nokey VARCHAR(1),  
+PRIMARY KEY (pk)  
+);
+INSERT INTO t1 VALUES (19,5,'h'),(20,5,'h');
+CREATE TABLE t2 (col_int_nokey INT);
+INSERT INTO t2 VALUES (1),(2);
+CREATE VIEW v1 AS
+SELECT col_varchar_nokey, COUNT( col_varchar_nokey )
+FROM t1
+WHERE col_int_nokey <= 141 AND pk <= 4
+;
+SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+col_int_nokey
+# shouldn't crash
+EXPLAIN SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<derived3>	ALL	NULL	NULL	NULL	NULL	2	Using where
+3	DERIVED	t1	range	PRIMARY	PRIMARY	4	NULL	1	Using index condition; Using where
+DROP TABLE t1,t2,h1;
+DROP VIEW v1;
 CREATE TABLE t1 (a int, index(a)) engine=heap min_rows=10 max_rows=100;
 insert into t1 values(1);
 select data_length,index_length from information_schema.tables where table_schema="test" and table_name="t1";
diff --git a/mysql-test/suite/heap/heap.test b/mysql-test/suite/heap/heap.test
index ef2527beeb0..b504661d0ff 100644
--- a/mysql-test/suite/heap/heap.test
+++ b/mysql-test/suite/heap/heap.test
@@ -510,6 +510,41 @@ DELETE FROM t1 WHERE ts = 1 AND color = 'GREEN';
 SELECT * from t1;
 DROP TABLE t1;
 
+--echo #
+--echo # BUG#11825482: Broken key length calculation for btree index
+--echo #
+CREATE TABLE h1 (f1 VARCHAR(1), f2 INT NOT NULL,
+                 UNIQUE KEY h1i (f1,f2) USING BTREE ) ENGINE=HEAP;
+INSERT INTO h1 VALUES(NULL,0),(NULL,1);
+SELECT 'wrong' as 'result' FROM dual WHERE ('h', 0) NOT IN (SELECT * FROM h1);
+
+CREATE TABLE t1 (  
+  pk int NOT NULL,  
+  col_int_nokey INT,  
+  col_varchar_nokey VARCHAR(1),  
+  PRIMARY KEY (pk)  
+);
+
+INSERT INTO t1 VALUES (19,5,'h'),(20,5,'h');
+  
+CREATE TABLE t2 (col_int_nokey INT);
+
+INSERT INTO t2 VALUES (1),(2);
+
+CREATE VIEW v1 AS
+  SELECT col_varchar_nokey, COUNT( col_varchar_nokey )
+  FROM t1
+  WHERE col_int_nokey <= 141 AND pk <= 4
+;
+
+SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+--echo # shouldn't crash
+EXPLAIN SELECT col_int_nokey FROM t2
+WHERE ('h', 0) NOT IN ( SELECT * FROM v1);
+
+DROP TABLE t1,t2,h1;
+DROP VIEW v1;
 # End of 5.1 tests
 
 #