MDEV-6738: use_stat_table + histograms crashing optimizer

- When EITS code calls store_key_image_to_rec(), it should follow its 
  calling convention (which is counter-intuitive)

4 files changed, 32 insertions, 3 deletions

diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result
index 0dc678f0c0d..013fb1d876c 100644
--- a/mysql-test/r/selectivity_innodb.result
+++ b/mysql-test/r/selectivity_innodb.result
@@ -1408,6 +1408,16 @@ SELECT t1_2.b, t1_1.a FROM t1 AS t1_1 STRAIGHT_JOIN t1 AS t1_2 ON ( t1_2.a = t1_
 );
 a	b	i
 DROP TABLE t1,t2;
+# 
+# MDEV-6738: use_stat_table + histograms crashing optimizer
+# 
+set use_stat_tables='preferably';
+set optimizer_use_condition_selectivity=4;
+# Need innodb because there is a special kind of field_bit for non-myisam tables
+create table t1(col1 int, col2 bit(1) DEFAULT NULL) engine=innodb;
+select * from t1 where col2 != true;
+col1	col2
+drop table t1;
 #
 # End of 10.0 tests
 #
diff --git a/mysql-test/t/selectivity_innodb.test b/mysql-test/t/selectivity_innodb.test
index 60b0da0d5d1..5674cb5c006 100644
--- a/mysql-test/t/selectivity_innodb.test
+++ b/mysql-test/t/selectivity_innodb.test
@@ -65,6 +65,20 @@ SELECT * FROM t1, t2 WHERE ( 't', 'o' ) IN (
 );
 DROP TABLE t1,t2;
 
+--echo # 
+--echo # MDEV-6738: use_stat_table + histograms crashing optimizer
+--echo # 
+
+set use_stat_tables='preferably';
+set optimizer_use_condition_selectivity=4;
+
+--echo # Need innodb because there is a special kind of field_bit for non-myisam tables
+create table t1(col1 int, col2 bit(1) DEFAULT NULL) engine=innodb;
+
+select * from t1 where col2 != true;
+
+drop table t1;
+
 --echo #
 --echo # End of 10.0 tests
 --echo #
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 07a563d8090..ee26f3745b2 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -3737,6 +3737,11 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond)
       field  Field which key image should be stored
       ptr    Field value in key format
       len    Length of the value, in bytes
+  
+  ATTENTION
+    len is the length of the value not counting the NULL-byte (at the same
+    time, ptr points to the key image, which starts with NULL-byte for 
+    nullable columns)
 
   DESCRIPTION
     Copy the field value from its key image to the table record. The source
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index e1b197f4a63..a38df24cfb7 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -3531,7 +3531,7 @@ double get_column_range_cardinality(Field *field,
         if (hist->is_available())
         {
           store_key_image_to_rec(field, (uchar *) min_endp->key,
-                                 min_endp->length);
+                                 field->key_length());
           double pos= field->pos_in_interval(col_stats->min_value,
                                              col_stats->max_value);
           res= col_non_nulls * 
@@ -3555,7 +3555,7 @@ double get_column_range_cardinality(Field *field,
       if (min_endp && !(field->null_ptr && min_endp->key[0]))
       {
         store_key_image_to_rec(field, (uchar *) min_endp->key,
-                               min_endp->length);
+                               field->key_length());
         min_mp_pos= field->pos_in_interval(col_stats->min_value,
                                            col_stats->max_value);
       }
@@ -3564,7 +3564,7 @@ double get_column_range_cardinality(Field *field,
       if (max_endp)
       {
         store_key_image_to_rec(field, (uchar *) max_endp->key,
-                               max_endp->length);
+                               field->key_length());
         max_mp_pos= field->pos_in_interval(col_stats->min_value,
                                            col_stats->max_value);
       }