Fixed bug mdev-4363.

When calculating the selectivity of a range in the function
get_column_range_cardinality a check whether NULL values are
included into into the range must be done.

6 files changed, 127 insertions, 7 deletions

diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result
index 781df332f77..2ab143d6b14 100644
--- a/mysql-test/r/selectivity.result
+++ b/mysql-test/r/selectivity.result
@@ -882,7 +882,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 FLUSH TABLES;
-SET optimizer_use_condition_selectivity=3;
+set optimizer_use_condition_selectivity=3;
 EXPLAIN EXTENDED
 SELECT * FROM t1, t2 WHERE EXISTS ( SELECT 1 FROM t1, t2 ) AND a != b OR a <= 4;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -895,4 +895,44 @@ Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t2`.`b` AS `b` from `test`.`t1`
 set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
 DROP TABLE t1,t2;
 set use_stat_tables=@save_use_stat_tables;
+#
+# Bug mdev-4363: selectivity of the condition a IS NULL OR IS NOT NULL   
+#                with optimizer_use_condition_selectivity=3
+#
+set use_stat_tables = PREFERABLY;
+CREATE TABLE t1 (a int);
+INSERT INTO t1 VALUES 
+(1),(7),(4),(7),(NULL),(2),(NULL),(4),(NULL),(NULL),(1),(3),(8),(8);
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+FLUSH TABLE t1;
+set optimizer_use_condition_selectivity=3;
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	28.57	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where isnull(`test`.`t1`.`a`)
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NOT NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	71.43	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (`test`.`t1`.`a` is not null)
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a IS NOT NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (isnull(`test`.`t1`.`a`) or (`test`.`t1`.`a` is not null))
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a < 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	69.39	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (isnull(`test`.`t1`.`a`) or (`test`.`t1`.`a` < 5))
+set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set use_stat_tables=@save_use_stat_tables;
diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result
index e65e61af4c1..2a639e922af 100644
--- a/mysql-test/r/selectivity_innodb.result
+++ b/mysql-test/r/selectivity_innodb.result
@@ -890,7 +890,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 FLUSH TABLES;
-SET optimizer_use_condition_selectivity=3;
+set optimizer_use_condition_selectivity=3;
 EXPLAIN EXTENDED
 SELECT * FROM t1, t2 WHERE EXISTS ( SELECT 1 FROM t1, t2 ) AND a != b OR a <= 4;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -903,6 +903,46 @@ Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t2`.`b` AS `b` from `test`.`t1`
 set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
 DROP TABLE t1,t2;
 set use_stat_tables=@save_use_stat_tables;
+#
+# Bug mdev-4363: selectivity of the condition a IS NULL OR IS NOT NULL   
+#                with optimizer_use_condition_selectivity=3
+#
+set use_stat_tables = PREFERABLY;
+CREATE TABLE t1 (a int);
+INSERT INTO t1 VALUES 
+(1),(7),(4),(7),(NULL),(2),(NULL),(4),(NULL),(NULL),(1),(3),(8),(8);
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+FLUSH TABLE t1;
+set optimizer_use_condition_selectivity=3;
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	28.57	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where isnull(`test`.`t1`.`a`)
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NOT NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	71.43	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (`test`.`t1`.`a` is not null)
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a IS NOT NULL;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (isnull(`test`.`t1`.`a`) or (`test`.`t1`.`a` is not null))
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a < 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	14	69.39	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (isnull(`test`.`t1`.`a`) or (`test`.`t1`.`a` < 5))
+set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
+DROP TABLE t1;
+set use_stat_tables=@save_use_stat_tables;
 set use_stat_tables=@save_use_stat_tables;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 SET SESSION STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test
index f601d4f656a..d8ab19ac28a 100644
--- a/mysql-test/t/selectivity.test
+++ b/mysql-test/t/selectivity.test
@@ -483,7 +483,7 @@ INSERT INTO t2 VALUES (4),(5),(2),(5),(1),(1),(2);
 ANALYZE TABLE t1, t2;
 FLUSH TABLES;
 
-SET optimizer_use_condition_selectivity=3;
+set optimizer_use_condition_selectivity=3;
 
 EXPLAIN EXTENDED
 SELECT * FROM t1, t2 WHERE EXISTS ( SELECT 1 FROM t1, t2 ) AND a != b OR a <= 4;
@@ -495,4 +495,36 @@ DROP TABLE t1,t2;
 set use_stat_tables=@save_use_stat_tables;
 
 
+--echo #
+--echo # Bug mdev-4363: selectivity of the condition a IS NULL OR IS NOT NULL   
+--echo #                with optimizer_use_condition_selectivity=3
+--echo #
+
+set use_stat_tables = PREFERABLY;
+
+CREATE TABLE t1 (a int);
+INSERT INTO t1 VALUES 
+  (1),(7),(4),(7),(NULL),(2),(NULL),(4),(NULL),(NULL),(1),(3),(8),(8);
+
+ANALYZE TABLE t1;
+FLUSH TABLE t1;
+
+set optimizer_use_condition_selectivity=3;
+
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL;
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NOT NULL;
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a IS NOT NULL;
+EXPLAIN EXTENDED
+SELECT * FROM t1 WHERE a IS NULL OR a < 5;
+
+set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity;
+
+DROP TABLE t1;
+
+set use_stat_tables=@save_use_stat_tables;
+
+
 set use_stat_tables=@save_use_stat_tables;
diff --git a/sql/opt_range.cc b/sql/opt_range.cc
index 035ea3635e2..d101b2fe91b 100644
--- a/sql/opt_range.cc
+++ b/sql/opt_range.cc
@@ -3303,7 +3303,8 @@ double records_in_column_ranges(PARAM *param, uint idx,
     key_range *min_endp, *max_endp;
     min_endp= range.start_key.length? &range.start_key : NULL;
     max_endp= range.end_key.length? &range.end_key : NULL;
-    rows= get_column_range_cardinality(field, min_endp, max_endp);
+    rows= get_column_range_cardinality(field, min_endp, max_endp,
+                                       range.range_flag);
     if (HA_POS_ERROR == rows)
     {
       total_rows= HA_POS_ERROR;
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index 37c3a93ee08..efd5af7adbf 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -3339,7 +3339,8 @@ double get_column_avg_frequency(Field * field)
 
 double get_column_range_cardinality(Field *field,
                                     key_range *min_endp,
-                                    key_range *max_endp)
+                                    key_range *max_endp,
+                                    uint range_flag)
 {
   double res;
   TABLE *table= field->table;
@@ -3353,12 +3354,15 @@ double get_column_range_cardinality(Field *field,
 
   double col_non_nulls= tab_records - col_nulls;
 
+  bool nulls_incl= field->null_ptr && min_endp && min_endp->key[0] &&
+                   !(range_flag & NEAR_MIN);
+
   if (col_non_nulls < 1)
     res= 0;
   else if (min_endp && max_endp && min_endp->length == max_endp->length &&
            !memcmp(min_endp->key, max_endp->key, min_endp->length))
   { 
-    if (field->null_ptr && min_endp->key[0])
+    if (nulls_incl)
     {
       /* This is null single point range */
       res= col_nulls;
@@ -3416,6 +3420,8 @@ double get_column_range_cardinality(Field *field,
     }
     else
       res= col_non_nulls;
+    if (nulls_incl)
+      res+= col_nulls;
   }
   return res;
 }
diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h
index 14ffe3351d0..ea3dba3215a 100644
--- a/sql/sql_statistics.h
+++ b/sql/sql_statistics.h
@@ -105,7 +105,8 @@ double get_column_avg_frequency(Field * field);
 
 double get_column_range_cardinality(Field *field,
                                     key_range *min_endp,
-                                    key_range *max_endp);
+                                    key_range *max_endp,
+                                    uint range_flag);
 
 class Histogram
 {