Correctly decode string field values for pos_in_interval_for_string call

3 files changed, 42 insertions, 20 deletions

diff --git a/mysql-test/main/statistics_json.result b/mysql-test/main/statistics_json.result
index aa7b7c08a31..102fcfff194 100644
--- a/mysql-test/main/statistics_json.result
+++ b/mysql-test/main/statistics_json.result
@@ -2452,12 +2452,12 @@ test	t1_json	a	a-0	a-9	0.0000	3.0000	1.0000	100	JSON_HB	{
 }
 explain extended select * from t1_json where a between 'a-3a' and 'zzzzzzzzz';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	60.27	Using where
+1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	59.41	Using where
 Warnings:
 Note	1003	select `test`.`t1_json`.`a` AS `a` from `test`.`t1_json` where `test`.`t1_json`.`a` between 'a-3a' and 'zzzzzzzzz'
 analyze select * from t1_json where a between 'a-3a' and 'zzzzzzzzz';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
-1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	10.00	60.27	60.00	Using where
+1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	10.00	59.41	60.00	Using where
 explain extended select * from t1_json where a < 'b-1a';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	99.01	Using where
@@ -3179,7 +3179,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_f
 1	SIMPLE	Country	ALL	NULL	NULL	NULL	NULL	239	239.00	25.49	25.52	Using where
 analyze select * from Country use index () where Code < 'BBC';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
-1	SIMPLE	Country	ALL	NULL	NULL	NULL	NULL	239	239.00	5.88	7.11	Using where
+1	SIMPLE	Country	ALL	NULL	NULL	NULL	NULL	239	239.00	7.84	7.11	Using where
 set histogram_type=@save_histogram_type;
 set histogram_size=@save_histogram_size;
 DROP SCHEMA world;
diff --git a/sql/field.h b/sql/field.h
index e4224fc1e51..2db81c61b75 100644
--- a/sql/field.h
+++ b/sql/field.h
@@ -1511,11 +1511,20 @@ public:
     if (null_ptr)
       null_ptr=ADD_TO_PTR(null_ptr,ptr_diff,uchar*);
   }
+
+  /*
+    Copy the Field's value to buff. The value will be in table->record[]
+    format.
+  */
   void get_image(uchar *buff, uint length, CHARSET_INFO *cs) const
   { get_image(buff, length, ptr, cs); }
   virtual void get_image(uchar *buff, uint length,
                          const uchar *ptr_arg, CHARSET_INFO *cs) const
     { memcpy(buff,ptr_arg,length); }
+
+  /*
+    Set Field's value to the value in *buf.
+  */
   virtual void set_image(const uchar *buff,uint length, CHARSET_INFO *cs)
     { memcpy(ptr,buff,length); }
 
diff --git a/sql/opt_histogram_json.cc b/sql/opt_histogram_json.cc
index 5f4e0eca585..2817a257838 100644
--- a/sql/opt_histogram_json.cc
+++ b/sql/opt_histogram_json.cc
@@ -220,43 +220,48 @@ error:
 
 
 static
-void store_key_image_to_rec_no_null(Field *field, const uchar *ptr)
+void store_key_image_to_rec_no_null(Field *field, const char *ptr, uint len)
 {
   MY_BITMAP *old_map= dbug_tmp_use_all_columns(field->table,
                                     &field->table->write_set);
-  field->set_key_image(ptr, field->key_length());
+  field->set_key_image((const uchar*)ptr, len);
   dbug_tmp_restore_column_map(&field->table->write_set, old_map);
 }
 
 
 static
-double position_in_interval(Field *field, const  uchar *key,
+double position_in_interval(Field *field, const  uchar *key, uint key_len,
                             const std::string& left, const std::string& right)
 {
   double res;
   if (field->pos_through_val_str())
   {
-    uint32 min_len= uint2korr(left.data());
-    uint32 max_len= uint2korr(right.data());
-    uint32 midp_len= uint2korr(key);
+    StringBuffer<64> buf1, buf2, buf3;
+    String empty_buf1, empty_buf2, empty_buf3;
+
+    store_key_image_to_rec_no_null(field, left.data(), left.size());
+    String *min_str= field->val_str(&buf1, &empty_buf1);
+
+    store_key_image_to_rec_no_null(field, right.data(), right.size());
+    String *max_str= field->val_str(&buf2, &empty_buf2);
+
+    store_key_image_to_rec_no_null(field, (const char*)key, key_len);
+    String *midp_str= field->val_str(&buf3, &empty_buf3);
 
     res= pos_in_interval_for_string(field->charset(),
-           key + HA_KEY_BLOB_LENGTH,
-           midp_len,
-           (const uchar*)left.data() + HA_KEY_BLOB_LENGTH,
-           min_len,
-           (const uchar*)right.data() + HA_KEY_BLOB_LENGTH,
-           max_len);
+           (const uchar*)midp_str->ptr(), midp_str->length(),
+           (const uchar*)min_str->ptr(), min_str->length(),
+           (const uchar*)max_str->ptr(), max_str->length());
   }
   else
   {
-    store_key_image_to_rec_no_null(field, (const uchar*)left.data());
+    store_key_image_to_rec_no_null(field, left.data(), field->key_length());
     double min_val_real= field->val_real();
     
-    store_key_image_to_rec_no_null(field, (const uchar*)right.data());
+    store_key_image_to_rec_no_null(field, right.data(), field->key_length());
     double max_val_real= field->val_real();
 
-    store_key_image_to_rec_no_null(field, key);
+    store_key_image_to_rec_no_null(field, (const char*)key, field->key_length());
     double midp_val_real= field->val_real();
 
     res= pos_in_interval_for_double(midp_val_real, min_val_real, max_val_real);
@@ -318,13 +323,17 @@ double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp,
   {
     bool exclusive_endp= (min_endp->flag == HA_READ_AFTER_KEY)? true: false;
     const uchar *min_key= min_endp->key;
+    uint min_key_len= min_endp->length;
     if (field->real_maybe_null())
+    {
       min_key++;
+      min_key_len--;
+    }
 
     // Find the leftmost bucket that contains the lookup value.
     // (If the lookup value is to the left of all buckets, find bucket #0)
     int idx= find_bucket(field, min_key, exclusive_endp);
-    double min_sel= position_in_interval(field, (const uchar*)min_key,
+    double min_sel= position_in_interval(field, min_key, min_key_len,
                                          histogram_bounds[idx],
                                          histogram_bounds[idx+1]);
     min= idx*width + min_sel*width;
@@ -338,11 +347,15 @@ double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp,
     DBUG_ASSERT(!(field->null_ptr && max_endp->key[0]));
     bool inclusive_endp= (max_endp->flag == HA_READ_AFTER_KEY)? true: false;
     const uchar *max_key= max_endp->key;
+    uint max_key_len= max_endp->length;
     if (field->real_maybe_null())
+    {
       max_key++;
+      max_key_len--;
+    }
 
     int idx= find_bucket(field, max_key, inclusive_endp);
-    double max_sel= position_in_interval(field, (const uchar*)max_key,
+    double max_sel= position_in_interval(field, max_key, max_key_len,
                                          histogram_bounds[idx],
                                          histogram_bounds[idx+1]);
     max= idx*width + max_sel*width;