Followup fix10.5-varun

2 files changed, 41 insertions, 17 deletions

diff --git a/sql/filesort.cc b/sql/filesort.cc
index 70926d7d449..40db6e0070a 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -2199,7 +2199,11 @@ sortlength(THD *thd, Sort_keys *sort_keys, bool *allow_packing_for_sortkeys)
   length=0;
   uint nullable_cols=0;
 
-  sort_keys->reset_parameters();
+  if (!sort_keys->is_first_execution())
+  {
+    *allow_packing_for_sortkeys= sort_keys->using_packed_sortkeys();
+    return sort_keys->get_sort_length_with_memcmp_values();
+  }
 
   for (SORT_FIELD *sortorder= sort_keys->begin();
        sortorder != sort_keys->end();
@@ -2262,6 +2266,8 @@ sortlength(THD *thd, Sort_keys *sort_keys, bool *allow_packing_for_sortkeys)
   // add bytes for nullable_cols
   sort_keys->increment_original_sort_length(nullable_cols);
   *allow_packing_for_sortkeys= allow_packing_for_keys;
+  sort_keys->set_sort_length_with_memcmp_values(length + nullable_cols);
+  sort_keys->set_first_execution(false);
   DBUG_PRINT("info",("sort_length: %d",length));
   return length + nullable_cols;
 }
@@ -2520,7 +2526,7 @@ void Sort_param::try_to_pack_sortkeys()
     return;
 
   const uint sz= Sort_keys::size_of_length_field;
-  uint sort_len= sort_keys->get_sort_length();
+  uint sort_len= sort_keys->get_sort_length_with_original_values();
 
   /*
     Heuristic introduced, skip packing sort keys if saving less than 128 bytes
diff --git a/sql/sql_sort.h b/sql/sql_sort.h
index 9ebd2bcfa03..6951d4d190a 100644
--- a/sql/sql_sort.h
+++ b/sql/sql_sort.h
@@ -258,7 +258,9 @@ public:
   Sort_keys_array(arr, count),
   m_using_packed_sortkeys(false),
   size_of_packable_fields(0),
-  sort_length(0)
+  sort_length_with_original_values(0),
+  sort_length_with_memcmp_values(0),
+  first_execution(true)
   {
     DBUG_ASSERT(!is_null());
   }
@@ -280,14 +282,24 @@ public:
     return size_of_packable_fields;
   }
 
-  void set_sort_length(uint len)
+  void set_sort_length_with_original_values(uint len)
   {
-    sort_length= len;
+    sort_length_with_original_values= len;
   }
 
-  uint get_sort_length()
+  uint get_sort_length_with_original_values()
   {
-    return sort_length;
+    return sort_length_with_original_values;
+  }
+
+  void set_sort_length_with_memcmp_values(uint len)
+  {
+    sort_length_with_memcmp_values= len;
+  }
+
+  uint get_sort_length_with_memcmp_values()
+  {
+    return sort_length_with_memcmp_values;
   }
 
   static void store_sortkey_length(uchar *p, uint sz)
@@ -307,15 +319,11 @@ public:
 
   void increment_original_sort_length(uint len)
   {
-    sort_length+= len;
+    sort_length_with_original_values+= len;
   }
 
-  void reset_parameters()
-  {
-    m_using_packed_sortkeys= false;
-    size_of_packable_fields= 0;
-    sort_length= 0;
-  }
+  bool is_first_execution() { return first_execution; }
+  void set_first_execution(bool val) { first_execution= val; }
 
   static const uint size_of_length_field= 4;
 
@@ -324,10 +332,20 @@ private:
   uint size_of_packable_fields;     // Total length bytes for packable columns
 
   /*
-    The length that would be needed if we stored non-packed mem-comparable
-    images of fields?
+    The sort length for all the keyparts storing the original values
+  */
+  uint sort_length_with_original_values;
+
+  /*
+    The sort length for all the keyparts storing the mem-comparable images
+  */
+  uint sort_length_with_memcmp_values;
+
+  /*
+    TRUE       if it is the first invocation of filesort
+    FALSE      otherwise.
   */
-  uint sort_length;
+  bool first_execution;
 };