MDEV-24015: SQL Error (1038): Out of sort memory when enough memory for the sort buffer is provided

For a correlated subquery filesort is executed multiple times.
During each execution, sortlength() computed total sort key length in
Sort_keys::sort_length, without resetting it first.

Eventually Sort_keys::sort_length got larger than @@sort_buffer_size, which
caused filesort() to be aborted with error.

Fixed by making sortlength() to compute lengths only during the first
invocation. Subsequent invocations return pre-computed values.

1 files changed, 38 insertions, 12 deletions

diff --git a/sql/sql_sort.h b/sql/sql_sort.h
index 40f0c5ede5f..a474d7c25e9 100644
--- a/sql/sql_sort.h
+++ b/sql/sql_sort.h
@@ -255,10 +255,12 @@ class Sort_keys :public Sql_alloc,
 {
 public:
   Sort_keys(SORT_FIELD* arr, size_t count):
-  Sort_keys_array(arr, count),
-  m_using_packed_sortkeys(false),
-  size_of_packable_fields(0),
-  sort_length(0)
+    Sort_keys_array(arr, count),
+    m_using_packed_sortkeys(false),
+    size_of_packable_fields(0),
+    sort_length_with_original_values(0),
+    sort_length_with_memcmp_values(0),
+    parameters_computed(false)
   {
     DBUG_ASSERT(!is_null());
   }
@@ -280,14 +282,24 @@ public:
     return size_of_packable_fields;
   }
 
-  void set_sort_length(uint len)
+  void set_sort_length_with_original_values(uint len)
   {
-    sort_length= len;
+    sort_length_with_original_values= len;
   }
 
-  uint get_sort_length()
+  uint get_sort_length_with_original_values()
   {
-    return sort_length;
+    return sort_length_with_original_values;
+  }
+
+  void set_sort_length_with_memcmp_values(uint len)
+  {
+    sort_length_with_memcmp_values= len;
+  }
+
+  uint get_sort_length_with_memcmp_values()
+  {
+    return sort_length_with_memcmp_values;
   }
 
   static void store_sortkey_length(uchar *p, uint sz)
@@ -307,9 +319,12 @@ public:
 
   void increment_original_sort_length(uint len)
   {
-    sort_length+= len;
+    sort_length_with_original_values+= len;
   }
 
+  bool is_parameters_computed() { return parameters_computed; }
+  void set_parameters_computed(bool val) { parameters_computed= val; }
+
   static const uint size_of_length_field= 4;
 
 private:
@@ -317,10 +332,21 @@ private:
   uint size_of_packable_fields;     // Total length bytes for packable columns
 
   /*
-    The length that would be needed if we stored non-packed mem-comparable
-    images of fields?
+    The sort length for all the keyparts storing the original values
+  */
+  uint sort_length_with_original_values;
+
+  /*
+    The sort length for all the keyparts storing the mem-comparable images
+  */
+  uint sort_length_with_memcmp_values;
+
+  /*
+    TRUE       parameters(like sort_length_* , size_of_packable_field)
+               are computed
+    FALSE      otherwise.
   */
-  uint sort_length;
+  bool parameters_computed;
 };