MDEV-25994 Crash with union of my_decimal type in ORDER BY clausebb-10.2-MDEV-25994

During execution of Type_handler_decimal_result::make_sort_key_part()
the call to item->val_decimal_result() may fail and return NULL pointer.
Such case was not analyzed which led to the crash.
This commit adds necessary checks to the case described above and also
adds an indicator of successful execution for make_sort_key() functions
(return value = FALSE for success and TRUE failure)

5 files changed, 80 insertions, 33 deletions

diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result
index d1f2d067643..b21e350ac4a 100644
--- a/mysql-test/r/order_by.result
+++ b/mysql-test/r/order_by.result
@@ -3508,4 +3508,24 @@ DELETE FROM t1 ORDER BY c;
 DROP TABLE t1;
 SET @@SESSION.max_sort_length=DEFAULT;
 SET sql_mode=DEFAULT;
+#
+# MDEV-25994 Crash with union of my_decimal type in ORDER BY clause
+#
+CREATE TABLE t1 (v1 INTEGER) ;
+INSERT INTO t1 (v1) VALUES (8);
+UPDATE t1 SET v1 = 1 ORDER BY (SELECT 1.1 UNION SELECT -1);
+ERROR 21000: Subquery returns more than 1 row
+# This one must be successful
+UPDATE t1 SET v1 = 2 ORDER BY (SELECT 1 UNION SELECT 1);
+UPDATE t1 SET v1 = 3 ORDER BY (SELECT 'a' UNION SELECT 'b');
+ERROR 21000: Subquery returns more than 1 row
+# Insert some more data
+INSERT INTO t1 (v1) VALUES (8),(9),(100),(-234),(46584),(0);
+UPDATE t1 SET v1 = v1+1 ORDER BY (SELECT 100.122 UNION SELECT -189.2);
+ERROR 21000: Subquery returns more than 1 row
+# This one must be successful
+UPDATE t1 SET v1 = v1-200 ORDER BY (SELECT 1 UNION SELECT 1);
+UPDATE t1 SET v1 = v1 ORDER BY (SELECT 'abc' UNION SELECT 'bbb');
+ERROR 21000: Subquery returns more than 1 row
+DROP TABLE t1;
 # End of 10.2 tests
diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test
index 7cb24b7d03b..2187786deb7 100644
--- a/mysql-test/t/order_by.test
+++ b/mysql-test/t/order_by.test
@@ -2331,5 +2331,30 @@ DROP TABLE t1;
 SET @@SESSION.max_sort_length=DEFAULT;
 SET sql_mode=DEFAULT;
 
+--echo #
+--echo # MDEV-25994 Crash with union of my_decimal type in ORDER BY clause
+--echo #
+
+CREATE TABLE t1 (v1 INTEGER) ;
+INSERT INTO t1 (v1) VALUES (8);
+--error ER_SUBQUERY_NO_1_ROW
+UPDATE t1 SET v1 = 1 ORDER BY (SELECT 1.1 UNION SELECT -1);
+--echo # This one must be successful
+UPDATE t1 SET v1 = 2 ORDER BY (SELECT 1 UNION SELECT 1);
+--error ER_SUBQUERY_NO_1_ROW
+UPDATE t1 SET v1 = 3 ORDER BY (SELECT 'a' UNION SELECT 'b');
+
+-- echo # Insert some more data
+INSERT INTO t1 (v1) VALUES (8),(9),(100),(-234),(46584),(0);
+--error ER_SUBQUERY_NO_1_ROW
+UPDATE t1 SET v1 = v1+1 ORDER BY (SELECT 100.122 UNION SELECT -189.2);
+--echo # This one must be successful
+UPDATE t1 SET v1 = v1-200 ORDER BY (SELECT 1 UNION SELECT 1);
+--error ER_SUBQUERY_NO_1_ROW
+UPDATE t1 SET v1 = v1 ORDER BY (SELECT 'abc' UNION SELECT 'bbb');
+
+
+DROP TABLE t1;
+
 
 --echo # End of 10.2 tests
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 8b019caf8f5..3e87ad1d35d 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -1009,7 +1009,7 @@ static inline void store_length(uchar *to, uint length, uint pack_length)
 }
 
 
-void
+bool
 Type_handler_string_result::make_sort_key(uchar *to, Item *item,
                                           const SORT_FIELD_ATTR *sort_field,
                                           Sort_param *param) const
@@ -1025,23 +1025,13 @@ Type_handler_string_result::make_sort_key(uchar *to, Item *item,
   String *res= item->str_result(&tmp);
   if (!res)
   {
-    if (maybe_null)
-      memset(to - 1, 0, sort_field->length + 1);
-    else
+    if (unlikely(!maybe_null))
     {
-      /* purecov: begin deadcode */
-      /*
-        This should only happen during extreme conditions if we run out
-        of memory or have an item marked not null when it can be null.
-        This code is here mainly to avoid a hard crash in this case.
-      */
-      DBUG_ASSERT(0);
-      DBUG_PRINT("warning",
-                 ("Got null on something that shouldn't be null"));
-      memset(to, 0, sort_field->length);	// Avoid crash
-      /* purecov: end */
+      // This indicates an error during item->str_result() call
+      return true;
     }
-    return;
+    memset(to - 1, 0, sort_field->length + 1);
+    return false;
   }
 
   if (use_strnxfrm(cs))
@@ -1077,10 +1067,11 @@ Type_handler_string_result::make_sort_key(uchar *to, Item *item,
     char fill_char= ((cs->state & MY_CS_BINSORT) ? (char) 0 : ' ');
     cs->cset->fill(cs, (char *)to+length,diff,fill_char);
   }
+  return false;
 }
 
 
-void
+bool
 Type_handler_int_result::make_sort_key(uchar *to, Item *item,
                                        const SORT_FIELD_ATTR *sort_field,
                                        Sort_param *param) const
@@ -1088,10 +1079,11 @@ Type_handler_int_result::make_sort_key(uchar *to, Item *item,
   longlong value= item->val_int_result();
   make_sort_key_longlong(to, item->maybe_null, item->null_value,
                          item->unsigned_flag, value);
+  return false;
 }
 
 
-void
+bool
 Type_handler_temporal_result::make_sort_key(uchar *to, Item *item,
                                             const SORT_FIELD_ATTR *sort_field,
                                             Sort_param *param) const
@@ -1103,10 +1095,13 @@ Type_handler_temporal_result::make_sort_key(uchar *to, Item *item,
     DBUG_ASSERT(item->null_value);
     make_sort_key_longlong(to, item->maybe_null, true,
                            item->unsigned_flag, 0);
+    // If both flags set, it is NOT an error. Otherwise it is an error
+    return !(item->maybe_null && item->null_value);
   }
   else
     make_sort_key_longlong(to, item->maybe_null, false,
                            item->unsigned_flag, pack_time(&buf));
+  return false;
 }
 
 
@@ -1141,7 +1136,7 @@ Type_handler::make_sort_key_longlong(uchar *to,
 }
 
 
-void
+bool
 Type_handler_decimal_result::make_sort_key(uchar *to, Item *item,
                                            const SORT_FIELD_ATTR *sort_field,
                                            Sort_param *param) const
@@ -1152,17 +1147,23 @@ Type_handler_decimal_result::make_sort_key(uchar *to, Item *item,
     if (item->null_value)
     {
       memset(to, 0, sort_field->length + 1);
-      return;
+      return false;
     }
     *to++= 1;
   }
-  my_decimal2binary(E_DEC_FATAL_ERROR, dec_val, to,
+  else if (item->null_value)
+  {
+    // Error during item->val_decimal_result()
+    DBUG_ASSERT(!dec_val);
+    return true;
+  }
+  return my_decimal2binary(E_DEC_FATAL_ERROR, dec_val, to,
                     item->max_length - (item->decimals ? 1 : 0),
                     item->decimals);
 }
 
 
-void
+bool
 Type_handler_real_result::make_sort_key(uchar *to, Item *item,
                                         const SORT_FIELD_ATTR *sort_field,
                                         Sort_param *param) const
@@ -1173,11 +1174,12 @@ Type_handler_real_result::make_sort_key(uchar *to, Item *item,
     if (item->null_value)
     {
       memset(to, 0, sort_field->length + 1);
-      return;
+      return false;
     }
     *to++= 1;
   }
   change_double_for_sort(value, to);
+  return false;
 }
 
 
diff --git a/sql/item.h b/sql/item.h
index 6b4ca89f3c7..6b114ac00a1 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -932,10 +932,10 @@ public:
   Item_result result_type() const { return type_handler()->result_type(); }
   /* ... while cmp_type() specifies how it should be compared */
   Item_result cmp_type() const { return type_handler()->cmp_type(); }
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const
   {
-    type_handler()->make_sort_key(to, item, sort_field, param);
+    return type_handler()->make_sort_key(to, item, sort_field, param);
   }
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
diff --git a/sql/sql_type.h b/sql/sql_type.h
index 42090037ead..61a9f9ada90 100644
--- a/sql/sql_type.h
+++ b/sql/sql_type.h
@@ -86,7 +86,7 @@ public:
   virtual Field *make_conversion_table_field(TABLE *TABLE,
                                              uint metadata,
                                              const Field *target) const= 0;
-  virtual void make_sort_key(uchar *to, Item *item,
+  virtual bool make_sort_key(uchar *to, Item *item,
                              const SORT_FIELD_ATTR *sort_field,
                              Sort_param *param) const= 0;
   virtual void sortlength(THD *thd,
@@ -103,7 +103,7 @@ public:
   Item_result result_type() const { return REAL_RESULT; }
   Item_result cmp_type() const { return REAL_RESULT; }
   virtual ~Type_handler_real_result() {}
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const;
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
@@ -118,7 +118,7 @@ public:
   Item_result cmp_type() const { return DECIMAL_RESULT; }
   virtual ~Type_handler_decimal_result() {};
   Field *make_num_distinct_aggregator_field(MEM_ROOT *, const Item *) const;
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const;
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
@@ -133,7 +133,7 @@ public:
   Item_result cmp_type() const { return INT_RESULT; }
   virtual ~Type_handler_int_result() {}
   Field *make_num_distinct_aggregator_field(MEM_ROOT *, const Item *) const;
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const;
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
@@ -147,7 +147,7 @@ public:
   Item_result result_type() const { return STRING_RESULT; }
   Item_result cmp_type() const { return TIME_RESULT; }
   virtual ~Type_handler_temporal_result() {}
-  void make_sort_key(uchar *to, Item *item,  const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item,  const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const;
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
@@ -164,7 +164,7 @@ public:
   const Type_handler *
   type_handler_adjusted_to_max_octet_length(uint max_octet_length,
                                             CHARSET_INFO *cs) const;
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const;
   void sortlength(THD *thd,
                   const Type_std_attributes *item,
@@ -565,10 +565,10 @@ public:
   {
     return m_type_handler->make_conversion_table_field(table, metadata, target);
   }
-  void make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
+  bool make_sort_key(uchar *to, Item *item, const SORT_FIELD_ATTR *sort_field,
                      Sort_param *param) const
   {
-    m_type_handler->make_sort_key(to, item, sort_field, param);
+    return m_type_handler->make_sort_key(to, item, sort_field, param);
   }
   void sortlength(THD *thd,
                   const Type_std_attributes *item,