Bug #40761: Assert on sum function on

            IF(..., CAST(longtext AS UNSIGNED), signed_val)
            (was: LEFT JOIN on inline view crashes server)

Select from a LONGTEXT column wrapped with an expression
like "IF(..., CAST(longtext_column AS UNSIGNED), smth_signed)"
failed an assertion or crashed the server. IFNULL function was
affected too.

LONGTEXT column item has a maximum length of 32^2-1 bytes,
at the same time this is a maximum possible length of any
MySQL item. CAST(longtext_column AS UNSIGNED) returns some
unsigned numeric result of length 32^2-1, so the result of
IF/IFNULL function of this number and some other signed number
will have text length of (32^2-1)+1=32^2 (one byte for the
minus sign) - there is integer overflow, and the length is
equal to zero. That caused assert/crash.

The bug has been fixed by the same solution as in the CASE
function implementation.

1 files changed, 22 insertions, 28 deletions

diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index 3b1d18b4252..759e912cc82 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -2069,21 +2069,23 @@ Item_func_ifnull::fix_length_and_dec()
 {
   agg_result_type(&hybrid_type, args, 2);
   maybe_null=args[1]->maybe_null;
-  decimals= max(args[0]->decimals, args[1]->decimals);
-  unsigned_flag= args[0]->unsigned_flag && args[1]->unsigned_flag;
 
   if (hybrid_type == DECIMAL_RESULT || hybrid_type == INT_RESULT) 
   {
-    int len0= args[0]->max_length - args[0]->decimals
-      - (args[0]->unsigned_flag ? 0 : 1);
-
-    int len1= args[1]->max_length - args[1]->decimals
-      - (args[1]->unsigned_flag ? 0 : 1);
-
-    max_length= max(len0, len1) + decimals + (unsigned_flag ? 0 : 1);
+    max_length= 0;
+    decimals= 0;
+    unsigned_flag= TRUE;
+    agg_num_lengths(args[0]);
+    agg_num_lengths(args[1]);
+    max_length= my_decimal_precision_to_length(max_length + decimals, decimals,
+                                               unsigned_flag);
   }
   else
+  {
     max_length= max(args[0]->max_length, args[1]->max_length);
+    decimals= max(args[0]->decimals, args[1]->decimals);
+    unsigned_flag=args[0]->unsigned_flag && args[1]->unsigned_flag;
+  }
 
   switch (hybrid_type) {
   case STRING_RESULT:
@@ -2238,8 +2240,6 @@ void
 Item_func_if::fix_length_and_dec()
 {
   maybe_null=args[1]->maybe_null || args[2]->maybe_null;
-  decimals= max(args[1]->decimals, args[2]->decimals);
-  unsigned_flag=args[1]->unsigned_flag && args[2]->unsigned_flag;
 
   enum Item_result arg1_type=args[1]->result_type();
   enum Item_result arg2_type=args[2]->result_type();
@@ -2276,16 +2276,20 @@ Item_func_if::fix_length_and_dec()
   if ((cached_result_type == DECIMAL_RESULT )
       || (cached_result_type == INT_RESULT))
   {
-    int len1= args[1]->max_length - args[1]->decimals
-      - (args[1]->unsigned_flag ? 0 : 1);
-
-    int len2= args[2]->max_length - args[2]->decimals
-      - (args[2]->unsigned_flag ? 0 : 1);
-
-    max_length=max(len1, len2) + decimals + (unsigned_flag ? 0 : 1);
+    max_length= 0;
+    decimals= 0;
+    unsigned_flag= TRUE;
+    agg_num_lengths(args[1]);
+    agg_num_lengths(args[2]);
+    max_length= my_decimal_precision_to_length(max_length + decimals, decimals,
+                                               unsigned_flag);
   }
   else
+  {
     max_length= max(args[1]->max_length, args[2]->max_length);
+    decimals= max(args[1]->decimals, args[2]->decimals);
+    unsigned_flag=args[1]->unsigned_flag && args[2]->unsigned_flag;
+  }
 }
 
 
@@ -2633,16 +2637,6 @@ void Item_func_case::agg_str_lengths(Item* arg)
 }
 
 
-void Item_func_case::agg_num_lengths(Item *arg)
-{
-  uint len= my_decimal_length_to_precision(arg->max_length, arg->decimals,
-                                           arg->unsigned_flag) - arg->decimals;
-  set_if_bigger(max_length, len); 
-  set_if_bigger(decimals, arg->decimals);
-  unsigned_flag= unsigned_flag && arg->unsigned_flag; 
-}
-
-
 void Item_func_case::fix_length_and_dec()
 {
   Item **agg;