Merge mysql.com:/home/hf/work/8663/my50-8663

into  mysql.com:/home/hf/work/8663/my51-8663

3 files changed, 22 insertions, 1 deletions

diff --git a/mysql-test/r/bigint.result b/mysql-test/r/bigint.result
index a70c6b3df7d..8d831ba800b 100644
--- a/mysql-test/r/bigint.result
+++ b/mysql-test/r/bigint.result
@@ -352,6 +352,16 @@ select c1 mod 50 as result from t1;
 result
 6
 drop table t1;
+select cast(19999999999999999999 as signed);
+cast(19999999999999999999 as signed)
+9223372036854775807
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+select cast(-19999999999999999999 as signed);
+cast(-19999999999999999999 as signed)
+-9223372036854775808
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
 create table t1 select -9223372036854775808 bi;
 describe t1;
 Field	Type	Null	Key	Default	Extra
diff --git a/mysql-test/t/bigint.test b/mysql-test/t/bigint.test
index c2c6f895545..52f34e58e97 100644
--- a/mysql-test/t/bigint.test
+++ b/mysql-test/t/bigint.test
@@ -288,6 +288,13 @@ insert into t1 values (10000002383263201056);
 select c1 mod 50 as result from t1;
 drop table t1;
 
+#
+# Bug #8663 cant use bgint unsigned as input to cast
+#
+
+select cast(19999999999999999999 as signed);
+select cast(-19999999999999999999 as signed);
+
 # Bug #28005 Partitions: can't use -9223372036854775808 
 create table t1 select -9223372036854775808 bi;
 describe t1;
diff --git a/strings/decimal.c b/strings/decimal.c
index b29017aa6cb..0768c8cd4ca 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1089,7 +1089,11 @@ int decimal2longlong(decimal_t *from, longlong *to)
     x=x*DIG_BASE - *buf++;
     if (unlikely(y < (LONGLONG_MIN/DIG_BASE) || x > y))
     {
-      *to= from->sign ? y : -y;
+      /*
+        the decimal is bigger than any possible integer
+        return border integer depending on the sign
+      */
+      *to= from->sign ? LONGLONG_MIN : LONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
   }