MDEV-22742 UBSAN: Many overflow issues in strings/decimal.c - runtime error: signed integer overflow: x * y cannot be represented in type 'long long int' (on optimized builds).bb-10.2-mdev-22742-hf

Avoid integer overflow, do the check before the calculation.

1 files changed, 14 insertions, 8 deletions

diff --git a/strings/decimal.c b/strings/decimal.c
index 9d18a9ce52a..6249d7e097a 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1128,13 +1128,16 @@ int decimal2ulonglong(const decimal_t *from, ulonglong *to)
 
   for (intg=from->intg; intg > 0; intg-=DIG_PER_DEC1)
   {
-    ulonglong y=x;
-    x=x*DIG_BASE + *buf++;
-    if (unlikely(y > ((ulonglong) ULONGLONG_MAX/DIG_BASE) || x < y))
+    if (unlikely (
+          x >= ULONGLONG_MAX/DIG_BASE &&
+          (x > ULONGLONG_MAX/DIG_BASE ||
+             *buf > (dec1) (ULONGLONG_MAX%DIG_BASE))))
     {
       *to=ULONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
+
+    x=x*DIG_BASE + *buf++;
   }
   *to=x;
   for (frac=from->frac; unlikely(frac > 0); frac-=DIG_PER_DEC1)
@@ -1151,23 +1154,26 @@ int decimal2longlong(const decimal_t *from, longlong *to)
 
   for (intg=from->intg; intg > 0; intg-=DIG_PER_DEC1)
   {
-    longlong y=x;
     /*
       Attention: trick!
       we're calculating -|from| instead of |from| here
       because |LONGLONG_MIN| > LONGLONG_MAX
       so we can convert -9223372036854775808 correctly
     */
-    x=x*DIG_BASE - *buf++;
-    if (unlikely(y < (LONGLONG_MIN/DIG_BASE) || x > y))
+    if (unlikely (
+          x <= LONGLONG_MIN/DIG_BASE &&
+          (x < LONGLONG_MIN/DIG_BASE ||
+            *buf > (dec1) (-(LONGLONG_MIN%DIG_BASE)))))
     {
       /*
-        the decimal is bigger than any possible integer
-        return border integer depending on the sign
+         the decimal is bigger than any possible integer
+         return border integer depending on the sign
       */
       *to= from->sign ? LONGLONG_MIN : LONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
+
+    x=x*DIG_BASE - *buf++;
   }
   /* boundary case: 9223372036854775808 */
   if (unlikely(from->sign==0 && x == LONGLONG_MIN))