Fix ldbl-96 fma (Inf, Inf, finite) (bug 23272).aaribaud/y2038-rfc-3-base

As reported in bug 23272, the ldbl-96 implementation of fma (fma for
double, in terms of ldbl-96 as the internal arithmetic type, as used
on 32-bit x86) is missing some of the special-case handling for
non-finite arguments, resulting in incorrect NaN results when the
first two arguments are infinities, the third is finite and so the
infinities go through the logic for finite arguments.  This patch
fixes it by handling all cases of non-finite arguments up front, with
additional fma tests for the problem cases being added to the
testsuite.

Tested for x86_64 and x86.

	[BZ #23272]
	* sysdeps/ieee754/ldbl-96/s_fma.c (__fma): Start by handling all
	cases of non-finite arguments.
	* math/libm-test-fma.inc (fma_test_data): Add more tests.

3 files changed, 46 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index 0a4c0ff6d9..f238908fe8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2018-06-11  Joseph Myers  <joseph@codesourcery.com>
+
+	[BZ #23272]
+	* sysdeps/ieee754/ldbl-96/s_fma.c (__fma): Start by handling all
+	cases of non-finite arguments.
+	* math/libm-test-fma.inc (fma_test_data): Add more tests.
+
 2018-06-10  John David Anglin  <danglin@gcc.gnu.org>
 
 	[BZ #23174]
diff --git a/math/libm-test-fma.inc b/math/libm-test-fma.inc
index 655267b507..5b29fb8201 100644
--- a/math/libm-test-fma.inc
+++ b/math/libm-test-fma.inc
@@ -113,6 +113,39 @@ static const struct test_fff_f_data fma_test_data[] =
     TEST_fff_f (fma, plus_infty, minus_infty, minus_infty, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
     TEST_fff_f (fma, minus_infty, plus_infty, minus_infty, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
 
+    TEST_fff_f (fma, plus_infty, plus_infty, plus_zero, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, minus_zero, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, min_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, -min_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, min_subnorm_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, -min_subnorm_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, max_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, plus_infty, -max_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, plus_zero, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, minus_zero, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, min_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, -min_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, min_subnorm_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, -min_subnorm_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, max_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, plus_infty, minus_infty, -max_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, plus_zero, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, minus_zero, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, min_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, -min_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, min_subnorm_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, -min_subnorm_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, max_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, plus_infty, -max_value, minus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, plus_zero, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, minus_zero, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, min_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, -min_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, min_subnorm_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, -min_subnorm_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, max_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+    TEST_fff_f (fma, minus_infty, minus_infty, -max_value, plus_infty, NO_INEXACT_EXCEPTION|ERRNO_UNCHANGED),
+
     AUTO_TESTS_fff_f (fma),
   };
 
diff --git a/sysdeps/ieee754/ldbl-96/s_fma.c b/sysdeps/ieee754/ldbl-96/s_fma.c
index f7f4dfd28d..986879cda5 100644
--- a/sysdeps/ieee754/ldbl-96/s_fma.c
+++ b/sysdeps/ieee754/ldbl-96/s_fma.c
@@ -32,14 +32,12 @@
 double
 __fma (double x, double y, double z)
 {
-  if (__glibc_unlikely (isinf (z)))
-    {
-      /* If z is Inf, but x and y are finite, the result should be
-	 z rather than NaN.  */
-      if (isfinite (x) && isfinite (y))
-	return (z + x) + y;
-      return (x * y) + z;
-    }
+  if (__glibc_unlikely (!isfinite (x) || !isfinite (y)))
+    return x * y + z;
+  else if (__glibc_unlikely (!isfinite (z)))
+    /* If z is Inf, but x and y are finite, the result should be z
+       rather than NaN.  */
+    return (z + x) + y;
 
   /* Ensure correct sign of exact 0 + 0.  */
   if (__glibc_unlikely ((x == 0 || y == 0) && z == 0))