diff options
Diffstat (limited to 'lib/builtins/fp_lib.h')
| -rw-r--r-- | lib/builtins/fp_lib.h | 288 |
1 files changed, 140 insertions, 148 deletions
diff --git a/lib/builtins/fp_lib.h b/lib/builtins/fp_lib.h index a0e19ab6a..83c3081aa 100644 --- a/lib/builtins/fp_lib.h +++ b/lib/builtins/fp_lib.h @@ -1,9 +1,8 @@ //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,22 +20,22 @@ #ifndef FP_LIB_HEADER #define FP_LIB_HEADER -#include <stdint.h> -#include <stdbool.h> -#include <limits.h> #include "int_lib.h" #include "int_math.h" +#include <limits.h> +#include <stdbool.h> +#include <stdint.h> // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in // 32-bit mode. #if defined(__FreeBSD__) && defined(__i386__) -# include <sys/param.h> -# if __FreeBSD_version < 903000 // v9.3 -# define uint64_t unsigned long long -# define int64_t long long -# undef UINT64_C -# define UINT64_C(c) (c ## ULL) -# endif +#include <sys/param.h> +#if __FreeBSD_version < 903000 // v9.3 +#define uint64_t unsigned long long +#define int64_t long long +#undef UINT64_C +#define UINT64_C(c) (c##ULL) +#endif #endif #if defined SINGLE_PRECISION @@ -47,15 +46,13 @@ typedef float fp_t; #define REP_C UINT32_C #define significandBits 23 -static __inline int rep_clz(rep_t a) { - return __builtin_clz(a); -} +static __inline int rep_clz(rep_t a) { return __builtin_clz(a); } // 32x32 --> 64 bit multiply static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product = (uint64_t)a*b; - *hi = product >> 32; - *lo = product; + const uint64_t product = (uint64_t)a * b; + *hi = product >> 32; + *lo = product; } COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); @@ -69,12 +66,12 @@ typedef double fp_t; static __inline int rep_clz(rep_t a) { #if defined __LP64__ - return __builtin_clzl(a); + return __builtin_clzl(a); #else - if (a & REP_C(0xffffffff00000000)) - return __builtin_clz(a >> 32); - else - return 32 + __builtin_clz(a & REP_C(0xffffffff)); + if (a & REP_C(0xffffffff00000000)) + return __builtin_clz(a >> 32); + else + return 32 + __builtin_clz(a & REP_C(0xffffffff)); #endif } @@ -85,17 +82,17 @@ static __inline int rep_clz(rep_t a) { // many 64-bit platforms have this operation, but they tend to have hardware // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - // Each of the component 32x32 -> 64 products - const uint64_t plolo = loWord(a) * loWord(b); - const uint64_t plohi = loWord(a) * hiWord(b); - const uint64_t philo = hiWord(a) * loWord(b); - const uint64_t phihi = hiWord(a) * hiWord(b); - // Sum terms that contribute to lo in a way that allows us to get the carry - const uint64_t r0 = loWord(plolo); - const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); - *lo = r0 + (r1 << 32); - // Sum terms contributing to hi with the carry from lo - *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; + // Each of the component 32x32 -> 64 products + const uint64_t plolo = loWord(a) * loWord(b); + const uint64_t plohi = loWord(a) * hiWord(b); + const uint64_t philo = hiWord(a) * loWord(b); + const uint64_t phihi = hiWord(a) * hiWord(b); + // Sum terms that contribute to lo in a way that allows us to get the carry + const uint64_t r0 = loWord(plolo); + const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); + *lo = r0 + (r1 << 32); + // Sum terms contributing to hi with the carry from lo + *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; } #undef loWord #undef hiWord @@ -114,32 +111,34 @@ typedef long double fp_t; #define significandBits 112 static __inline int rep_clz(rep_t a) { - const union - { - __uint128_t ll; + const union { + __uint128_t ll; #if _YUGA_BIG_ENDIAN - struct { uint64_t high, low; } s; + struct { + uint64_t high, low; + } s; #else - struct { uint64_t low, high; } s; + struct { + uint64_t low, high; + } s; #endif - } uu = { .ll = a }; + } uu = {.ll = a}; - uint64_t word; - uint64_t add; + uint64_t word; + uint64_t add; - if (uu.s.high){ - word = uu.s.high; - add = 0; - } - else{ - word = uu.s.low; - add = 64; - } - return __builtin_clzll(word) + add; + if (uu.s.high) { + word = uu.s.high; + add = 0; + } else { + word = uu.s.low; + add = 64; + } + return __builtin_clzll(word) + add; } -#define Word_LoMask UINT64_C(0x00000000ffffffff) -#define Word_HiMask UINT64_C(0xffffffff00000000) +#define Word_LoMask UINT64_C(0x00000000ffffffff) +#define Word_HiMask UINT64_C(0xffffffff00000000) #define Word_FullMask UINT64_C(0xffffffffffffffff) #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask) #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask) @@ -151,55 +150,41 @@ static __inline int rep_clz(rep_t a) { // floating-point, so we don't bother with a special case for them here. static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { - const uint64_t product11 = Word_1(a) * Word_1(b); - const uint64_t product12 = Word_1(a) * Word_2(b); - const uint64_t product13 = Word_1(a) * Word_3(b); - const uint64_t product14 = Word_1(a) * Word_4(b); - const uint64_t product21 = Word_2(a) * Word_1(b); - const uint64_t product22 = Word_2(a) * Word_2(b); - const uint64_t product23 = Word_2(a) * Word_3(b); - const uint64_t product24 = Word_2(a) * Word_4(b); - const uint64_t product31 = Word_3(a) * Word_1(b); - const uint64_t product32 = Word_3(a) * Word_2(b); - const uint64_t product33 = Word_3(a) * Word_3(b); - const uint64_t product34 = Word_3(a) * Word_4(b); - const uint64_t product41 = Word_4(a) * Word_1(b); - const uint64_t product42 = Word_4(a) * Word_2(b); - const uint64_t product43 = Word_4(a) * Word_3(b); - const uint64_t product44 = Word_4(a) * Word_4(b); - - const __uint128_t sum0 = (__uint128_t)product44; - const __uint128_t sum1 = (__uint128_t)product34 + - (__uint128_t)product43; - const __uint128_t sum2 = (__uint128_t)product24 + - (__uint128_t)product33 + - (__uint128_t)product42; - const __uint128_t sum3 = (__uint128_t)product14 + - (__uint128_t)product23 + - (__uint128_t)product32 + - (__uint128_t)product41; - const __uint128_t sum4 = (__uint128_t)product13 + - (__uint128_t)product22 + - (__uint128_t)product31; - const __uint128_t sum5 = (__uint128_t)product12 + - (__uint128_t)product21; - const __uint128_t sum6 = (__uint128_t)product11; - - const __uint128_t r0 = (sum0 & Word_FullMask) + - ((sum1 & Word_LoMask) << 32); - const __uint128_t r1 = (sum0 >> 64) + - ((sum1 >> 32) & Word_FullMask) + - (sum2 & Word_FullMask) + - ((sum3 << 32) & Word_HiMask); - - *lo = r0 + (r1 << 64); - *hi = (r1 >> 64) + - (sum1 >> 96) + - (sum2 >> 64) + - (sum3 >> 32) + - sum4 + - (sum5 << 32) + - (sum6 << 64); + const uint64_t product11 = Word_1(a) * Word_1(b); + const uint64_t product12 = Word_1(a) * Word_2(b); + const uint64_t product13 = Word_1(a) * Word_3(b); + const uint64_t product14 = Word_1(a) * Word_4(b); + const uint64_t product21 = Word_2(a) * Word_1(b); + const uint64_t product22 = Word_2(a) * Word_2(b); + const uint64_t product23 = Word_2(a) * Word_3(b); + const uint64_t product24 = Word_2(a) * Word_4(b); + const uint64_t product31 = Word_3(a) * Word_1(b); + const uint64_t product32 = Word_3(a) * Word_2(b); + const uint64_t product33 = Word_3(a) * Word_3(b); + const uint64_t product34 = Word_3(a) * Word_4(b); + const uint64_t product41 = Word_4(a) * Word_1(b); + const uint64_t product42 = Word_4(a) * Word_2(b); + const uint64_t product43 = Word_4(a) * Word_3(b); + const uint64_t product44 = Word_4(a) * Word_4(b); + + const __uint128_t sum0 = (__uint128_t)product44; + const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43; + const __uint128_t sum2 = + (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42; + const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 + + (__uint128_t)product32 + (__uint128_t)product41; + const __uint128_t sum4 = + (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31; + const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21; + const __uint128_t sum6 = (__uint128_t)product11; + + const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32); + const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) + + (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask); + + *lo = r0 + (r1 << 64); + *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 + + (sum5 << 32) + (sum6 << 64); } #undef Word_1 #undef Word_2 @@ -213,58 +198,65 @@ static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined. #endif -#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || defined(CRT_LDBL_128BIT) -#define typeWidth (sizeof(rep_t)*CHAR_BIT) -#define exponentBits (typeWidth - significandBits - 1) -#define maxExponent ((1 << exponentBits) - 1) -#define exponentBias (maxExponent >> 1) +#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \ + defined(CRT_LDBL_128BIT) +#define typeWidth (sizeof(rep_t) * CHAR_BIT) +#define exponentBits (typeWidth - significandBits - 1) +#define maxExponent ((1 << exponentBits) - 1) +#define exponentBias (maxExponent >> 1) -#define implicitBit (REP_C(1) << significandBits) +#define implicitBit (REP_C(1) << significandBits) #define significandMask (implicitBit - 1U) -#define signBit (REP_C(1) << (significandBits + exponentBits)) -#define absMask (signBit - 1U) -#define exponentMask (absMask ^ significandMask) -#define oneRep ((rep_t)exponentBias << significandBits) -#define infRep exponentMask -#define quietBit (implicitBit >> 1) -#define qnanRep (exponentMask | quietBit) +#define signBit (REP_C(1) << (significandBits + exponentBits)) +#define absMask (signBit - 1U) +#define exponentMask (absMask ^ significandMask) +#define oneRep ((rep_t)exponentBias << significandBits) +#define infRep exponentMask +#define quietBit (implicitBit >> 1) +#define qnanRep (exponentMask | quietBit) static __inline rep_t toRep(fp_t x) { - const union { fp_t f; rep_t i; } rep = {.f = x}; - return rep.i; + const union { + fp_t f; + rep_t i; + } rep = {.f = x}; + return rep.i; } static __inline fp_t fromRep(rep_t x) { - const union { fp_t f; rep_t i; } rep = {.i = x}; - return rep.f; + const union { + fp_t f; + rep_t i; + } rep = {.i = x}; + return rep.f; } static __inline int normalize(rep_t *significand) { - const int shift = rep_clz(*significand) - rep_clz(implicitBit); - *significand <<= shift; - return 1 - shift; + const int shift = rep_clz(*significand) - rep_clz(implicitBit); + *significand <<= shift; + return 1 - shift; } static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { - *hi = *hi << count | *lo >> (typeWidth - count); - *lo = *lo << count; + *hi = *hi << count | *lo >> (typeWidth - count); + *lo = *lo << count; } -static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int count) { - if (count < typeWidth) { - const bool sticky = *lo << (typeWidth - count); - *lo = *hi << (typeWidth - count) | *lo >> count | sticky; - *hi = *hi >> count; - } - else if (count < 2*typeWidth) { - const bool sticky = *hi << (2*typeWidth - count) | *lo; - *lo = *hi >> (count - typeWidth) | sticky; - *hi = 0; - } else { - const bool sticky = *hi | *lo; - *lo = sticky; - *hi = 0; - } +static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, + unsigned int count) { + if (count < typeWidth) { + const bool sticky = *lo << (typeWidth - count); + *lo = *hi << (typeWidth - count) | *lo >> count | sticky; + *hi = *hi >> count; + } else if (count < 2 * typeWidth) { + const bool sticky = *hi << (2 * typeWidth - count) | *lo; + *lo = *hi >> (count - typeWidth) | sticky; + *hi = 0; + } else { + const bool sticky = *hi | *lo; + *lo = sticky; + *hi = 0; + } } // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids @@ -280,9 +272,9 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { // 2) 0.0 returns -inf if (exp == maxExponent) { if (((rep & signBit) == 0) || (x != x)) { - return x; // NaN or +inf: return x + return x; // NaN or +inf: return x } else { - return -x; // -inf: return -x + return -x; // -inf: return -x } } else if (x == 0.0) { // 0.0: return -inf @@ -291,13 +283,13 @@ static __inline fp_t __compiler_rt_logbX(fp_t x) { if (exp != 0) { // Normal number - return exp - exponentBias; // Unbias exponent + return exp - exponentBias; // Unbias exponent } else { // Subnormal number; normalize and repeat rep &= absMask; const int shift = 1 - normalize(&rep); exp = (rep & exponentMask) >> significandBits; - return exp - exponentBias - shift; // Unbias exponent + return exp - exponentBias - shift; // Unbias exponent } } #endif @@ -311,17 +303,17 @@ static __inline fp_t __compiler_rt_logb(fp_t x) { return __compiler_rt_logbX(x); } #elif defined(QUAD_PRECISION) - #if defined(CRT_LDBL_128BIT) +#if defined(CRT_LDBL_128BIT) static __inline fp_t __compiler_rt_logbl(fp_t x) { return __compiler_rt_logbX(x); } - #else +#else // The generic implementation only works for ieee754 floating point. For other // floating point types, continue to rely on the libm implementation for now. static __inline long double __compiler_rt_logbl(long double x) { return crt_logbl(x); } - #endif +#endif #endif #endif // FP_LIB_HEADER |