glibc/sysdeps/ieee754/dbl-64/e_fmod.c
Adhemerval Zanella Netto 34b9f8bc17 math: Improve fmod
This uses a new algorithm similar to already proposed earlier [1].
With x = mx * 2^ex and y = my * 2^ey (mx, my, ex, ey being integers),
the simplest implementation is:

   mx * 2^ex == 2 * mx * 2^(ex - 1)

   while (ex > ey)
     {
       mx *= 2;
       --ex;
       mx %= my;
     }

With mx/my being mantissa of double floating pointer, on each step the
argument reduction can be improved 11 (which is sizeo of uint64_t minus
MANTISSA_WIDTH plus the signal bit):

   while (ex > ey)
     {
       mx << 11;
       ex -= 11;
       mx %= my;
     }  */

The implementation uses builtin clz and ctz, along with shifts to
convert hx/hy back to doubles.  Different than the original patch,
this path assume modulo/divide operation is slow, so use multiplication
with invert values.

I see the following performance improvements using fmod benchtests
(result only show the 'mean' result):

  Architecture     | Input           | master   | patch
  -----------------|-----------------|----------|--------
  x86_64 (Ryzen 9) | subnormals      | 19.1584  | 12.5049
  x86_64 (Ryzen 9) | normal          | 1016.51  | 296.939
  x86_64 (Ryzen 9) | close-exponents | 18.4428  | 16.0244
  aarch64 (N1)     | subnormal       | 11.153   | 6.81778
  aarch64 (N1)     | normal          | 528.649  | 155.62
  aarch64 (N1)     | close-exponents | 11.4517  | 8.21306

I also see similar improvements on arm-linux-gnueabihf when running on
the N1 aarch64 chips, where it a lot of soft-fp implementation (for
modulo, clz, ctz, and multiplication):

  Architecture     | Input           | master   | patch
  -----------------|-----------------|----------|--------
  armhf (N1)       | subnormal       | 15.908   | 15.1083
  armhf (N1)       | normal          | 837.525  | 244.833
  armhf (N1)       | close-exponents | 16.2111  | 21.8182

Instead of using the math_private.h definitions, I used the
math_config.h instead which is used on newer math implementations.

Co-authored-by: kirill <kirill.okhotnikov@gmail.com>

[1] https://sourceware.org/pipermail/libc-alpha/2020-November/119794.html
Reviewed-by: Wilco Dijkstra  <Wilco.Dijkstra@arm.com>
2023-04-03 16:36:24 -03:00

157 lines
4.3 KiB
C

/* Floating-point remainder function.
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <libm-alias-finite.h>
#include <math.h>
#include "math_config.h"
/* With x = mx * 2^ex and y = my * 2^ey (mx, my, ex, ey being integers), the
simplest implementation is:
mx * 2^ex == 2 * mx * 2^(ex - 1)
or
while (ex > ey)
{
mx *= 2;
--ex;
mx %= my;
}
With the mathematical equivalence of:
r == x % y == (x % (N * y)) % y
And with mx/my being mantissa of double floating point number (which uses
less bits than the storage type), on each step the argument reduction can
be improved by 11 (which is the size of uint64_t minus MANTISSA_WIDTH plus
the signal bit):
mx * 2^ex == 2^11 * mx * 2^(ex - 11)
or
while (ex > ey)
{
mx << 11;
ex -= 11;
mx %= my;
} */
double
__ieee754_fmod (double x, double y)
{
uint64_t hx = asuint64 (x);
uint64_t hy = asuint64 (y);
uint64_t sx = hx & SIGN_MASK;
/* Get |x| and |y|. */
hx ^= sx;
hy &= ~SIGN_MASK;
/* Special cases:
- If x or y is a Nan, NaN is returned.
- If x is an inifinity, a NaN is returned.
- If y is zero, Nan is returned.
- If x is +0/-0, and y is not zero, +0/-0 is returned. */
if (__glibc_unlikely (hy == 0 || hx >= EXPONENT_MASK || hy > EXPONENT_MASK))
return (x * y) / (x * y);
if (__glibc_unlikely (hx <= hy))
{
if (hx < hy)
return x;
return asdouble (sx);
}
int ex = hx >> MANTISSA_WIDTH;
int ey = hy >> MANTISSA_WIDTH;
/* Common case where exponents are close: ey >= -907 and |x/y| < 2^52, */
if (__glibc_likely (ey > MANTISSA_WIDTH && ex - ey <= EXPONENT_WIDTH))
{
uint64_t mx = (hx & MANTISSA_MASK) | (MANTISSA_MASK + 1);
uint64_t my = (hy & MANTISSA_MASK) | (MANTISSA_MASK + 1);
uint64_t d = (ex == ey) ? (mx - my) : (mx << (ex - ey)) % my;
return make_double (d, ey - 1, sx);
}
/* Special case, both x and y are subnormal. */
if (__glibc_unlikely (ex == 0 && ey == 0))
return asdouble (sx | hx % hy);
/* Convert |x| and |y| to 'mx + 2^ex' and 'my + 2^ey'. Assume that hx is
not subnormal by conditions above. */
uint64_t mx = get_mantissa (hx) | (MANTISSA_MASK + 1);
ex--;
uint64_t my = get_mantissa (hy) | (MANTISSA_MASK + 1);
int lead_zeros_my = EXPONENT_WIDTH;
if (__glibc_likely (ey > 0))
ey--;
else
{
my = hy;
lead_zeros_my = clz_uint64 (my);
}
/* Assume hy != 0 */
int tail_zeros_my = ctz_uint64 (my);
int sides_zeroes = lead_zeros_my + tail_zeros_my;
int exp_diff = ex - ey;
int right_shift = exp_diff < tail_zeros_my ? exp_diff : tail_zeros_my;
my >>= right_shift;
exp_diff -= right_shift;
ey += right_shift;
int left_shift = exp_diff < EXPONENT_WIDTH ? exp_diff : EXPONENT_WIDTH;
mx <<= left_shift;
exp_diff -= left_shift;
mx %= my;
if (__glibc_unlikely (mx == 0))
return asdouble (sx);
if (exp_diff == 0)
return make_double (mx, ey, sx);
/* Assume modulo/divide operation is slow, so use multiplication with invert
values. */
uint64_t inv_hy = UINT64_MAX / my;
while (exp_diff > sides_zeroes) {
exp_diff -= sides_zeroes;
uint64_t hd = (mx * inv_hy) >> (BIT_WIDTH - sides_zeroes);
mx <<= sides_zeroes;
mx -= hd * my;
while (__glibc_unlikely (mx > my))
mx -= my;
}
uint64_t hd = (mx * inv_hy) >> (BIT_WIDTH - exp_diff);
mx <<= exp_diff;
mx -= hd * my;
while (__glibc_unlikely (mx > my))
mx -= my;
return make_double (mx, ey, sx);
}
libm_alias_finite (__ieee754_fmod, __fmod)