aarch64: Add vector implementations of asin routines

This commit is contained in:
Joe Ramsay 2023-11-03 12:12:19 +00:00 committed by Szabolcs Nagy
parent d1dcb565a1
commit 9bed498418
13 changed files with 407 additions and 1 deletions

View File

@ -1,4 +1,5 @@
libmvec-supported-funcs = cos \
libmvec-supported-funcs = asin \
cos \
exp \
exp10 \
exp2 \

View File

@ -18,6 +18,10 @@ libmvec {
_ZGVsMxv_sinf;
}
GLIBC_2.39 {
_ZGVnN4v_asinf;
_ZGVnN2v_asin;
_ZGVsMxv_asinf;
_ZGVsMxv_asin;
_ZGVnN4v_exp10f;
_ZGVnN2v_exp10;
_ZGVsMxv_exp10f;

View File

@ -0,0 +1,113 @@
/* Double-precision AdvSIMD inverse sin
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "v_math.h"
#include "poly_advsimd_f64.h"
static const struct data
{
float64x2_t poly[12];
float64x2_t pi_over_2;
uint64x2_t abs_mask;
} data = {
/* Polynomial approximation of (asin(sqrt(x)) - sqrt(x)) / (x * sqrt(x))
on [ 0x1p-106, 0x1p-2 ], relative error: 0x1.c3d8e169p-57. */
.poly = { V2 (0x1.555555555554ep-3), V2 (0x1.3333333337233p-4),
V2 (0x1.6db6db67f6d9fp-5), V2 (0x1.f1c71fbd29fbbp-6),
V2 (0x1.6e8b264d467d6p-6), V2 (0x1.1c5997c357e9dp-6),
V2 (0x1.c86a22cd9389dp-7), V2 (0x1.856073c22ebbep-7),
V2 (0x1.fd1151acb6bedp-8), V2 (0x1.087182f799c1dp-6),
V2 (-0x1.6602748120927p-7), V2 (0x1.cfa0dd1f9478p-6), },
.pi_over_2 = V2 (0x1.921fb54442d18p+0),
.abs_mask = V2 (0x7fffffffffffffff),
};
#define AllMask v_u64 (0xffffffffffffffff)
#define One (0x3ff0000000000000)
#define Small (0x3e50000000000000) /* 2^-12. */
#if WANT_SIMD_EXCEPT
static float64x2_t VPCS_ATTR NOINLINE
special_case (float64x2_t x, float64x2_t y, uint64x2_t special)
{
return v_call_f64 (asin, x, y, special);
}
#endif
/* Double-precision implementation of vector asin(x).
For |x| < Small, approximate asin(x) by x. Small = 2^-12 for correct
rounding. If WANT_SIMD_EXCEPT = 0, Small = 0 and we proceed with the
following approximation.
For |x| in [Small, 0.5], use an order 11 polynomial P such that the final
approximation is an odd polynomial: asin(x) ~ x + x^3 P(x^2).
The largest observed error in this region is 1.01 ulps,
_ZGVnN2v_asin (0x1.da9735b5a9277p-2) got 0x1.ed78525a927efp-2
want 0x1.ed78525a927eep-2.
For |x| in [0.5, 1.0], use same approximation with a change of variable
asin(x) = pi/2 - (y + y * z * P(z)), with z = (1-x)/2 and y = sqrt(z).
The largest observed error in this region is 2.69 ulps,
_ZGVnN2v_asin (0x1.044ac9819f573p-1) got 0x1.110d7e85fdd5p-1
want 0x1.110d7e85fdd53p-1. */
float64x2_t VPCS_ATTR V_NAME_D1 (asin) (float64x2_t x)
{
const struct data *d = ptr_barrier (&data);
float64x2_t ax = vabsq_f64 (x);
#if WANT_SIMD_EXCEPT
/* Special values need to be computed with scalar fallbacks so
that appropriate exceptions are raised. */
uint64x2_t special
= vcgtq_u64 (vsubq_u64 (vreinterpretq_u64_f64 (ax), v_u64 (Small)),
v_u64 (One - Small));
if (__glibc_unlikely (v_any_u64 (special)))
return special_case (x, x, AllMask);
#endif
uint64x2_t a_lt_half = vcltq_f64 (ax, v_f64 (0.5));
/* Evaluate polynomial Q(x) = y + y * z * P(z) with
z = x ^ 2 and y = |x| , if |x| < 0.5
z = (1 - |x|) / 2 and y = sqrt(z), if |x| >= 0.5. */
float64x2_t z2 = vbslq_f64 (a_lt_half, vmulq_f64 (x, x),
vfmsq_n_f64 (v_f64 (0.5), ax, 0.5));
float64x2_t z = vbslq_f64 (a_lt_half, ax, vsqrtq_f64 (z2));
/* Use a single polynomial approximation P for both intervals. */
float64x2_t z4 = vmulq_f64 (z2, z2);
float64x2_t z8 = vmulq_f64 (z4, z4);
float64x2_t z16 = vmulq_f64 (z8, z8);
float64x2_t p = v_estrin_11_f64 (z2, z4, z8, z16, d->poly);
/* Finalize polynomial: z + z * z2 * P(z2). */
p = vfmaq_f64 (z, vmulq_f64 (z, z2), p);
/* asin(|x|) = Q(|x|) , for |x| < 0.5
= pi/2 - 2 Q(|x|), for |x| >= 0.5. */
float64x2_t y = vbslq_f64 (a_lt_half, p, vfmsq_n_f64 (d->pi_over_2, p, 2.0));
/* Copy sign. */
return vbslq_f64 (d->abs_mask, y, x);
}

View File

@ -0,0 +1,86 @@
/* Double-precision SVE inverse sin
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "sv_math.h"
#include "poly_sve_f64.h"
static const struct data
{
float64_t poly[12];
float64_t pi_over_2f;
} data = {
/* Polynomial approximation of (asin(sqrt(x)) - sqrt(x)) / (x * sqrt(x))
on [ 0x1p-106, 0x1p-2 ], relative error: 0x1.c3d8e169p-57. */
.poly = { 0x1.555555555554ep-3, 0x1.3333333337233p-4,
0x1.6db6db67f6d9fp-5, 0x1.f1c71fbd29fbbp-6,
0x1.6e8b264d467d6p-6, 0x1.1c5997c357e9dp-6,
0x1.c86a22cd9389dp-7, 0x1.856073c22ebbep-7,
0x1.fd1151acb6bedp-8, 0x1.087182f799c1dp-6,
-0x1.6602748120927p-7, 0x1.cfa0dd1f9478p-6, },
.pi_over_2f = 0x1.921fb54442d18p+0,
};
#define P(i) sv_f64 (d->poly[i])
/* Double-precision SVE implementation of vector asin(x).
For |x| in [0, 0.5], use an order 11 polynomial P such that the final
approximation is an odd polynomial: asin(x) ~ x + x^3 P(x^2).
The largest observed error in this region is 0.52 ulps,
_ZGVsMxv_asin(0x1.d95ae04998b6cp-2) got 0x1.ec13757305f27p-2
want 0x1.ec13757305f26p-2.
For |x| in [0.5, 1.0], use same approximation with a change of variable
asin(x) = pi/2 - (y + y * z * P(z)), with z = (1-x)/2 and y = sqrt(z).
The largest observed error in this region is 2.69 ulps,
_ZGVsMxv_asin(0x1.044ac9819f573p-1) got 0x1.110d7e85fdd5p-1
want 0x1.110d7e85fdd53p-1. */
svfloat64_t SV_NAME_D1 (asin) (svfloat64_t x, const svbool_t pg)
{
const struct data *d = ptr_barrier (&data);
svuint64_t sign = svand_x (pg, svreinterpret_u64 (x), 0x8000000000000000);
svfloat64_t ax = svabs_x (pg, x);
svbool_t a_ge_half = svacge (pg, x, 0.5);
/* Evaluate polynomial Q(x) = y + y * z * P(z) with
z = x ^ 2 and y = |x| , if |x| < 0.5
z = (1 - |x|) / 2 and y = sqrt(z), if |x| >= 0.5. */
svfloat64_t z2 = svsel (a_ge_half, svmls_x (pg, sv_f64 (0.5), ax, 0.5),
svmul_x (pg, x, x));
svfloat64_t z = svsqrt_m (ax, a_ge_half, z2);
/* Use a single polynomial approximation P for both intervals. */
svfloat64_t z4 = svmul_x (pg, z2, z2);
svfloat64_t z8 = svmul_x (pg, z4, z4);
svfloat64_t z16 = svmul_x (pg, z8, z8);
svfloat64_t p = sv_estrin_11_f64_x (pg, z2, z4, z8, z16, d->poly);
/* Finalize polynomial: z + z * z2 * P(z2). */
p = svmla_x (pg, z, svmul_x (pg, z, z2), p);
/* asin(|x|) = Q(|x|) , for |x| < 0.5
= pi/2 - 2 Q(|x|), for |x| >= 0.5. */
svfloat64_t y = svmad_m (a_ge_half, p, sv_f64 (-2.0), d->pi_over_2f);
/* Copy sign. */
return svreinterpret_f64 (svorr_x (pg, svreinterpret_u64 (y), sign));
}

View File

@ -0,0 +1,104 @@
/* Single-precision AdvSIMD inverse sin
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "v_math.h"
#include "poly_advsimd_f32.h"
static const struct data
{
float32x4_t poly[5];
float32x4_t pi_over_2f;
} data = {
/* Polynomial approximation of (asin(sqrt(x)) - sqrt(x)) / (x * sqrt(x)) on
[ 0x1p-24 0x1p-2 ] order = 4 rel error: 0x1.00a23bbp-29 . */
.poly = { V4 (0x1.55555ep-3), V4 (0x1.33261ap-4), V4 (0x1.70d7dcp-5),
V4 (0x1.b059dp-6), V4 (0x1.3af7d8p-5) },
.pi_over_2f = V4 (0x1.921fb6p+0f),
};
#define AbsMask 0x7fffffff
#define Half 0x3f000000
#define One 0x3f800000
#define Small 0x39800000 /* 2^-12. */
#if WANT_SIMD_EXCEPT
static float32x4_t VPCS_ATTR NOINLINE
special_case (float32x4_t x, float32x4_t y, uint32x4_t special)
{
return v_call_f32 (asinf, x, y, special);
}
#endif
/* Single-precision implementation of vector asin(x).
For |x| < Small, approximate asin(x) by x. Small = 2^-12 for correct
rounding. If WANT_SIMD_EXCEPT = 0, Small = 0 and we proceed with the
following approximation.
For |x| in [Small, 0.5], use order 4 polynomial P such that the final
approximation is an odd polynomial: asin(x) ~ x + x^3 P(x^2).
The largest observed error in this region is 0.83 ulps,
_ZGVnN4v_asinf (0x1.ea00f4p-2) got 0x1.fef15ep-2 want 0x1.fef15cp-2.
For |x| in [0.5, 1.0], use same approximation with a change of variable
asin(x) = pi/2 - (y + y * z * P(z)), with z = (1-x)/2 and y = sqrt(z).
The largest observed error in this region is 2.41 ulps,
_ZGVnN4v_asinf (0x1.00203ep-1) got 0x1.0c3a64p-1 want 0x1.0c3a6p-1. */
float32x4_t VPCS_ATTR V_NAME_F1 (asin) (float32x4_t x)
{
const struct data *d = ptr_barrier (&data);
uint32x4_t ix = vreinterpretq_u32_f32 (x);
uint32x4_t ia = vandq_u32 (ix, v_u32 (AbsMask));
#if WANT_SIMD_EXCEPT
/* Special values need to be computed with scalar fallbacks so
that appropriate fp exceptions are raised. */
uint32x4_t special
= vcgtq_u32 (vsubq_u32 (ia, v_u32 (Small)), v_u32 (One - Small));
if (__glibc_unlikely (v_any_u32 (special)))
return special_case (x, x, v_u32 (0xffffffff));
#endif
float32x4_t ax = vreinterpretq_f32_u32 (ia);
uint32x4_t a_lt_half = vcltq_u32 (ia, v_u32 (Half));
/* Evaluate polynomial Q(x) = y + y * z * P(z) with
z = x ^ 2 and y = |x| , if |x| < 0.5
z = (1 - |x|) / 2 and y = sqrt(z), if |x| >= 0.5. */
float32x4_t z2 = vbslq_f32 (a_lt_half, vmulq_f32 (x, x),
vfmsq_n_f32 (v_f32 (0.5), ax, 0.5));
float32x4_t z = vbslq_f32 (a_lt_half, ax, vsqrtq_f32 (z2));
/* Use a single polynomial approximation P for both intervals. */
float32x4_t p = v_horner_4_f32 (z2, d->poly);
/* Finalize polynomial: z + z * z2 * P(z2). */
p = vfmaq_f32 (z, vmulq_f32 (z, z2), p);
/* asin(|x|) = Q(|x|) , for |x| < 0.5
= pi/2 - 2 Q(|x|), for |x| >= 0.5. */
float32x4_t y
= vbslq_f32 (a_lt_half, p, vfmsq_n_f32 (d->pi_over_2f, p, 2.0));
/* Copy sign. */
return vbslq_f32 (v_u32 (AbsMask), y, x);
}

View File

@ -0,0 +1,78 @@
/* Single-precision SVE inverse sin
Copyright (C) 2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "sv_math.h"
#include "poly_sve_f32.h"
static const struct data
{
float32_t poly[5];
float32_t pi_over_2f;
} data = {
/* Polynomial approximation of (asin(sqrt(x)) - sqrt(x)) / (x * sqrt(x)) on
[ 0x1p-24 0x1p-2 ] order = 4 rel error: 0x1.00a23bbp-29 . */
.poly = { 0x1.55555ep-3, 0x1.33261ap-4, 0x1.70d7dcp-5, 0x1.b059dp-6,
0x1.3af7d8p-5, },
.pi_over_2f = 0x1.921fb6p+0f,
};
/* Single-precision SVE implementation of vector asin(x).
For |x| in [0, 0.5], use order 4 polynomial P such that the final
approximation is an odd polynomial: asin(x) ~ x + x^3 P(x^2).
The largest observed error in this region is 0.83 ulps,
_ZGVsMxv_asinf (0x1.ea00f4p-2) got 0x1.fef15ep-2
want 0x1.fef15cp-2.
For |x| in [0.5, 1.0], use same approximation with a change of variable
asin(x) = pi/2 - (y + y * z * P(z)), with z = (1-x)/2 and y = sqrt(z).
The largest observed error in this region is 2.41 ulps,
_ZGVsMxv_asinf (-0x1.00203ep-1) got -0x1.0c3a64p-1
want -0x1.0c3a6p-1. */
svfloat32_t SV_NAME_F1 (asin) (svfloat32_t x, const svbool_t pg)
{
const struct data *d = ptr_barrier (&data);
svuint32_t sign = svand_x (pg, svreinterpret_u32 (x), 0x80000000);
svfloat32_t ax = svabs_x (pg, x);
svbool_t a_ge_half = svacge (pg, x, 0.5);
/* Evaluate polynomial Q(x) = y + y * z * P(z) with
z = x ^ 2 and y = |x| , if |x| < 0.5
z = (1 - |x|) / 2 and y = sqrt(z), if |x| >= 0.5. */
svfloat32_t z2 = svsel (a_ge_half, svmls_x (pg, sv_f32 (0.5), ax, 0.5),
svmul_x (pg, x, x));
svfloat32_t z = svsqrt_m (ax, a_ge_half, z2);
/* Use a single polynomial approximation P for both intervals. */
svfloat32_t p = sv_horner_4_f32_x (pg, z2, d->poly);
/* Finalize polynomial: z + z * z2 * P(z2). */
p = svmla_x (pg, z, svmul_x (pg, z, z2), p);
/* asin(|x|) = Q(|x|) , for |x| < 0.5
= pi/2 - 2 Q(|x|), for |x| >= 0.5. */
svfloat32_t y = svmad_m (a_ge_half, p, sv_f32 (-2.0), d->pi_over_2f);
/* Copy sign. */
return svreinterpret_f32 (svorr_x (pg, svreinterpret_u32 (y), sign));
}

View File

@ -49,6 +49,7 @@ typedef __SVBool_t __sv_bool_t;
# define __vpcs __attribute__ ((__aarch64_vector_pcs__))
__vpcs __f32x4_t _ZGVnN4v_asinf (__f32x4_t);
__vpcs __f32x4_t _ZGVnN4v_cosf (__f32x4_t);
__vpcs __f32x4_t _ZGVnN4v_expf (__f32x4_t);
__vpcs __f32x4_t _ZGVnN4v_exp10f (__f32x4_t);
@ -59,6 +60,7 @@ __vpcs __f32x4_t _ZGVnN4v_log2f (__f32x4_t);
__vpcs __f32x4_t _ZGVnN4v_sinf (__f32x4_t);
__vpcs __f32x4_t _ZGVnN4v_tanf (__f32x4_t);
__vpcs __f64x2_t _ZGVnN2v_asin (__f64x2_t);
__vpcs __f64x2_t _ZGVnN2v_cos (__f64x2_t);
__vpcs __f64x2_t _ZGVnN2v_exp (__f64x2_t);
__vpcs __f64x2_t _ZGVnN2v_exp10 (__f64x2_t);
@ -74,6 +76,7 @@ __vpcs __f64x2_t _ZGVnN2v_tan (__f64x2_t);
#ifdef __SVE_VEC_MATH_SUPPORTED
__sv_f32_t _ZGVsMxv_asinf (__sv_f32_t, __sv_bool_t);
__sv_f32_t _ZGVsMxv_cosf (__sv_f32_t, __sv_bool_t);
__sv_f32_t _ZGVsMxv_expf (__sv_f32_t, __sv_bool_t);
__sv_f32_t _ZGVsMxv_exp10f (__sv_f32_t, __sv_bool_t);
@ -84,6 +87,7 @@ __sv_f32_t _ZGVsMxv_log2f (__sv_f32_t, __sv_bool_t);
__sv_f32_t _ZGVsMxv_sinf (__sv_f32_t, __sv_bool_t);
__sv_f32_t _ZGVsMxv_tanf (__sv_f32_t, __sv_bool_t);
__sv_f64_t _ZGVsMxv_asin (__sv_f64_t, __sv_bool_t);
__sv_f64_t _ZGVsMxv_cos (__sv_f64_t, __sv_bool_t);
__sv_f64_t _ZGVsMxv_exp (__sv_f64_t, __sv_bool_t);
__sv_f64_t _ZGVsMxv_exp10 (__sv_f64_t, __sv_bool_t);

View File

@ -23,6 +23,7 @@
#define VEC_TYPE float64x2_t
VPCS_VECTOR_WRAPPER (asin_advsimd, _ZGVnN2v_asin)
VPCS_VECTOR_WRAPPER (cos_advsimd, _ZGVnN2v_cos)
VPCS_VECTOR_WRAPPER (exp_advsimd, _ZGVnN2v_exp)
VPCS_VECTOR_WRAPPER (exp10_advsimd, _ZGVnN2v_exp10)

View File

@ -32,6 +32,7 @@
return svlastb_f64 (svptrue_b64 (), mr); \
}
SVE_VECTOR_WRAPPER (asin_sve, _ZGVsMxv_asin)
SVE_VECTOR_WRAPPER (cos_sve, _ZGVsMxv_cos)
SVE_VECTOR_WRAPPER (exp_sve, _ZGVsMxv_exp)
SVE_VECTOR_WRAPPER (exp10_sve, _ZGVsMxv_exp10)

View File

@ -23,6 +23,7 @@
#define VEC_TYPE float32x4_t
VPCS_VECTOR_WRAPPER (asinf_advsimd, _ZGVnN4v_asinf)
VPCS_VECTOR_WRAPPER (cosf_advsimd, _ZGVnN4v_cosf)
VPCS_VECTOR_WRAPPER (expf_advsimd, _ZGVnN4v_expf)
VPCS_VECTOR_WRAPPER (exp10f_advsimd, _ZGVnN4v_exp10f)

View File

@ -32,6 +32,7 @@
return svlastb_f32 (svptrue_b32 (), mr); \
}
SVE_VECTOR_WRAPPER (asinf_sve, _ZGVsMxv_asinf)
SVE_VECTOR_WRAPPER (cosf_sve, _ZGVsMxv_cosf)
SVE_VECTOR_WRAPPER (expf_sve, _ZGVsMxv_expf)
SVE_VECTOR_WRAPPER (exp10f_sve, _ZGVsMxv_exp10f)

View File

@ -46,11 +46,19 @@ double: 1
float: 1
ldouble: 1
Function: "asin_advsimd":
double: 2
float: 2
Function: "asin_downward":
double: 1
float: 1
ldouble: 2
Function: "asin_sve":
double: 2
float: 2
Function: "asin_towardzero":
double: 1
float: 1

View File

@ -14,16 +14,20 @@ GLIBC_2.38 _ZGVsMxv_log F
GLIBC_2.38 _ZGVsMxv_logf F
GLIBC_2.38 _ZGVsMxv_sin F
GLIBC_2.38 _ZGVsMxv_sinf F
GLIBC_2.39 _ZGVnN2v_asin F
GLIBC_2.39 _ZGVnN2v_exp10 F
GLIBC_2.39 _ZGVnN2v_exp2 F
GLIBC_2.39 _ZGVnN2v_log10 F
GLIBC_2.39 _ZGVnN2v_log2 F
GLIBC_2.39 _ZGVnN2v_tan F
GLIBC_2.39 _ZGVnN4v_asinf F
GLIBC_2.39 _ZGVnN4v_exp10f F
GLIBC_2.39 _ZGVnN4v_exp2f F
GLIBC_2.39 _ZGVnN4v_log10f F
GLIBC_2.39 _ZGVnN4v_log2f F
GLIBC_2.39 _ZGVnN4v_tanf F
GLIBC_2.39 _ZGVsMxv_asin F
GLIBC_2.39 _ZGVsMxv_asinf F
GLIBC_2.39 _ZGVsMxv_exp10 F
GLIBC_2.39 _ZGVsMxv_exp10f F
GLIBC_2.39 _ZGVsMxv_exp2 F