mirror/eigen
2
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2024-12-21 07:19:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

32-bit floats/ints, 64-bit doubles pass packetmath tests, complex 32/64-bit remaining

Browse Source
This commit is contained in:
Konstantinos Margaritis 2014-09-19 21:05:01 +00:00
parent 470aa15c35
commit 60663a510a
2 changed files with 171 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
Eigen/src/Core/arch/AltiVec/Complex.h
View File

@ -7,8 +7,8 @@
// Public License v. 2.0. If a copy of the MPL was not distributed // Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#ifndef EIGEN_COMPLEX_ALTIVEC_H #ifndef EIGEN_COMPLEX32_ALTIVEC_H
#define EIGEN_COMPLEX_ALTIVEC_H #define EIGEN_COMPLEX32_ALTIVEC_H
namespace Eigen { namespace Eigen {
@ -21,16 +21,6 @@ static Packet4ui p4ui_CONJ_XOR = vec_mergeh((Packet4ui)p4f_ZERO_, (Packet4ui)p4
static Packet2ul p2ul_CONJ_XOR = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui)p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 }; static Packet2ul p2ul_CONJ_XOR = (Packet2ul) vec_sld((Packet4ui) p2d_ZERO_, (Packet4ui)p2l_ZERO, 8);//{ 0x8000000000000000, 0x0000000000000000 };
#endif #endif
static Packet16uc p16uc_COMPLEX_RE = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
static Packet16uc p16uc_COMPLEX_IM = vec_sld(p16uc_DUPLICATE, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
static Packet16uc p16uc_COMPLEX_REV = vec_sld(p16uc_REVERSE, p16uc_REVERSE, 8);//{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
static Packet16uc p16uc_COMPLEX_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8);//{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_PSET_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_COMPLEX_RE, (Packet4ui)p16uc_COMPLEX_IM);//{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_PSET_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_COMPLEX_RE, (Packet4ui)p16uc_COMPLEX_IM);//{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
static Packet16uc p16uc_COMPLEX_MASK16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8);//{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
static Packet16uc p16uc_COMPLEX_TRANSPOSE_0 = vec_add(p16uc_PSET_HI, p16uc_COMPLEX_MASK16);//{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
static Packet16uc p16uc_COMPLEX_TRANSPOSE_1 = vec_add(p16uc_PSET_LO, p16uc_COMPLEX_MASK16);//{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
//---------- float ---------- //---------- float ----------
struct Packet2cf struct Packet2cf
{ {
@ -71,7 +61,7 @@ template<> EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<flo
res.v = pload<Packet4f>((const float *)&from); res.v = pload<Packet4f>((const float *)&from);
else else
res.v = ploadu<Packet4f>((const float *)&from); res.v = ploadu<Packet4f>((const float *)&from);
res.v = vec_perm(res.v, res.v, p16uc_PSET_HI); res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI);
return res; return res;
} }
@ -100,17 +90,24 @@ template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, con
{ {
Packet4f v1, v2; Packet4f v1, v2;
std::cout << "a: " << a.v << ", b: " << b.v << std::endl;
// Permute and multiply the real parts of a and b // Permute and multiply the real parts of a and b
v1 = vec_perm(a.v, a.v, p16uc_COMPLEX_RE); v1 = vec_perm(a.v, a.v, p16uc_PSET32_WODD);
std::cout << "v1 = a(re): " << v1 << std::endl;
// Get the imaginary parts of a // Get the imaginary parts of a
v2 = vec_perm(a.v, a.v, p16uc_COMPLEX_IM); v2 = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN);
std::cout << "v2 = a(im): " << v2 << std::endl;
// multiply a_re * b // multiply a_re * b
v1 = vec_madd(v1, b.v, p4f_ZERO); v1 = vec_madd(v1, b.v, p4f_ZERO);
std::cout << "v1+ b : " << v1 << std::endl;
// multiply a_im * b and get the conjugate result // multiply a_im * b and get the conjugate result
v2 = vec_madd(v2, b.v, p4f_ZERO); v2 = vec_madd(v2, b.v, p4f_ZERO);
std::cout << "v2+ b : " << v2 << std::endl;
v2 = (Packet4f) vec_xor((Packet4ui)v2, p4ui_CONJ_XOR); v2 = (Packet4f) vec_xor((Packet4ui)v2, p4ui_CONJ_XOR);
std::cout << "~v2 : " << v2 << std::endl;
// permute back to a proper order // permute back to a proper order
v2 = vec_perm(v2, v2, p16uc_COMPLEX_REV); v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV);
std::cout << "v2 : " << v2 << std::endl;
return Packet2cf(vec_add(v1, v2)); return Packet2cf(vec_add(v1, v2));
} }
@ -144,7 +141,7 @@ template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Pack
template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) template<> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a)
{ {
Packet4f rev_a; Packet4f rev_a;
rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX_REV2); rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2);
return Packet2cf(rev_a); return Packet2cf(rev_a);
} }
@ -185,7 +182,11 @@ struct palign_impl<Offset,Packet2cf>
{ {
if (Offset==1) if (Offset==1)
{ {
#ifdef _BIG_ENDIAN
first.v = vec_sld(first.v, second.v, 8); first.v = vec_sld(first.v, second.v, 8);
#else
first.v = vec_sld(second.v, first.v, 8);
#endif
} }
} }
}; };
@ -228,18 +229,18 @@ template<> EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, con
// TODO optimize it for AltiVec // TODO optimize it for AltiVec
Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b); Packet2cf res = conj_helper<Packet2cf,Packet2cf,false,true>().pmul(a,b);
Packet4f s = vec_madd(b.v, b.v, p4f_ZERO); Packet4f s = vec_madd(b.v, b.v, p4f_ZERO);
return Packet2cf(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_COMPLEX_REV)))); return Packet2cf(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_COMPLEX32_REV))));
} }
template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& x) template<> EIGEN_STRONG_INLINE Packet2cf pcplxflip<Packet2cf>(const Packet2cf& x)
{ {
return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX_REV)); return Packet2cf(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV));
} }
EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf,2>& kernel) EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2cf,2>& kernel)
{ {
Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_COMPLEX_TRANSPOSE_0); Packet4f tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI);
kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_COMPLEX_TRANSPOSE_1); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO);
kernel.packet[0].v = tmp; kernel.packet[0].v = tmp;
} }
@ -289,7 +290,7 @@ template<> EIGEN_STRONG_INLINE Packet1cd pset1<Packet1cd>(const std::complex<dou
res.v = pload<Packet2d>((const double *)&from); res.v = pload<Packet2d>((const double *)&from);
else else
res.v = ploadu<Packet2d>((const double *)&from); res.v = ploadu<Packet2d>((const double *)&from);
res.v = vec_perm(res.v, res.v, p16uc_PSET_HI); res.v = vec_perm(res.v, res.v, p16uc_PSET64_HI);
return res; return res;
} }
template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, DenseIndex stride) template<> EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, DenseIndex stride)
@ -317,9 +318,9 @@ template<> EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, con
Packet2d a_re, a_im, v1, v2; Packet2d a_re, a_im, v1, v2;
// Permute and multiply the real parts of a and b // Permute and multiply the real parts of a and b
a_re = vec_perm(a.v, a.v, p16uc_PSET_HI); a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI);
// Get the imaginary parts of a // Get the imaginary parts of a
a_im = vec_perm(a.v, a.v, p16uc_PSET_HI); a_im = vec_perm(a.v, a.v, p16uc_PSET64_HI);
// multiply a_re * b // multiply a_re * b
v1 = vec_madd(a_re, b.v, p2d_ZERO_); v1 = vec_madd(a_re, b.v, p2d_ZERO_);
// multiply a_im * b and get the conjugate result // multiply a_im * b and get the conjugate result
@ -353,7 +354,7 @@ template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Pac
template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) template<> EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a)
{ {
Packet2d rev_a; Packet2d rev_a;
rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX_REV2); rev_a = vec_perm(a.v, a.v, p16uc_COMPLEX32_REV2);
return Packet1cd(rev_a); return Packet1cd(rev_a);
} }
@ -432,18 +433,18 @@ template<> EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, con
// TODO optimize it for AltiVec // TODO optimize it for AltiVec
Packet1cd res = conj_helper<Packet1cd,Packet1cd,false,true>().pmul(a,b); Packet1cd res = conj_helper<Packet1cd,Packet1cd,false,true>().pmul(a,b);
Packet2d s = vec_madd(b.v, b.v, p2d_ZERO_); Packet2d s = vec_madd(b.v, b.v, p2d_ZERO_);
return Packet1cd(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_COMPLEX_REV)))); return Packet1cd(pdiv(res.v, vec_add(s,vec_perm(s, s, p16uc_COMPLEX32_REV))));
} }
template<> EIGEN_STRONG_INLINE Packet1cd pcplxflip<Packet1cd>(const Packet1cd& x) template<> EIGEN_STRONG_INLINE Packet1cd pcplxflip<Packet1cd>(const Packet1cd& x)
{ {
return Packet1cd(vec_perm(x.v, x.v, p16uc_COMPLEX_REV)); return Packet1cd(vec_perm(x.v, x.v, p16uc_COMPLEX32_REV));
} }
EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd,2>& kernel) EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd,2>& kernel)
{ {
Packet2d tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_COMPLEX_TRANSPOSE_0); Packet2d tmp = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_HI);
kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_COMPLEX_TRANSPOSE_1); kernel.packet[1].v = vec_perm(kernel.packet[0].v, kernel.packet[1].v, p16uc_TRANSPOSE64_LO);
kernel.packet[0].v = tmp; kernel.packet[0].v = tmp;
} }
@ -451,4 +452,4 @@ EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd,2>& kernel)
} // end namespace Eigen } // end namespace Eigen
#endif // EIGEN_COMPLEX_ALTIVEC_H #endif // EIGEN_COMPLEX32_ALTIVEC_H

221
Eigen/src/Core/arch/AltiVec/PacketMath.h
View File

@ -62,21 +62,6 @@ typedef __vector unsigned char Packet16uc;
#define DST_CHAN 1 #define DST_CHAN 1
#define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride)) #define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride))
// Handle endianness properly while loading constants
// Define global static constants:
#ifdef _BIG_ENDIAN
static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
static Packet16uc p16uc_REVERSE = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3};
static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0); //{ 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15}
static Packet16uc p16uc_DUPLICATE = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7};
#else
static Packet4f p4f_COUNTDOWN = { 3.0, 2.0, 1.0, 0.0 };
static Packet4i p4i_COUNTDOWN = { 3, 2, 1, 0 };
static Packet16uc p16uc_REVERSE = { 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15 };
static Packet16uc p16uc_FORWARD = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3};
static Packet16uc p16uc_DUPLICATE = { 4,5,6,7, 4,5,6,7, 0,1,2,3, 0,1,2,3 };
#endif // _BIG_ENDIAN
// These constants are endian-agnostic // These constants are endian-agnostic
static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0} static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
@ -87,6 +72,40 @@ static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1}
static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0} static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0}
static Packet4f p4f_ZERO_ = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000} static Packet4f p4f_ZERO_ = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000}
static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
// Handle endianness properly while loading constants
// Define global static constants:
#ifdef _BIG_ENDIAN
static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
#else
// Mask alignment
#define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
#define _EIGEN_ALIGNED_PTR(x) ((ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT)
static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
#endif // _BIG_ENDIAN
static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16); //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16); //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
template<> struct packet_traits<float> : default_packet_traits template<> struct packet_traits<float> : default_packet_traits
{ {
typedef Packet4f type; typedef Packet4f type;
@ -121,7 +140,19 @@ template<> struct packet_traits<int> : default_packet_traits
template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; typedef Packet4f half; }; template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; typedef Packet4f half; };
template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; typedef Packet4i half; }; template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; typedef Packet4i half; };
/*
inline std::ostream & operator <<(std::ostream & s, const Packet16uc & v)
{
union {
Packet16uc v;
unsigned char n[16];
} vt;
vt.v = v;
for (int i=0; i< 16; i++)
s << (int)vt.n[i] << ", ";
return s;
}
inline std::ostream & operator <<(std::ostream & s, const Packet4f & v) inline std::ostream & operator <<(std::ostream & s, const Packet4f & v)
{ {
union { union {
@ -154,7 +185,7 @@ inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v)
s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
return s; return s;
} }
/*
inline std::ostream & operator <<(std::ostream & s, const Packetbi & v) inline std::ostream & operator <<(std::ostream & s, const Packetbi & v)
{ {
union { union {
@ -164,8 +195,8 @@ inline std::ostream & operator <<(std::ostream & s, const Packetbi & v)
vt.v = v; vt.v = v;
s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3]; s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
return s; return s;
} }*/
*/
// Need to define them first or we get specialization after instantiation errors // Need to define them first or we get specialization after instantiation errors
template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vec_ld(0, from); } template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vec_ld(0, from); }
@ -370,12 +401,12 @@ template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
{ {
EIGEN_DEBUG_ALIGNED_LOAD EIGEN_DEBUG_ALIGNED_LOAD
return (Packet4i) vec_vsx_ld((long)from & 15, from); // align the data return (Packet4i) vec_vsx_ld((long)from & 15, (const int*) _EIGEN_ALIGNED_PTR(from));
} }
template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
{ {
EIGEN_DEBUG_ALIGNED_LOAD EIGEN_DEBUG_ALIGNED_LOAD
return (Packet4f) vec_vsx_ld((long)from & 15, from); // align the data return (Packet4f) vec_vsx_ld((long)from & 15, (const float*) _EIGEN_ALIGNED_PTR(from));
} }
#endif #endif
@ -384,16 +415,17 @@ template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
Packet4f p; Packet4f p;
if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from); if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from);
else p = ploadu<Packet4f>(from); else p = ploadu<Packet4f>(from);
return vec_perm(p, p, p16uc_DUPLICATE); return vec_perm(p, p, p16uc_DUPLICATE32_HI);
} }
template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
{ {
Packet4i p; Packet4i p;
if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from); if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from);
else p = ploadu<Packet4i>(from); else p = ploadu<Packet4i>(from);
return vec_perm(p, p, p16uc_DUPLICATE); return vec_perm(p, p, p16uc_DUPLICATE32_HI);
} }
#ifdef _BIG_ENDIAN
template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from)
{ {
EIGEN_DEBUG_UNALIGNED_STORE EIGEN_DEBUG_UNALIGNED_STORE
@ -430,6 +462,19 @@ template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& f
vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first
vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part
} }
#else
// We also need ot redefine little endian loading of Packet4i/Packet4f using VSX
template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from)
{
EIGEN_DEBUG_ALIGNED_STORE
vec_vsx_st(from, (long)to & 15, (int*) _EIGEN_ALIGNED_PTR(to));
}
template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from)
{
EIGEN_DEBUG_ALIGNED_STORE
vec_vsx_st(from, (long)to & 15, (float*) _EIGEN_ALIGNED_PTR(to));
}
#endif
template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); } template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); }
template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); } template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { vec_dstt(addr, DST_CTRL(2,2,32), DST_CHAN); }
@ -437,8 +482,8 @@ template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { vec_dst
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; } template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; }
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; } template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; vec_st(a, 0, x); return x[0]; }
template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return (Packet4f)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE); } template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { return (Packet4f)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE32); }
template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return (Packet4i)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE); } template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { return (Packet4i)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE32); }
template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vec_abs(a); }
template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); }
@ -485,7 +530,11 @@ template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
{ {
Packet4i sum; Packet4i sum;
sum = vec_sums(a, p4i_ZERO); sum = vec_sums(a, p4i_ZERO);
#ifdef _BIG_ENDIAN
sum = vec_sld(sum, p4i_ZERO, 12); sum = vec_sld(sum, p4i_ZERO, 12);
#else
sum = vec_sld(p4i_ZERO, sum, 4);
#endif
return pfirst(sum); return pfirst(sum);
} }
@ -572,8 +621,25 @@ struct palign_impl<Offset,Packet4f>
{ {
static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second) static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
{ {
if (Offset!=0) #ifdef _BIG_ENDIAN
first = vec_sld(first, second, Offset*4); switch (Offset % 4) {
case 1:
first = vec_sld(first, second, 4); break;
case 2:
first = vec_sld(first, second, 8); break;
case 3:
first = vec_sld(first, second, 12); break;
}
#else
switch (Offset % 4) {
case 1:
first = vec_sld(second, first, 12); break;
case 2:
first = vec_sld(second, first, 8); break;
case 3:
first = vec_sld(second, first, 4); break;
}
#endif
} }
}; };
@ -582,8 +648,25 @@ struct palign_impl<Offset,Packet4i>
{ {
static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second) static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
{ {
if (Offset!=0) #ifdef _BIG_ENDIAN
first = vec_sld(first, second, Offset*4); switch (Offset % 4) {
case 1:
first = vec_sld(first, second, 4); break;
case 2:
first = vec_sld(first, second, 8); break;
case 3:
first = vec_sld(first, second, 12); break;
}
#else
switch (Offset % 4) {
case 1:
first = vec_sld(second, first, 12); break;
case 2:
first = vec_sld(second, first, 8); break;
case 3:
first = vec_sld(second, first, 4); break;
}
#endif
} }
}; };
@ -625,24 +708,15 @@ static Packet2d p2l_ONE = { 1, 1 };
static Packet2d p2d_ONE = { 1.0, 1.0 }; static Packet2d p2d_ONE = { 1.0, 1.0 };
static Packet2d p2d_ZERO = { 0.0, 0.0 }; static Packet2d p2d_ZERO = { 0.0, 0.0 };
static Packet2d p2d_ZERO_ = { (double) 0x8000000000000000, (double) 0x8000000000000000 }; static Packet2d p2d_ZERO_ = { (double) 0x8000000000000000, (double) 0x8000000000000000 };
static Packet2d p2d_COUNTDOWN = { 1.0, 0.0 }; static Packet2d p2d_COUNTDOWN = { 0.0, 1.0 };
#ifdef _BIG_ENDIAN
static Packet16uc p16uc_SPLATQ1 = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
static Packet16uc p16uc_SPLATQ2 = { 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
#else
static Packet16uc p16uc_SPLATQ1 = { 4,5,6,7, 0,1,2,3, 4,5,6,7, 0,1,3,4 };
static Packet16uc p16uc_SPLATQ2 = { 12,13,14,15, 8,9,10,11, 12,13,14,15, 8,9,10,11 };
#endif
static EIGEN_STRONG_INLINE Packet2d vec_splat_dbl(Packet2d& a, int index) static EIGEN_STRONG_INLINE Packet2d vec_splat_dbl(Packet2d& a, int index)
{ {
switch (index) { switch (index) {
case 0: case 0:
return (Packet2d) vec_perm(a, a, p16uc_SPLATQ1); return (Packet2d) vec_perm(a, a, p16uc_PSET64_HI);
case 1: case 1:
return (Packet2d) vec_perm(a, a, p16uc_SPLATQ2); return (Packet2d) vec_perm(a, a, p16uc_PSET64_LO);
} }
return a; return a;
} }
@ -665,17 +739,17 @@ template<> struct packet_traits<double> : default_packet_traits
template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; typedef Packet2d half; }; template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; typedef Packet2d half; };
/*
inline std::ostream & operator <<(std::ostream & s, const Packet2d & v) inline std::ostream & operator <<(std::ostream & s, const Packet2d & v)
{ {
union { union {
Packetdf v; Packet2d v;
double n[2]; double n[2];
} vt; } vt;
vt.v = v; vt.v = v;
s << vt.n[0] << ", " << vt.n[1]; s << vt.n[0] << ", " << vt.n[1];
return s; return s;
}*/ }
// Need to define them first or we get specialization after instantiation errors // Need to define them first or we get specialization after instantiation errors
template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d) vec_ld(0, (const float *) from); } //FIXME template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d) vec_ld(0, (const float *) from); } //FIXME
@ -698,7 +772,7 @@ pbroadcast4<Packet2d>(const double *a,
a1 = vec_splat_dbl(a1, 1); a1 = vec_splat_dbl(a1, 1);
a3 = pload<Packet2d>(a+2); a3 = pload<Packet2d>(a+2);
a2 = vec_splat_dbl(a3, 0); a2 = vec_splat_dbl(a3, 0);
a3 = vec_splat_dbl(a1, 1); a3 = vec_splat_dbl(a3, 1);
} }
template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, DenseIndex stride) template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, DenseIndex stride)
{ {
@ -755,43 +829,30 @@ template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const
template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); } template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); }
template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
{ {
EIGEN_DEBUG_ALIGNED_LOAD EIGEN_DEBUG_ALIGNED_LOAD
return (Packet2d) vec_vsx_ld((int)((size_t)(from) & 15), (const float *)from); // align the data return (Packet2d) vec_vsx_ld((long)from & 15, (const float*) _EIGEN_ALIGNED_PTR(from));
} }
template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
{ {
Packet2d p; Packet2d p;
if((ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from); if((ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from);
else p = ploadu<Packet2d>(from); else p = ploadu<Packet2d>(from);
return vec_perm(p, p, p16uc_DUPLICATE); return vec_perm(p, p, p16uc_PSET64_HI);
} }
template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from)
{ {
EIGEN_DEBUG_UNALIGNED_STORE EIGEN_DEBUG_ALIGNED_STORE
// Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html vec_vsx_st((Packet4f)from, (long)to & 15, (float*) _EIGEN_ALIGNED_PTR(to));
// Warning: not thread safe!
Packet16uc MSQ, LSQ, edges;
Packet16uc edgeAlign, align;
MSQ = vec_ld(0, (unsigned char *)to); // most significant quadword
LSQ = vec_ld(15, (unsigned char *)to); // least significant quadword
edgeAlign = vec_lvsl(0, to); // permute map to extract edges
edges=vec_perm(LSQ,MSQ,edgeAlign); // extract the edges
align = vec_lvsr( 0, to ); // permute map to misalign data
MSQ = vec_perm(edges,(Packet16uc)from,align); // misalign the data (MSQ)
LSQ = vec_perm((Packet16uc)from,edges,align); // misalign the data (LSQ)
vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first
vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part
} }
template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { vec_dstt((const float *) addr, DST_CTRL(2,2,32), DST_CHAN); } template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { vec_dstt((const float *) addr, DST_CTRL(2,2,32), DST_CHAN); }
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; } template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; }
template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return (Packet2d)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE); } template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { return (Packet2d)vec_perm((Packet16uc)a,(Packet16uc)a, p16uc_REVERSE64); }
template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); }
@ -807,9 +868,10 @@ template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
{ {
Packet2d v[2], sum; Packet2d v[2], sum;
v[0] = (Packet2d) vec_sld((Packet4ui) vecs[0], (Packet4ui) vecs[1], 8); v[0] = vec_add(vecs[0], (Packet2d) vec_sld((Packet4ui) vecs[0], (Packet4ui) vecs[0], 8));
v[1] = (Packet2d) vec_sld((Packet4ui) vecs[1], (Packet4ui) vecs[0], 8); v[1] = vec_add(vecs[1], (Packet2d) vec_sld((Packet4ui) vecs[1], (Packet4ui) vecs[1], 8));
sum = vec_add(v[0], v[1]);
sum = (Packet2d) vec_sld((Packet4ui) v[1], (Packet4ui) v[0], 8);
return sum; return sum;
} }
@ -817,26 +879,19 @@ template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
// mul // mul
template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
{ {
Packet2d prod; return pfirst(pmul(a, (Packet2d)vec_sld((Packet4ui) a, (Packet4ui) a, 8)));
prod = pmul(a, (Packet2d)vec_sld((Packet4ui) a, (Packet4ui)a, 8));
return pfirst(pmul(prod, (Packet2d)vec_sld((Packet4ui) prod, (Packet4ui) prod, 4)));
} }
// min // min
template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
{ {
Packet2d b, res; return pfirst(vec_min(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8)));
b = vec_min(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8));
res = vec_min(b, (Packet2d) vec_sld((Packet4ui) b, (Packet4ui) b, 4));
return pfirst(res);
} }
// max // max
template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
{ {
Packet2d res; return pfirst(vec_max(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8)));
res = vec_max(a, (Packet2d) vec_sld((Packet4ui) a, (Packet4ui) a, 8));
return pfirst(res);
} }
template<int Offset> template<int Offset>
@ -844,16 +899,20 @@ struct palign_impl<Offset,Packet2d>
{ {
static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second) static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
{ {
if (Offset!=0) if (Offset == 1)
first = (Packet2d) vec_sld((Packet4ui) first, (Packet4ui) second, Offset*8); #ifdef _BIG_ENDIAN
first = (Packet2d) vec_sld((Packet4ui) first, (Packet4ui) second, 8);
#else
first = (Packet2d) vec_sld((Packet4ui) second, (Packet4ui) first, 8);
#endif
} }
}; };
EIGEN_DEVICE_FUNC inline void EIGEN_DEVICE_FUNC inline void
ptranspose(PacketBlock<Packet2d,2>& kernel) { ptranspose(PacketBlock<Packet2d,2>& kernel) {
Packet2d t0, t1; Packet2d t0, t1;
t0 = (Packet2d) vec_sld((Packet4ui) kernel.packet[0], (Packet4ui) kernel.packet[1], 8); t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
t1 = (Packet2d) vec_sld((Packet4ui) kernel.packet[1], (Packet4ui) kernel.packet[0], 8); t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
kernel.packet[0] = t0; kernel.packet[0] = t0;
kernel.packet[1] = t1; kernel.packet[1] = t1;
} }