mirror/eigen
2
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2025-03-01 18:26:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

add support for vectorized conjugated products

Browse Source
This commit is contained in:
Gael Guennebaud 2010-07-06 19:10:24 +02:00
parent 291fef5760
commit d6454788d9
12 changed files with 246 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
Eigen/src/Core/arch/SSE/Complex.h
View File

@ -79,9 +79,9 @@ template<> EIGEN_STRONG_INLINE Packet2cf ei_pmul<Packet2cf>(const Packet2cf& a,
{
// TODO optimize it for SSE3 and 4
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x00000000,0x80000000,0x00000000));
return Packet2cf(_mm_add_ps(_mm_mul_ps(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(a.v), 0xa0)), b.v),
_mm_xor_ps(_mm_mul_ps(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(a.v), 0xf5)),
_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(b.v), 0xb1 ))), mask)));
return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
}
// template<> EIGEN_STRONG_INLINE Packet2cf ei_pmadd<Packet2cf>(const Packet2cf& a, const Packet2cf& b, const Packet2cf& c)
@ -157,4 +157,46 @@ struct ei_palign_impl<Offset,Packet2cf>
}
};
template<> struct ei_conj_helper<Packet2cf, Packet2cf, false,true>
{
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
{ return ei_padd(pmul(x,y),c); }
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
{
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
return Packet2cf(_mm_add_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
}
};
template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,false>
{
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
{ return ei_padd(pmul(x,y),c); }
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
{
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
}
};
template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,true>
{
EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
{ return ei_padd(pmul(x,y),c); }
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
{
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
return Packet2cf(_mm_sub_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
}
};
#endif // EIGEN_COMPLEX_SSE_H

1
Eigen/src/Core/arch/SSE/PacketMath.h
View File

@ -58,6 +58,7 @@ template<> struct ei_is_arithmetic<__m128d> { enum { ret = true }; };
#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
const Packet4i ei_p4i_##NAME = ei_pset1<int>(X)
template<> struct ei_packet_traits<float> : ei_default_packet_traits
{
typedef Packet4f type; enum {size=4};

215
Eigen/src/Core/products/GeneralBlockPanelKernel.h
View File

@ -134,13 +134,13 @@ inline void computeProductBlockingSizes(std::ptrdiff_t& k, std::ptrdiff_t& m, st
}
#ifdef EIGEN_HAS_FUSE_CJMADD
#define CJMADD(A,B,C,T) C = cj.pmadd(A,B,C);
#define MADD(CJ,A,B,C,T) C = CJ.pmadd(A,B,C);
#else
#define CJMADD(A,B,C,T) T = B; T = cj.pmul(A,T); C = ei_padd(C,T);
#define MADD(CJ,A,B,C,T) T = B; T = CJ.pmul(A,T); C = ei_padd(C,T);
#endif
// optimized GEneral packed Block * packed Panel product kernel
template<typename Scalar, typename Index, int mr, int nr, typename Conj>
template<typename Scalar, typename Index, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
struct ei_gebp_kernel
{
void operator()(Scalar* res, Index resStride, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols,
@ -150,7 +150,8 @@ struct ei_gebp_kernel
enum { PacketSize = ei_packet_traits<Scalar>::size };
if(strideA==-1) strideA = depth;
if(strideB==-1) strideB = depth;
Conj cj;
ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
ei_conj_helper<PacketType,PacketType,ConjugateLhs,ConjugateRhs> pcj;
Index packet_cols = (cols/nr) * nr;
const Index peeled_mc = (rows/mr)*mr;
const Index peeled_mc2 = peeled_mc + (rows-peeled_mc >= PacketSize ? PacketSize : 0);
@ -263,38 +264,38 @@ EIGEN_ASM_COMMENT("mybegin");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C1,T0);
CJMADD(A1,B0,C5,B0);
MADD(pcj,A0,B0,C1,T0);
MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[2*PacketSize]);
A1 = ei_pload(&blA[3*PacketSize]);
B0 = ei_pload(&blB[2*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[3*PacketSize]);
CJMADD(A0,B0,C1,T0);
CJMADD(A1,B0,C5,B0);
MADD(pcj,A0,B0,C1,T0);
MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[4*PacketSize]);
A1 = ei_pload(&blA[5*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[5*PacketSize]);
CJMADD(A0,B0,C1,T0);
CJMADD(A1,B0,C5,B0);
MADD(pcj,A0,B0,C1,T0);
MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[6*PacketSize]);
A1 = ei_pload(&blA[7*PacketSize]);
B0 = ei_pload(&blB[6*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[7*PacketSize]);
CJMADD(A0,B0,C1,T0);
CJMADD(A1,B0,C5,B0);
MADD(pcj,A0,B0,C1,T0);
MADD(pcj,A1,B0,C5,B0);
EIGEN_ASM_COMMENT("myend");
}
else
@ -309,59 +310,59 @@ EIGEN_ASM_COMMENT("mybegin");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C0,T0);
MADD(pcj,A0,B0,C0,T0);
B2 = ei_pload(&blB[2*PacketSize]);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A1,B0,C4,B0);
B3 = ei_pload(&blB[3*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
CJMADD(A0,B1,C1,T0);
CJMADD(A1,B1,C5,B1);
MADD(pcj,A0,B1,C1,T0);
MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[5*PacketSize]);
CJMADD(A0,B2,C2,T0);
CJMADD(A1,B2,C6,B2);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[6*PacketSize]);
CJMADD(A0,B3,C3,T0);
MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[2*PacketSize]);
CJMADD(A1,B3,C7,B3);
MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[3*PacketSize]);
B3 = ei_pload(&blB[7*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[8*PacketSize]);
CJMADD(A0,B1,C1,T0);
CJMADD(A1,B1,C5,B1);
MADD(pcj,A0,B1,C1,T0);
MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[9*PacketSize]);
CJMADD(A0,B2,C2,T0);
CJMADD(A1,B2,C6,B2);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[10*PacketSize]);
CJMADD(A0,B3,C3,T0);
MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[4*PacketSize]);
CJMADD(A1,B3,C7,B3);
MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[5*PacketSize]);
B3 = ei_pload(&blB[11*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[12*PacketSize]);
CJMADD(A0,B1,C1,T0);
CJMADD(A1,B1,C5,B1);
MADD(pcj,A0,B1,C1,T0);
MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[13*PacketSize]);
CJMADD(A0,B2,C2,T0);
CJMADD(A1,B2,C6,B2);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[14*PacketSize]);
CJMADD(A0,B3,C3,T0);
MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[6*PacketSize]);
CJMADD(A1,B3,C7,B3);
MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[7*PacketSize]);
B3 = ei_pload(&blB[15*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
CJMADD(A0,B1,C1,T0);
CJMADD(A1,B1,C5,B1);
CJMADD(A0,B2,C2,T0);
CJMADD(A1,B2,C6,B2);
CJMADD(A0,B3,C3,T0);
CJMADD(A1,B3,C7,B3);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
MADD(pcj,A0,B1,C1,T0);
MADD(pcj,A1,B1,C5,B1);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A1,B2,C6,B2);
MADD(pcj,A0,B3,C3,T0);
MADD(pcj,A1,B3,C7,B3);
EIGEN_ASM_COMMENT("myend");
}
@ -381,11 +382,11 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C1,T0);
CJMADD(A1,B0,C5,B0);
MADD(pcj,A0,B0,C1,T0);
MADD(pcj,A1,B0,C5,B0);
}
else
{
@ -399,16 +400,16 @@ EIGEN_ASM_COMMENT("myend");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C0,T0);
MADD(pcj,A0,B0,C0,T0);
B2 = ei_pload(&blB[2*PacketSize]);
CJMADD(A1,B0,C4,B0);
MADD(pcj,A1,B0,C4,B0);
B3 = ei_pload(&blB[3*PacketSize]);
CJMADD(A0,B1,C1,T0);
CJMADD(A1,B1,C5,B1);
CJMADD(A0,B2,C2,T0);
CJMADD(A1,B2,C6,B2);
CJMADD(A0,B3,C3,T0);
CJMADD(A1,B3,C7,B3);
MADD(pcj,A0,B1,C1,T0);
MADD(pcj,A1,B1,C5,B1);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A1,B2,C6,B2);
MADD(pcj,A0,B3,C3,T0);
MADD(pcj,A1,B3,C7,B3);
}
blB += nr*PacketSize;
@ -468,23 +469,23 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[2*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[1*PacketSize]);
B1 = ei_pload(&blB[3*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[4*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[2*PacketSize]);
B1 = ei_pload(&blB[5*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[6*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[3*PacketSize]);
B1 = ei_pload(&blB[7*PacketSize]);
CJMADD(A0,B0,C0,B0);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B0,C0,B0);
MADD(pcj,A0,B1,C1,B1);
}
else
{
@ -494,41 +495,41 @@ EIGEN_ASM_COMMENT("myend");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B2 = ei_pload(&blB[2*PacketSize]);
B3 = ei_pload(&blB[3*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[5*PacketSize]);
CJMADD(A0,B2,C2,B2);
MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[6*PacketSize]);
CJMADD(A0,B3,C3,B3);
MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[1*PacketSize]);
B3 = ei_pload(&blB[7*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[8*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[9*PacketSize]);
CJMADD(A0,B2,C2,B2);
MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[10*PacketSize]);
CJMADD(A0,B3,C3,B3);
MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[2*PacketSize]);
B3 = ei_pload(&blB[11*PacketSize]);
CJMADD(A0,B0,C0,B0);
MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[12*PacketSize]);
CJMADD(A0,B1,C1,B1);
MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[13*PacketSize]);
CJMADD(A0,B2,C2,B2);
MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[14*PacketSize]);
CJMADD(A0,B3,C3,B3);
MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[3*PacketSize]);
B3 = ei_pload(&blB[15*PacketSize]);
CJMADD(A0,B0,C0,B0);
CJMADD(A0,B1,C1,B1);
CJMADD(A0,B2,C2,B2);
CJMADD(A0,B3,C3,B3);
MADD(pcj,A0,B0,C0,B0);
MADD(pcj,A0,B1,C1,B1);
MADD(pcj,A0,B2,C2,B2);
MADD(pcj,A0,B3,C3,B3);
}
blB += 4*nr*PacketSize;
@ -546,9 +547,9 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
CJMADD(A0,B0,C0,T0);
MADD(pcj,A0,B0,C0,T0);
B0 = ei_pload(&blB[1*PacketSize]);
CJMADD(A0,B0,C1,T0);
MADD(pcj,A0,B0,C1,T0);
}
else
{
@ -563,10 +564,10 @@ EIGEN_ASM_COMMENT("myend");
B2 = ei_pload(&blB[2*PacketSize]);
B3 = ei_pload(&blB[3*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A0,B1,C1,T1);
CJMADD(A0,B2,C2,T0);
CJMADD(A0,B3,C3,T1);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A0,B1,C1,T1);
MADD(pcj,A0,B2,C2,T0);
MADD(pcj,A0,B3,C3,T1);
}
blB += nr*PacketSize;
@ -598,9 +599,9 @@ EIGEN_ASM_COMMENT("myend");
A0 = blA[k];
B0 = blB[0*PacketSize];
CJMADD(A0,B0,C0,T0);
MADD(cj,A0,B0,C0,T0);
B0 = blB[1*PacketSize];
CJMADD(A0,B0,C1,T0);
MADD(cj,A0,B0,C1,T0);
}
else
{
@ -615,10 +616,10 @@ EIGEN_ASM_COMMENT("myend");
B2 = blB[2*PacketSize];
B3 = blB[3*PacketSize];
CJMADD(A0,B0,C0,T0);
CJMADD(A0,B1,C1,T1);
CJMADD(A0,B2,C2,T0);
CJMADD(A0,B3,C3,T1);
MADD(cj,A0,B0,C0,T0);
MADD(cj,A0,B1,C1,T1);
MADD(cj,A0,B2,C2,T0);
MADD(cj,A0,B3,C3,T1);
}
blB += nr*PacketSize;
@ -664,8 +665,8 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
CJMADD(A0,B0,C0,T0);
CJMADD(A1,B0,C4,T1);
MADD(pcj,A0,B0,C0,T0);
MADD(pcj,A1,B0,C4,T1);
blB += PacketSize;
blA += mr;
@ -686,8 +687,7 @@ EIGEN_ASM_COMMENT("myend");
for(Index k=0; k<depth; k++)
{
PacketType T0;
CJMADD(ei_pload(blA), ei_pload(blB), C0, T0);
//C0 = cj.pmadd(ei_pload(blA), ei_pload(blB), C0);
MADD(pcj,ei_pload(blA), ei_pload(blB), C0, T0);
blB += PacketSize;
blA += PacketSize;
}
@ -704,7 +704,12 @@ EIGEN_ASM_COMMENT("myend");
// FIXME directly use blockB ??
const Scalar* blB = unpackedB;
for(Index k=0; k<depth; k++)
C0 = cj.pmadd(blA[k], blB[k*PacketSize], C0);
{
#ifndef EIGEN_HAS_FUSE_CJMADD
Scalar T0;
#endif
MADD(cj,blA[k], blB[k*PacketSize], C0, T0);
}
res[(j2+0)*resStride + i] += C0;
}
}

70
Eigen/src/Core/products/GeneralMatrixVector.h
View File

@ -48,19 +48,20 @@ void ei_cache_friendly_product_colmajor_times_vector(
ei_pstore(&res[j], \
ei_padd(ei_pload(&res[j]), \
ei_padd( \
ei_padd(cj.pmul(EIGEN_CAT(ei_ploa , A0)(&lhs0[j]), ptmp0), \
cj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs1[j]), ptmp1)), \
ei_padd(cj.pmul(EIGEN_CAT(ei_ploa , A2)(&lhs2[j]), ptmp2), \
cj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs3[j]), ptmp3)) )))
ei_conj_helper<ConjugateLhs,ConjugateRhs> cj;
if(ConjugateRhs)
alpha = ei_conj(alpha);
ei_padd(pcj.pmul(EIGEN_CAT(ei_ploa , A0)(&lhs0[j]), ptmp0), \
pcj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs1[j]), ptmp1)), \
ei_padd(pcj.pmul(EIGEN_CAT(ei_ploa , A2)(&lhs2[j]), ptmp2), \
pcj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs3[j]), ptmp3)) )))
typedef typename NumTraits<Scalar>::Real RealScalar;
typedef typename ei_packet_traits<Scalar>::type Packet;
const Index PacketSize = sizeof(Packet)/sizeof(Scalar);
ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
ei_conj_helper<Packet,Packet,ConjugateLhs,ConjugateRhs> pcj;
if(ConjugateRhs)
alpha = ei_conj(alpha);
enum { AllAligned = 0, EvenAligned, FirstAligned, NoneAligned };
const Index columnsAtOnce = 4;
const Index peels = 2;
@ -168,19 +169,19 @@ void ei_cache_friendly_product_colmajor_times_vector(
A00 = ei_pload (&lhs0[j]);
A10 = ei_pload (&lhs0[j+PacketSize]);
A00 = cj.pmadd(A00, ptmp0, ei_pload(&res[j]));
A10 = cj.pmadd(A10, ptmp0, ei_pload(&res[j+PacketSize]));
A00 = pcj.pmadd(A00, ptmp0, ei_pload(&res[j]));
A10 = pcj.pmadd(A10, ptmp0, ei_pload(&res[j+PacketSize]));
A00 = cj.pmadd(A01, ptmp1, A00);
A00 = pcj.pmadd(A01, ptmp1, A00);
A01 = ei_pload(&lhs1[j-1+2*PacketSize]); ei_palign<1>(A11,A01);
A00 = cj.pmadd(A02, ptmp2, A00);
A00 = pcj.pmadd(A02, ptmp2, A00);
A02 = ei_pload(&lhs2[j-2+2*PacketSize]); ei_palign<2>(A12,A02);
A00 = cj.pmadd(A03, ptmp3, A00);
A00 = pcj.pmadd(A03, ptmp3, A00);
ei_pstore(&res[j],A00);
A03 = ei_pload(&lhs3[j-3+2*PacketSize]); ei_palign<3>(A13,A03);
A10 = cj.pmadd(A11, ptmp1, A10);
A10 = cj.pmadd(A12, ptmp2, A10);
A10 = cj.pmadd(A13, ptmp3, A10);
A10 = pcj.pmadd(A11, ptmp1, A10);
A10 = pcj.pmadd(A12, ptmp2, A10);
A10 = pcj.pmadd(A13, ptmp3, A10);
ei_pstore(&res[j+PacketSize],A10);
}
}
@ -225,10 +226,10 @@ void ei_cache_friendly_product_colmajor_times_vector(
// process aligned result's coeffs
if ((size_t(lhs0+alignedStart)%sizeof(Packet))==0)
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
ei_pstore(&res[j], cj.pmadd(ei_pload(&lhs0[j]), ptmp0, ei_pload(&res[j])));
ei_pstore(&res[j], pcj.pmadd(ei_pload(&lhs0[j]), ptmp0, ei_pload(&res[j])));
else
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
ei_pstore(&res[j], cj.pmadd(ei_ploadu(&lhs0[j]), ptmp0, ei_pload(&res[j])));
ei_pstore(&res[j], pcj.pmadd(ei_ploadu(&lhs0[j]), ptmp0, ei_pload(&res[j])));
}
// process remaining scalars (or all if no explicit vectorization)
@ -261,17 +262,18 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
#define _EIGEN_ACCUMULATE_PACKETS(A0,A13,A2) {\
Packet b = ei_pload(&rhs[j]); \
ptmp0 = cj.pmadd(EIGEN_CAT(ei_ploa,A0) (&lhs0[j]), b, ptmp0); \
ptmp1 = cj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs1[j]), b, ptmp1); \
ptmp2 = cj.pmadd(EIGEN_CAT(ei_ploa,A2) (&lhs2[j]), b, ptmp2); \
ptmp3 = cj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs3[j]), b, ptmp3); }
ei_conj_helper<ConjugateLhs,ConjugateRhs> cj;
ptmp0 = pcj.pmadd(EIGEN_CAT(ei_ploa,A0) (&lhs0[j]), b, ptmp0); \
ptmp1 = pcj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs1[j]), b, ptmp1); \
ptmp2 = pcj.pmadd(EIGEN_CAT(ei_ploa,A2) (&lhs2[j]), b, ptmp2); \
ptmp3 = pcj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs3[j]), b, ptmp3); }
typedef typename NumTraits<Scalar>::Real RealScalar;
typedef typename ei_packet_traits<Scalar>::type Packet;
const Index PacketSize = sizeof(Packet)/sizeof(Scalar);
ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
ei_conj_helper<Packet,Packet,ConjugateLhs,ConjugateRhs> pcj;
enum { AllAligned=0, EvenAligned=1, FirstAligned=2, NoneAligned=3 };
const Index rowsAtOnce = 4;
const Index peels = 2;
@ -386,19 +388,19 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
A12 = ei_pload(&lhs2[j-2+PacketSize]); ei_palign<2>(A02,A12);
A13 = ei_pload(&lhs3[j-3+PacketSize]); ei_palign<3>(A03,A13);
ptmp0 = cj.pmadd(ei_pload (&lhs0[j]), b, ptmp0);
ptmp1 = cj.pmadd(A01, b, ptmp1);
ptmp0 = pcj.pmadd(ei_pload (&lhs0[j]), b, ptmp0);
ptmp1 = pcj.pmadd(A01, b, ptmp1);
A01 = ei_pload(&lhs1[j-1+2*PacketSize]); ei_palign<1>(A11,A01);
ptmp2 = cj.pmadd(A02, b, ptmp2);
ptmp2 = pcj.pmadd(A02, b, ptmp2);
A02 = ei_pload(&lhs2[j-2+2*PacketSize]); ei_palign<2>(A12,A02);
ptmp3 = cj.pmadd(A03, b, ptmp3);
ptmp3 = pcj.pmadd(A03, b, ptmp3);
A03 = ei_pload(&lhs3[j-3+2*PacketSize]); ei_palign<3>(A13,A03);
b = ei_pload(&rhs[j+PacketSize]);
ptmp0 = cj.pmadd(ei_pload (&lhs0[j+PacketSize]), b, ptmp0);
ptmp1 = cj.pmadd(A11, b, ptmp1);
ptmp2 = cj.pmadd(A12, b, ptmp2);
ptmp3 = cj.pmadd(A13, b, ptmp3);
ptmp0 = pcj.pmadd(ei_pload (&lhs0[j+PacketSize]), b, ptmp0);
ptmp1 = pcj.pmadd(A11, b, ptmp1);
ptmp2 = pcj.pmadd(A12, b, ptmp2);
ptmp3 = pcj.pmadd(A13, b, ptmp3);
}
}
for (Index j = peeledSize; j<alignedSize; j+=PacketSize)
@ -447,10 +449,10 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
// process aligned rhs coeffs
if ((size_t(lhs0+alignedStart)%sizeof(Packet))==0)
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
ptmp0 = cj.pmadd(ei_pload(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
ptmp0 = pcj.pmadd(ei_pload(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
else
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
ptmp0 = cj.pmadd(ei_ploadu(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
ptmp0 = pcj.pmadd(ei_ploadu(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
tmp0 += ei_predux(ptmp0);
}

4
Eigen/src/Core/products/SelfadjointMatrixMatrix.h
View File

@ -270,7 +270,7 @@ struct ei_product_selfadjoint_matrix<Scalar,Index,LhsStorageOrder,true,Conjugate
Scalar* allocatedBlockB = ei_aligned_stack_new(Scalar, sizeB);
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_symm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder==RowMajor?ColMajor:RowMajor, true> pack_lhs_transposed;
@ -353,7 +353,7 @@ struct ei_product_selfadjoint_matrix<Scalar,Index,LhsStorageOrder,false,Conjugat
Scalar* allocatedBlockB = ei_aligned_stack_new(Scalar, sizeB);
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_symm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;

13
Eigen/src/Core/products/SelfadjointMatrixVector.h
View File

@ -46,8 +46,11 @@ static EIGEN_DONT_INLINE void ei_product_selfadjoint_vector(
FirstTriangular = IsRowMajor == IsLower
};
ei_conj_helper<NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0;
ei_conj_helper<NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
ei_conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0;
ei_conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> pcj0;
ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1;
Scalar cjAlpha = ConjugateRhs ? ei_conj(alpha) : alpha;
@ -121,9 +124,9 @@ static EIGEN_DONT_INLINE void ei_product_selfadjoint_vector(
Packet Bi = ei_ploadu(rhsIt); rhsIt += PacketSize; // FIXME should be aligned in most cases
Packet Xi = ei_pload (resIt);
Xi = cj0.pmadd(A0i,ptmp0, cj0.pmadd(A1i,ptmp1,Xi));
ptmp2 = cj1.pmadd(A0i, Bi, ptmp2);
ptmp3 = cj1.pmadd(A1i, Bi, ptmp3);
Xi = pcj0.pmadd(A0i,ptmp0, pcj0.pmadd(A1i,ptmp1,Xi));
ptmp2 = pcj1.pmadd(A0i, Bi, ptmp2);
ptmp3 = pcj1.pmadd(A1i, Bi, ptmp3);
ei_pstore(resIt,Xi); resIt += PacketSize;
}
for (size_t i=alignedEnd; i<endi; i++)

15
Eigen/src/Core/products/SelfadjointProduct.h
View File

@ -32,7 +32,7 @@
**********************************************************************/
// forward declarations (defined at the end of this file)
template<typename Scalar, typename Index, int mr, int nr, typename Conj, int UpLo>
template<typename Scalar, typename Index, int mr, int nr, bool ConjLhs, bool ConjRhs, int UpLo>
struct ei_sybb_kernel;
/* Optimized selfadjoint product (_SYRK) */
@ -84,12 +84,15 @@ struct ei_selfadjoint_product<Scalar, Index, MatStorageOrder, ColMajor, AAT, UpL
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
// note that the actual rhs is the transpose/adjoint of mat
typedef ei_conj_helper<NumTraits<Scalar>::IsComplex && !AAT, NumTraits<Scalar>::IsComplex && AAT> Conj;
enum {
ConjLhs = NumTraits<Scalar>::IsComplex && !AAT,
ConjRhs = NumTraits<Scalar>::IsComplex && AAT
};
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conj> gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjLhs, ConjRhs> gebp_kernel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,MatStorageOrder==RowMajor ? ColMajor : RowMajor> pack_rhs;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,MatStorageOrder, false> pack_lhs;
ei_sybb_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conj, UpLo> sybb;
ei_sybb_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjLhs, ConjRhs, UpLo> sybb;
for(Index k2=0; k2<depth; k2+=kc)
{
@ -163,7 +166,7 @@ SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo>
// while the selfadjoint block overlapping the diagonal is evaluated into a
// small temporary buffer which is then accumulated into the result using a
// triangular traversal.
template<typename Scalar, typename Index, int mr, int nr, typename Conj, int UpLo>
template<typename Scalar, typename Index, int mr, int nr, bool ConjLhs, bool ConjRhs, int UpLo>
struct ei_sybb_kernel
{
enum {
@ -172,7 +175,7 @@ struct ei_sybb_kernel
};
void operator()(Scalar* res, Index resStride, const Scalar* blockA, const Scalar* blockB, Index size, Index depth, Scalar* workspace)
{
ei_gebp_kernel<Scalar, Index, mr, nr, Conj> gebp_kernel;
ei_gebp_kernel<Scalar, Index, mr, nr, ConjLhs, ConjRhs> gebp_kernel;
Matrix<Scalar,BlockSize,BlockSize,ColMajor> buffer;
// let's process the block per panel of actual_mc x BlockSize,

6
Eigen/src/Core/products/TriangularMatrixMatrix.h
View File

@ -129,7 +129,7 @@ struct ei_product_triangular_matrix_matrix<Scalar,Index,Mode,true,
triangularBuffer.setZero();
triangularBuffer.diagonal().setOnes();
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
@ -254,10 +254,10 @@ struct ei_product_triangular_matrix_matrix<Scalar,Index,Mode,false,
triangularBuffer.setZero();
triangularBuffer.diagonal().setOnes();
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,true> pack_rhs_panel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,false,true> pack_rhs_panel;
for(Index k2=IsLower ? 0 : depth;
IsLower ? k2<depth : k2>0;

8
Eigen/src/Core/products/TriangularSolverMatrix.h
View File

@ -74,9 +74,9 @@ struct ei_triangular_solve_matrix<Scalar,Index,OnTheLeft,Mode,Conjugate,TriStora
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_conj_if<Conjugate> conj;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<Conjugate,false> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conjugate, false> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,TriStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr, ColMajor, true> pack_rhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr, ColMajor, false, true> pack_rhs;
for(Index k2=IsLower ? 0 : size;
IsLower ? k2<size : k2>0;
@ -212,9 +212,9 @@ struct ei_triangular_solve_matrix<Scalar,Index,OnTheRight,Mode,Conjugate,TriStor
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_conj_if<Conjugate> conj;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<false,Conjugate> > gebp_kernel;
ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, false, Conjugate> gebp_kernel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,true> pack_rhs_panel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,false,true> pack_rhs_panel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr, ColMajor, false, true> pack_lhs_panel;
for(Index k2=IsLower ? size : 0;

45
Eigen/src/Core/util/BlasUtil.h
View File

@ -29,12 +29,7 @@
// implement and control fast level 2 and level 3 BLAS-like routines.
// forward declarations
// Provides scalar/packet-wise product and product with accumulation
// with optional conjugation of the arguments.
template<bool ConjLhs, bool ConjRhs> struct ei_conj_helper;
template<typename Scalar, typename Index, int mr, int nr, typename Conj = ei_conj_helper<false,false> >
template<typename Scalar, typename Index, int mr, int nr, bool ConjugateLhs=false, bool ConjugateRhs=false>
struct ei_gebp_kernel;
template<typename Scalar, typename Index, int nr, int StorageOrder, bool Conjugate = false, bool PanelMode=false>
@ -58,42 +53,40 @@ template<bool ConjugateLhs, bool ConjugateRhs, typename Scalar, typename Index,
static void ei_cache_friendly_product_rowmajor_times_vector(
const Scalar* lhs, Index lhsStride, const Scalar* rhs, Index rhsSize, ResType& res, Scalar alpha);
template<> struct ei_conj_helper<false,false>
template<typename Scalar> struct ei_conj_helper<Scalar,Scalar,false,false>
{
template<typename T>
EIGEN_STRONG_INLINE T pmadd(const T& x, const T& y, const T& c) const { return ei_pmadd(x,y,c); }
template<typename T>
EIGEN_STRONG_INLINE T pmul(const T& x, const T& y) const { return ei_pmul(x,y); }
EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const { return ei_pmadd(x,y,c); }
EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const { return ei_pmul(x,y); }
};
template<> struct ei_conj_helper<false,true>
template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, false,true>
{
template<typename T> std::complex<T>
pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
typedef std::complex<RealScalar> Scalar;
EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
{ return std::complex<T>(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_imag(x)*ei_real(y) - ei_real(x)*ei_imag(y)); }
EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
{ return Scalar(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_imag(x)*ei_real(y) - ei_real(x)*ei_imag(y)); }
};
template<> struct ei_conj_helper<true,false>
template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, true,false>
{
template<typename T> std::complex<T>
pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
typedef std::complex<RealScalar> Scalar;
EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
{ return std::complex<T>(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
{ return Scalar(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
};
template<> struct ei_conj_helper<true,true>
template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, true,true>
{
template<typename T> std::complex<T>
pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
typedef std::complex<RealScalar> Scalar;
EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
{ return std::complex<T>(ei_real(x)*ei_real(y) - ei_imag(x)*ei_imag(y), - ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
{ return Scalar(ei_real(x)*ei_real(y) - ei_imag(x)*ei_imag(y), - ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
};
// Lightweight helper class to access matrix coefficients.

4
Eigen/src/Core/util/ForwardDeclarations.h
View File

@ -106,6 +106,10 @@ template<typename Lhs, typename Rhs,
int ProductType = ei_product_type<Lhs,Rhs>::value>
struct ProductReturnType;
// Provides scalar/packet-wise product and product with accumulation
// with optional conjugation of the arguments.
template<typename LhsScalar, typename RhsScalar, bool ConjLhs, bool ConjRhs> struct ei_conj_helper;
template<typename Scalar> struct ei_scalar_sum_op;
template<typename Scalar> struct ei_scalar_difference_op;
template<typename Scalar> struct ei_scalar_product_op;

10
Eigen/src/Jacobi/Jacobi.h
View File

@ -324,7 +324,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
const Packet pc = ei_pset1(Scalar(j.c()));
const Packet ps = ei_pset1(Scalar(j.s()));
ei_conj_helper<NumTraits<Scalar>::IsComplex,false> cj;
ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex,false> pcj;
for(Index i=0; i<alignedStart; ++i)
{
@ -343,7 +343,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
{
Packet xi = ei_pload(px);
Packet yi = ei_pload(py);
ei_pstore(px, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
ei_pstore(px, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
ei_pstore(py, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
px += PacketSize;
py += PacketSize;
@ -358,8 +358,8 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
Packet xi1 = ei_ploadu(px+PacketSize);
Packet yi = ei_pload (py);
Packet yi1 = ei_pload (py+PacketSize);
ei_pstoreu(px, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
ei_pstoreu(px+PacketSize, ei_padd(ei_pmul(pc,xi1),cj.pmul(ps,yi1)));
ei_pstoreu(px, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
ei_pstoreu(px+PacketSize, ei_padd(ei_pmul(pc,xi1),pcj.pmul(ps,yi1)));
ei_pstore (py, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
ei_pstore (py+PacketSize, ei_psub(ei_pmul(pc,yi1),ei_pmul(ps,xi1)));
px += Peeling*PacketSize;
@ -369,7 +369,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
{
Packet xi = ei_ploadu(x+peelingEnd);
Packet yi = ei_pload (y+peelingEnd);
ei_pstoreu(x+peelingEnd, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
ei_pstoreu(x+peelingEnd, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
ei_pstore (y+peelingEnd, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
}
}