mirror/eigen
2
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2025-02-23 18:20:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix incorrect NEON native fp16 multiplication.

Browse Source
This commit is contained in:
Antonio Sánchez 2022-12-19 20:46:44 +00:00 committed by Rasmus Munk Larsen
parent dd85d26946
commit bb6675caf7
3 changed files with 40 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Eigen/src/Core/arch/NEON/GeneralBlockPanelKernel.h
View File

@ -2,7 +2,7 @@
namespace Eigen {
namespace internal {
#if EIGEN_ARCH_ARM && EIGEN_COMP_CLANG
// Clang seems to excessively spill registers in the GEBP kernel on 32-bit arm.
@ -218,7 +218,9 @@ struct gebp_traits <half,half,false,false,Architecture::NEON>
EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const
{
loadRhs(b,dest);
// If LHS is a Packet8h, we cannot correctly mimic a ploadquad of the RHS
// using a single scalar value.
eigen_assert(false && "Cannot loadRhsQuad for a scalar RHS.");
}
EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/, const FixedInt<0>&) const

11
Eigen/src/Core/products/GeneralBlockPanelKernel.h
View File

@ -751,6 +751,9 @@ void loadQuadToDoublePacket(const Scalar* b, DoublePacket<RealPacket>& dest,
template<typename Packet> struct unpacket_traits<DoublePacket<Packet> > {
typedef DoublePacket<typename unpacket_traits<Packet>::half> half;
enum{
size = 2 * unpacket_traits<Packet>::size
};
};
// template<typename Packet>
// DoublePacket<Packet> pmadd(const DoublePacket<Packet> &a, const DoublePacket<Packet> &b)
@ -2490,7 +2493,13 @@ void gebp_kernel<LhsScalar,RhsScalar,Index,DataMapper,mr,nr,ConjugateLhs,Conjuga
// nr (which is currently 4) for the return type.
const int SResPacketHalfSize = unpacket_traits<typename unpacket_traits<SResPacket>::half>::size;
const int SResPacketQuarterSize = unpacket_traits<typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half>::size;
if ((SwappedTraits::LhsProgress % 4) == 0 &&
// The following code assumes we can load SRhsPacket in such a way that
// it multiplies blocks of 4 elements in SLhsPacket. This is not the
// case for some customized kernels (i.e. NEON fp16). If the assumption
// fails, drop down to the scalar path.
constexpr bool kCanLoadSRhsQuad = (unpacket_traits<SLhsPacket>::size < 4) || (unpacket_traits<SRhsPacket>::size % (unpacket_traits<SLhsPacket>::size / 4)) == 0;
if (kCanLoadSRhsQuad &&
(SwappedTraits::LhsProgress % 4) == 0 &&
(SwappedTraits::LhsProgress<=16) &&
(SwappedTraits::LhsProgress!=8 || SResPacketHalfSize==nr) &&
(SwappedTraits::LhsProgress!=16 || SResPacketQuarterSize==nr))

27
test/product_small.cpp
View File

@ -281,6 +281,25 @@ void product_small_regressions()
}
}
template<typename T>
void product_sweep(int max_m, int max_k, int max_n) {
using Matrix = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>;
for (int m = 1; m < max_m; ++m) {
for (int n = 1; n < max_n; ++n) {
Matrix C = Matrix::Zero(m, n);
Matrix Cref = Matrix::Zero(m, n);
for (int k = 1; k < max_k; ++k) {
Matrix A = Matrix::Random(m, k);
Matrix B = Matrix::Random(k, n);
C = A * B;
Cref.setZero();
ref_prod(Cref, A, B);
VERIFY_IS_APPROX(C, Cref);
}
}
}
}
EIGEN_DECLARE_TEST(product_small)
{
for(int i = 0; i < g_repeat; i++) {
@ -290,7 +309,7 @@ EIGEN_DECLARE_TEST(product_small)
CALL_SUBTEST_3( product(Matrix3d()) );
CALL_SUBTEST_4( product(Matrix4d()) );
CALL_SUBTEST_5( product(Matrix4f()) );
CALL_SUBTEST_50( product(Matrix<bfloat16, 3, 2>()) );
CALL_SUBTEST_10( product(Matrix<bfloat16, 3, 2>()) );
CALL_SUBTEST_6( product1x1<0>() );
CALL_SUBTEST_11( test_lazy_l1<float>() );
@ -317,6 +336,12 @@ EIGEN_DECLARE_TEST(product_small)
CALL_SUBTEST_6( bug_1311<5>() );
CALL_SUBTEST_9( test_dynamic_bool() );
// Commonly specialized vectorized types.
CALL_SUBTEST_50( product_sweep<float>(10, 10, 10) );
CALL_SUBTEST_51( product_sweep<double>(10, 10, 10) );
CALL_SUBTEST_52( product_sweep<Eigen::half>(10, 10, 10) );
CALL_SUBTEST_53( product_sweep<Eigen::bfloat16>(10, 10, 10) );
}
CALL_SUBTEST_6( product_small_regressions<0>() );