Fix homogeneous() for 1x1 matrix: in this case, homogeneous follows the storage order guaranteeing that v.transpose().homogeneous() == v.homogeneous().transpose()

This commit is contained in:
Gael Guennebaud 2015-06-08 15:40:51 +02:00
parent cbe3a1a83e
commit 274b1f5d7e

View File

@ -410,7 +410,8 @@ template<typename Derived> class MatrixBase
ScalarMultipleReturnType operator*(const UniformScaling<Scalar>& s) const;
// put this as separate enum value to work around possible GCC 4.3 bug (?)
enum { HomogeneousReturnTypeDirection = ColsAtCompileTime==1?Vertical:Horizontal };
enum { HomogeneousReturnTypeDirection = ColsAtCompileTime==1&&RowsAtCompileTime==1 ? ((internal::traits<Derived>::Flags&RowMajorBit)==RowMajorBit ? Horizontal : Vertical)
: ColsAtCompileTime==1 ? Vertical : Horizontal };
typedef Homogeneous<Derived, HomogeneousReturnTypeDirection> HomogeneousReturnType;
HomogeneousReturnType homogeneous() const;