Merged in chtz/eigen-maxsizevector (pull request PR-490)

Let MaxSizeVector respect alignment of objects

Approved-by: Rasmus Munk Larsen <rmlarsen@google.com>

Eigen/src/Core/util/ConfigureVectorization.h

@@ -39,15 +39,19 @@
  */
 #if (defined EIGEN_CUDACC)
   #define EIGEN_ALIGN_TO_BOUNDARY(n) __align__(n)
+  #define EIGEN_ALIGNOF(x) __alignof(x)
 #elif EIGEN_COMP_GNUC || EIGEN_COMP_PGI || EIGEN_COMP_IBM || EIGEN_COMP_ARM
   #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n)))
+  #define EIGEN_ALIGNOF(x) __alignof(x)
 #elif EIGEN_COMP_MSVC
   #define EIGEN_ALIGN_TO_BOUNDARY(n) __declspec(align(n))
+  #define EIGEN_ALIGNOF(x) __alignof(x)
 #elif EIGEN_COMP_SUNCC
   // FIXME not sure about this one:
   #define EIGEN_ALIGN_TO_BOUNDARY(n) __attribute__((aligned(n)))
+  #define EIGEN_ALIGNOF(x) __alignof(x)
 #else
-  #error Please tell me what is the equivalent of __attribute__((aligned(n))) for your compiler
+  #error Please tell me what is the equivalent of __attribute__((aligned(n))) and __alignof(x) for your compiler
 #endif
 
 // If the user explicitly disable vectorization, then we also disable alignment

Eigen/src/Core/util/Memory.h

@@ -96,11 +96,12 @@ inline void throw_std_bad_alloc()
 /** \internal Like malloc, but the returned pointer is guaranteed to be 16-byte aligned.
   * Fast, but wastes 16 additional bytes of memory. Does not throw any exception.
   */
-inline void* handmade_aligned_malloc(std::size_t size)
+inline void* handmade_aligned_malloc(std::size_t size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES)
 {
-  void *original = std::malloc(size+EIGEN_DEFAULT_ALIGN_BYTES);
+  eigen_assert(alignment >= sizeof(void*) && (alignment & -alignment) == alignment && "Alignment must be at least sizeof(void*) and a power of 2");
+  void *original = std::malloc(size+alignment);
   if (original == 0) return 0;
-  void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + EIGEN_DEFAULT_ALIGN_BYTES);
+  void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(alignment-1))) + alignment);
   *(reinterpret_cast<void**>(aligned) - 1) = original;
   return aligned;
 }

unsupported/Eigen/CXX11/src/ThreadPool/EventCount.h

@@ -169,7 +169,9 @@ class EventCount {
 
   class Waiter {
     friend class EventCount;
-    std::atomic<Waiter*> next;
+    // Align to 128 byte boundary to prevent false sharing with other Waiter
+    // objects in the same vector.
+    EIGEN_ALIGN_TO_BOUNDARY(128) std::atomic<Waiter*> next;
     std::mutex mu;
     std::condition_variable cv;
     uint64_t epoch;
@@ -179,9 +181,6 @@ class EventCount {
       kWaiting,
       kSignaled,
     };
-    // Pad past 128 byte boundary to prevent false sharing with other Waiter
-    // objects in the same vector.
-    char pad_[128];
   };
 
  private:

unsupported/Eigen/CXX11/src/util/MaxSizeVector.h

@@ -29,12 +29,13 @@ namespace Eigen {
   */
 template <typename T>
 class MaxSizeVector {
+  static const size_t alignment = EIGEN_PLAIN_ENUM_MAX(EIGEN_ALIGNOF(T), sizeof(void*));
  public:
   // Construct a new MaxSizeVector, reserve n elements.
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
   explicit MaxSizeVector(size_t n)
       : reserve_(n), size_(0),
-        data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) {
+        data_(static_cast<T*>(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) {
   }
 
   // Construct a new MaxSizeVector, reserve and resize to n.
@@ -42,7 +43,7 @@ class MaxSizeVector {
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
   MaxSizeVector(size_t n, const T& init)
       : reserve_(n), size_(n),
-        data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) {
+        data_(static_cast<T*>(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) {
     size_t i = 0;
     EIGEN_TRY
     {
@@ -52,7 +53,7 @@ class MaxSizeVector {
     {
       // Construction failed, destruct in reverse order:
       for(; (i+1) > 0; --i) { data_[i-1].~T(); }
-      internal::aligned_free(data_);
+      internal::handmade_aligned_free(data_);
       EIGEN_THROW;
     }
   }
@@ -62,7 +63,7 @@ class MaxSizeVector {
     for (size_t i = size_; i > 0; --i) {
       data_[i-1].~T();
     }
-    internal::aligned_free(data_);
+    internal::handmade_aligned_free(data_);
   }
 
   void resize(size_t n) {

unsupported/test/CMakeLists.txt

@@ -142,6 +142,7 @@ ei_add_test(cxx11_tensor_sugar)
 ei_add_test(cxx11_tensor_roundings)
 ei_add_test(cxx11_tensor_layout_swap)
 ei_add_test(cxx11_tensor_io)
+ei_add_test(cxx11_maxsizevector)
 endif()
 
 if(EIGEN_TEST_CXX11)

unsupported/test/cxx11_maxsizevector.cpp

@@ -0,0 +1,77 @@
+#include "main.h"
+
+#include <exception>  // std::exception
+
+#include <unsupported/Eigen/CXX11/Tensor>
+
+struct Foo
+{
+  static Index object_count;
+  static Index object_limit;
+  EIGEN_ALIGN_TO_BOUNDARY(128) int dummy;
+
+  Foo(int x=0) : dummy(x)
+  {
+#ifdef EIGEN_EXCEPTIONS
+    // TODO: Is this the correct way to handle this?
+    if (Foo::object_count > Foo::object_limit) { std::cout << "\nThrow!\n"; throw Foo::Fail(); }
+#endif
+    std::cout << '+';
+    ++Foo::object_count;
+    eigen_assert((internal::UIntPtr(this) & (127)) == 0);
+  }
+  Foo(const Foo&)
+  {
+    std::cout << 'c';
+    ++Foo::object_count;
+    eigen_assert((internal::UIntPtr(this) & (127)) == 0);
+  }
+
+  ~Foo()
+  {
+    std::cout << '~';
+    --Foo::object_count;
+  }
+
+  class Fail : public std::exception {};
+};
+
+Index Foo::object_count = 0;
+Index Foo::object_limit = 0;
+
+
+
+EIGEN_DECLARE_TEST(cxx11_maxsizevector)
+{
+  typedef MaxSizeVector<Foo> VectorX;
+  Foo::object_count = 0;
+  for(int r = 0; r < g_repeat; r++) {
+    Index rows = internal::random<Index>(3,30);
+    Foo::object_limit = internal::random<Index>(0, rows - 2);
+    std::cout << "object_limit = " << Foo::object_limit << std::endl;
+    bool exception_raised = false;
+#ifdef EIGEN_EXCEPTIONS
+    try
+    {
+#endif
+      std::cout <<       "\nVectorX m(" << rows << ");\n";
+      VectorX vect(rows);
+      for(int i=0; i<rows; ++i)
+          vect.push_back(Foo());
+#ifdef EIGEN_EXCEPTIONS
+      VERIFY(false);  // not reached if exceptions are enabled
+    }
+    catch (const Foo::Fail&) { exception_raised = true; }
+    VERIFY(exception_raised);
+#endif
+    VERIFY_IS_EQUAL(Index(0), Foo::object_count);
+
+    {
+      Foo::object_limit = rows+1;
+      VectorX vect2(rows, Foo());
+      VERIFY_IS_EQUAL(Foo::object_count, rows);
+    }
+    VERIFY_IS_EQUAL(Index(0), Foo::object_count);
+    std::cout << '\n';
+  }
+}