Initial version of a small script to help tracking performance regressions

bench/perf_monitoring/gemm/changesets.txt

@@ -0,0 +1,25 @@
+3.0.1
+3.1.1
+3.2.0
+3.2.4
+5745:37f59e65eb6c
+5891:d8652709345d
+5893:24b4dc92c6d3
+5895:997c2ef9fc8b
+5904:e1eafd14eaa1
+5908:f8ee3c721251
+5921:ca808bb456b0
+5927:8b1001f9e3ac
+5937:5a4ca1ad8c53
+5949:f3488f4e45b2
+5969:e09031dccfd9
+5992:4a429f5e0483
+before-evaluators
+6334:f6a45e5b8b7c
+6639:c9121c60b5c7
+6655:06f163b5221f
+6677:700e023044e7
+6681:11d31dafb0e3
+6844:039efd86b75c
+6845:7333ed40c6ef
+6911:6192dd812d84

bench/perf_monitoring/gemm/gemm.cpp

@@ -0,0 +1,67 @@
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <Eigen/Core>
+#include "../../BenchTimer.h"
+using namespace Eigen;
+
+#ifndef SCALAR
+#error SCALAR must be defined
+#endif
+
+typedef SCALAR Scalar;
+
+typedef Matrix<Scalar,Dynamic,Dynamic> Mat;
+
+EIGEN_DONT_INLINE
+void gemm(const Mat &A, const Mat &B, Mat &C)
+{
+  C.noalias() += A * B;
+}
+
+EIGEN_DONT_INLINE
+double bench(long m, long n, long k)
+{
+  Mat A(m,k);
+  Mat B(k,n);
+  Mat C(m,n);
+  A.setRandom();
+  B.setRandom();
+  C.setZero();
+  
+  BenchTimer t;
+  
+  double up = 1e8*4/sizeof(Scalar);
+  double tm0 = 4, tm1 = 10;
+  if(NumTraits<Scalar>::IsComplex)
+  {
+    up /= 4;
+    tm0 = 2;
+    tm1 = 4;
+  }
+  
+  double flops = 2. * m * n * k;
+  long rep = std::max(1., std::min(100., up/flops) );
+  long tries = std::max(tm0, std::min(tm1, up/flops) );
+  
+  BENCH(t, tries, rep, gemm(A,B,C));
+  
+  return 1e-9 * rep * flops / t.best();
+}
+
+int main(int argc, char **argv)
+{
+  std::vector<double> results;
+  
+  std::ifstream settings("settings.txt");
+  long m, n, k;
+  while(settings >> m >> n >> k)
+  {
+    //std::cerr << "  Testing " << m << " " << n << " " << k << std::endl;
+    results.push_back( bench(m, n, k) );
+  }
+  
+  std::cout << RowVectorXd::Map(results.data(), results.size());
+  
+  return 0;
+}

bench/perf_monitoring/gemm/make_plot.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# base name of the bench
+# it reads $1.out
+# and generates $1.pdf
+WHAT=$1
+
+header="rev "
+while read line
+do
+  if [ ! -z '$line' ]; then
+    header="$header  \"$line\""
+  fi
+done < settings.txt
+
+echo $header > $WHAT.out.header
+cat $WHAT.out >> $WHAT.out.header
+
+
+echo "set title '$WHAT'" > $WHAT.gnuplot
+echo "set key autotitle columnhead outside " >> $WHAT.gnuplot
+echo "set xtics rotate 1" >> $WHAT.gnuplot
+
+echo "set term pdf color rounded enhanced fontscale 0.35 size 7in,5in" >> $WHAT.gnuplot
+echo set output "'"$WHAT.pdf"'" >> $WHAT.gnuplot
+
+col=`cat settings.txt | wc -l`
+echo "plot for [col=2:$col+1] '$WHAT.out.header' using 0:col:xticlabels(1) with lines" >> $WHAT.gnuplot
+echo " " >>  $WHAT.gnuplot
+
+gnuplot -persist < $WHAT.gnuplot
+
+# generate a png file
+# convert -background white -density 120 -rotate 90 -resize 800 +dither -colors 256 -quality 0 $WHAT.ps -background white -flatten  .$WHAT.png
+
+# clean
+rm $WHAT.out.header $WHAT.gnuplot

bench/perf_monitoring/gemm/run_gemm.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+if [ ! -d "eigen_src" ]; then
+  hg clone https://bitbucket.org/eigen/eigen eigen_src
+fi
+
+if [ ! -z '$CXX' ]; then
+  CXX=g++
+fi
+
+rm sgemm.out
+rm dgemm.out
+rm cgemm.out
+
+function test_current 
+{
+  rev=$1
+  scalar=$2
+  name=$3
+  
+  if $CXX -O2 -DNDEBUG -march=native $CXX_FLAGS -I eigen_src gemm.cpp -DSCALAR=$scalar -o $name; then
+    res=`./$name`
+    echo $res
+    echo "$rev $res" >> $name.out
+  else
+    echo "Compilation failed, skip rev $rev"
+  fi
+}
+
+while read rev
+do
+  if [ ! -z '$rev' ]; then
+    echo "Testing rev $rev"
+    cd eigen_src
+    hg up -C $rev
+    actual_rev=`hg identify | cut -f1 -d' '`
+    cd ..
+    
+    test_current $actual_rev float                  sgemm
+    test_current $actual_rev double                 dgemm
+    test_current $actual_rev "std::complex<double>" cgemm
+  fi
+  
+done < changesets.txt
+
+echo "Float:"
+cat sgemm.out
+echo ""
+
+echo "Double:"
+cat dgemm.out
+echo ""
+
+echo "Complex:"
+cat cgemm.out
+echo ""
+
+./make_plot.sh sgemm
+./make_plot.sh dgemm
+./make_plot.sh cgemm
+
+

bench/perf_monitoring/gemm/settings.txt

@@ -0,0 +1,11 @@
+8 8 8
+9 9 9
+239 239 239
+240 240 240
+2400 24 24
+24 2400 24
+24 24 2400
+24 2400 2400
+2400 24 2400
+2400 2400 24
+2400 2400 2400