From 03ec601ff745ec9bb2902eadc08ebfdd2834976d Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 20 Feb 2015 19:20:34 +0100 Subject: [PATCH] Initial version of a small script to help tracking performance regressions --- bench/perf_monitoring/gemm/changesets.txt | 25 +++++++++ bench/perf_monitoring/gemm/gemm.cpp | 67 +++++++++++++++++++++++ bench/perf_monitoring/gemm/make_plot.sh | 37 +++++++++++++ bench/perf_monitoring/gemm/run_gemm.sh | 62 +++++++++++++++++++++ bench/perf_monitoring/gemm/settings.txt | 11 ++++ 5 files changed, 202 insertions(+) create mode 100644 bench/perf_monitoring/gemm/changesets.txt create mode 100644 bench/perf_monitoring/gemm/gemm.cpp create mode 100755 bench/perf_monitoring/gemm/make_plot.sh create mode 100755 bench/perf_monitoring/gemm/run_gemm.sh create mode 100644 bench/perf_monitoring/gemm/settings.txt diff --git a/bench/perf_monitoring/gemm/changesets.txt b/bench/perf_monitoring/gemm/changesets.txt new file mode 100644 index 000000000..291fd55b4 --- /dev/null +++ b/bench/perf_monitoring/gemm/changesets.txt @@ -0,0 +1,25 @@ +3.0.1 +3.1.1 +3.2.0 +3.2.4 +5745:37f59e65eb6c +5891:d8652709345d +5893:24b4dc92c6d3 +5895:997c2ef9fc8b +5904:e1eafd14eaa1 +5908:f8ee3c721251 +5921:ca808bb456b0 +5927:8b1001f9e3ac +5937:5a4ca1ad8c53 +5949:f3488f4e45b2 +5969:e09031dccfd9 +5992:4a429f5e0483 +before-evaluators +6334:f6a45e5b8b7c +6639:c9121c60b5c7 +6655:06f163b5221f +6677:700e023044e7 +6681:11d31dafb0e3 +6844:039efd86b75c +6845:7333ed40c6ef +6911:6192dd812d84 \ No newline at end of file diff --git a/bench/perf_monitoring/gemm/gemm.cpp b/bench/perf_monitoring/gemm/gemm.cpp new file mode 100644 index 000000000..72eb9cab6 --- /dev/null +++ b/bench/perf_monitoring/gemm/gemm.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include "../../BenchTimer.h" +using namespace Eigen; + +#ifndef SCALAR +#error SCALAR must be defined +#endif + +typedef SCALAR Scalar; + +typedef Matrix Mat; + +EIGEN_DONT_INLINE +void gemm(const Mat &A, const Mat &B, Mat &C) +{ + C.noalias() += A * B; +} + +EIGEN_DONT_INLINE +double bench(long m, long n, long k) +{ + Mat A(m,k); + Mat B(k,n); + Mat C(m,n); + A.setRandom(); + B.setRandom(); + C.setZero(); + + BenchTimer t; + + double up = 1e8*4/sizeof(Scalar); + double tm0 = 4, tm1 = 10; + if(NumTraits::IsComplex) + { + up /= 4; + tm0 = 2; + tm1 = 4; + } + + double flops = 2. * m * n * k; + long rep = std::max(1., std::min(100., up/flops) ); + long tries = std::max(tm0, std::min(tm1, up/flops) ); + + BENCH(t, tries, rep, gemm(A,B,C)); + + return 1e-9 * rep * flops / t.best(); +} + +int main(int argc, char **argv) +{ + std::vector results; + + std::ifstream settings("settings.txt"); + long m, n, k; + while(settings >> m >> n >> k) + { + //std::cerr << " Testing " << m << " " << n << " " << k << std::endl; + results.push_back( bench(m, n, k) ); + } + + std::cout << RowVectorXd::Map(results.data(), results.size()); + + return 0; +} diff --git a/bench/perf_monitoring/gemm/make_plot.sh b/bench/perf_monitoring/gemm/make_plot.sh new file mode 100755 index 000000000..609c471f9 --- /dev/null +++ b/bench/perf_monitoring/gemm/make_plot.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# base name of the bench +# it reads $1.out +# and generates $1.pdf +WHAT=$1 + +header="rev " +while read line +do + if [ ! -z '$line' ]; then + header="$header \"$line\"" + fi +done < settings.txt + +echo $header > $WHAT.out.header +cat $WHAT.out >> $WHAT.out.header + + +echo "set title '$WHAT'" > $WHAT.gnuplot +echo "set key autotitle columnhead outside " >> $WHAT.gnuplot +echo "set xtics rotate 1" >> $WHAT.gnuplot + +echo "set term pdf color rounded enhanced fontscale 0.35 size 7in,5in" >> $WHAT.gnuplot +echo set output "'"$WHAT.pdf"'" >> $WHAT.gnuplot + +col=`cat settings.txt | wc -l` +echo "plot for [col=2:$col+1] '$WHAT.out.header' using 0:col:xticlabels(1) with lines" >> $WHAT.gnuplot +echo " " >> $WHAT.gnuplot + +gnuplot -persist < $WHAT.gnuplot + +# generate a png file +# convert -background white -density 120 -rotate 90 -resize 800 +dither -colors 256 -quality 0 $WHAT.ps -background white -flatten .$WHAT.png + +# clean +rm $WHAT.out.header $WHAT.gnuplot \ No newline at end of file diff --git a/bench/perf_monitoring/gemm/run_gemm.sh b/bench/perf_monitoring/gemm/run_gemm.sh new file mode 100755 index 000000000..4d87ff656 --- /dev/null +++ b/bench/perf_monitoring/gemm/run_gemm.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +if [ ! -d "eigen_src" ]; then + hg clone https://bitbucket.org/eigen/eigen eigen_src +fi + +if [ ! -z '$CXX' ]; then + CXX=g++ +fi + +rm sgemm.out +rm dgemm.out +rm cgemm.out + +function test_current +{ + rev=$1 + scalar=$2 + name=$3 + + if $CXX -O2 -DNDEBUG -march=native $CXX_FLAGS -I eigen_src gemm.cpp -DSCALAR=$scalar -o $name; then + res=`./$name` + echo $res + echo "$rev $res" >> $name.out + else + echo "Compilation failed, skip rev $rev" + fi +} + +while read rev +do + if [ ! -z '$rev' ]; then + echo "Testing rev $rev" + cd eigen_src + hg up -C $rev + actual_rev=`hg identify | cut -f1 -d' '` + cd .. + + test_current $actual_rev float sgemm + test_current $actual_rev double dgemm + test_current $actual_rev "std::complex" cgemm + fi + +done < changesets.txt + +echo "Float:" +cat sgemm.out +echo "" + +echo "Double:" +cat dgemm.out +echo "" + +echo "Complex:" +cat cgemm.out +echo "" + +./make_plot.sh sgemm +./make_plot.sh dgemm +./make_plot.sh cgemm + + diff --git a/bench/perf_monitoring/gemm/settings.txt b/bench/perf_monitoring/gemm/settings.txt new file mode 100644 index 000000000..6ef690708 --- /dev/null +++ b/bench/perf_monitoring/gemm/settings.txt @@ -0,0 +1,11 @@ +8 8 8 +9 9 9 +239 239 239 +240 240 240 +2400 24 24 +24 2400 24 +24 24 2400 +24 2400 2400 +2400 24 2400 +2400 2400 24 +2400 2400 2400