From ddcb186427412838baf4cf18000e97b03aee9ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Kuzn=C3=ADk?= Date: Wed, 14 Nov 2018 13:27:27 +0000 Subject: [PATCH] Update ITS#8752 regression test --- tests/data/regressions/its8752/its8752 | 280 ++++++++++++++++-- tests/data/regressions/its8752/slapd.conf.mmr | 9 +- 2 files changed, 268 insertions(+), 21 deletions(-) diff --git a/tests/data/regressions/its8752/its8752 b/tests/data/regressions/its8752/its8752 index 4d2d49241d..e719c770d2 100755 --- a/tests/data/regressions/its8752/its8752 +++ b/tests/data/regressions/its8752/its8752 @@ -36,10 +36,21 @@ if test $dtest = N; then exit 0 fi +# This mimics the scenario where a single server has been used until now (no +# syncprov either, so no contextCSN) and we convert it to a delta-MMR setup: +# 1. stop the server (note that there is likely no contextCSN in the DB at this point) +# 2. configure all servers to delta-replicate from each other and start them up +# - empty servers will start with a refresh of the main DB +# - when the refresh is successful they should change over to replicating the log +# 3. keep making changes on all servers to see things still work + echo "This test tracks a case where slapd deadlocks during a significant write load" echo "See http://www.openldap.org/its/index.cgi/?findid=8752 for more information." MMR=4 +iterations=20000 +check_sync_every=100 +MAPSIZE=`expr 100 \* 1024 \* 1024` XDIR=$TESTDIR/srv mkdir -p $TESTDIR @@ -66,7 +77,7 @@ if test $WAIT != 0 ; then fi KILLPIDS="$PID" -sleep 1 +sleep $SLEEP0 echo "Using ldapsearch to check that slapd is running..." for i in 0 1 2 3 4 5; do @@ -76,8 +87,8 @@ for i in 0 1 2 3 4 5; do if test $RC = 0 ; then break fi - echo "Waiting 5 seconds for slapd to start..." - sleep 5 + echo "Waiting $SLEEP1 seconds for slapd to start..." + sleep $SLEEP1 done if test $RC != 0 ; then @@ -149,7 +160,7 @@ fi echo "Stopping slapd and reworking configuration for MMR..." -test $KILLSERVERS != no && kill -HUP $KILLPIDS +kill -HUP $KILLPIDS wait $KILLPIDS KILLPIDS= @@ -160,7 +171,7 @@ while [ $n -le $MMR ]; do MYCONF=`eval echo '$CONF'$n` echo "Starting provider slapd on TCP/IP URI $MYURI" . $CONFFILTER $BACKEND $MONITORDB < $ITSDIR/slapd.conf.mmr > $TESTDIR/slapd.conf - sed -e "s/MMR/$n/g" $TESTDIR/slapd.conf > $MYCONF + sed -e "s/MMR/$n/g" -e "s/wronglog/log/" -e "s/@MAPSIZE@/$MAPSIZE/" $TESTDIR/slapd.conf > $MYCONF j=1 while [ $j -le $MMR ]; do MMCURI=`eval echo '$URI'$j` @@ -178,7 +189,7 @@ while [ $n -le $MMR ]; do read foo fi KILLPIDS="$PID $KILLPIDS" - sleep 1 + sleep $SLEEP0 echo "Using ldapsearch to check that provider slapd is running..." for i in 0 1 2 3 4 5; do @@ -188,10 +199,10 @@ while [ $n -le $MMR ]; do if test $RC = 0 ; then break fi - echo "Waiting 5 seconds for slapd to start..." - sleep 5 + echo "Waiting $SLEEP1 seconds for slapd to start..." + sleep $SLEEP1 done - + if test $RC != 0 ; then echo "ldapsearch failed ($RC)!" test $KILLSERVERS != no && kill -HUP $KILLPIDS @@ -203,7 +214,7 @@ done echo "Setting up accesslog on each master..." n=1 while [ $n -le $MMR ]; do - echo "Modifying dn: cn=Elmer_Fudd,ou=People,dc=example,dc=com on master $n" + echo "Modifying dn: cn=Elmer_Fudd,ou=People,$BASEDN on master $n" MYURI=`eval echo '$URI'$n` $LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \ $TESTOUT 2>&1 << EOMODS @@ -213,14 +224,68 @@ changetype: modify replace: sn sn: Fudd EOMODS - sleep 5 + RC=$? + if test $RC != 0; then + echo "ldapmodify failed ($RC)" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 + fi + sleep $SLEEP0 n=`expr $n + 1` done -echo "Sleeping 30 seconds to ensure replication is reconciled between all nodes" -echo "The next step of the test will perform 20000 random write operations and may take some time." +#echo "Letting server 1 establish its own contextCSN..." +#echo "Modifying dn: cn=Elmer_Fudd,ou=People,$BASEDN on $URI1" +#$LDAPMODIFY -v -D "$MANAGERDN" -H $URI1 -w $PASSWD > \ +# $TESTOUT 2>&1 << EOMODS +#dn: cn=Elmer_Fudd,ou=People,$BASEDN +#changetype: modify +#replace: sn +#sn: Fudd +#EOMODS + +for i in 0 1 2 3 4 5; do + j=1 + while [ $j -le $MMR ]; do + MYURI=`eval echo '$URI'$j` + $LDAPSEARCH -b "$BASEDN" -H "$MYURI" \ + '*' '+' >"$TESTDIR/server$j.out" 2>&1 + RC=$? + if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC + fi + $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt" + j=`expr $j + 1` + done + + in_sync=1 + j=1 + while [ $j -lt $MMR ]; do + k=$j + j=`expr $j + 1` + $CMP "$TESTDIR/server$k.flt" "$TESTDIR/server$j.flt" > $CMPOUT + if test $? != 0 ; then + in_sync=0 + fi + done + if test $in_sync = 1; then + break + fi + + echo "Waiting $SLEEP1 seconds for servers to catch up..." + sleep $SLEEP1 +done + +if test $in_sync = 0; then + echo "Servers did not replicate in time" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + +echo "The next step of the test will perform $iterations random write operations and may take some time." echo "As this test is for a deadlock, it will take manual intervention to exit the test if one occurs." -sleep 30 echo "Starting random master/entry modifications..." DN1="cn=Elmer_Fudd,ou=People,$BASEDN" @@ -236,17 +301,17 @@ DN4="cn=Daffy_Duck,ou=People,$BASEDN" VAL4="Duck" n=1 -while [ $n -le 20000 ]; do +while [ $n -le $iterations ]; do seed=`date +%N|sed s/...$//` rvalue=`echo|awk "BEGIN {srand($seed) -{print int(1+rand()*4)}}"` +{print int(1+rand()*$MMR)}}"` MYURI=`eval echo '$URI'$rvalue` seed=`date +%N|sed s/...$//` rvalue=`echo|awk "BEGIN {srand($seed) {print int(1+rand()*4)}}"` MYDN=`eval echo '$DN'$rvalue` MYVAL=`eval echo '$VAL'$rvalue` - echo "Modifying $MYURI entry $MYDN with value $MYVAL iteration $n of 20000" + echo "Modifying $MYURI entry $MYDN with value $MYVAL iteration $n of $iterations" $LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \ $TESTOUT 2>&1 << EOMODS @@ -255,6 +320,187 @@ changetype: modify replace: sn sn: $MYVAL EOMODS + RC=$? + if test $RC != 0; then + echo "ldapmodify failed ($RC)" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 + fi + + if [ "$check_sync_every" -gt 0 ] && [ `expr $n % $check_sync_every` = 0 ]; then + i=1 + echo "Checking replication status before we start iteration $n..." + for i in 0 1 2 3 4 5; do + j=1 + while [ $j -le $MMR ]; do + MYURI=`eval echo '$URI'$j` + echo "Reading database from server $j..." + $LDAPSEARCH -b "$BASEDN" -H "$MYURI" \ + '*' '+' >"$TESTDIR/server$j.out" 2>&1 + RC=$? + if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC + fi + $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt" + j=`expr $j + 1` + done + + in_sync=1 + j=1 + while [ $j -lt $MMR ]; do + k=`expr $j + 1` + $CMP "$TESTDIR/server$j.flt" "$TESTDIR/server$k.flt" > $CMPOUT + if test $? != 0 ; then + in_sync=0 + fi + j=$k + done + if test $in_sync = 1; then + break + fi + + echo "Waiting $SLEEP1 seconds for servers to catch up..." + sleep $SLEEP1 + done + + if test $in_sync = 0; then + echo "Servers did not replicate in time" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 + fi + fi + n=`expr $n + 1` +done + +kill -HUP $KILLPIDS +wait $KILLPIDS + +echo "The next step of the test will perform $iterations random write operations and may take some time." +echo "As this test is for a deadlock, it will take manual intervention to exit the test if one occurs." + +echo "Starting servers again, this time with the wrong logbase setting..." +KILLPIDS= +n=1 +while [ $n -le $MMR ]; do + MYURI=`eval echo '$URI'$n` + MYLOG=`eval echo '$LOG'$n` + MYCONF=`eval echo '$CONF'$n` + echo "Starting provider slapd on TCP/IP URI $MYURI" + . $CONFFILTER $BACKEND $MONITORDB < $ITSDIR/slapd.conf.mmr > $TESTDIR/slapd.conf + sed -e "s/MMR/$n/g" -e "s/@MAPSIZE@/$MAPSIZE/" $TESTDIR/slapd.conf > $MYCONF + j=1 + while [ $j -le $MMR ]; do + MMCURI=`eval echo '$URI'$j` + sed -e "s|MMC${j}|${MMCURI}|" $MYCONF > $TESTDIR/slapd.conf + mv $TESTDIR/slapd.conf $MYCONF + j=`expr $j + 1` + done + if [ -f $TESTDIR/slapd.conf ]; then + rm -f $TESTDIR/slapd.conf + fi + $SLAPD -f $MYCONF -h $MYURI -d $LVL $TIMING >> $MYLOG 2>&1 & + PID=$! + if test $WAIT != 0 ; then + echo PID $PID + read foo + fi + KILLPIDS="$PID $KILLPIDS" + sleep $SLEEP0 + + echo "Using ldapsearch to check that provider slapd is running..." + for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "" -H $MYURI \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting $SLEEP1 seconds for slapd to start..." + sleep $SLEEP1 + done + + if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC + fi + n=`expr $n + 1` + done + + echo "Starting random master/entry modifications..." + n=1 + while [ $n -le $iterations ]; do + seed=`date +%N|sed s/...$//` + rvalue=`echo|awk "BEGIN {srand($seed) + {print int(1+rand()*$MMR)}}"` + MYURI=`eval echo '$URI'$rvalue` + seed=`date +%N|sed s/...$//` + rvalue=`echo|awk "BEGIN {srand($seed) + {print int(1+rand()*4)}}"` + MYDN=`eval echo '$DN'$rvalue` + MYVAL=`eval echo '$VAL'$rvalue` + echo "Modifying $MYURI entry $MYDN with value $MYVAL iteration $n of $iterations" + $LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \ + $TESTOUT 2>&1 << EOMODS + +dn: $MYDN +changetype: modify +replace: sn +sn: $MYVAL +EOMODS +RC=$? +if test $RC != 0; then + echo "ldapmodify failed ($RC)" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + +if [ "$check_sync_every" -gt 0 ] && [ `expr $n % $check_sync_every` = 0 ]; then + i=1 + echo "Checking replication status before we start iteration $n..." + for i in 0 1 2 3 4 5; do + j=1 + while [ $j -le $MMR ]; do + MYURI=`eval echo '$URI'$j` + echo "Reading database from server $j..." + $LDAPSEARCH -b "$BASEDN" -H "$MYURI" \ + '*' '+' >"$TESTDIR/server$j.out" 2>&1 + RC=$? + if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC + fi + $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt" + j=`expr $j + 1` + done + + in_sync=1 + j=1 + while [ $j -lt $MMR ]; do + k=`expr $j + 1` + $CMP "$TESTDIR/server$j.flt" "$TESTDIR/server$k.flt" > $CMPOUT + if test $? != 0 ; then + in_sync=0 + fi + j=$k + done + if test $in_sync = 1; then + break + fi + + echo "Waiting $SLEEP1 seconds for servers to catch up..." + sleep $SLEEP1 + done + + if test $in_sync = 0; then + echo "Servers did not replicate in time" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 + fi + fi n=`expr $n + 1` done diff --git a/tests/data/regressions/its8752/slapd.conf.mmr b/tests/data/regressions/its8752/slapd.conf.mmr index 99cad423f0..578c9905d0 100644 --- a/tests/data/regressions/its8752/slapd.conf.mmr +++ b/tests/data/regressions/its8752/slapd.conf.mmr @@ -57,7 +57,7 @@ syncrepl scope=sub schemachecking=on retry="5 10 60 +" - logbase="cn=accesslog" + logbase="cn=log" logfilter="(&(objectClass=auditWriteObject)(reqResult=0))" syncdata=accesslog sizeLimit=unlimited @@ -75,7 +75,7 @@ syncrepl scope=sub schemachecking=on retry="5 10 60 +" - logbase="cn=accesslog" + logbase="cn=log" logfilter="(&(objectClass=auditWriteObject)(reqResult=0))" syncdata=accesslog sizeLimit=unlimited @@ -93,7 +93,7 @@ syncrepl scope=sub schemachecking=on retry="5 10 60 +" - logbase="cn=accesslog" + logbase="cn=log" logfilter="(&(objectClass=auditWriteObject)(reqResult=0))" syncdata=accesslog sizeLimit=unlimited @@ -111,7 +111,7 @@ syncrepl scope=sub schemachecking=on retry="5 10 60 +" - logbase="cn=accesslog" + logbase="cn=log" logfilter="(&(objectClass=auditWriteObject)(reqResult=0))" syncdata=accesslog sizeLimit=unlimited @@ -135,6 +135,7 @@ rootdn "cn=Manager,dc=example,dc=com" #~null~#directory @TESTDIR@/srvMMR/db.2 #indexdb#index objectClass eq #indexdb#index entryCSN,entryUUID,reqEnd,reqResult,reqStart eq +#mdb#maxsize @MAPSIZE@ overlay syncprov syncprov-reloadhint true