Add persist support

This commit is contained in:
Howard Chu 2019-02-28 13:24:13 +00:00
parent aba073e171
commit 47102fcced
5 changed files with 527 additions and 14 deletions

View File

@ -428,6 +428,7 @@ init_syncrepl(syncinfo_t *si)
for ( i = 0; dsee_descs[ i ] != NULL; i++ ) {
attrs[ n++ ] = ch_strdup ( dsee_descs[i]->ad_cname.bv_val );
}
attrs[n] = NULL;
}
si->si_attrs = attrs;
@ -541,7 +542,7 @@ ldap_sync_search(
}
ldap_msgfree( res );
if ( gotfirst && gotlast ) {
if ( !si->si_lastchange || si->si_lastchange < first )
if ( si->si_lastchange < first || (!si->si_lastchange && !si->si_refreshDone ))
si->si_logstate = SYNCLOG_FALLBACK;
/* if we're in logging mode, it will update si_lastchange itself */
if ( si->si_logstate == SYNCLOG_FALLBACK )
@ -628,7 +629,16 @@ ldap_sync_search(
#endif
#ifdef DO_DSEE
if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
ctrls[0] = NULL;
if ( si->si_logstate == SYNCLOG_LOGGING && si->si_type == LDAP_SYNC_REFRESH_AND_PERSIST ) {
c[0].ldctl_oid = LDAP_CONTROL_PERSIST_REQUEST;
c[0].ldctl_iscritical = 0;
rc = ldap_create_persistentsearch_control_value( si->si_ld, LDAP_CONTROL_PERSIST_ENTRY_CHANGE_ADD,
0, 1, &c[0].ldctl_value );
ctrls[0] = &c[0];
ctrls[1] = NULL;
} else {
ctrls[0] = NULL;
}
} else
#endif
{
@ -1090,12 +1100,30 @@ do_syncrep2(
break;
}
#endif
ldap_get_entry_controls( si->si_ld, msg, &rctrls );
ldap_get_dn_ber( si->si_ld, msg, NULL, &bdn );
if (!bdn.bv_len) {
bdn.bv_val = empty;
bdn.bv_len = sizeof(empty)-1;
}
#ifdef DO_DSEE
if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
if ( si->si_logstate == SYNCLOG_LOGGING ) {
rc = syncrepl_message_to_op( si, op, msg );
if ( rc )
si->si_logstate = SYNCLOG_FALLBACK;
goto logerr;
if ( si->si_type == LDAP_SYNC_REFRESH_AND_PERSIST && rctrls ) {
LDAPControl **next = NULL;
/* The notification control is only sent during persist phase */
rctrlp = ldap_control_find( LDAP_CONTROL_PERSIST_ENTRY_CHANGE_NOTICE, rctrls, &next );
if ( rctrlp ) {
if ( !si->si_refreshDone )
si->si_refreshDone = 1;
if ( si->si_refreshDone )
syncrepl_dsee_update( si, op );
}
}
} else {
syncstate = DSEE_SYNC_ADD;
rc = syncrepl_message_to_entry( si, op, msg,
@ -1111,12 +1139,6 @@ do_syncrep2(
break;
}
#endif
ldap_get_entry_controls( si->si_ld, msg, &rctrls );
ldap_get_dn_ber( si->si_ld, msg, NULL, &bdn );
if (!bdn.bv_len) {
bdn.bv_val = empty;
bdn.bv_len = sizeof(empty)-1;
}
/* we can't work without the control */
if ( rctrls ) {
LDAPControl **next = NULL;
@ -1269,7 +1291,9 @@ do_syncrep2(
syncCookie.ctxcsn )
{
rc = syncrepl_updateCookie( si, op, &syncCookie, 0 );
} else switch ( rc ) {
} else
logerr:
switch ( rc ) {
case LDAP_ALREADY_EXISTS:
case LDAP_NO_SUCH_OBJECT:
case LDAP_NO_SUCH_ATTRIBUTE:
@ -1365,8 +1389,13 @@ do_syncrep2(
if ( si->si_syncdata == SYNCDATA_CHANGELOG && err == LDAP_SUCCESS ) {
rc = syncrepl_dsee_update( si, op );
if ( rc == LDAP_SUCCESS ) {
rc = -2; /* schedule a re-poll */
si->si_logstate = SYNCLOG_LOGGING;
if ( si->si_logstate == SYNCLOG_FALLBACK ) {
si->si_logstate = SYNCLOG_LOGGING;
si->si_refreshDone = 1;
rc = LDAP_SYNC_REFRESH_REQUIRED;
} else {
rc = -2;
}
}
goto done;
}

View File

@ -0,0 +1,63 @@
# slave slapd config -- for testing of SYNC replication
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 1998-2018 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
include @SCHEMADIR@/core.schema
include @SCHEMADIR@/cosine.schema
include @SCHEMADIR@/inetorgperson.schema
include @SCHEMADIR@/openldap.schema
include @SCHEMADIR@/nis.schema
include @SCHEMADIR@/dsee.schema
#
pidfile @TESTDIR@/slapd.4.pid
argsfile @TESTDIR@/slapd.4.args
#mod#modulepath ../servers/slapd/back-@BACKEND@/
#mod#moduleload back_@BACKEND@.la
#monitormod#modulepath ../servers/slapd/back-monitor/
#monitormod#moduleload back_monitor.la
#######################################################################
# consumer database definitions
#######################################################################
database @BACKEND@
suffix "dc=example,dc=com"
rootdn "cn=Replica,dc=example,dc=com"
rootpw secret
#null#bind on
#~null~#directory @TESTDIR@/db.4.a
#indexdb#index objectClass eq
#indexdb#index cn,sn,uid pres,eq,sub
#indexdb#index entryUUID,entryCSN eq
#ndb#dbname db_4
#ndb#include @DATADIR@/ndb.conf
# Don't change syncrepl spec yet
syncrepl rid=1
provider=@URI1@
binddn="cn=Directory Manager"
bindmethod=simple
credentials=secret21
searchbase="dc=example,dc=com"
filter="(objectClass=*)"
schemachecking=off
scope=sub
type=refreshAndPersist
logbase="cn=changelog"
syncdata=changelog
retry="3 5 300 5"
updateref @URI1@
#monitor#database monitor

View File

@ -237,8 +237,8 @@ cn: ITD Staff
objectclass: groupOfUniqueNames
objectclass: domainRelatedObject
uniquemember: cn=Manager,dc=example,dc=com
uniquemember: cn=Bjorn Jensen,OU=Information Technology Division,ou=People,dc=example,dc=com
uniquemember: cn=James A Jones 2,ou=Information Technology Division,ou=People,dc=example,dc=com
uniquemember: cn=Bjorn Jensen,OU=Information Technology Division,ou=People,dc=example,dc=com
uniquemember: cn=James A Jones 2,ou=Information Technology Division,ou=People,dc=example,dc=com
uniquemember: cn=John Doe,ou=Information Technology Division,ou=People,dc=example,dc=com
associatedDomain: test.openldap.org

View File

@ -108,6 +108,7 @@ P2SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist2.conf
P3SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist3.conf
DIRSYNC1CONF=$DATADIR/slapd-dirsync1.conf
DSEESYNC1CONF=$DATADIR/slapd-dsee-slave1.conf
DSEESYNC2CONF=$DATADIR/slapd-dsee-slave2.conf
REFSLAVECONF=$DATADIR/slapd-ref-slave.conf
SCHEMACONF=$DATADIR/slapd-schema.conf
TLSCONF=$DATADIR/slapd-tls.conf

View File

@ -0,0 +1,420 @@
#! /bin/sh
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 1998-2019 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
echo "running defines.sh"
. $SRCDIR/scripts/defines.sh
OPATTRS="creatorsName createTimestamp modifiersName modifyTimestamp"
if test -z `which dsadm`; then
echo "DSEE dsadm not in path, test skipped"
exit 0
fi
mkdir -p $TESTDIR $DBDIR4
#
# Test replication:
# - start provider
# - start consumer
# - populate over ldap
# - perform some modifies and deleted
# - attempt to modify the consumer (referral or chain)
# - retrieve database over ldap and compare against expected results
#
DSEEPW=secret21
DSEEDN="cn=Directory Manager"
DSEEPWF=$TESTDIR/dseepw
echo "secret21" > $DSEEPWF
echo "Setting up DSEE provider slapd on TCP/IP port $PORT1..."
dsadm create -p $PORT1 -w $DSEEPWF $DBDIR1
dsadm start $DBDIR1
dsconf create-suffix -c -p $PORT1 -w $DSEEPWF $BASEDN
dsconf set-server-prop -p $PORT1 -w $DSEEPWF moddn-enabled:on
dsconf set-server-prop -p $PORT1 -w $DSEEPWF retro-cl-enabled:on
dsadm restart $DBDIR1
PID=`basename $DBDIR1/locks/server/*`
KILLPIDS="$PID"
sleep 1
echo "Using ldapsearch to check that provider slapd is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Starting consumer slapd on TCP/IP port $PORT4..."
. $CONFFILTER $BACKEND $MONITORDB < $DSEESYNC2CONF > $CONF4
$SLAPD -f $CONF4 -h $URI4 -d $LVL $TIMING > $LOG4 2>&1 &
SLAVEPID=$!
if test $WAIT != 0 ; then
echo SLAVEPID $SLAVEPID
read foo
fi
KILLPIDS="$KILLPIDS $SLAVEPID"
sleep 1
echo "Using ldapsearch to check that consumer slapd is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT4 \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
# using LDIFDIRSYNCNOCP to avoid custom OpenLDAP schema
echo "Using ldapadd to populate the provider directory..."
$LDAPADD -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW < \
$LDIFDIRSYNCNOCP > /dev/null 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
echo "Using ldapsearch to read all the entries from the provider..."
$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
-D "$DSEEDN" -w "$DSEEPW" \
'(objectclass=*)' '*' $OPATTRS > $MASTEROUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at provider ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Using ldapsearch to read all the entries from the consumer..."
$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT4 \
'(objectclass=*)' '*' $OPATTRS > $SLAVEOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at consumer ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Filtering provider results..."
$LDIFFILTER -s a < $MASTEROUT > $MASTERFLT
echo "Filtering consumer results..."
$LDIFFILTER -s a < $SLAVEOUT > $SLAVEFLT
echo "Comparing retrieved entries from provider and consumer..."
$CMP $MASTERFLT $SLAVEFLT > $CMPOUT
if test $? != 0 ; then
echo "test failed - provider and consumer databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
echo "Stopping the provider, sleeping 10 seconds and restarting it..."
kill -HUP "$PID"
wait $PID
sleep 10
echo "RESTART" >> $LOG1
dsadm start $DBDIR1
PID=`basename $DBDIR1/locks/server/*`
if test $WAIT != 0 ; then
echo PID $PID
read foo
fi
KILLPIDS="$PID $SLAVEPID"
sleep 1
echo "Using ldapsearch to check that provider slapd is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
echo "Waiting $SLEEP1 seconds for consumer to reconnect..."
sleep $SLEEP1
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Using ldapmodify to modify provider directory..."
#
# Do some modifications
#
$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \
$TESTOUT 2>&1 << EOMODS
dn: cn=James A Jones 1, ou=Alumni Association, ou=People, dc=example,dc=com
changetype: modify
add: carLicense
carLicense: Orange Juice
-
delete: sn
sn: Jones
-
add: sn
sn: Jones
dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modify
replace: carLicense
carLicense: Iced Tea
dn: cn=ITD Staff,ou=Groups,dc=example,dc=com
changetype: modify
delete: uniquemember
uniquemember: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com
uniquemember: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
-
add: uniquemember
uniquemember: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com
uniquemember: cn=James A Jones 1,ou=Alumni Association,ou=People,dc=example,dc=com
dn: cn=All Staff,ou=Groups,dc=example,dc=com
changetype: modify
delete: description
dn: cn=Gern Jensen,ou=Information Technology Division,ou=People,dc=example,dc=com
changetype: add
objectclass: inetOrgPerson
cn: Gern Jensen
sn: Jensen
uid: gjensen
title: Chief Investigator, ITD
postaladdress: ITD $ 535 W. William St $ Ann Arbor, MI 48103
seealso: cn=All Staff,ou=Groups,dc=example,dc=com
carLicense: Coffee
homepostaladdress: 844 Brown St. Apt. 4 $ Ann Arbor, MI 48104
description: Very odd
facsimiletelephonenumber: +1 313 555 7557
facsimiletelephonenumber: +1 313 555 9998
facsimiletelephonenumber: +1 313 555 9999
telephonenumber: +1 313 555 8343
mail: gjensen@mailgw.example.com
homephone: +1 313 555 8844
# modify attribute with no matching rule (ITS#6458)
dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modify
replace: facsimiletelephonenumber
facsimiletelephonenumber: +1 313 555 9998
facsimiletelephonenumber: +1 313 555 9999
dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modify
replace: facsimiletelephonenumber
facsimiletelephonenumber: +1 313 555 9998
facsimiletelephonenumber: +1 313 555 9999
facsimiletelephonenumber: +1 313 555 7557
dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modify
replace: facsimiletelephonenumber
facsimiletelephonenumber: +1 313 555 9998
facsimiletelephonenumber: +1 313 555 9999
dn: ou=Retired,ou=People,dc=example,dc=com
changetype: add
objectclass: organizationalUnit
ou: Retired
dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: add
objectclass: inetOrgPerson
cn: Rosco P. Coltrane
sn: Coltrane
uid: rosco
description: Fat tycoon
dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modrdn
newrdn: cn=Rosco P. Coltrane
deleteoldrdn: 1
newsuperior: ou=Retired,ou=People,dc=example,dc=com
dn: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: delete
dn: ou=testdomain1,dc=example,dc=com
changetype: modrdn
newrdn: ou=itsdomain1
deleteoldrdn: 1
dn: ou=itsdomain1,dc=example,dc=com
changetype: modify
replace: description
description: Example, Inc. ITS test domain
dn: ou=testdomain2,dc=example,dc=com
changetype: modrdn
newrdn: ou=itsdomain2
deleteoldrdn: 1
EOMODS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
## ldappasswd test removed, not supported on DSEE
echo "Stopping consumer to test recovery..."
kill -HUP $SLAVEPID
wait $SLAVEPID
echo "Modifying more entries on the provider..."
$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW >> \
$TESTOUT 2>&1 << EOMODS
dn: cn=Rosco P. Coltrane, ou=Retired, ou=People, dc=example,dc=com
changetype: delete
dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
changetype: modify
add: carLicense
carLicense: Mad Dog 20/20
dn: cn=Rosco P. Coltrane,ou=Retired,ou=People,dc=example,dc=com
changetype: add
objectclass: inetOrgPerson
sn: Coltrane
uid: rosco
cn: Rosco P. Coltrane
dn: ou=itsdomain2,dc=example,dc=com
changetype: modify
replace: description
description: Example, Inc. itsdomain2 test domain
# rename with a newly added newSuperior while the consumer is down (ITS#6472)
dn: ou=New Branch,dc=example,dc=com
changetype: add
objectClass: organizationalUnit
ou: New Branch
dn: cn=Dorothy Stevens, ou=Alumni Association, ou=People, dc=example,dc=com
changetype: modrdn
newrdn: cn=Dorothy Stevens
deleteoldrdn: 0
newsuperior: ou=New Branch,dc=example,dc=com
EOMODS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Restarting consumer..."
echo "RESTART" >> $LOG4
$SLAPD -f $CONF4 -h $URI4 -d $LVL $TIMING >> $LOG4 2>&1 &
SLAVEPID=$!
if test $WAIT != 0 ; then
echo SLAVEPID $SLAVEPID
read foo
fi
KILLPIDS="$PID $SLAVEPID"
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
echo "Using ldapsearch to read all the entries from the provider..."
$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
-D "$DSEEDN" -w "$DSEEPW" \
'(objectclass=*)' '*' $OPATTRS > $MASTEROUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at provider ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Using ldapsearch to read all the entries from the consumer..."
$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT4 \
'(objectclass=*)' '*' $OPATTRS > $SLAVEOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at consumer ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Filtering provider results..."
$LDIFFILTER -s a < $MASTEROUT > $MASTERFLT
echo "Filtering consumer results..."
$LDIFFILTER -s a < $SLAVEOUT > $SLAVEFLT
echo "Comparing retrieved entries from provider and consumer..."
$CMP $MASTERFLT $SLAVEFLT > $CMPOUT
if test $? != 0 ; then
echo "test failed - provider and consumer databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
test $KILLSERVERS != no && kill -HUP $KILLPIDS
echo ">>>>> Test succeeded"
test $KILLSERVERS != no && wait
exit 0