Added linearindex option for slapindex. Improves performance when database

size exceeds dbcache size by minimizing cache thrashing.

doc/man/man5/slapd-bdb.5

@@ -109,6 +109,14 @@ may be specified to disallow use of this index by named subtypes.
 Note: changing index settings requires rebuilding indices, see
 .BR slapindex (8).
 .TP
+.B linearindex
+Tell slapindex to index one attribute at a time. By default, all indexed
+attributes in an entry are processed at the same time. With this option,
+each indexed attribute is processed individually, using multiple passes
+through the entire database. This option improves slapindex performance
+when the database size exceeds the dbcache size. When the dbcache is
+large enough, this option is not needed and will decrease performance.
+.TP
 .B lockdetect {oldest|youngest|fewest|random|default}
 Specify which transaction to abort when a deadlock is detected.
 The default is the same as

servers/slapd/back-bdb/back-bdb.h

@@ -163,6 +163,7 @@ struct bdb_info {
 	Avlnode		*bi_attrs;
 	void		*bi_search_stack;
 	int		bi_search_stack_depth;
+	int		bi_linear_index;
 
 	int			bi_txn_cp;
 	u_int32_t	bi_txn_cp_min;

servers/slapd/back-bdb/config.c

@@ -69,6 +69,10 @@ bdb_db_config(
 		slapMode & SLAP_TOOL_MODE ) {
 		bdb->bi_dbenv_xflags |= DB_TXN_NOT_DURABLE;
 
+	/* slapindex algorithm tuning */
+	} else if ( strcasecmp( argv[0], "linearindex" ) == 0 ) {
+		bdb->bi_linear_index = 1;
+
 	/* transaction checkpoint configuration */
 	} else if ( strcasecmp( argv[0], "checkpoint" ) == 0 ) {
 		if ( argc < 3 ) {

servers/slapd/back-bdb/tools.c

@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include <ac/string.h>
 
+#define AVL_INTERNAL
 #include "back-bdb.h"
 #include "external.h"
 
@@ -35,6 +36,8 @@ static dn_id hbuf[HOLE_SIZE], *holes = hbuf;
 static unsigned nhmax = HOLE_SIZE;
 static unsigned nholes;
 
+Avlnode *index_attrs, index_dummy;
+
 int bdb_tool_entry_open(
 	BackendDB *be, int mode )
 {
@@ -79,6 +82,8 @@ int bdb_tool_entry_close(
 	return 0;
 }
 
+static int bdb_reindex_cmp(const void *a, const void *b) { return 0; }
+
 ID bdb_tool_entry_next(
 	BackendDB *be )
 {
@@ -90,6 +95,7 @@ ID bdb_tool_entry_next(
 	assert( slapMode & SLAP_TOOL_MODE );
 	assert( bdb != NULL );
 	
+	/* Initialization */
 	if (cursor == NULL) {
 		rc = bdb->bi_id2entry->bdi_db->cursor(
 			bdb->bi_id2entry->bdi_db, NULL, &cursor,
@@ -102,7 +108,22 @@ ID bdb_tool_entry_next(
 	rc = cursor->c_get( cursor, &key, &data, DB_NEXT );
 
 	if( rc != 0 ) {
-		return NOID;
+		/* If we're doing linear indexing and there are more attrs to
+		 * index, and we're at the end of the database, start over.
+		 */
+		if ( bdb->bi_attrs == &index_dummy ) {
+			if ( index_attrs && rc == DB_NOTFOUND ) {
+				/* optional - do a checkpoint here? */
+				index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
+				rc = cursor->c_get( cursor, &key, &data, DB_FIRST );
+			}
+			if ( rc ) {
+				bdb->bi_attrs = NULL;
+				return NOID;
+			}
+		} else {
+			return NOID;
+		}
 	}
 
 	if( data.data == NULL ) {
@@ -354,7 +375,8 @@ ID bdb_tool_entry_put(
 		goto done;
 	}
 
-	rc = bdb_index_entry_add( &op, tid, e );
+	if ( !bdb->bi_linear_index )
+		rc = bdb_index_entry_add( &op, tid, e );
 	if( rc != 0 ) {
 		snprintf( text->bv_val, text->bv_len,
 				"index_entry_add failed: %s (%d)",
@@ -424,6 +446,20 @@ int bdb_tool_entry_reindex(
 		(long) id, 0, 0 );
 #endif
 
+	/* No indexes configured, nothing to do. Could return an
+	 * error here to shortcut things.
+	 */
+	if (!bi->bi_attrs) {
+		return 0;
+	}
+
+	/* Get the first attribute to index */
+	if (bi->bi_linear_index && !index_attrs && bi->bi_attrs != &index_dummy) {
+		index_attrs = bi->bi_attrs;
+		bi->bi_attrs = &index_dummy;
+		index_dummy.avl_data = avl_delete(&index_attrs, NULL, bdb_reindex_cmp);
+	}
+
 	e = bdb_tool_entry_get( be, id );
 
 	if( e == NULL ) {
@@ -472,7 +508,7 @@ int bdb_tool_entry_reindex(
 	op.o_tmpmemctx = NULL;
 	op.o_tmpmfuncs = &ch_mfuncs;
 
-#ifndef BDB_HIER
+#if 0 /* ndef BDB_HIER */
 	/* add dn2id indices */
 	rc = bdb_dn2id_add( &op, tid, NULL, e );
 	if( rc != 0 && rc != DB_KEYEXIST ) {