mirror/openldap
2
0
Fork 0
mirror of https://git.openldap.org/openldap/openldap.git synced 2025-01-30 13:30:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

ITS#8977 make IDL size configurable

Browse Source
This commit is contained in:
Howard Chu 2019-02-15 14:37:51 +00:00
parent 17f1e32b65
commit c8b806b676
9 changed files with 105 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/man/man5/slapd-mdb.5
View File

@ -20,6 +20,18 @@ supports subtree renames. It is both more space-efficient and more
execution-efficient than the \fBbdb\fP backend, while being overall
much simpler to manage.
.SH CONFIGURATION
These
.B slapd.conf
options apply to the \fBmdb\fP backend.
That is, they must follow a "backend mdb" line and
come before any subsequent "backend" or "database" lines.
.TP
.BI idlexp \ <exp>
Specify a power of 2 for the maximum size of an index slot.
The default is 16, yielding a maximum slot size of 2^16 or 65536.
Once set, this option applies to every \fBmdb\fP database instance.
.LP
These
.B slapd.conf
options apply to the \fBmdb\fP backend database.

43
servers/slapd/back-mdb/config.c
View File

@ -22,13 +22,16 @@
#include <ac/errno.h>
#include "back-mdb.h"
#include "idl.h"
#include "config.h"
#include "lutil.h"
#include "ldap_rq.h"
static ConfigDriver mdb_cf_gen;
static ConfigDriver mdb_bk_cfg;
enum {
MDB_CHKPT = 1,
@ -41,9 +44,15 @@ enum {
MDB_MODE,
MDB_SSTACK,
MDB_MULTIVAL,
MDB_IDLEXP,
};
static ConfigTable mdbcfg[] = {
{ "idlexp", "log", 2, 2, 0, ARG_UINT|ARG_MAGIC|MDB_IDLEXP,
mdb_bk_cfg, "( OLcfgBkAt:12.1 NAME 'olcBkMdbIdlExp' "
"DESC 'Power of 2 used to set IDL size' "
"EQUALITY integerMatch "
"SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL },
{ "directory", "dir", 2, 2, 0, ARG_STRING|ARG_MAGIC|MDB_DIRECTORY,
mdb_cf_gen, "( OLcfgDbAt:0.1 NAME 'olcDbDirectory' "
"DESC 'Directory for database content' "
@ -112,17 +121,24 @@ static ConfigTable mdbcfg[] = {
};
static ConfigOCs mdbocs[] = {
{
"( OLcfgBkOc:12.1 "
"NAME 'olcMdbBkConfig' "
"DESC 'MDB backend configuration' "
"SUP olcBackendConfig "
"MAY olcBkMdbIdlExp )",
Cft_Backend, mdbcfg },
{
"( OLcfgDbOc:12.1 "
"NAME 'olcMdbConfig' "
"DESC 'MDB backend configuration' "
"DESC 'MDB database configuration' "
"SUP olcDatabaseConfig "
"MUST olcDbDirectory "
"MAY ( olcDbCheckpoint $ olcDbEnvFlags $ "
"olcDbNoSync $ olcDbIndex $ olcDbMaxReaders $ olcDbMaxSize $ "
"olcDbMode $ olcDbSearchStack $ olcDbMaxEntrySize $ olcDbRtxnSize $ "
"olcDbMultival ) )",
Cft_Database, mdbcfg },
Cft_Database, mdbcfg+1 },
{ NULL, 0, NULL }
};
@ -135,6 +151,29 @@ static slap_verbmasks mdb_envflags[] = {
{ BER_BVNULL, 0 }
};
static int
mdb_bk_cfg( ConfigArgs *c )
{
int rc = 0;
if ( c->op == SLAP_CONFIG_EMIT ) {
if ( MDB_idl_logn != MDB_IDL_LOGN )
c->value_int = MDB_idl_logn;
else
rc = 1;
} else if ( c->op == LDAP_MOD_DELETE ) {
MDB_idl_logn = 0;
mdb_idl_reset();
} else {
if ( c->value_int >= MDB_IDL_LOGN && c->value_int < sizeof(int) * CHAR_BIT ) {
MDB_idl_logn = c->value_int;
mdb_idl_reset();
} else {
rc = 1;
}
}
return rc;
}
/* perform periodic syncs */
static void *
mdb_checkpoint( void *ctx, void *arg )

2
servers/slapd/back-mdb/dn2id.c
View File

@ -664,7 +664,7 @@ mdb_idscope(
ptr += data.mv_size - sizeof(ID);
memcpy( &id, ptr, sizeof(ID) );
if ( id == base ) {
if ( res[0] >= MDB_IDL_DB_SIZE-1 ) {
if ( res[0] >= MDB_idl_db_max ) {
/* too many aliases in scope. Fallback to range */
MDB_IDL_RANGE( res, MDB_IDL_FIRST( ids ), MDB_IDL_LAST( ids ));
goto leave;

4
servers/slapd/back-mdb/filterindex.c
View File

@ -270,7 +270,7 @@ comp_list_candidates(
continue;
}
MDB_IDL_ZERO( save );
rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_IDL_UM_SIZE );
rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_idl_um_size );
if ( rc != 0 ) {
if ( ftype == LDAP_COMP_FILTER_AND ) {
@ -580,7 +580,7 @@ list_candidates(
}
MDB_IDL_ZERO( save );
rc = mdb_filter_candidates( op, rtxn, f, save, tmp,
save+MDB_IDL_UM_SIZE );
save+MDB_idl_um_size );
if ( rc != 0 ) {
if ( ftype == LDAP_FILTER_AND ) {

29
servers/slapd/back-mdb/idl.c
View File

@ -22,6 +22,12 @@
#include "back-mdb.h"
#include "idl.h"
unsigned int MDB_idl_logn = MDB_IDL_LOGN;
unsigned int MDB_idl_db_size = 1 << MDB_IDL_LOGN;
unsigned int MDB_idl_um_size = 1 << (MDB_IDL_LOGN+1);
unsigned int MDB_idl_db_max = (1 << MDB_IDL_LOGN) - 1;
unsigned int MDB_idl_um_max = (1 << (MDB_IDL_LOGN+1) - 1);
#define IDL_MAX(x,y) ( (x) > (y) ? (x) : (y) )
#define IDL_MIN(x,y) ( (x) < (y) ? (x) : (y) )
#define IDL_CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) )
@ -67,6 +73,17 @@ static void idl_dump( ID *ids )
#endif /* IDL_DEBUG > 1 */
#endif /* IDL_DEBUG > 0 */
void mdb_idl_reset()
{
if ( !MDB_idl_logn )
MDB_idl_logn = MDB_IDL_LOGN;
MDB_idl_db_size = 1 << MDB_idl_logn;
MDB_idl_um_size = 1 << (MDB_idl_logn+1);
MDB_idl_db_max = MDB_idl_db_size - 1;
MDB_idl_um_max = MDB_idl_um_size - 1;
}
unsigned mdb_idl_search( ID *ids, ID id )
{
#define IDL_BINARY_SEARCH 1
@ -160,7 +177,7 @@ int mdb_idl_insert( ID *ids, ID id )
return -1;
}
if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
if ( ++ids[0] >= MDB_idl_db_max ) {
if( id < ids[1] ) {
ids[1] = id;
ids[2] = ids[ids[0]-1];
@ -447,7 +464,7 @@ mdb_idl_insert_keys(
err = "c_count";
goto fail;
}
if ( count >= MDB_IDL_DB_MAX ) {
if ( count >= MDB_idl_db_max ) {
/* No room, convert to a range */
lo = *i;
rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP );
@ -784,7 +801,7 @@ over: ida = IDL_MIN( MDB_IDL_FIRST(a), MDB_IDL_FIRST(b) );
/* The distinct elements of a are cat'd to b */
while( ida != NOID || idb != NOID ) {
if ( ida < idb ) {
if( ++cursorc > MDB_IDL_UM_MAX ) {
if( ++cursorc > MDB_idl_um_max ) {
goto over;
}
b[cursorc] = ida;
@ -949,7 +966,7 @@ int mdb_idl_append_one( ID *ids, ID id )
}
}
ids[0]++;
if ( ids[0] >= MDB_IDL_UM_MAX ) {
if ( ids[0] >= MDB_idl_um_max ) {
ids[0] = NOID;
ids[2] = id;
} else {
@ -977,7 +994,7 @@ int mdb_idl_append( ID *a, ID *b )
ida = MDB_IDL_LAST( a );
idb = MDB_IDL_LAST( b );
if ( MDB_IDL_IS_RANGE( a ) || MDB_IDL_IS_RANGE(b) ||
a[0] + b[0] >= MDB_IDL_UM_MAX ) {
a[0] + b[0] >= MDB_idl_um_max ) {
a[2] = IDL_MAX( ida, idb );
a[1] = IDL_MIN( a[1], b[1] );
a[0] = NOID;
@ -1239,7 +1256,7 @@ int mdb_id2l_insert( ID2L ids, ID2 *id )
return -1;
}
if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
if ( ids[0].mid >= MDB_idl_um_max ) {
/* too big */
return -2;

16
servers/slapd/back-mdb/idl.h
View File

@ -21,13 +21,11 @@
* limiting factors: sizeof(ID), thread stack size
*/
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN)
#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1))
#define MDB_IDL_UM_SIZEOF (MDB_IDL_UM_SIZE * sizeof(ID))
#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE-1)
#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE-1)
extern unsigned int MDB_idl_logn;
extern unsigned int MDB_idl_db_size;
extern unsigned int MDB_idl_um_size;
extern unsigned int MDB_idl_db_max;
extern unsigned int MDB_idl_um_max;
#define MDB_IDL_IS_RANGE(ids) ((ids)[0] == NOID)
#define MDB_IDL_RANGE_SIZE (3)
@ -97,6 +95,10 @@ typedef struct IdScopes {
} IdScopes;
LDAP_BEGIN_DECL
/** Reset IDL params after changing logn */
void mdb_idl_reset();
/** Search for an ID in an ID2L.
* @param[in] ids The ID2L to search.
* @param[in] id The ID to search for.

2
servers/slapd/back-mdb/init.c
View File

@ -68,7 +68,7 @@ mdb_db_init( BackendDB *be, ConfigReply *cr )
mdb->mi_multi_lo = UINT_MAX;
be->be_private = mdb;
be->be_cf_ocs = be->bd_info->bi_cf_ocs;
be->be_cf_ocs = be->bd_info->bi_cf_ocs+1;
#ifndef MDB_MULTIPLE_SUFFIXES
SLAP_DBFLAGS( be ) |= SLAP_DBFLAG_ONE_SUFFIX;

22
servers/slapd/back-mdb/search.c
View File

@ -142,11 +142,11 @@ static int search_aliases(
Filter af;
aliases = stack; /* IDL of all aliases in the database */
curscop = aliases + MDB_IDL_DB_SIZE; /* Aliases in the current scope */
visited = curscop + MDB_IDL_DB_SIZE; /* IDs we've seen in this search */
newsubs = visited + MDB_IDL_DB_SIZE; /* New subtrees we've added */
oldsubs = newsubs + MDB_IDL_DB_SIZE; /* Subtrees added previously */
tmp = oldsubs + MDB_IDL_DB_SIZE; /* Scratch space for deref_base() */
curscop = aliases + MDB_idl_db_size; /* Aliases in the current scope */
visited = curscop + MDB_idl_db_size; /* IDs we've seen in this search */
newsubs = visited + MDB_idl_db_size; /* New subtrees we've added */
oldsubs = newsubs + MDB_idl_db_size; /* Subtrees added previously */
tmp = oldsubs + MDB_idl_db_size; /* Scratch space for deref_base() */
af.f_choice = LDAP_FILTER_EQUALITY;
af.f_ava = &aa_alias;
@ -297,7 +297,7 @@ static ID2 *scope_chunk_get( Operation *op )
ldap_pvt_thread_pool_getkey( op->o_threadctx, (void *)scope_chunk_get,
(void *)&ret, NULL );
if ( !ret ) {
ret = ch_malloc( MDB_IDL_UM_SIZE * sizeof( ID2 ));
ret = ch_malloc( MDB_idl_um_size * sizeof( ID2 ));
} else {
void *r2 = ret[0].mval.mv_data;
ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)scope_chunk_get,
@ -416,8 +416,8 @@ mdb_search( Operation *op, SlapReply *rs )
struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
ID id, cursor, nsubs, ncand, cscope;
ID lastid = NOID;
ID candidates[MDB_IDL_UM_SIZE];
ID iscopes[MDB_IDL_DB_SIZE];
ID candidates[MDB_idl_um_size];
ID iscopes[MDB_idl_db_size];
ID2 *scopes;
void *stack;
Entry *e = NULL, *base = NULL;
@ -1289,7 +1289,7 @@ static void *search_stack( Operation *op )
}
if ( !ret ) {
ret = ch_malloc( mdb->mi_search_stack_depth * MDB_IDL_UM_SIZE
ret = ch_malloc( mdb->mi_search_stack_depth * MDB_idl_um_size
* sizeof( ID ) );
if ( op->o_threadctx ) {
ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)search_stack,
@ -1366,7 +1366,7 @@ static int search_candidates(
/* Allocate IDL stack, plus 1 more for former tmp */
if ( depth+1 > mdb->mi_search_stack_depth ) {
stack = ch_malloc( (depth + 1) * MDB_IDL_UM_SIZE * sizeof( ID ) );
stack = ch_malloc( (depth + 1) * MDB_idl_um_size * sizeof( ID ) );
}
if( op->ors_deref & LDAP_DEREF_SEARCHING ) {
@ -1377,7 +1377,7 @@ static int search_candidates(
if ( rc == LDAP_SUCCESS ) {
rc = mdb_filter_candidates( op, isc->mt, f, ids,
stack, stack+MDB_IDL_UM_SIZE );
stack, stack+MDB_idl_um_size );
}
if ( depth+1 > mdb->mi_search_stack_depth ) {

10
servers/slapd/back-mdb/tools.c
View File

@ -1268,14 +1268,14 @@ mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
ID id, nid;
/* Freshly allocated, ignore it */
if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
if ( !ic->head && ic->count <= MDB_idl_db_size ) {
return 0;
}
key.mv_data = ic->kstr.bv_val;
key.mv_size = ic->kstr.bv_len;
if ( ic->count > MDB_IDL_DB_SIZE ) {
if ( ic->count > MDB_idl_db_size ) {
while ( ic->flags & WAS_FOUND ) {
rc = mdb_cursor_get( mc, &key, data, MDB_SET );
if ( rc ) {
@ -1438,7 +1438,7 @@ int mdb_tool_idl_add(
ic->flags |= WAS_FOUND;
nid = *(ID *)data.mv_data;
if ( nid == 0 ) {
ic->count = MDB_IDL_DB_SIZE+1;
ic->count = MDB_idl_db_size+1;
ic->flags |= WAS_RANGE;
} else {
size_t count;
@ -1451,11 +1451,11 @@ int mdb_tool_idl_add(
}
}
/* are we a range already? */
if ( ic->count > MDB_IDL_DB_SIZE ) {
if ( ic->count > MDB_idl_db_size ) {
ic->last = id;
continue;
/* Are we at the limit, and converting to a range? */
} else if ( ic->count == MDB_IDL_DB_SIZE ) {
} else if ( ic->count == MDB_idl_db_size ) {
if ( ic->head ) {
ic->tail->next = ax->ai_flist;
ax->ai_flist = ic->head;