mirror/openldap
2
0
Fork 0
mirror of https://git.openldap.org/openldap/openldap.git synced 2025-01-18 11:05:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

adaptive caching implementation

Browse Source
This commit is contained in:
Jong Hyuk Choi 2004-12-14 17:22:22 +00:00
parent d6410c9207
commit 273052b573
7 changed files with 181 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
servers/slapd/back-bdb/back-bdb.h
View File

@ -114,6 +114,10 @@ typedef struct bdb_entry_info {
#endif
Entry *bei_e;
Avlnode *bei_kids;
#ifdef SLAP_ZONE_ALLOC
struct bdb_info *bei_bdb;
int bei_zseq;
#endif
ldap_pvt_thread_mutex_t bei_kids_mutex;
struct bdb_entry_info *bei_lrunext; /* for cache lru list */
@ -135,6 +139,9 @@ typedef struct bdb_cache {
ldap_pvt_thread_rdwr_t c_rwlock;
ldap_pvt_thread_mutex_t lru_mutex;
u_int32_t c_locker; /* used by lru cleaner */
#ifdef SLAP_ZONE_ALLOC
void *c_zctx;
#endif
} Cache;
#define CACHE_READ_LOCK 0

65
servers/slapd/back-bdb/cache.c
View File

@ -230,6 +230,9 @@ bdb_entryinfo_add_internal(
#ifdef BDB_HIER
ei2->bei_rdn = ei->bei_rdn;
#endif
#ifdef SLAP_ZONE_ALLOC
ei2->bei_bdb = bdb;
#endif
/* Add to cache ID tree */
if (avl_insert( &bdb->bi_cache.c_idtree, ei2, bdb_id_cmp, avl_dup_error )) {
@ -393,6 +396,9 @@ hdb_cache_find_parent(
ein->bei_nrdn = ei.bei_nrdn;
ein->bei_rdn = ei.bei_rdn;
ein->bei_ckids = ei.bei_ckids;
#ifdef SLAP_ZONE_ALLOC
ein->bei_bdb = bdb;
#endif
ei.bei_ckids = 0;
/* This node is not fully connected yet */
@ -530,27 +536,32 @@ bdb_cache_lru_add(
elprev = elru->bei_lruprev;
/* Too many probes, not enough idle, give up */
if (i > 10) break;
if (i > 10)
break;
/* If we can successfully writelock it, then
* the object is idle.
*/
if ( bdb_cache_entry_db_lock( bdb->bi_dbenv, bdb->bi_cache.c_locker, elru, 1, 1,
lockp ) == 0 ) {
if ( bdb_cache_entry_db_lock( bdb->bi_dbenv,
bdb->bi_cache.c_locker, elru, 1, 1, lockp ) == 0 ) {
int stop = 0;
/* If there's no entry, or this node is in
* the process of linking into the cache,
* or this node is being deleted, skip it.
*/
if ( !elru->bei_e || (elru->bei_state &
if ( !elru->bei_e || (elru->bei_state &
( CACHE_ENTRY_NOT_LINKED | CACHE_ENTRY_DELETED ))) {
bdb_cache_entry_db_unlock( bdb->bi_dbenv, lockp );
continue;
}
LRU_DELETE( &bdb->bi_cache, elru );
elru->bei_e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( bdb, elru->bei_e, elru->bei_zseq );
#else
bdb_entry_return( elru->bei_e );
#endif
elru->bei_e = NULL;
ldap_pvt_thread_rdwr_wlock( &bdb->bi_cache.c_rwlock );
--bdb->bi_cache.c_cursize;
@ -606,6 +617,9 @@ bdb_cache_find_id(
ei.bei_id = id;
#ifdef SLAP_ZONE_ALLOC
slap_zh_rlock(bdb->bi_cache.c_zctx);
#endif
/* If we weren't given any info, see if we have it already cached */
if ( !*eip ) {
again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
@ -649,7 +663,11 @@ again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
&ep->e_nname, eip );
if ( *eip ) islocked = 1;
if ( rc ) {
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( bdb, ep, (*eip)->bei_zseq );
#else
bdb_entry_return( ep );
#endif
ep = NULL;
}
}
@ -665,7 +683,15 @@ again: ldap_pvt_thread_rdwr_rlock( &bdb->bi_cache.c_rwlock );
rc = DB_NOTFOUND;
} else {
/* Make sure only one thread tries to load the entry */
load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
load1:
#ifdef SLAP_ZONE_ALLOC
if ((*eip)->bei_e && !slap_zn_validate(
bdb->bi_cache.c_zctx, (*eip)->bei_e, (*eip)->bei_zseq)) {
(*eip)->bei_e = NULL;
(*eip)->bei_zseq = 0;
}
#endif
if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
load = 1;
(*eip)->bei_state |= CACHE_ENTRY_LOADING;
}
@ -711,6 +737,9 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
bdb_fix_dn( ep, 0 );
#endif
(*eip)->bei_e = ep;
#ifdef SLAP_ZONE_ALLOC
(*eip)->bei_zseq = *((ber_len_t *)ep - 2);
#endif
ep = NULL;
}
(*eip)->bei_state ^= CACHE_ENTRY_LOADING;
@ -771,7 +800,11 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
bdb_cache_entryinfo_unlock( *eip );
}
if ( ep ) {
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( bdb, ep, (*eip)->bei_zseq );
#else
bdb_entry_return( ep );
#endif
}
if ( rc == 0 ) {
@ -800,6 +833,12 @@ load1: if ( !(*eip)->bei_e && !((*eip)->bei_state & CACHE_ENTRY_LOADING)) {
}
}
#ifdef SLAP_ZONE_ALLOC
if (rc == 0 && (*eip)->bei_e) {
slap_zn_rlock(bdb->bi_cache.c_zctx, (*eip)->bei_e);
}
slap_zh_runlock(bdb->bi_cache.c_zctx);
#endif
return rc;
}
@ -865,7 +904,11 @@ bdb_cache_add(
/* bdb_csn_commit can cause this when adding the database root entry */
if ( new->bei_e ) {
new->bei_e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( bdb, new->bei_e, new->bei_zseq );
#else
bdb_entry_return( new->bei_e );
#endif
}
new->bei_e = e;
e->e_private = new;
@ -1043,7 +1086,11 @@ bdb_cache_delete_cleanup(
{
if ( ei->bei_e ) {
ei->bei_e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq );
#else
bdb_entry_return( ei->bei_e );
#endif
ei->bei_e = NULL;
}
@ -1123,7 +1170,11 @@ bdb_entryinfo_release( void *data )
}
if ( ei->bei_e ) {
ei->bei_e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( ei->bei_bdb, ei->bei_e, ei->bei_zseq );
#else
bdb_entry_return( ei->bei_e );
#endif
}
bdb_cache_entryinfo_destroy( ei );
}
@ -1310,7 +1361,11 @@ bdb_cache_delete_entry(
if ( ei->bei_e && !(ei->bei_state & CACHE_ENTRY_NOT_LINKED )) {
LRU_DELETE( &bdb->bi_cache, ei );
ei->bei_e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return( bdb, ei->bei_e, ei->bei_zseq );
#else
bdb_entry_return( ei->bei_e );
#endif
ei->bei_e = NULL;
--bdb->bi_cache.c_cursize;
}

71
servers/slapd/back-bdb/id2entry.c
View File

@ -121,7 +121,11 @@ int bdb_id2entry(
DBT2bv( &data, &bv );
rc = entry_decode( &bv, e );
#ifdef SLAP_ZONE_ALLOC
rc = entry_decode(&bv, e, bdb->bi_cache.c_zctx);
#else
rc = entry_decode(&bv, e);
#endif
if( rc == 0 ) {
(*e)->e_id = id;
@ -129,8 +133,13 @@ int bdb_id2entry(
/* only free on error. On success, the entry was
* decoded in place.
*/
ch_free( data.data );
#ifndef SLAP_ZONE_ALLOC
ch_free(data.data);
#endif
}
#ifdef SLAP_ZONE_ALLOC
ch_free(data.data);
#endif
return rc;
}
@ -157,9 +166,23 @@ int bdb_id2entry_delete(
return rc;
}
#ifdef SLAP_ZONE_ALLOC
int bdb_entry_return(
Entry *e )
struct bdb_info *bdb,
Entry *e,
int zseq
)
#else
int bdb_entry_return(
Entry *e
)
#endif
{
#ifdef SLAP_ZONE_ALLOC
if (!slap_zn_validate(bdb->bi_cache.c_zctx, e, zseq)) {
return 0;
}
#endif
/* Our entries are allocated in two blocks; the data comes from
* the db itself and the Entry structure and associated pointers
* are allocated in entry_decode. The db data pointer is saved
@ -168,10 +191,11 @@ int bdb_entry_return(
* is when an entry has been modified, in which case we also need
* to free e_attrs.
*/
#ifdef LDAP_COMP_MATCH
comp_tree_free( e->e_attrs );
#endif
if( !e->e_bv.bv_val ) { /* A regular entry, from do_add */
if( !e->e_bv.bv_val ) { /* Entry added by do_add */
entry_free( e );
return 0;
}
@ -187,6 +211,7 @@ int bdb_entry_return(
e->e_name.bv_val = NULL;
e->e_nname.bv_val = NULL;
}
#ifndef SLAP_ZONE_ALLOC
#ifndef BDB_HIER
/* In tool mode the e_bv buffer is realloc'd, leave it alone */
if( !(slapMode & SLAP_TOOL_MODE) ) {
@ -194,18 +219,24 @@ int bdb_entry_return(
}
#else
free( e->e_bv.bv_val );
#endif
#endif /* BDB_HIER */
#endif /* !SLAP_ZONE_ALLOC */
#ifdef SLAP_ZONE_ALLOC
slap_zn_free( e, bdb->bi_cache.c_zctx );
#else
free( e );
#endif
return 0;
}
int bdb_entry_release(
Operation *o,
Operation *op,
Entry *e,
int rw )
{
struct bdb_info *bdb = (struct bdb_info *) o->o_bd->be_private;
struct bdb_info *bdb = (struct bdb_info *) op->o_bd->be_private;
struct bdb_op_info *boi = NULL;
/* slapMode : SLAP_SERVER_MODE, SLAP_TOOL_MODE,
@ -214,11 +245,15 @@ int bdb_entry_release(
if ( slapMode == SLAP_SERVER_MODE ) {
/* If not in our cache, just free it */
if ( !e->e_private ) {
#ifdef SLAP_ZONE_ALLOC
return bdb_entry_return( bdb, e, -1 );
#else
return bdb_entry_return( e );
#endif
}
/* free entry and reader or writer lock */
if ( o ) {
boi = (struct bdb_op_info *)o->o_private;
if ( op ) {
boi = (struct bdb_op_info *)op->o_private;
}
/* lock is freed with txn */
if ( !boi || boi->boi_txn ) {
@ -231,20 +266,32 @@ int bdb_entry_release(
bdb_cache_return_entry_rw( bdb->bi_dbenv, &bdb->bi_cache,
e, rw, &bli->bli_lock );
prev->bli_next = bli->bli_next;
o->o_tmpfree( bli, o->o_tmpmemctx );
op->o_tmpfree( bli, op->o_tmpmemctx );
break;
}
}
if ( !boi->boi_locks ) {
o->o_tmpfree( boi, o->o_tmpmemctx );
o->o_private = NULL;
op->o_tmpfree( boi, op->o_tmpmemctx );
op->o_private = NULL;
}
}
} else {
#ifdef SLAP_ZONE_ALLOC
int zseq = -1;
if (e->e_private != NULL) {
BEI(e)->bei_e = NULL;
zseq = BEI(e)->bei_zseq;
}
#else
if (e->e_private != NULL)
BEI(e)->bei_e = NULL;
#endif
e->e_private = NULL;
#ifdef SLAP_ZONE_ALLOC
bdb_entry_return ( bdb, e, zseq );
#else
bdb_entry_return ( e );
#endif
}
return 0;

10
servers/slapd/back-bdb/init.c
View File

@ -192,6 +192,16 @@ bdb_db_open( BackendDB *be )
/* One long-lived TXN per thread, two TXNs per write op */
bdb->bi_dbenv->set_tx_max( bdb->bi_dbenv, connection_pool_max * 3 );
#ifdef SLAP_ZONE_ALLOC
if ( bdb->bi_cache.c_maxsize ) {
bdb->bi_cache.c_zctx = slap_zn_mem_create(
SLAP_ZONE_INITSIZE,
SLAP_ZONE_MAXSIZE,
SLAP_ZONE_DELTA,
SLAP_ZONE_SIZE);
}
#endif
if ( bdb->bi_idl_cache_max_size ) {
bdb->bi_idl_tree = NULL;
ldap_pvt_thread_rdwr_init( &bdb->bi_idl_tree_rwlock );

9
servers/slapd/back-bdb/proto-bdb.h
View File

@ -189,11 +189,14 @@ int bdb_id2entry_delete(
DB_TXN *tid,
Entry *e);
#ifdef SLAP_ZONE_ALLOC
#else
int bdb_id2entry(
BackendDB *be,
DB_TXN *tid,
ID id,
Entry **e);
#endif
#define bdb_entry_free BDB_SYMBOL(entry_free)
#define bdb_entry_return BDB_SYMBOL(entry_return)
@ -201,7 +204,11 @@ int bdb_id2entry(
#define bdb_entry_get BDB_SYMBOL(entry_get)
void bdb_entry_free ( Entry *e );
#ifdef SLAP_ZONE_ALLOC
int bdb_entry_return( struct bdb_info *bdb, Entry *e, int seqno );
#else
int bdb_entry_return( Entry *e );
#endif
BI_entry_release_rw bdb_entry_release;
BI_entry_get_rw bdb_entry_get;
@ -427,7 +434,7 @@ void bdb_unlocked_cache_return_entry_rw( Cache *cache, Entry *e, int rw );
#define bdb_cache_add BDB_SYMBOL(cache_add)
#define bdb_cache_children BDB_SYMBOL(cache_children)
#define bdb_cache_delete BDB_SYMBOL(cache_delete)
#define bdb_cache_delete_cleanup BDB_SYMBOL(cache_delete_cleanup)
#define bdb_cache_delete_cleanup BDB_SYMBOL(cache_delete_cleanup)
#define bdb_cache_find_id BDB_SYMBOL(cache_find_id)
#define bdb_cache_find_info BDB_SYMBOL(cache_find_info)
#define bdb_cache_find_ndn BDB_SYMBOL(cache_find_ndn)

34
servers/slapd/back-bdb/search.c
View File

@ -842,6 +842,9 @@ dn2entry_retry:
? get_entry_referrals( op, matched )
: NULL;
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, matched);
#endif
bdb_cache_return_entry_r (bdb->bi_dbenv, &bdb->bi_cache,
matched, &lock);
matched = NULL;
@ -853,6 +856,9 @@ dn2entry_retry:
}
} else {
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, matched);
#endif
rs->sr_ref = referral_rewrite( default_referral,
NULL, &sop->o_req_dn, sop->oq_search.rs_scope );
}
@ -882,6 +888,9 @@ dn2entry_retry:
ber_dupbv( &matched_dn, &e->e_name );
erefs = get_entry_referrals( op, e );
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock );
e = NULL;
@ -913,6 +922,9 @@ dn2entry_retry:
( test_filter( op, e, get_assertion( op )) != LDAP_COMPARE_TRUE ))
{
rs->sr_err = LDAP_ASSERTION_FAILED;
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
send_ldap_result( sop, rs );
return 1;
}
@ -930,6 +942,9 @@ dn2entry_retry:
base.e_nname = realbase;
base.e_id = e->e_id;
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
if ( e != &e_root ) {
bdb_cache_return_entry_r(bdb->bi_dbenv, &bdb->bi_cache, e, &lock);
}
@ -1230,7 +1245,7 @@ loop_begin:
#ifdef BDB_PSEARCH
if (!IS_PSEARCH) {
#endif
id2entry_retry:
fetch_entry_retry:
/* get the entry with reader lock */
ei = NULL;
rs->sr_err = bdb_cache_find_id( op, ltid,
@ -1244,7 +1259,7 @@ id2entry_retry:
} else if ( rs->sr_err == DB_LOCK_DEADLOCK
|| rs->sr_err == DB_LOCK_NOTGRANTED )
{
goto id2entry_retry;
goto fetch_entry_retry;
}
if ( ei && rs->sr_err == LDAP_SUCCESS ) {
@ -1471,6 +1486,9 @@ id2entry_retry:
) {
#ifdef BDB_PSEARCH
if (!IS_PSEARCH) {
#endif
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r( bdb->bi_dbenv,
&bdb->bi_cache, e, &lock );
@ -1670,6 +1688,9 @@ post_search_no_entry:
case -1: /* connection closed */
#ifdef BDB_PSEARCH
if (!IS_PSEARCH) {
#endif
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r(bdb->bi_dbenv,
&bdb->bi_cache, e, &lock);
@ -1697,6 +1718,9 @@ loop_continue:
if (!IS_PSEARCH) {
if (!(IS_POST_SEARCH &&
entry_sync_state == LDAP_SYNC_DELETE)) {
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r( bdb->bi_dbenv,
&bdb->bi_cache, e , &lock );
if ( sop->o_nocaching ) {
@ -1705,6 +1729,9 @@ loop_continue:
}
}
#else
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r( bdb->bi_dbenv,
&bdb->bi_cache, e , &lock );
#endif
@ -1867,6 +1894,9 @@ done:
if( !IS_PSEARCH && e != NULL ) {
/* free reader lock */
#ifdef SLAP_ZONE_ALLOC
slap_zn_runlock(bdb->bi_cache.c_zctx, e);
#endif
bdb_cache_return_entry_r( bdb->bi_dbenv, &bdb->bi_cache, e, &lock );
}
ber_bvfree( search_context_csn );

5
servers/slapd/back-bdb/tools.c
View File

@ -176,7 +176,12 @@ Entry* bdb_tool_entry_get( BackendDB *be, ID id )
#ifndef BDB_HIER
DBT2bv( &data, &bv );
#ifdef SLAP_ZONE_ALLOC
/* FIXME: will add ctx later */
rc = entry_decode( &bv, &e, NULL );
#else
rc = entry_decode( &bv, &e );
#endif
if( rc == LDAP_SUCCESS ) {
e->e_id = id;