parallelize dobind/search (ITS#4519)

servers/slapd/back-meta/back-meta.h

@@ -257,8 +257,8 @@ typedef struct metainfo_t {
 	unsigned		mi_flags;
 #define	li_flags		mi_flags
 /* uses flags as defined in <back-ldap/back-ldap.h> */
-#define	META_BACK_F_ONERR_STOP		0x00010000U
+#define	META_BACK_F_ONERR_STOP		(0x00010000U)
-#define	META_BACK_F_DEFER_ROOTDN_BIND	0x00020000U
+#define	META_BACK_F_DEFER_ROOTDN_BIND	(0x00020000U)
 
 #define	META_BACK_ONERR_STOP(mi)	( (mi)->mi_flags & META_BACK_F_ONERR_STOP )
 #define	META_BACK_ONERR_CONTINUE(mi)	( !META_BACK_ONERR_CONTINUE( (mi) ) )

servers/slapd/back-meta/conn.c

@@ -531,21 +531,42 @@ meta_back_retry(
 		if ( rc == LDAP_SUCCESS ) {
 			rc = meta_back_single_dobind( op, rs, mcp, candidate,
 				sendok, mt->mt_nretries, 0 );
+
+			Debug( LDAP_DEBUG_ANY,
+				"%s meta_back_retry[%d]: "
+				"meta_back_single_dobind=%d\n",
+				op->o_log_prefix, candidate, rc );
+			if ( rc == LDAP_SUCCESS ) {
+				if ( be_isroot( op )  ) {
+					LDAP_BACK_CONN_ISBOUND_SET( msc );
+				} else {
+					LDAP_BACK_CONN_ISANON_SET( msc );
+				}
+			}
         	}
 	}
 
 	if ( rc != LDAP_SUCCESS ) {
+		SlapReply		*candidates = meta_back_candidates_get( op );
+
+		candidates[ candidate ].sr_err = rc;
+
 		if ( *mcp != NULL ) {
 			if ( binding ) {
 				LDAP_BACK_CONN_BINDING_CLEAR( msc );
 			}
 			LDAP_BACK_CONN_TAINTED_SET( mc );
-			meta_back_release_conn_lock( op, mc, 0 );
+			/* only release if mandatory; otherwise
-			*mcp = NULL;
+			 * let the caller do what's best before
+			 * releasing */
+			if ( META_BACK_ONERR_STOP( mi ) ) {
+				meta_back_release_conn_lock( op, mc, 0 );
+				*mcp = NULL;
+			}
 		}
 
 		if ( sendok ) {
-			rs->sr_err = LDAP_UNAVAILABLE;
+			rs->sr_err = rc;
 			rs->sr_text = NULL;
 			send_ldap_result( op, rs );
 		}

servers/slapd/back-meta/search.c

@@ -40,6 +40,7 @@
  * NOTREADY means the search on that target has not been initialized yet
  */
 #define	META_MSGID_IGNORE	(-1)
+#define	META_MSGID_NEED_BIND	(-2)
 
 static int
 meta_send_entry(
@@ -52,9 +53,175 @@ meta_send_entry(
 typedef enum meta_search_candidate_t {
 	META_SEARCH_ERR = -1,
 	META_SEARCH_NOT_CANDIDATE,
-	META_SEARCH_CANDIDATE
+	META_SEARCH_CANDIDATE,
+	META_SEARCH_BINDING,
+	META_SEARCH_NEED_BIND
 } meta_search_candidate_t;
 
+/*
+ * meta_search_dobind_init()
+ *
+ * initiates bind for a candidate target of a search.
+ */
+static meta_search_candidate_t
+meta_search_dobind_init(
+	Operation		*op,
+	SlapReply		*rs,
+	metaconn_t		**mcp,
+	int			candidate,
+	SlapReply		*candidates )
+{
+	metaconn_t		*mc = *mcp;
+	metainfo_t		*mi = ( metainfo_t * )op->o_bd->be_private;
+	metatarget_t		*mt = &mi->mi_targets[ candidate ];
+	metasingleconn_t	*msc = &mc->mc_conns[ candidate ];
+
+	char			*binddn = "";
+	struct berval		cred = BER_BVC( "" );
+
+	int			rc;
+	int			nretries = 1;
+
+	Debug( LDAP_DEBUG_TRACE, "%s >>> meta_search_dobind_init[%d]\n", op->o_log_prefix, candidate, 0 );
+
+	meta_search_candidate_t	retcode;
+
+	/*
+	 * all the targets are already bound as pseudoroot
+	 */
+	if ( mc->mc_authz_target == META_BOUND_ALL ) {
+		return META_SEARCH_CANDIDATE;
+	}
+
+	assert( msc->msc_ld != NULL );
+
+	ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+	if ( LDAP_BACK_CONN_ISBOUND( msc ) || LDAP_BACK_CONN_ISANON( msc ) ) {
+		ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+		return META_SEARCH_CANDIDATE;
+	}
+
+	if ( LDAP_BACK_CONN_BINDING( msc ) ) {
+		ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+		candidates[ candidate ].sr_msgid = META_MSGID_NEED_BIND;
+		return META_SEARCH_NEED_BIND;
+	}
+
+	LDAP_BACK_CONN_BINDING_SET( msc );
+	ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+
+	if ( be_isroot( op ) && !BER_BVISNULL( &mt->mt_pseudorootdn ) ) {
+		binddn = mt->mt_pseudorootdn.bv_val;
+		cred = mt->mt_pseudorootpw;
+	}
+
+	/*
+	 * Otherwise an anonymous bind is performed
+	 * (note: if the target was already bound, the anonymous
+	 * bind clears the previous bind).
+	 */
+	if ( !BER_BVISNULL( &msc->msc_bound_ndn ) ) {
+		ber_memfree( msc->msc_bound_ndn.bv_val );
+		BER_BVZERO( &msc->msc_bound_ndn );
+	}
+		
+	if ( LDAP_BACK_SAVECRED( mi ) && !BER_BVISNULL( &msc->msc_cred ) ) {
+		/* destroy sensitive data */
+		memset( msc->msc_cred.bv_val, 0, msc->msc_cred.bv_len );
+		ber_memfree( msc->msc_cred.bv_val );
+		BER_BVZERO( &msc->msc_cred );
+	}
+
+retry:;
+	rc = ldap_sasl_bind( msc->msc_ld, binddn, LDAP_SASL_SIMPLE, &cred,
+			NULL, NULL, &candidates[ candidate ].sr_msgid );
+	switch ( rc ) {
+	case LDAP_SUCCESS:
+		return META_SEARCH_BINDING;
+
+	case LDAP_SERVER_DOWN:
+		if ( nretries && meta_back_retry( op, rs, mcp, candidate, LDAP_BACK_DONTSEND ) ) {
+			nretries = 0;
+			goto retry;
+		}
+
+		if ( *mcp == NULL ) {
+			retcode = META_SEARCH_ERR;
+			rs->sr_err = LDAP_UNAVAILABLE;
+			break;
+		}
+		/* fall thru */
+
+	default:
+		rs->sr_err = rc;
+		rc = slap_map_api2result( rs );
+
+		ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+		LDAP_BACK_CONN_BINDING_CLEAR( msc );
+		if ( META_BACK_ONERR_STOP( mi ) ) {
+			LDAP_BACK_CONN_TAINTED_SET( mc );
+			meta_back_release_conn( op, mc );
+			*mcp = NULL;
+
+			retcode = META_SEARCH_ERR;
+
+		} else {
+			candidates[ candidate ].sr_msgid = META_MSGID_IGNORE;
+			retcode = META_SEARCH_NOT_CANDIDATE;
+		}
+		ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+		break;
+	}
+
+	return retcode;
+}
+
+static meta_search_candidate_t
+meta_search_dobind_result(
+	Operation		*op,
+	SlapReply		*rs,
+	metaconn_t		**mcp,
+	int			candidate,
+	SlapReply		*candidates,
+	LDAPMessage		*res )
+{
+	metainfo_t		*mi = ( metainfo_t * )op->o_bd->be_private;
+	metaconn_t		*mc = *mcp;
+	metasingleconn_t	*msc = &mc->mc_conns[ candidate ];
+
+	meta_search_candidate_t	retcode = META_SEARCH_NOT_CANDIDATE;
+	int			rc;
+
+	 rc = ldap_parse_result( msc->msc_ld, res,
+		&candidates[ candidate ].sr_err,
+		NULL, NULL, NULL, NULL, 1 );
+	if ( rc == LDAP_SUCCESS ) {
+		rc = slap_map_api2result( &candidates[ candidate ] );
+	}
+
+	ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+	LDAP_BACK_CONN_BINDING_CLEAR( msc );
+	if ( rc != LDAP_SUCCESS ) {
+		if ( META_BACK_ONERR_STOP( mi ) ) {
+	        	LDAP_BACK_CONN_TAINTED_SET( mc );
+			meta_back_release_conn( op, mc );
+			*mcp = NULL;
+			retcode = META_SEARCH_ERR;
+		}
+
+	} else {
+		if ( be_isroot( op ) ) {
+			LDAP_BACK_CONN_ISBOUND_SET( msc );
+		} else {
+			LDAP_BACK_CONN_ISANON_SET( msc );
+		}
+		retcode = META_SEARCH_CANDIDATE;
+	}
+	ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+
+	return retcode;
+}
+
 static meta_search_candidate_t
 meta_back_search_start(
 	Operation		*op,
@@ -79,6 +246,8 @@ meta_back_search_start(
 	struct timeval		tv, *tvp = NULL;
 	int			nretries = 1;
 
+	Debug( LDAP_DEBUG_TRACE, "%s >>> meta_back_search_start[%d]\n", op->o_log_prefix, candidate, 0 );
+
 	/* should we check return values? */
 	if ( op->ors_deref != -1 ) {
 		ldap_set_option( msc->msc_ld, LDAP_OPT_DEREF,
@@ -157,6 +326,15 @@ meta_back_search_start(
 		}
 	}
 
+	/* initiate dobind */
+	retcode = meta_search_dobind_init( op, rs, mcp, candidate, candidates );
+
+	Debug( LDAP_DEBUG_TRACE, "%s <<< meta_search_dobind_init[%d]=%d\n", op->o_log_prefix, candidate, retcode );
+
+	if ( retcode != META_SEARCH_CANDIDATE ) {
+		return retcode;
+	}
+
 	/*
 	 * Rewrite the search base, if required
 	 */
@@ -280,7 +458,7 @@ meta_back_search( Operation *op, SlapReply *rs )
 	 * to map attrs and maybe rewrite value
 	 */
 	mc = meta_back_getconn( op, rs, NULL, LDAP_BACK_SENDERR );
-	if ( !mc || !meta_back_dobind( op, rs, mc, LDAP_BACK_SENDERR ) ) {
+	if ( !mc ) {
 		return rs->sr_err;
 	}
 
@@ -309,11 +487,17 @@ meta_back_search( Operation *op, SlapReply *rs )
 			break;
 
 		case META_SEARCH_CANDIDATE:
+		case META_SEARCH_BINDING:
+		case META_SEARCH_NEED_BIND:
 			candidates[ i ].sr_type = REP_INTERMEDIATE;
 			++ncandidates;
 			break;
 
 		case META_SEARCH_ERR:
+			savepriv = op->o_private;
+			op->o_private = (void *)i;
+			send_ldap_result( op, rs );
+			op->o_private = savepriv;
 			rc = -1;
 			goto finish;
 		}
@@ -389,6 +573,19 @@ meta_back_search( Operation *op, SlapReply *rs )
 	for ( rc = 0; ncandidates > 0; ) {
 		int	gotit = 0, doabandon = 0;
 
+		/* check time limit */
+		if ( op->ors_tlimit != SLAP_NO_LIMIT
+				&& slap_get_time() > stoptime )
+		{
+			doabandon = 1;
+			rc = rs->sr_err = LDAP_TIMELIMIT_EXCEEDED;
+			savepriv = op->o_private;
+			op->o_private = (void *)i;
+			send_ldap_result( op, rs );
+			op->o_private = savepriv;
+			goto finish;
+		}
+
 		for ( i = 0; i < mi->mi_ntargets; i++ ) {
 			metasingleconn_t	*msc = &mc->mc_conns[ i ];
 
@@ -396,6 +593,68 @@ meta_back_search( Operation *op, SlapReply *rs )
 				continue;
 			}
 
+			if ( candidates[ i ].sr_msgid == META_MSGID_NEED_BIND ) {
+				/* initiate dobind */
+				switch ( meta_search_dobind_init( op, rs, &mc, i, candidates ) )
+				{
+				case META_SEARCH_BINDING:
+				case META_SEARCH_NEED_BIND:
+					break;
+
+				case META_SEARCH_NOT_CANDIDATE:
+					/*
+					 * When no candidates are left,
+					 * the outer cycle finishes
+					 */
+					candidates[ i ].sr_msgid = META_MSGID_IGNORE;
+					--ncandidates;
+					break;
+
+				case META_SEARCH_ERR:
+					if ( META_BACK_ONERR_STOP( mi ) ) {
+						savepriv = op->o_private;
+						op->o_private = (void *)i;
+						send_ldap_result( op, rs );
+						op->o_private = savepriv;
+						goto finish;
+					}
+					break;
+
+				case META_SEARCH_CANDIDATE:
+					switch ( meta_back_search_start( op, rs, &dc, &mc, i, candidates ) )
+					{
+					case META_SEARCH_CANDIDATE:
+						goto get_result;
+
+						/* means that failed but onerr == continue */
+					case META_SEARCH_NOT_CANDIDATE:
+					case META_SEARCH_ERR:
+						candidates[ i ].sr_msgid = META_MSGID_IGNORE;
+						--ncandidates;
+
+						if ( META_BACK_ONERR_STOP( mi ) ) {
+							savepriv = op->o_private;
+							op->o_private = (void *)i;
+							send_ldap_result( op, rs );
+							op->o_private = savepriv;
+							goto finish;
+						}
+						break;
+
+					default:
+						/* impossible */
+						assert( 0 );
+						break;
+					}
+
+				default:
+					/* impossible */
+					assert( 0 );
+					break;
+				}
+				continue;
+			}
+
 			/* check for abandon */
 			if ( op->o_abandon ) {
 				break;
@@ -416,19 +675,6 @@ get_result:;
 				/* FIXME: res should not need to be freed */
 				assert( res == NULL );
 
-				/* check time limit */
-				if ( op->ors_tlimit != SLAP_NO_LIMIT
-						&& slap_get_time() > stoptime )
-				{
-					doabandon = 1;
-					rc = rs->sr_err = LDAP_TIMELIMIT_EXCEEDED;
-					savepriv = op->o_private;
-					op->o_private = (void *)i;
-					send_ldap_result( op, rs );
-					op->o_private = savepriv;
-					goto finish;
-				}
-
 				continue;
 
 			} else if ( rc == -1 ) {
@@ -443,7 +689,22 @@ really_bad:;
 						case META_SEARCH_CANDIDATE:
 							goto get_result;
 
+							/* means that failed but onerr == continue */
 						case META_SEARCH_NOT_CANDIDATE:
+							candidates[ i ].sr_msgid = META_MSGID_IGNORE;
+							--ncandidates;
+
+							if ( META_BACK_ONERR_STOP( mi ) ) {
+								savepriv = op->o_private;
+								op->o_private = (void *)i;
+								send_ldap_result( op, rs );
+								op->o_private = savepriv;
+								goto finish;
+							}
+							break;
+
+						case META_SEARCH_BINDING:
+						case META_SEARCH_NEED_BIND:
 							assert( 0 );
 
 						default:
@@ -759,6 +1020,38 @@ really_bad:;
 				candidates[ i ].sr_msgid = META_MSGID_IGNORE;
 				--ncandidates;
 
+			} else if ( rc == LDAP_RES_BIND ) {
+				meta_search_candidate_t	retcode;
+
+				retcode = meta_search_dobind_result( op, rs, &mc, i, candidates, res );
+				if ( retcode == META_SEARCH_CANDIDATE ) {
+					retcode = meta_back_search_start( op, rs, &dc, &mc, i, candidates );
+				}
+
+				switch ( retcode ) {
+				case META_SEARCH_CANDIDATE:
+					goto get_result;
+
+					/* means that failed but onerr == continue */
+				case META_SEARCH_NOT_CANDIDATE:
+				case META_SEARCH_ERR:
+					candidates[ i ].sr_msgid = META_MSGID_IGNORE;
+					--ncandidates;
+
+					if ( META_BACK_ONERR_STOP( mi ) ) {
+						savepriv = op->o_private;
+						op->o_private = (void *)i;
+						send_ldap_result( op, rs );
+						op->o_private = savepriv;
+						goto finish;
+					}
+					break;
+
+				default:
+					assert( 0 );
+					break;
+				}
+
 			} else {
 				assert( 0 );
 				goto really_bad;