Ticket 616 - High contention on computed attribute lock

Bug Description:
	On search request, some requested attributes are computed:
		- subschemasubentry
		- nscpentrywsi
		- numsubordinates
		- hassubordinates
	This is done with a linked list of registered callback.
	When sending the results, each requested attribute is evaluated against all the callbacks.
	When going through the callbacks list the thread hold the lock on the list.
	Currently all the server callbacks are registered at startup time with only
	one thread registering them (main).

	A custom plugin may register custom callback using slapi_compute_add_evaluator

	The important point is that callback are only added to the list.
	Also on standard deployment without custom plugin adding new computed attribute,
	this list is built at startup.

	The problem on computed attribute is that it can create contention between search threads
	trying to evaluate/compute the requested attributes

	The same description applies to computed filters. For example with 'views' or
	filters containing "subordinates", the filter is recomputed.
	This using a linked list of callbacks registered during startup.
	A custom plugin may register custom callback using slapi_compute_add_search_rewriter.

	In short:
		On standard deployment computed filters and computed attributes lists are unchanged
	after startup.
		Computed filters/attribute are only added to the lists.

Fix Description:

	The fix consist to not protect the access to computed filter/attribute unless a
	computed filter/attribute is added after startup.

	The fix also contains a improvement of rsearch to support filters larger than 256 char

	The fix also contains reworks according to review feedback.
	Basically using a temporary variable to hold the actual value of require_*

	The benefit is limited to 2% in terms of search throughput (8*4core machine)

https://fedorahosted.org/389/ticket/616

Reviewed by: Rich Megginson (thanks Rich !)

Platforms tested: 1*4cores fedora and 8*4cores RHEL6.4

Flag Day: no

Doc impact: no

ldap/servers/slapd/computed.c

@@ -44,6 +44,7 @@
 /* Handles computed attributes for entries as they're returned to the client */
 
 #include "slap.h"
+#include "proto-slap.h"
 
 
 /* Structure used to pass the context needed for completing a computed attribute operation */
@@ -61,8 +62,11 @@ struct _compute_evaluator {
 };
 typedef struct _compute_evaluator compute_evaluator;
 
+static PRBool startup_completed = PR_FALSE;
+
 static compute_evaluator *compute_evaluators = NULL;
 static Slapi_RWLock *compute_evaluators_lock = NULL;
+static PRBool require_compute_evaluator_lock = PR_FALSE;
 
 static int
 compute_stock_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi_compute_output_t outputfn);
@@ -75,6 +79,7 @@ typedef struct _compute_rewriter compute_rewriter;
 
 static compute_rewriter *compute_rewriters = NULL;
 static Slapi_RWLock *compute_rewriters_lock = NULL;
+static PRBool require_compute_rewriters_lock = PR_FALSE;
 
 /* Function called by evaluators to have the value output */
 static int
@@ -83,17 +88,36 @@ compute_output_callback(computed_attr_context *c,Slapi_Attr *a , Slapi_Entry *e)
 	return encode_attr (c->pb, c->ber, e, a, c->attrsonly, c->requested_type);
 }
 
+static
+int compute_call_evaluators_nolock(computed_attr_context *c,slapi_compute_output_t outfn,char *type,Slapi_Entry *e)
+{
+        int rc = -1;
+        compute_evaluator *current = NULL;
+        
+        for (current = compute_evaluators; (current != NULL) && (-1 == rc); current = current->next) {
+                rc = (*(current->function))(c,type,e,outfn);
+        }
+        return rc;
+}
+
 static int
 compute_call_evaluators(computed_attr_context *c,slapi_compute_output_t outfn,char *type,Slapi_Entry *e)
 {
 	int rc = -1;
-	compute_evaluator *current = NULL;
+        int need_lock = require_compute_evaluator_lock;
+        
 	/* Walk along the list (locked) calling the evaluator functions util one says yes, an error happens, or we finish */
-	slapi_rwlock_rdlock(compute_evaluators_lock);
-	for (current = compute_evaluators; (current != NULL) && (-1 == rc); current = current->next) {
-		rc = (*(current->function))(c,type,e,outfn);
-	}
-	slapi_rwlock_unlock(compute_evaluators_lock);
+
+        if (need_lock) {
+                slapi_rwlock_rdlock(compute_evaluators_lock);
+        }
+       
+        rc = compute_call_evaluators_nolock(c, outfn, type, e);
+        
+        if (need_lock) {
+                slapi_rwlock_unlock(compute_evaluators_lock);
+        }   
+
 	return rc;
 }
 
@@ -132,25 +156,54 @@ compute_stock_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi
 	return rc; /* I see no ships */
 }
 
+static void
+compute_add_evaluator_nolock(slapi_compute_callback_t function, compute_evaluator *new_eval)
+{
+    new_eval->next = compute_evaluators;
+    new_eval->function = function;
+    compute_evaluators = new_eval;
+}
 int slapi_compute_add_evaluator(slapi_compute_callback_t function)
 {
 	int rc = 0;
 	compute_evaluator *new_eval = NULL;
 	PR_ASSERT(NULL != function);
 	PR_ASSERT(NULL != compute_evaluators_lock);
-	slapi_rwlock_wrlock(compute_evaluators_lock);
+        if (startup_completed) {
+            /* We are now in multi-threaded and we still add
+             * a attribute evaluator.
+             * switch to use locking mechanimsm
+             */
+            require_compute_evaluator_lock = PR_TRUE;
+        }
+
 	new_eval = (compute_evaluator *)slapi_ch_calloc(1,sizeof (compute_evaluator));
 	if (NULL == new_eval) {
 		rc = ENOMEM;
 	} else {
-		new_eval->next = compute_evaluators;
-		new_eval->function = function;
-		compute_evaluators = new_eval;
+                int need_lock = require_compute_evaluator_lock;
+                
+                if (need_lock) {
+                    slapi_rwlock_wrlock(compute_evaluators_lock);
+                }
+                
+                compute_add_evaluator_nolock(function, new_eval);
+                
+                if (need_lock) {
+                    slapi_rwlock_unlock(compute_evaluators_lock);
+                }
 	}
-	slapi_rwlock_unlock(compute_evaluators_lock);
+
 	return rc;
 }
 
+/* Called when */
+void
+compute_plugins_started()
+{
+    startup_completed = PR_TRUE;
+}
+
 /* Call this on server startup, before the first LDAP operation is serviced */
 int compute_init()
 {
@@ -200,6 +253,14 @@ int compute_terminate()
 	return 0;
 }
 
+static void
+compute_add_search_rewrite_nolock(slapi_search_rewrite_callback_t function, compute_rewriter *new_rewriter)
+{
+    new_rewriter->next = compute_rewriters;
+    new_rewriter->function = function;
+    compute_rewriters = new_rewriter;
+}
+
 /* Functions dealing with re-writing of search filters */
 
 int slapi_compute_add_search_rewriter(slapi_search_rewrite_callback_t function)
@@ -208,36 +269,66 @@ int slapi_compute_add_search_rewriter(slapi_search_rewrite_callback_t function)
 	compute_rewriter *new_rewriter = NULL;
 	PR_ASSERT(NULL != function);
 	PR_ASSERT(NULL != compute_rewriters_lock);
+        if (startup_completed) {
+            /* We are now in multi-threaded and we still add
+             * a filter rewriter.
+             * switch to use locking mechanimsm
+             */
+            require_compute_rewriters_lock = PR_TRUE;
+        }
 	new_rewriter = (compute_rewriter *)slapi_ch_calloc(1,sizeof (compute_rewriter));
 	if (NULL == new_rewriter) {
 		rc = ENOMEM;
 	} else {
-		slapi_rwlock_wrlock(compute_rewriters_lock);
-		new_rewriter->next = compute_rewriters;
-		new_rewriter->function = function;
-		compute_rewriters = new_rewriter;
-		slapi_rwlock_unlock(compute_rewriters_lock);
+                int need_lock = require_compute_rewriters_lock;
+                
+		if (need_lock) {
+                    slapi_rwlock_wrlock(compute_rewriters_lock);
+                }
+                
+                compute_add_search_rewrite_nolock(function, new_rewriter);
+                
+                if (need_lock) {
+                    slapi_rwlock_unlock(compute_rewriters_lock);
+                }
 	}
 	return rc;
 }
 
-int	compute_rewrite_search_filter(Slapi_PBlock *pb)
+static
+int compute_rewrite_search_filter_nolock(Slapi_PBlock *pb)
+{
+        int rc = -1;
+        compute_rewriter *current = NULL;
+        
+        for (current = compute_rewriters; (current != NULL) && (-1 == rc); current = current->next) {
+                rc = (*(current->function))(pb);
+                /* Meaning of the return code :
+                   -1 : keep looking
+                    0 : rewrote OK
+                    1 : refuse to do this search
+                    2 : operations error
+                */
+        }
+        return(rc);
+}
+
+int compute_rewrite_search_filter(Slapi_PBlock *pb)
 {
 	/* Iterate through the listed rewriters until one says it matched */
 	int rc = -1;
-	compute_rewriter *current = NULL;
+        int need_lock = require_compute_rewriters_lock;
+
 	/* Walk along the list (locked) calling the evaluator functions util one says yes, an error happens, or we finish */
-	slapi_rwlock_rdlock(compute_rewriters_lock);
-	for (current = compute_rewriters; (current != NULL) && (-1 == rc); current = current->next) {
-		rc = (*(current->function))(pb);
-		/* Meaning of the return code :
-		 -1 : keep looking
-		  0 : rewrote OK
-		  1 : refuse to do this search
-		  2 : operations error
-		 */
-	}
-	slapi_rwlock_unlock(compute_rewriters_lock);
+        if (need_lock) {
+                slapi_rwlock_rdlock(compute_rewriters_lock);
+        }
+        
+        rc = compute_rewrite_search_filter_nolock(pb);
+        
+        if (need_lock) {
+                slapi_rwlock_unlock(compute_rewriters_lock);
+        }
 	return rc;
 
 }

ldap/servers/slapd/main.c

@@ -1178,7 +1178,8 @@ main( int argc, char **argv)
 		pw_exp_init ();
 
 		plugin_print_lists();
-		plugin_startall(argc, argv, 1 /* Start Backends */, 1 /* Start Globals */); 
+		plugin_startall(argc, argv, 1 /* Start Backends */, 1 /* Start Globals */);
+                compute_plugins_started();
 		if (housekeeping_start((time_t)0, NULL) == NULL) {
 			return_value = 1;
 			goto cleanup;

ldap/servers/slapd/proto-slap.h

@@ -251,6 +251,7 @@ void do_compare( Slapi_PBlock *pb );
 int compute_attribute(char *type, Slapi_PBlock *pb,BerElement *ber,Slapi_Entry *e,int attrsonly,char *requested_type);
 int compute_init();
 int compute_terminate();
+void compute_plugins_started();
 
 
 /*

ldap/servers/slapd/tools/rsearch/nametable.c

@@ -152,8 +152,8 @@ int nt_load(NameTable *nt, const char *filename)
     if (!fd) return 0;
 
     while (PR_Available(fd) > 0) {
-	char temp[256], *s;
-	if (PR_GetLine(fd, temp, 256)) break;
+	char temp[4096], *s;
+	if (PR_GetLine(fd, temp, sizeof(temp))) break;
 	s = strdup(temp);
 	if (!s) break;
 	if (!nt_push(nt, s)) break;

ldap/servers/slapd/tools/rsearch/searchthread.c

@@ -285,7 +285,7 @@ static int st_bind(SearchThread *st)
                     return 0;
                 }
             } else if (uid) {
-                char filterBuffer[100];
+                char filterBuffer[4096];
                 char *pFilter;
 				char *filterTemplate = (uidFilter) ? uidFilter : "(uid=%s)";
                 struct timeval timeout;