Trac Ticket #346 - Slow ldapmodify operation time for large quantities of multi-valued attribute values

https://fedorahosted.org/389/ticket/346

Fix description:
1) Set SLAPI_ATTR_FLAG_NORMALIZED to a_flag for the already
   normalized dn values.  Then, let slapi_attr_value_find
   pass the flag to plugin_call_syntax_filter_ava, where
   dn normalization is skipped based on the knowledge.
2) To avoid the confusion, rename value_normalize_value and
   valuearray_normalize_value to value_dn_normalize_value and
   valuearray_dn_normalize_value, respectively.
3) In slapi_has8thBit, instead of checking the 8th bit one byte
   by one byte, check 4 bytes at one time.
(cherry picked from commit c0151f78509c35250095da6e2785842337963008)
(cherry picked from commit 89f13859bf753ab8fc858a5a005cdce920be2359)

ldap/servers/plugins/syntaxes/nameoptuid.c

@@ -170,8 +170,7 @@ nameoptuid_filter_ava(
 			syntax |= SYNTAX_NORM_FILT;
 		}
 	}
-	return( string_filter_ava( bvfilter, bvals, syntax,
-	    ftype, retVal ) );
+	return( string_filter_ava( bvfilter, bvals, syntax, ftype, retVal ) );
 }
 
 

ldap/servers/slapd/attr.c

@@ -411,7 +411,11 @@ slapi_attr_value_find( const Slapi_Attr *a, const struct berval *v )
 
 	ava.ava_type = a->a_type;
 	ava.ava_value = *v;
-	ava.ava_private = NULL;
+	if (a->a_flags & SLAPI_ATTR_FLAG_NORMALIZED) {
+	    ava.ava_private = &a->a_flags;
+	} else {
+	    ava.ava_private = NULL;
+	}
 	return(plugin_call_syntax_filter_ava( a, LDAP_FILTER_EQUALITY, &ava ));
 }
 

ldap/servers/slapd/attrlist.c

@@ -291,7 +291,8 @@ int attrlist_replace(Slapi_Attr **alist, const char *type, struct berval **vals)
         attrlist_find_or_create(alist, type, &a);
         valuearray_init_bervalarray(vals, &values);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
-            valuearray_normalize_value(values);
+            valuearray_dn_normalize_value(values);
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
         }
         rc = attr_replace(*a, values);
     }
@@ -317,7 +318,8 @@ int attrlist_replace_with_flags(Slapi_Attr **alist, const char *type, struct ber
         attrlist_find_or_create(alist, type, &a);
         valuearray_init_bervalarray_with_flags(vals, &values, flags);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
-            valuearray_normalize_value(values);
+            valuearray_dn_normalize_value(values);
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
         }
         rc = attr_replace(*a, values);
     }

ldap/servers/slapd/entry.c

@@ -2132,7 +2132,6 @@ slapi_entry_dup( const Slapi_Entry *e )
 
 	/* Copy flags as well */
 	ec->e_flags = e->e_flags;
-	
 	ENTRY_DUMP(ec,"slapi_entry_dup");
 	return( ec );
 }
@@ -3359,7 +3358,8 @@ slapi_entry_add_values_sv(Slapi_Entry *e,
 		Slapi_Attr **alist= &e->e_attrs;
 		attrlist_find_or_create(alist, type, &a);
 		if (slapi_attr_is_dn_syntax_attr(*a)) {
-			valuearray_normalize_value(vals);
+			valuearray_dn_normalize_value(vals);
+			(*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
 		}
 		rc= attr_add_valuearray(*a,vals,slapi_entry_get_dn_const(e));
     }

ldap/servers/slapd/entrywsi.c

@@ -441,7 +441,8 @@ entry_add_present_values_wsi(Slapi_Entry *e, const char *type, struct berval **b
 		a->a_flags |= flags;
 		/* Check if the type of the to-be-added values has DN syntax or not. */
 		if (slapi_attr_is_dn_syntax_attr(a)) {
-			valuearray_normalize_value(valuestoadd);
+			valuearray_dn_normalize_value(valuestoadd);
+			a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
 		}
 		if(urp)
 		{
@@ -562,7 +563,8 @@ entry_delete_present_values_wsi(Slapi_Entry *e, const char *type, struct berval
 			/* Check if the type of the to-be-deleted values has DN syntax 
 			 * or not. */
 			if (slapi_attr_is_dn_syntax_attr(a)) {
-				valuearray_normalize_value(valuestodelete);
+				valuearray_dn_normalize_value(valuestodelete);
+				a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
 			}
 			if(urp)
 			{

ldap/servers/slapd/slapi-private.h

@@ -369,7 +369,7 @@ const CSN *value_get_csn( const Slapi_Value *value, CSNType t );
 const CSNSet *value_get_csnset ( const Slapi_Value *value);
 Slapi_Value *value_remove_csn( Slapi_Value *value, CSNType t);
 int value_contains_csn( const Slapi_Value *value, CSN *csn);
-int value_normalize_value(Slapi_Value *value);
+int value_dn_normalize_value(Slapi_Value *value);
 
 /* dn.c */
 /* this functions should only be used for dns allocated on the stack */
@@ -835,7 +835,7 @@ void valuearray_add_value_fast(Slapi_Value ***vals, Slapi_Value *addval, int nva
 void valuearray_add_valuearray( Slapi_Value ***vals, Slapi_Value **addvals, PRUint32 flags );
 void valuearray_add_valuearray_fast( Slapi_Value ***vals, Slapi_Value **addvals, int nvals, int naddvals, int *maxvals, int exact, int passin );
 int valuearray_find(const Slapi_Attr *a, Slapi_Value **va, const Slapi_Value *v);
-int valuearray_normalize_value(Slapi_Value **vals);
+int valuearray_dn_normalize_value(Slapi_Value **vals);
 
 /*
  * proxyauth.c

ldap/servers/slapd/utf8compare.c

@@ -60,14 +60,27 @@ typedef struct sUpperLowerTbl {
 int
 slapi_has8thBit(unsigned char *s)
 {
-    unsigned char *p, *tail;
-    tail = s + strlen((char *)s);
-    for (p = s; p < tail; p++) {
+#define MY8THBITWIDTH 4 /* sizeof(PRUint32) */
+#define MY8THBITFILTER 0x80808080
+    unsigned char *p, *stail, *ltail;
+    PRUint32 *uip;
+    size_t len = strlen((const char *)s);
+    ltail = s + len;
+    stail = ltail - (len % MY8THBITWIDTH);
+    for (p = s; p < stail; p += MY8THBITWIDTH) {
+        uip = (PRUint32 *)p;
+        if (MY8THBITFILTER & *uip) {
+             return 1;
+        }
+    }
+    for (; p < ltail; p++) {
         if (0x80 & *p) {
              return 1;
         }
     }
     return 0;
+#undef MY8THBITWIDTH
+#undef MY8THBITFILTER
 }
 
 /*

ldap/servers/slapd/value.c

@@ -581,7 +581,7 @@ value_dump( const Slapi_Value *value, const char *text)
 #endif
 
 int
-value_normalize_value(Slapi_Value *value)
+value_dn_normalize_value(Slapi_Value *value)
 {
 	Slapi_DN *sdn = NULL;
 	int rc = 0;
@@ -595,6 +595,7 @@ value_normalize_value(Slapi_Value *value)
 		value->bv.bv_val = slapi_ch_strdup(slapi_sdn_get_dn(sdn));
 		value->bv.bv_len = slapi_sdn_get_ndn_len(sdn);
 		slapi_sdn_free(&sdn);
+		slapi_value_set_flags(value, SLAPI_ATTR_FLAG_NORMALIZED);
 	} else {
 		rc = 1;
 		slapi_ch_free((void **)&sdn); /* free just Slapi_DN */

ldap/servers/slapd/valueset.c

@@ -1444,13 +1444,13 @@ valueset_update_csn_for_valuearray(Slapi_ValueSet *vs, const Slapi_Attr *a, Slap
 }
 
 int
-valuearray_normalize_value(Slapi_Value **vals)
+valuearray_dn_normalize_value(Slapi_Value **vals)
 {
 	int rc = 0;
 	Slapi_Value **vp = NULL;
 
 	for (vp = vals; vp && *vp; vp++) {
-		rc |= value_normalize_value(*vp);
+		rc |= value_dn_normalize_value(*vp);
 	}
 
 	return rc;