Trac Ticket #346 - Slow ldapmodify operation time for large

                   quantities of multi-valued attribute values

https://fedorahosted.org/389/ticket/346

Fix Description:
Fix in "commit c0151f78509c35250095da6e2785842337963008" takes
advantage of the knowledge on normalized attribute DNs.  When
generating index keys from the attribute value, it has to be
case normalized, but the attribute DN is normalized w/o the
cases lowered.  This patch introduces attribute flags to
distinguish the 2 cases:
  SLAPI_ATTR_FLAG_NORMALIZED_CES - normalized but not case-
                                   normalized.
  SLAPI_ATTR_FLAG_NORMALIZED_CIS - case-normalized
And SLAPI_ATTR_FLAG_NORMALIZED is the combination of the 2
macros.
(cherry picked from commit 63d68b122dc5efe6cd0e73873e26c38ba8a2d2f4)

ldap/servers/plugins/syntaxes/string.c

@@ -448,21 +448,31 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		for ( bvlp = bvals, nbvlp = nbvals; bvlp && *bvlp; bvlp++, nbvlp++ )
 		{
+			unsigned long value_flags = slapi_value_get_flags(*bvlp);
 			c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
 			/* if the NORMALIZED flag is set, skip normalizing */
-			if (!(slapi_value_get_flags(*bvlp) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			if (!(value_flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
 				/* 3rd arg: 1 - trim leading blanks */
 				value_normalize_ext( c, syntax, 1, &alt );
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			} else if ((syntax & SYNTAX_DN) &&
+			           (value_flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+				/* This dn value is normalized, but not case-normalized. */
+				slapi_dn_ignore_case(c);
+				/* This dn value is case-normalized */
+				value_flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
 			}
 			if (alt) {
 				slapi_ch_free_string(&c);
-		    	*nbvlp = slapi_value_new_string_passin(alt);
+				*nbvlp = slapi_value_new_string_passin(alt);
 				alt = NULL;
 			} else {
-		    	*nbvlp = slapi_value_new_string_passin(c);
+				*nbvlp = slapi_value_new_string_passin(c);
+				c = NULL;
 			}
 			/* new value is normalized */
-			slapi_value_set_flags(*nbvlp, slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+			slapi_value_set_flags(*nbvlp, value_flags);
 		}
 		*ivals = nbvals;
 		break;
@@ -570,8 +580,9 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		bvdup= slapi_value_new(); 
 		for ( bvlp = bvals; bvlp && *bvlp; bvlp++ ) {
+			unsigned long value_flags = slapi_value_get_flags(*bvlp);
 			/* 3rd arg: 1 - trim leading blanks */
-			if (!(slapi_value_get_flags(*bvlp) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			if (!(value_flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
 				c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
 				value_normalize_ext( c, syntax, 1, &alt );
 				if (alt) {
@@ -583,6 +594,17 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 					c = NULL;
 				}
 				bvp = slapi_value_get_berval(bvdup);
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			} else if ((syntax & SYNTAX_DN) &&
+			           (value_flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+				/* This dn value is normalized, but not case-normalized. */
+				c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
+				slapi_dn_ignore_case(c);
+				slapi_value_set_string_passin(bvdup, c);
+				c = NULL;
+				/* This dn value is case-normalized */
+				value_flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
 			} else {
 				bvp = slapi_value_get_berval(*bvlp);
 			}
@@ -595,7 +617,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRBEGIN]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 
@@ -608,7 +630,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRMIDDLE]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 
@@ -621,7 +643,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				buf[substrlens[INDEX_SUBSTREND] - 1] = '$';
 				buf[substrlens[INDEX_SUBSTREND]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 		}
@@ -674,8 +696,16 @@ string_assertion2keys_ava(
                 alt = NULL;
             }
             tmpval->bv.bv_len=strlen(tmpval->bv.bv_val);
+            flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+        } else if ((syntax & SYNTAX_DN) &&
+                   (flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+            /* This dn value is normalized, but not case-normalized. */
+            slapi_dn_ignore_case(tmpval->bv.bv_val);
+            /* This dn value is case-normalized */
+            flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+            flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
         }
-        slapi_value_set_flags(tmpval, flags|SLAPI_ATTR_FLAG_NORMALIZED);
+        slapi_value_set_flags(tmpval, flags);
         break;
 	case LDAP_FILTER_EQUALITY:
 		(*ivals) = (Slapi_Value **) slapi_ch_malloc( 2 * sizeof(Slapi_Value *) );
@@ -689,8 +719,16 @@ string_assertion2keys_ava(
 				(*ivals)[0]->bv.bv_len = strlen( (*ivals)[0]->bv.bv_val );
 				alt = NULL;
 			}
-			slapi_value_set_flags((*ivals)[0], flags|SLAPI_ATTR_FLAG_NORMALIZED);
-		}
+			flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+		} else if ((syntax & SYNTAX_DN) &&
+		           (flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+            /* This dn value is normalized, but not case-normalized. */
+			slapi_dn_ignore_case((*ivals)[0]->bv.bv_val);
+			/* This dn value is case-normalized */
+			flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+			flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
+		}
+		slapi_value_set_flags((*ivals)[0], flags);
 		(*ivals)[1] = NULL;
 		break;
 

ldap/servers/slapd/attrlist.c

@@ -292,7 +292,7 @@ int attrlist_replace(Slapi_Attr **alist, const char *type, struct berval **vals)
         valuearray_init_bervalarray(vals, &values);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
             valuearray_dn_normalize_value(values);
-            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
         }
         rc = attr_replace(*a, values);
     }
@@ -319,7 +319,7 @@ int attrlist_replace_with_flags(Slapi_Attr **alist, const char *type, struct ber
         valuearray_init_bervalarray_with_flags(vals, &values, flags);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
             valuearray_dn_normalize_value(values);
-            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
         }
         rc = attr_replace(*a, values);
     }

ldap/servers/slapd/back-ldbm/index.c

@@ -455,7 +455,9 @@ index_addordel_entry(
                     /* skip "entrydn" */
                     continue;
                 } else {
-                    slapi_values_set_flags(svals, SLAPI_ATTR_FLAG_NORMALIZED);
+                    /* entrydn is case-normalized */
+                    slapi_values_set_flags(svals,
+                                           SLAPI_ATTR_FLAG_NORMALIZED_CIS);
                 }
             }
             result = index_addordel_values_sv( be, type, svals, NULL,

ldap/servers/slapd/back-ldbm/ldbm_add.c

@@ -1189,7 +1189,7 @@ add_update_entrydn_operational_attributes(struct backentry *ep)
     bv.bv_val = (void*)backentry_get_ndn(ep);
     bv.bv_len = strlen( bv.bv_val );
     entry_replace_values_with_flags( ep->ep_entry, LDBM_ENTRYDN_STR, bvp,
-                                     SLAPI_ATTR_FLAG_NORMALIZED );
+                                     SLAPI_ATTR_FLAG_NORMALIZED_CIS );
 }
 
 /*

ldap/servers/slapd/entry.c

@@ -3533,7 +3533,7 @@ slapi_entry_add_values_sv(Slapi_Entry *e,
 		attrlist_find_or_create(alist, type, &a);
 		if (slapi_attr_is_dn_syntax_attr(*a)) {
 			valuearray_dn_normalize_value(vals);
-			(*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			(*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 		}
 		rc= attr_add_valuearray(*a,vals,slapi_entry_get_dn_const(e));
     }

ldap/servers/slapd/entrywsi.c

@@ -442,7 +442,7 @@ entry_add_present_values_wsi(Slapi_Entry *e, const char *type, struct berval **b
 		/* Check if the type of the to-be-added values has DN syntax or not. */
 		if (slapi_attr_is_dn_syntax_attr(a)) {
 			valuearray_dn_normalize_value(valuestoadd);
-			a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 		}
 		if(urp)
 		{
@@ -564,7 +564,7 @@ entry_delete_present_values_wsi(Slapi_Entry *e, const char *type, struct berval
 			 * or not. */
 			if (slapi_attr_is_dn_syntax_attr(a)) {
 				valuearray_dn_normalize_value(valuestodelete);
-				a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+				a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 			}
 			if(urp)
 			{

ldap/servers/slapd/slapi-plugin.h

@@ -177,7 +177,15 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
  * \see slapi_value_set_flags()
  * \see slapi_values_set_flags()
  */
-#define SLAPI_ATTR_FLAG_NORMALIZED	0x0200	/* the attr value is normalized */
+/* the attr value is normalized */
+#define SLAPI_ATTR_FLAG_NORMALIZED \
+        (SLAPI_ATTR_FLAG_NORMALIZED_CES|SLAPI_ATTR_FLAG_NORMALIZED_CIS)
+#define SLAPI_ATTR_FLAG_NORMALIZED_CES 0x0200 /* the attr value is normalized,
+                                                 but not case-normalized.
+                                                 Used for DN. */
+#define SLAPI_ATTR_FLAG_NORMALIZED_CIS 0x0400 /* the attr value is normalized 
+                                                 including case.
+                                                 Used for DN. */
 
 /**
  * Flag to indicate that the attribute value is not exposed if specified.

ldap/servers/slapd/value.c

@@ -595,7 +595,7 @@ value_dn_normalize_value(Slapi_Value *value)
 		value->bv.bv_val = slapi_ch_strdup(slapi_sdn_get_dn(sdn));
 		value->bv.bv_len = slapi_sdn_get_ndn_len(sdn);
 		slapi_sdn_free(&sdn);
-		slapi_value_set_flags(value, SLAPI_ATTR_FLAG_NORMALIZED);
+		slapi_value_set_flags(value, SLAPI_ATTR_FLAG_NORMALIZED_CES);
 	} else {
 		rc = 1;
 		slapi_ch_free((void **)&sdn); /* free just Slapi_DN */