Ticket 346 - version 4 Slow ldapmodify operation time

	for large quantities of multi-valued attribute values

Bug Description:  The reason for the performance degradation is that for
		operations like add and delete of values and for updating the index for
		each values a check has tobe done if it exists in the current attribute.
		if the number of values grows the seaqrch time increases

Fix Description:  Keep a secondary array of the indexes of the valuearray which is sorted.
		To locate a value, a binary search can be used.
		A design doc is available at: http://port389.org/wiki/Static_group_performance

https://fedorahosted.org/389/ticket/346

Reviewed by: RichM

ldap/servers/slapd/attr.c

@@ -365,11 +365,9 @@ Slapi_Attr *
 slapi_attr_dup(const Slapi_Attr *attr)
 {
 	Slapi_Attr *newattr= slapi_attr_new();
-	Slapi_Value **present_va= valueset_get_valuearray(&attr->a_present_values); /* JCM Mucking about inside the value set */
-	Slapi_Value **deleted_va= valueset_get_valuearray(&attr->a_deleted_values); /* JCM Mucking about inside the value set */
 	slapi_attr_init(newattr, attr->a_type);
-	valueset_add_valuearray( &newattr->a_present_values, present_va );
-	valueset_add_valuearray( &newattr->a_deleted_values, deleted_va );
+	slapi_valueset_set_valueset( &newattr->a_deleted_values,  &attr->a_deleted_values );
+	slapi_valueset_set_valueset( &newattr->a_present_values,  &attr->a_present_values );
 	newattr->a_deletioncsn= csn_dup(attr->a_deletioncsn);
 	return newattr;
 }
@@ -835,43 +833,11 @@ attr_add_valuearray(Slapi_Attr *a, Slapi_Value **vals, const char *dn)
     }
 
     /*
-     * determine whether we should use an AVL tree of values or not
+     * add values and check for duplicate values
      */
-    for ( i = 0; vals[i] != NULL; i++ ) ;
-    numofvals = i;
-
-    /*
-     * detect duplicate values
-     */
-    if ( numofvals > 1 ) {
-        /*
-         * Several values to add: use an AVL tree to detect duplicates.
-         */
-        LDAPDebug( LDAP_DEBUG_TRACE,
-                   "slapi_entry_add_values: using an AVL tree to "
-                   "detect duplicate values\n", 0, 0, 0 );
-
-        if (valueset_isempty(&a->a_present_values)) {
-            /* if the attribute contains no values yet, just check the
-             * input vals array for duplicates
-             */
-            Avlnode *vtree = NULL;
-            rc= valuetree_add_valuearray(a, vals, &vtree, &duplicate_index);
-            valuetree_free(&vtree);
-            was_present_null = 1;
-        } else {
-            /* the attr and vals both contain values, check intersection */
-            rc= valueset_intersectswith_valuearray(&a->a_present_values, a, vals, &duplicate_index);
-        }
-
-    } else if ( !valueset_isempty(&a->a_present_values) ) {
-        /*
-         * One or no value to add: don't bother constructing
-         * an AVL tree, etc. since it probably isn't worth the time.
-         */
-        for ( i = 0; vals[i] != NULL; ++i ) {
-            if ( slapi_attr_value_find( a, slapi_value_get_berval(vals[i]) ) == 0 ) {
-                duplicate_index = i;
+    numofvals = valuearray_count(vals);
+    rc = slapi_valueset_add_attr_valuearray_ext (a, &a->a_present_values, vals, numofvals, SLAPI_VALUE_FLAG_DUPCHECK, &duplicate_index);
+    if ( rc != LDAP_SUCCESS) {
 #if defined(USE_OLD_UNHASHED)
                 if (is_type_forbidden(a->a_type)) {
                     /* If the attr is in the forbidden list
@@ -884,22 +850,12 @@ attr_add_valuearray(Slapi_Attr *a, Slapi_Value **vals, const char *dn)
 #else
                 rc = LDAP_TYPE_OR_VALUE_EXISTS;
 #endif
-                break;
             }
-        }
-    }
-
-    /*
-     * add values if no duplicates detected
-     */
-    if(rc==LDAP_SUCCESS) {
-        valueset_add_valuearray( &a->a_present_values, vals );
-    }
 
     /* In the case of duplicate value, rc == LDAP_TYPE_OR_VALUE_EXISTS or
      * LDAP_OPERATIONS_ERROR
      */
-    else if ( duplicate_index >= 0 ) {
+    if ( duplicate_index >= 0 ) {
         char bvvalcopy[BUFSIZ];
         char *duplicate_string = "null or non-ASCII";
 
@@ -940,7 +896,7 @@ attr_add_valuearray(Slapi_Attr *a, Slapi_Value **vals, const char *dn)
  */
 int attr_replace(Slapi_Attr *a, Slapi_Value **vals)
 {
-    return valueset_replace(a, &a->a_present_values, vals);
+    return valueset_replace_valuearray(a, &a->a_present_values, vals);
 }
 
 int 

ldap/servers/slapd/attrlist.c

@@ -129,7 +129,7 @@ attrlist_merge_valuearray(Slapi_Attr **alist, const char *type, Slapi_Value **va
 	Slapi_Attr	**a= NULL;
 	if (!vals) return;
 	attrlist_find_or_create(alist, type, &a);
-	valueset_add_valuearray( &(*a)->a_present_values, vals );
+	slapi_valueset_add_valuearray( *a, &(*a)->a_present_values, vals );
 }    
 
 

ldap/servers/slapd/attrsyntax.c

@@ -1047,6 +1047,10 @@ slapi_attr_is_dn_syntax_attr(Slapi_Attr *attr)
 	const char *syntaxoid = NULL;
 	int dn_syntax = 0; /* not DN, by default */
 
+	if (attr && attr->a_flags & SLAPI_ATTR_FLAG_SYNTAX_IS_DN)
+		/* it was checked before */
+		return(1);
+
 	if (attr && attr->a_plugin == NULL) {
  	    slapi_attr_init_syntax (attr);
  	}
@@ -1055,6 +1059,8 @@ slapi_attr_is_dn_syntax_attr(Slapi_Attr *attr)
 			dn_syntax = ((0 == strcmp(syntaxoid, NAMEANDOPTIONALUID_SYNTAX_OID))
 						 || (0 == strcmp(syntaxoid, DN_SYNTAX_OID)));
 		}
+		if (dn_syntax)
+			attr->a_flags |= SLAPI_ATTR_FLAG_SYNTAX_IS_DN;
 	}
 	return dn_syntax;
 }

ldap/servers/slapd/back-ldbm/import-threads.c

@@ -2639,7 +2639,7 @@ import_foreman(void *param)
 
                 /* Setting new entrydn attribute value */
                 slapi_attr_init(new_entrydn, "entrydn");
-                valueset_add_string(&new_entrydn->a_present_values,
+                valueset_add_string(new_entrydn, &new_entrydn->a_present_values,
                                     /* new_dn: duped in valueset_add_string */
                                     (const char *)new_dn,
                                     CSN_TYPE_UNKNOWN, NULL);

ldap/servers/slapd/back-ldbm/index.c

@@ -545,7 +545,7 @@ index_add_mods(
 
         for (curr_attr = newe->ep_entry->e_attrs; curr_attr != NULL; curr_attr = curr_attr->a_next) {
             if (slapi_attr_type_cmp( basetype, curr_attr->a_type, SLAPI_TYPE_CMP_BASE ) == 0) {
-                valueset_add_valuearray(all_vals, attr_get_present_values(curr_attr));
+                slapi_valueset_join_attr_valueset(curr_attr, all_vals, &curr_attr->a_present_values);
             }
         }
  
@@ -566,7 +566,7 @@ index_add_mods(
 
             for (curr_attr = olde->ep_entry->e_attrs; curr_attr != NULL; curr_attr = curr_attr->a_next) {
                 if (slapi_attr_type_cmp( mods[i]->mod_type, curr_attr->a_type, SLAPI_TYPE_CMP_EXACT ) == 0) {
-                    valueset_add_valuearray(mod_vals, attr_get_present_values(curr_attr));
+                	slapi_valueset_join_attr_valueset(curr_attr, mod_vals, &curr_attr->a_present_values);
                 }
             }
                                                                                                                             
@@ -584,7 +584,7 @@ index_add_mods(
                 slapi_entry_attr_find( olde->ep_entry, mods[i]->mod_type, &curr_attr );
                 if ( mods_valueArray != NULL ) {
                     for ( j = 0; mods_valueArray[j] != NULL; j++ ) {
-                        Slapi_Value *rval = valuearray_remove_value(curr_attr, evals, mods_valueArray[j]);
+                        Slapi_Value *rval = valueset_remove_value(curr_attr, all_vals, mods_valueArray[j]);
                         slapi_value_free( &rval );
                     }
                 }
@@ -593,12 +593,12 @@ index_add_mods(
                  * they don't exist, delete the equality index.
                  */
                 for ( j = 0; deleted_valueArray[j] != NULL; j++ ) {
-                    if (valuearray_find(curr_attr, evals, deleted_valueArray[j]) == -1) {
+                    if ( !slapi_valueset_find(curr_attr, all_vals, deleted_valueArray[j])) {
                         if (!(flags & BE_INDEX_EQUALITY)) {
                             flags |= BE_INDEX_EQUALITY;
                         }
                     } else {
-                        Slapi_Value *rval = valuearray_remove_value(curr_attr, deleted_valueArray, deleted_valueArray[j]);
+                        Slapi_Value *rval = valueset_remove_value(curr_attr, mod_vals, deleted_valueArray[j]);
                         slapi_value_free( &rval );
                         j--;
                         /* indicates there was some conflict */
@@ -637,8 +637,8 @@ index_add_mods(
                 if (curr_attr) { /* found the type */
                     for (j = 0; mods_valueArray[j] != NULL; j++) {
                         /* mods_valueArray[j] is in curr_attr ==> return 0 */
-                        if (slapi_attr_value_find(curr_attr,
-                                slapi_value_get_berval(mods_valueArray[j]))) {
+                        if ( !slapi_valueset_find(curr_attr, &curr_attr->a_present_values,
+                                mods_valueArray[j])) {
                             /* The value is NOT in newe, remove it. */
                             Slapi_Value *rval;
                             rval = valuearray_remove_value(curr_attr,
@@ -677,7 +677,7 @@ index_add_mods(
 
                 for (curr_attr = olde->ep_entry->e_attrs; curr_attr != NULL; curr_attr = curr_attr->a_next) {
                         if (slapi_attr_type_cmp( mods[i]->mod_type, curr_attr->a_type, SLAPI_TYPE_CMP_EXACT ) == 0) {
-                            valueset_add_valuearray(mod_vals, attr_get_present_values(curr_attr));
+                	    slapi_valueset_join_attr_valueset(curr_attr, mod_vals, &curr_attr->a_present_values);
                         }
                 }
 
@@ -694,14 +694,14 @@ index_add_mods(
                              * also exist in a subtype.
                              */
                             for (j = 0; deleted_valueArray && deleted_valueArray[j]; j++) {
-                                if ( valuearray_find(curr_attr, evals, deleted_valueArray[j]) == -1 ) {
+                    		if ( !slapi_valueset_find(curr_attr, all_vals, deleted_valueArray[j])) {
                                     /* If the equality flag isn't already set, set it */
                                     if (!(flags & BE_INDEX_EQUALITY)) {
                                         flags |= BE_INDEX_EQUALITY;
                                     }
                                 } else {
                                     /* Remove duplicate value from the mod list */
-                                    Slapi_Value *rval = valuearray_remove_value(curr_attr, deleted_valueArray, deleted_valueArray[j]);
+                                    Slapi_Value *rval = valueset_remove_value(curr_attr, mod_vals, deleted_valueArray[j]);
                                     slapi_value_free( &rval );
                                     j--;
                                 }
@@ -750,7 +750,7 @@ index_add_mods(
                     if (curr_attr) {
                         int found = 0;
                         for (j = 0; mods_valueArray[j] != NULL; j++ ) {
-                            if ( valuearray_find(curr_attr, evals, mods_valueArray[j]) > -1 ) {
+                    	    if ( slapi_valueset_find(curr_attr, all_vals, mods_valueArray[j])) {
                                 /* The same value found in evals. 
                                  * We don't touch the equality index. */
                                 found = 1;

ldap/servers/slapd/back-ldbm/ldbm_attr.c

@@ -437,7 +437,7 @@ ldbm_compute_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi_
 			Slapi_Attr our_attr;
 			slapi_attr_init(&our_attr, numsubordinates);
 			our_attr.a_flags = SLAPI_ATTR_FLAG_OPATTR;
-			valueset_add_string(&our_attr.a_present_values,"0",CSN_TYPE_UNKNOWN,NULL);
+			valueset_add_string(&our_attr, &our_attr.a_present_values,"0",CSN_TYPE_UNKNOWN,NULL);
 			rc = (*outputfn) (c, &our_attr, e);
 			attr_done(&our_attr);
 			return (rc);
@@ -454,9 +454,9 @@ ldbm_compute_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi_
 		rc = slapi_entry_attr_find( e, numsubordinates, &read_attr );
 		if ( (0 != rc) || slapi_entry_attr_hasvalue(e,numsubordinates,"0") ) {
 			/* If not, or present and zero, we return FALSE, otherwise TRUE */
-			valueset_add_string(&our_attr.a_present_values,"FALSE",CSN_TYPE_UNKNOWN,NULL);
+			valueset_add_string(&our_attr, &our_attr.a_present_values,"FALSE",CSN_TYPE_UNKNOWN,NULL);
 		} else {
-			valueset_add_string(&our_attr.a_present_values,"TRUE",CSN_TYPE_UNKNOWN,NULL);
+			valueset_add_string(&our_attr, &our_attr.a_present_values,"TRUE",CSN_TYPE_UNKNOWN,NULL);
 		}
 		rc = (*outputfn) (c, &our_attr, e);
 		attr_done(&our_attr);

ldap/servers/slapd/computed.c

@@ -148,7 +148,7 @@ compute_stock_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi
 		Slapi_Attr our_attr;
 		slapi_attr_init(&our_attr, subschemasubentry);
 		our_attr.a_flags = SLAPI_ATTR_FLAG_OPATTR;
-		valueset_add_string(&our_attr.a_present_values,SLAPD_SCHEMA_DN,CSN_TYPE_UNKNOWN,NULL);
+		valueset_add_string(&our_attr, &our_attr.a_present_values,SLAPD_SCHEMA_DN,CSN_TYPE_UNKNOWN,NULL);
 		rc = (*outputfn) (c, &our_attr, e);
 		attr_done(&our_attr);
 		return (rc);

ldap/servers/slapd/entry.c

@@ -237,8 +237,6 @@ str2entry_fast( const char *rawdn, const Slapi_RDN *srdn, char *s, int flags, in
 		int freeval = 0;
 		int value_state= VALUE_NOTFOUND;
 		int attr_state= ATTRIBUTE_NOTFOUND;
-		int maxvals;
-		int del_maxvals;
 
 		if ( *s == '\n' || *s == '\0' ) {
 			break;
@@ -280,9 +278,7 @@ str2entry_fast( const char *rawdn, const Slapi_RDN *srdn, char *s, int flags, in
 			slapi_ch_free_string(&ptype);
 			ptype=PL_strndup(type.bv_val, type.bv_len);
 			nvals = 0;
-			maxvals = 0;
 			del_nvals = 0;
-			del_maxvals = 0;
 			a = NULL;
 		}
 
@@ -503,25 +499,21 @@ str2entry_fast( const char *rawdn, const Slapi_RDN *srdn, char *s, int flags, in
 			if(value_state==VALUE_DELETED)
 			{
 				/* consumes the value */
-				valuearray_add_value_fast(
-					&(*a)->a_deleted_values.va, /* JCM .va is private */
-					svalue,
-					del_nvals,
-					&del_maxvals,
-					0/*!Exact*/,
-					1/*Passin*/ );
+				slapi_valueset_add_attr_value_ext(
+					*a,
+					&(*a)->a_deleted_values,
+ 					svalue,
+					SLAPI_VALUE_FLAG_PASSIN );
 				del_nvals++;
 			}
 			else
 			{
 				/* consumes the value */
-				valuearray_add_value_fast(
-					&(*a)->a_present_values.va, /* JCM .va is private */
-					svalue, 
-					nvals,
-					&maxvals, 
-					0 /*!Exact*/, 
-					1 /*Passin*/ );
+				slapi_valueset_add_attr_value_ext(
+					*a,
+					&(*a)->a_present_values,
+ 					svalue, 
+					SLAPI_VALUE_FLAG_PASSIN);
 				nvals++;
 			}
 			if(attributedeletioncsn!=NULL)
@@ -603,8 +595,8 @@ typedef struct _entry_attrs {
 typedef struct _str2entry_attr {
 	char *sa_type;
 	int sa_state;
- 	struct valuearrayfast sa_present_values;
- 	struct valuearrayfast sa_deleted_values;
+ 	struct slapi_value_set sa_present_values;
+ 	struct slapi_value_set sa_deleted_values;
 	int sa_numdups;
 	value_compare_fn_type sa_comparefn;
 	Avlnode *sa_vtree;
@@ -617,8 +609,8 @@ entry_attr_init(str2entry_attr *sa, const char *type, int state)
 {
     sa->sa_type= slapi_ch_strdup(type);
 	sa->sa_state= state;
-	valuearrayfast_init(&sa->sa_present_values,NULL);
-	valuearrayfast_init(&sa->sa_deleted_values,NULL);
+	slapi_valueset_init(&sa->sa_present_values);
+	slapi_valueset_init(&sa->sa_deleted_values);
     sa->sa_numdups= 0;
 	sa->sa_comparefn = NULL;
     sa->sa_vtree= NULL;
@@ -1106,66 +1098,20 @@ str2entry_dupcheck( const char *rawdn, char *s, int flags, int read_stateinfo )
 		{
 			/* 
 			 * for deleted values, we do not want to perform a dupcheck against
-			 * existing values. Also, we do not want to add it to the
-			 * avl tree (if one is being maintained)
-			 *
+			 * existing values.
 			 */
-			rc = 0; /* Presume no duplicate */
+			rc = slapi_valueset_add_attr_value_ext(&sa->sa_attr, &sa->sa_deleted_values,value, SLAPI_VALUE_FLAG_PASSIN);
 		}
-		else if ( !check_for_duplicate_values )
+		else
 		{
-			rc = LDAP_SUCCESS;	/* presume no duplicate */
-		} else {
-			/* For value dup checking, we either use brute-force, if there's a small number */
-			/* Or a tree-based approach if there's a large number. */
-			/* The tree code is expensive, which is why we don't use it unless there's many attributes */
-			rc = 0; /* Presume no duplicate */
-			if (fast_dup_check)
-			{
-				/* Fast dup-checking */
-				/* Do we now have so many values that we should switch to tree-based checking ? */
-				if (sa->sa_present_values.num > STR2ENTRY_VALUE_DUPCHECK_THRESHOLD)
-				{
-					/* Make the tree from the existing attr values */
-					rc= valuetree_add_valuearray( &sa->sa_attr, sa->sa_present_values.va, &sa->sa_vtree, NULL);
-					/* Check if the value already exists, in the tree. */
-					rc= valuetree_add_value( &sa->sa_attr, value, &sa->sa_vtree);
-					fast_dup_check = 0;
-				}
-				else
-				{
-					/* JCM - need an efficient valuearray function to do this */
-					/* Brute-force check */
-					for ( j = 0; j < sa->sa_present_values.num; j++ )/* JCM innards */
-					{
-						if (0 == sa->sa_comparefn(slapi_value_get_berval(value),slapi_value_get_berval(sa->sa_present_values.va[j])))/* JCM innards */
-						{
-							/* Oops---this value matches one already present */
-							rc = LDAP_TYPE_OR_VALUE_EXISTS;
-							break;
-						}
-					}
-				}
-			}
-			else
-			{
-				/* Check if the value already exists, in the tree. */
-				rc = valuetree_add_value( &sa->sa_attr, value, &sa->sa_vtree);
-			}
+			int flags = SLAPI_VALUE_FLAG_PASSIN;
+			if (check_for_duplicate_values) flags |= SLAPI_VALUE_FLAG_DUPCHECK;
+			rc = slapi_valueset_add_attr_value_ext(&sa->sa_attr, &sa->sa_present_values,value, flags);
 		}
 
 		if ( rc==LDAP_SUCCESS )
 		{
-			if(value_state==VALUE_DELETED)
-			{
-				valuearrayfast_add_value_passin(&sa->sa_deleted_values,value);
-				value= NULL; /* value was consumed */
-			}
-			else
-			{
-				valuearrayfast_add_value_passin(&sa->sa_present_values,value);
-				value= NULL; /* value was consumed */
-			}
+			value= NULL; /* value was consumed */
 			if(attributedeletioncsn!=NULL)
 			{
 				sa->sa_attributedeletioncsn= attributedeletioncsn;
@@ -1180,7 +1126,7 @@ str2entry_dupcheck( const char *rawdn, char *s, int flags, int read_stateinfo )
 		else
 		{
 		    /* Failure adding to value tree */
-		    LDAPDebug( LDAP_DEBUG_ANY, "str2entry_dupcheck: unexpected failure %d constructing value tree\n", rc, 0, 0 );
+		    LDAPDebug( LDAP_DEBUG_ANY, "str2entry_dupcheck: unexpected failure %d adding value\n", rc, 0, 0 );
 		    slapi_entry_free( e ); e = NULL;
 		    goto free_and_return;
 		}
@@ -1245,27 +1191,21 @@ str2entry_dupcheck( const char *rawdn, char *s, int flags, int read_stateinfo )
 			}
 			if(alist!=NULL)
 			{
-				int maxvals = 0;
 				Slapi_Attr **a= NULL;
 				attrlist_find_or_create_locking_optional(alist, sa->sa_type, &a, PR_FALSE);
-				valuearray_add_valuearray_fast( /* JCM should be calling a valueset function */
-					&(*a)->a_present_values.va, /* JCM .va is private */
-					sa->sa_present_values.va,
-					0, /* Currently there are no present values on the attribute */
-					sa->sa_present_values.num,
-					&maxvals,
-					1/*Exact*/,
-					1/*Passin*/);
+				slapi_valueset_add_attr_valuearray_ext(
+					*a,
+					&(*a)->a_present_values,
+ 					sa->sa_present_values.va,
+ 					sa->sa_present_values.num,
+					SLAPI_VALUE_FLAG_PASSIN, NULL);
 				sa->sa_present_values.num= 0; /* The values have been consumed */
-				maxvals = 0;
-				valuearray_add_valuearray_fast( /* JCM should be calling a valueset function */
-					&(*a)->a_deleted_values.va, /* JCM .va is private */
-					sa->sa_deleted_values.va,
-					0, /* Currently there are no deleted values on the attribute */
-					sa->sa_deleted_values.num,
-					&maxvals,
-					1/*Exact*/,
-					1/*Passin*/);
+				slapi_valueset_add_attr_valuearray_ext(
+					*a,
+					&(*a)->a_deleted_values,
+ 					sa->sa_deleted_values.va,
+ 					sa->sa_deleted_values.num,
+					SLAPI_VALUE_FLAG_PASSIN, NULL);
 				sa->sa_deleted_values.num= 0; /* The values have been consumed */
 				if(sa->sa_attributedeletioncsn!=NULL)
 				{
@@ -1312,9 +1252,8 @@ free_and_return:
     for ( i = 0; i < nattrs; i++ )
     {
 		slapi_ch_free((void **) &(attrs[ i ].sa_type));
-		valuearrayfast_done(&attrs[ i ].sa_present_values);
-		valuearrayfast_done(&attrs[ i ].sa_deleted_values);
-		valuetree_free( &attrs[ i ].sa_vtree );
+		slapi_ch_free((void **) &(attrs[ i ].sa_present_values.va));
+		slapi_ch_free((void **) &(attrs[ i ].sa_deleted_values.va));
 		attr_done( &attrs[ i ].sa_attr );
     }
 	if (tree_attr_checking)
@@ -1740,7 +1679,7 @@ entry2str_internal_put_attrlist( const Slapi_Attr *attrlist, int attr_state, int
 						 * never be seen by any client. It will never be moved to the 
 						 * present values and is only used to preserve the AD-csn
 						 */
-						valueset_add_string ((Slapi_ValueSet *)&a->a_deleted_values, "", CSN_TYPE_VALUE_DELETED, a->a_deletioncsn);
+						valueset_add_string (a, (Slapi_ValueSet *)&a->a_deleted_values, "", CSN_TYPE_VALUE_DELETED, a->a_deletioncsn);
 					}
 
 					entry2str_internal_put_valueset(a->a_type, a->a_deletioncsn, CSN_TYPE_ATTRIBUTE_DELETED, attr_state, &a->a_deleted_values, VALUE_DELETED, ecur, typebuf, typebuf_len, entry2str_ctrl);
@@ -2727,7 +2666,7 @@ slapi_entry_add_string(Slapi_Entry *e, const char *type, const char *value)
 {
 	Slapi_Attr **a= NULL;
 	attrlist_find_or_create(&e->e_attrs, type, &a);
-	valueset_add_string ( &(*a)->a_present_values, value, CSN_TYPE_UNKNOWN, NULL);
+	valueset_add_string ( *a, &(*a)->a_present_values, value, CSN_TYPE_UNKNOWN, NULL);
 	return 0;
 }
 

ldap/servers/slapd/entrywsi.c

@@ -477,7 +477,7 @@ entry_add_present_values_wsi_single_valued(Slapi_Entry *e, const char *type, str
 						SLAPI_VALUE_FLAG_IGNOREERROR |
 						SLAPI_VALUE_FLAG_PRESERVECSNSET, NULL);
 			valuearray_update_csn (valuestoadd,CSN_TYPE_VALUE_UPDATED,csn);
-			valueset_add_valuearray_ext(&a->a_present_values, valuestoadd, SLAPI_VALUE_FLAG_PASSIN);
+            		slapi_valueset_add_attr_valuearray_ext (a, &a->a_present_values, valuestoadd, valuearray_count(valuestoadd), SLAPI_VALUE_FLAG_PASSIN, NULL);
 			slapi_ch_free ( (void **)&valuestoadd );
 			/*
 			 * Now delete non-RDN values from a->a_present_values; and
@@ -514,7 +514,7 @@ entry_add_present_values_wsi_single_valued(Slapi_Entry *e, const char *type, str
 				Slapi_ValueSet vs;
 				/* Add each deleted value to the present list */
 				valuearray_update_csn(deletedvalues,CSN_TYPE_VALUE_UPDATED,csn);
-				valueset_add_valuearray_ext(&a->a_present_values, deletedvalues, SLAPI_VALUE_FLAG_PASSIN);
+            			slapi_valueset_add_attr_valuearray_ext (a, &a->a_present_values, deletedvalues, valuearray_count(deletedvalues), SLAPI_VALUE_FLAG_PASSIN, NULL);
 				/* Remove the deleted values from the values to add */
 				valueset_set_valuearray_passin(&vs,valuestoadd); 
 				valueset_remove_valuearray(&vs, a, deletedvalues, SLAPI_VALUE_FLAG_IGNOREERROR, &v);
@@ -609,7 +609,7 @@ entry_add_present_values_wsi_multi_valued(Slapi_Entry *e, const char *type, stru
 				Slapi_ValueSet vs;
 				/* Add each deleted value to the present list */
 				valuearray_update_csn(deletedvalues,CSN_TYPE_VALUE_UPDATED,csn);
-				valueset_add_valuearray_ext(&a->a_present_values, deletedvalues, SLAPI_VALUE_FLAG_PASSIN);
+            			slapi_valueset_add_attr_valuearray_ext (a, &a->a_present_values, deletedvalues, valuearray_count(deletedvalues), SLAPI_VALUE_FLAG_PASSIN, NULL);
 				/* Remove the deleted values from the values to add */
 				valueset_set_valuearray_passin(&vs,valuestoadd); 
 				valueset_remove_valuearray(&vs, a, deletedvalues, SLAPI_VALUE_FLAG_IGNOREERROR, &v);
@@ -744,7 +744,7 @@ entry_delete_present_values_wsi_single_valued(Slapi_Entry *e, const char *type,
 			valueset_update_csn_for_valuearray(&a->a_deleted_values, a, valuestodelete, CSN_TYPE_VALUE_DELETED, csn, &valuesupdated);
 			valuearray_free(&valuesupdated);
 			valuearray_update_csn(valuestodelete,CSN_TYPE_VALUE_DELETED,csn);
-			valueset_add_valuearray_ext(&a->a_deleted_values, valuestodelete, SLAPI_VALUE_FLAG_PASSIN);
+      			slapi_valueset_add_attr_valuearray_ext (a, &a->a_deleted_values, valuestodelete, valuearray_count(valuestodelete), SLAPI_VALUE_FLAG_PASSIN, NULL);
 			slapi_ch_free((void **)&valuestodelete);
 			resolve_attribute_state_single_valued(e, a, attr_state);
 			retVal= LDAP_SUCCESS;
@@ -838,7 +838,7 @@ entry_delete_present_values_wsi_multi_valued(Slapi_Entry *e, const char *type, s
 				valuearray_free(&valuesupdated);
 
 				valuearray_update_csn(valuestodelete,CSN_TYPE_VALUE_DELETED,csn);
-				valueset_add_valuearray_ext(&a->a_deleted_values, valuestodelete, SLAPI_VALUE_FLAG_PASSIN);
+            			slapi_valueset_add_attr_valuearray_ext (a, &a->a_deleted_values, valuestodelete, valuearray_count(valuestodelete), SLAPI_VALUE_FLAG_PASSIN, NULL);
 				/* all the elements in valuestodelete are passed;
 				 * should free valuestodelete only (don't call valuearray_free)
 				 * [622023] */
@@ -856,7 +856,8 @@ entry_delete_present_values_wsi_multi_valued(Slapi_Entry *e, const char *type, s
 				{
 					/* Add each deleted value to the deleted set */
 					valuearray_update_csn(deletedvalues,CSN_TYPE_VALUE_DELETED,csn);
-					valueset_add_valuearray_ext(&a->a_deleted_values, deletedvalues, SLAPI_VALUE_FLAG_PASSIN);
+       					slapi_valueset_add_attr_valuearray_ext (a,
+						 &a->a_deleted_values, deletedvalues, valuearray_count(deletedvalues), SLAPI_VALUE_FLAG_PASSIN, NULL);
 					slapi_ch_free((void **)&deletedvalues);
 					if(valueset_isempty(&a->a_present_values))
 					{

ldap/servers/slapd/proto-slap.h

@@ -167,13 +167,6 @@ int valuearray_first_value( Slapi_Value **va, Slapi_Value **v );
 
 void valuearrayfast_init(struct valuearrayfast *vaf,Slapi_Value **va);
 void valuearrayfast_done(struct valuearrayfast *vaf);
-void valuearrayfast_add_value(struct valuearrayfast *vaf,const Slapi_Value *v);
-void valuearrayfast_add_value_passin(struct valuearrayfast *vaf,Slapi_Value *v);
-void valuearrayfast_add_valuearrayfast(struct valuearrayfast *vaf,const struct valuearrayfast *vaf_add);
-
-int valuetree_add_value( const Slapi_Attr *sattr, const Slapi_Value *va, Avlnode **valuetreep);
-int valuetree_add_valuearray( const Slapi_Attr *sattr, Slapi_Value **va, Avlnode **valuetreep, int *duplicate_index);
-void valuetree_free( Avlnode **valuetreep );
 
 /* Valueset functions */
 
@@ -184,15 +177,17 @@ int valueset_remove_valuearray(Slapi_ValueSet *vs, const Slapi_Attr *a, Slapi_Va
 int valueset_purge(Slapi_ValueSet *vs, const CSN *csn);
 Slapi_Value **valueset_get_valuearray(const Slapi_ValueSet *vs);
 size_t valueset_size(const Slapi_ValueSet *vs);
+void slapi_valueset_add_valuearray(const Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value **addvals);
 void valueset_add_valuearray(Slapi_ValueSet *vs, Slapi_Value **addvals);
 void valueset_add_valuearray_ext(Slapi_ValueSet *vs, Slapi_Value **addvals, PRUint32 flags);
-void valueset_add_string(Slapi_ValueSet *vs, const char *s, CSNType t, const CSN *csn);
+void valueset_add_string(const Slapi_Attr *a, Slapi_ValueSet *vs, const char *s, CSNType t, const CSN *csn);
 void valueset_update_csn(Slapi_ValueSet *vs, CSNType t, const CSN *csn);
 void valueset_add_valueset(Slapi_ValueSet *vs1, const Slapi_ValueSet *vs2);
 int valueset_intersectswith_valuearray(Slapi_ValueSet *vs, const Slapi_Attr *a, Slapi_Value **values, int *duplicate_index);
 Slapi_ValueSet *valueset_dup(const Slapi_ValueSet *dupee);
 void valueset_remove_string(const Slapi_Attr *a, Slapi_ValueSet *vs, const char *s);
-int valueset_replace(Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value **vals);
+int valueset_replace_valuearray(Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value **vals);
+int valueset_replace_valuearray_ext(Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value **vals, int dupcheck);
 void valueset_update_csn_for_valuearray(Slapi_ValueSet *vs, const Slapi_Attr *a, Slapi_Value **valuestoupdate, CSNType t, const CSN *csn, Slapi_Value ***valuesupdated);
 void valueset_update_csn_for_valuearray_ext(Slapi_ValueSet *vs, const Slapi_Attr *a, Slapi_Value **valuestoupdate, CSNType t, const CSN *csn, Slapi_Value ***valuesupdated, int csnref_updated);
 void valueset_set_valuearray_byval(Slapi_ValueSet *vs, Slapi_Value **addvals);

ldap/servers/slapd/schema.c

@@ -5568,11 +5568,12 @@ va_locate_oc_val( Slapi_Value **va, const char *oc_name, const char *oc_oid )
  *		oc_unlock();
  */
 static void
-va_expand_one_oc( const char *dn, Slapi_Value ***vap, const char *ocs )
+va_expand_one_oc( const char *dn, const Slapi_Attr *a, Slapi_ValueSet *vs, const char *ocs )
 {
 	struct objclass	*this_oc, *sup_oc;
-	int				p,i;
-	Slapi_Value		**newva;
+	int p;
+	Slapi_Value **va = vs->va;
+
 
 	this_oc = oc_find_nolock( ocs );
   
@@ -5589,29 +5590,18 @@ va_expand_one_oc( const char *dn, Slapi_Value ***vap, const char *ocs )
 		return;			/* superior is unknown -- ignore */
 	}
 
-	p = va_locate_oc_val( *vap, sup_oc->oc_name, sup_oc->oc_oid );
+	p = va_locate_oc_val( va, sup_oc->oc_name, sup_oc->oc_oid );
 
 	if ( p != -1 ) {
 		return;			/* value already present -- done! */
 	}
   
-	/* parent was not found.  add to the end */
-	for ( i = 0; (*vap)[i] != NULL; i++ ) {
-		;
-	}
-  
-	/* prevent loops: stop if more than 1000 OC values are present */
-	if ( i > 1000 ) {
+	if ( slapi_valueset_count(vs) > 1000 ) {
 		return;
 	}
   
-	newva = (Slapi_Value **)slapi_ch_realloc( (char *)*vap,
-			( i + 2 )*sizeof(Slapi_Value *));
-  
-	newva[i] = slapi_value_new_string(sup_oc->oc_name);
-	newva[i+1] = NULL;
-  
-	*vap = newva;
+  	slapi_valueset_add_attr_value_ext(a, vs, slapi_value_new_string(sup_oc->oc_name), SLAPI_VALUE_FLAG_PASSIN);
+
 	LDAPDebug( LDAP_DEBUG_TRACE,
 			"Entry \"%s\": added missing objectClass value %s\n",
 			dn, sup_oc->oc_name, 0 );
@@ -5623,11 +5613,12 @@ va_expand_one_oc( const char *dn, Slapi_Value ***vap, const char *ocs )
  * All missing superior classes are added to the objectClass attribute, as
  * is 'top' if it is missing.
  */
-void
-slapi_schema_expand_objectclasses( Slapi_Entry *e )
+static void
+schema_expand_objectclasses_ext( Slapi_Entry *e, int lock)
 {
 	Slapi_Attr		*sa;
-	Slapi_Value		**va;
+	Slapi_Value		*v;
+	Slapi_ValueSet		*vs;
 	const char		*dn = slapi_entry_get_dn_const( e );
 	int				i;
 
@@ -5635,76 +5626,41 @@ slapi_schema_expand_objectclasses( Slapi_Entry *e )
 		return;		/* no OC values -- nothing to do */
 	}
 
-	va = attr_get_present_values( sa );
-
-	if ( va == NULL || va[0] == NULL ) {
+	vs = &sa->a_present_values;
+	if ( slapi_valueset_isempty(vs) ) {
 		return;		/* no OC values -- nothing to do */
 	}
 
-	oc_lock_read();
+	if (lock)
+		oc_lock_read();
 
 	/*
 	 * This loop relies on the fact that bv_expand_one_oc()
 	 * always adds to the end
 	 */
-	for ( i = 0; va[i] != NULL; ++i ) {
-		if ( NULL != slapi_value_get_string(va[i]) ) {
-			va_expand_one_oc( dn, &va, slapi_value_get_string(va[i]) );
-		}
+	i = slapi_valueset_first_value(vs,&v);
+	while ( v != NULL) {
+		if ( NULL != slapi_value_get_string(v) ) {
+			va_expand_one_oc( dn, sa, &sa->a_present_values, slapi_value_get_string(v) );
+ 		}
+		i = slapi_valueset_next_value(vs, i, &v);
 	}
   
 	/* top must always be present */
-	va_expand_one_oc( dn, &va, "top" );
-
-	/*
-	 * Reset the present values in the set because we may have realloc'd it.
-	 * Note that this is the counterpart to the attr_get_present_values()
-	 * call we made above... nothing new has been allocated, but sa holds
-	 * a pointer to the original (pre realloc) va.
-	 */
-	sa->a_present_values.va = va;
-
-	oc_unlock();
+	va_expand_one_oc( dn, sa, &sa->a_present_values, "top" );
+	if (lock)
+		oc_unlock();
+}
+void
+slapi_schema_expand_objectclasses( Slapi_Entry *e )
+{
+	schema_expand_objectclasses_ext( e, 1);
 }
 
 void
 schema_expand_objectclasses_nolock( Slapi_Entry *e )
 {
-	Slapi_Attr		*sa;
-	Slapi_Value		**va;
-	const char		*dn = slapi_entry_get_dn_const( e );
-	int				i;
-
-	if ( 0 != slapi_entry_attr_find( e, SLAPI_ATTR_OBJECTCLASS, &sa )) {
-		return;		/* no OC values -- nothing to do */
-	}
-
-	va = attr_get_present_values( sa );
-
-	if ( va == NULL || va[0] == NULL ) {
-		return;		/* no OC values -- nothing to do */
-	}
-
-	/*
-	 * This loop relies on the fact that bv_expand_one_oc()
-	 * always adds to the end
-	 */
-	for ( i = 0; va[i] != NULL; ++i ) {
-		if ( NULL != slapi_value_get_string(va[i]) ) {
-			va_expand_one_oc( dn, &va, slapi_value_get_string(va[i]) );
-		}
-	}
-  
-	/* top must always be present */
-	va_expand_one_oc( dn, &va, "top" );
-
-	/*
-	 * Reset the present values in the set because we may have realloc'd it.
-	 * Note that this is the counterpart to the attr_get_present_values()
-	 * call we made above... nothing new has been allocated, but sa holds
-	 * a pointer to the original (pre realloc) va.
-	 */
-	sa->a_present_values.va = va;
+	schema_expand_objectclasses_ext( e, 0);
 }
 
 /* lock to protect both objectclass and schema_dse */

ldap/servers/slapd/slap.h

@@ -450,8 +450,14 @@ struct slapi_value
  *		struct slapi_value_tree *vt;
  *	};
  */
+
+/* It is a useless layer, always use the valuarray fast version */
+#define VALUE_SORT_THRESHOLD 10
 struct slapi_value_set
 {
+	int num; /* The number of values in the array */
+	int max; /* The number of slots in the array */
+	int *sorted; /* sorted array of indices, if NULL va is not sorted */
 	struct slapi_value **va;
 };
 
@@ -530,6 +536,8 @@ typedef struct asyntaxinfo {
 #define SLAPI_ATTR_FLAG_NOLOCKING	0x0020	/* the init code doesn't lock the
 											   tables */
 #define SLAPI_ATTR_FLAG_KEEP		0x8000 /* keep when replacing all */
+#define SLAPI_ATTR_FLAG_SYNTAX_LOOKUP_DONE		0x010000 /* syntax lookup done, flag set */
+#define SLAPI_ATTR_FLAG_SYNTAX_IS_DN		0x020000 /* syntax lookup done, flag set */
 
 /* This is the type of the function passed into attr_syntax_enumerate_attrs */
 typedef int (*AttrEnumFunc)(struct asyntaxinfo *asi, void *arg);

ldap/servers/slapd/slapi-plugin.h

@@ -4658,6 +4658,7 @@ void slapi_ber_bvcpy(struct berval *bvd, const struct berval *bvs);
 #define SLAPI_VALUE_FLAG_IGNOREERROR	0x2
 #define SLAPI_VALUE_FLAG_PRESERVECSNSET	0x4
 #define SLAPI_VALUE_FLAG_USENEWVALUE	0x8	/* see valueset_remove_valuearray */
+#define SLAPI_VALUE_FLAG_DUPCHECK	0x10	/* used in valueset_add... */
 
 /**
  * Creates an empty \c Slapi_ValueSet structure.
@@ -4752,6 +4753,7 @@ void slapi_valueset_add_value(Slapi_ValueSet *vs, const Slapi_Value *addval);
  * \see slapi_valueset_next_value()
  */
 void slapi_valueset_add_value_ext(Slapi_ValueSet *vs, Slapi_Value *addval, unsigned long flags);
+int slapi_valueset_add_attr_value_ext(const Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value *addval, unsigned long flags);
 
 /**
  * Gets the first value in a \c Slapi_ValueSet structure.
@@ -4806,6 +4808,16 @@ int slapi_valueset_next_value( Slapi_ValueSet *vs, int index, Slapi_Value **v);
  */
 int slapi_valueset_count( const Slapi_ValueSet *vs);
 
+/**
+ * Checks if  a \c Slapi_ValueSet structure has values
+ *
+ * \param vs Pointer to the \c Slapi_ValueSet structure of which
+ *        you wish to get the count.
+ * \return 1 if there are no values contained in the \c Slapi_ValueSet structure.
+ * \return 0 if there are values contained in the \c Slapi_ValueSet structure.
+ */
+int slapi_valueset_isempty( const Slapi_ValueSet *vs);
+
 /**
  * Initializes a \c Slapi_ValueSet with copies of the values of a \c Slapi_Mod structure.
  *
@@ -4837,6 +4849,7 @@ void slapi_valueset_set_from_smod(Slapi_ValueSet *vs, Slapi_Mod *smod);
  * \see slapi_valueset_done()
  */
 void slapi_valueset_set_valueset(Slapi_ValueSet *vs1, const Slapi_ValueSet *vs2);
+void slapi_valueset_join_attr_valueset(const Slapi_Attr *a, Slapi_ValueSet *vs1, const Slapi_ValueSet *vs2);
 
 /**
  * Finds a requested value in a valueset.

ldap/servers/slapd/slapi-private.h

@@ -839,9 +839,12 @@ int charray_normdn_add(char ***chararray, char *dn, char *errstr);
  * the very least before we make them public.
  */
 void valuearray_add_value(Slapi_Value ***vals, const Slapi_Value *addval);
-void valuearray_add_value_fast(Slapi_Value ***vals, Slapi_Value *addval, int nvals, int *maxvals, int exact, int passin);
 void valuearray_add_valuearray( Slapi_Value ***vals, Slapi_Value **addvals, PRUint32 flags );
 void valuearray_add_valuearray_fast( Slapi_Value ***vals, Slapi_Value **addvals, int nvals, int naddvals, int *maxvals, int exact, int passin );
+Slapi_Value * valueset_find_sorted (const Slapi_Attr *a, const Slapi_ValueSet *vs, const Slapi_Value *v, int *index);
+int valueset_insert_value_to_sorted(const Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value *vi, int dupcheck);
+void valueset_array_to_sorted (const Slapi_Attr *a, Slapi_ValueSet *vs);
+int slapi_valueset_add_attr_valuearray_ext(const Slapi_Attr *a, Slapi_ValueSet *vs, Slapi_Value **addval, int nvals, unsigned long flags, int *dup_index);
 int valuearray_find(const Slapi_Attr *a, Slapi_Value **va, const Slapi_Value *v);
 int valuearray_dn_normalize_value(Slapi_Value **vals);