Ticket 1 - pre-normalize filter and pre-compile substring regex - and other optimizations

When processing large search filters which are applied to every entry in
the search result set, the filter is normalized anew each time a new entry
is tested.  For substring filters, a regular expression must be created,
compiled, and freed each time the substring filter is tested, in addition
to normalizing the values.  For example, if the search filter contains
1000 substring sub-filters, for each entry tested with the filter, this
will require 1000 filter normalizations followed by 1000 regex creation,
compilation, and cleanup.  If there are 1000 entries in the search result
set, this will require a million such operations.

The solution is to "pre-compile" the search filter - perform all necessary
normalizations and compiling of the regular expressions used in the
filter once we know the search will go through.

struct subfilt and struct ava have "private" members which weren't being
used for anything.  For subfilt, the private field is used to store the
pre-compiled regex to pass to the syntax filter code.  For ava, the
private field is used to store the flags to specify if the type and/or
value is normalized.

Try to avoid normalization wherever possible.  slapi_value has a v_flags
field which is used to specify if the value is normalized.  Check this
before we attempt to normalize a value.  If we are creating a new
slapi_value, set the normalized flag if the new value is already
normalized.  Have to make sure that Slapi_Value structures are always
initialized correctly.

When examining the filter string, do not convert it to lower case first -
just use strcasestr - note that even though the string may be utf8,
strcasestr will still work, because we are searching for ascii characters.
Use PL_strcasestr because the system strcasestr causes valgrind to
print uninitialized memory errors.

Eliminate some uses of sprintf where a simple char assignment will suffice.
Reviewed by: nhosoi (Thanks!)

ldap/servers/plugins/syntaxes/bitstring.c

@@ -59,6 +59,12 @@ static int bitstring_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char *
 		char *final, Slapi_Value ***ivals );
 static int bitstring_compare(struct berval	*v1, struct berval	*v2);
 static int bitstring_validate(struct berval *val);
+static void bitstring_normalize(
+	Slapi_PBlock *pb,
+	char *s,
+	int trim_spaces,
+	char **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Bit String", "bitstring", BITSTRING_SYNTAX_OID, 0 };
@@ -133,6 +139,8 @@ bitstring_init( Slapi_PBlock *pb )
 	    (void *) bitstring_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) bitstring_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) bitstring_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= bitstring_init %d\n", rc, 0, 0 );
@@ -148,7 +156,13 @@ bitstring_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CES,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CES;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -229,3 +243,13 @@ exit:
 	return rc;
 }
 
+static void bitstring_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CES, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/ces.c

@@ -65,6 +65,12 @@ static int ces_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **any,
 		char *final, Slapi_Value ***ivals );
 static int ces_compare(struct berval	*v1, struct berval	*v2);
 static int ia5_validate(struct berval *val);
+static void ces_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 2252 */
 static char *ia5_names[] = { "IA5String", "ces", "caseexactstring",
@@ -251,6 +257,8 @@ register_ces_like_plugin( Slapi_PBlock *pb, Slapi_PluginDesc *pdescp,
 		rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 		    (void *)validate_fn );
 	}
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+		    (void *)ces_normalize );
 
 	return( rc );
 }
@@ -292,7 +300,13 @@ ces_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CES, ftype,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CES;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax, ftype,
 									  retVal) );
 }
 
@@ -378,3 +392,14 @@ ia5_validate(
 exit:
 	return rc;
 }
+
+static void ces_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CES, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/cis.c

@@ -77,6 +77,12 @@ static int country_validate(struct berval *val);
 static int postal_validate(struct berval *val);
 static int oid_validate(struct berval *val);
 static int printable_validate(struct berval *val);
+static void cis_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /*
   Even though the official RFC 4517 says that the postal syntax
@@ -458,7 +464,8 @@ register_cis_like_plugin( Slapi_PBlock *pb, Slapi_PluginDesc *pdescp,
 		rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 		    (void *)validate_fn );
 	}
-
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) cis_normalize );
 	return( rc );
 }
 
@@ -557,7 +564,13 @@ cis_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS, ftype,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax, ftype,
 									  retVal ) );
 }
 
@@ -1083,3 +1096,14 @@ static int printable_validate(
 exit:
 	return( rc );
 }
+
+static void cis_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/deliverymethod.c

@@ -60,6 +60,12 @@ static int delivery_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **
 static int delivery_compare(struct berval	*v1, struct berval	*v2);
 static int delivery_validate(struct berval *val);
 static int pdm_validate(const char *start, const char *end);
+static void delivery_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Delivery Method", "delivery", DELIVERYMETHOD_SYNTAX_OID, 0 };
@@ -99,6 +105,8 @@ delivery_init( Slapi_PBlock *pb )
 	    (void *) delivery_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) delivery_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) delivery_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= delivery_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -113,7 +121,13 @@ delivery_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -315,3 +329,14 @@ pdm_validate(const char *start, const char *end)
 exit:
 	return rc;
 }
+
+static void delivery_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/dn.c

@@ -58,6 +58,12 @@ static int dn_assertion2keys_ava( Slapi_PBlock *pb, Slapi_Value *val,
 static int dn_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **any,
 	char *final, Slapi_Value ***ivals );
 static int dn_validate( struct berval *val );
+static void dn_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 2252 */
 static char *names[] = { "DN", DN_SYNTAX_OID, 0 };
@@ -133,6 +139,8 @@ dn_init( Slapi_PBlock *pb )
 	    (void *) DN_SYNTAX_OID );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) dn_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) dn_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= dn_init %d\n", rc, 0, 0 );
@@ -143,8 +151,13 @@ static int
 dn_filter_ava( Slapi_PBlock *pb, struct berval *bvfilter,
     Slapi_Value **bvals, int ftype, Slapi_Value **retVal )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS | SYNTAX_DN,
-	    ftype, retVal ) );
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS | SYNTAX_DN;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax, ftype, retVal ) );
 }
 
 static int
@@ -193,3 +206,13 @@ static int dn_validate( struct berval *val )
 	return rc;
 }
 
+static void dn_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS | SYNTAX_DN, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/facsimile.c

@@ -60,6 +60,12 @@ static int facsimile_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char *
 static int facsimile_compare(struct berval	*v1, struct berval	*v2);
 static int facsimile_validate(struct berval *val);
 static int fax_parameter_validate(const char *start, const char *end);
+static void facsimile_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Facsimile Telephone Number", "facsimile", FACSIMILE_SYNTAX_OID, 0 };
@@ -99,6 +105,8 @@ facsimile_init( Slapi_PBlock *pb )
 	    (void *) facsimile_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) facsimile_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) facsimile_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= facsimile_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -113,7 +121,13 @@ facsimile_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -321,3 +335,14 @@ fax_parameter_validate(const char *start, const char *end)
 exit:
 	return rc;
 }
+
+static void facsimile_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/guide.c

@@ -63,6 +63,12 @@ static int guide_validate(struct berval *val);
 static int criteria_validate(const char *start, const char *end);
 static int andterm_validate(const char *start, const char *end, const char **last);
 static int term_validate(const char *start, const char *end, const char **last);
+static void guide_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *guide_names[] = { "Guide", "guide", GUIDE_SYNTAX_OID, 0 };
@@ -109,6 +115,8 @@ guide_init( Slapi_PBlock *pb )
 	    (void *) guide_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) guide_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) guide_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= guide_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -160,7 +168,13 @@ guide_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -732,3 +746,14 @@ term_validate(const char *start, const char *end, const char **last)
 exit:
 	return rc;
 }
+
+static void guide_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/int.c

@@ -55,6 +55,12 @@ static int int_assertion2keys( Slapi_PBlock *pb, Slapi_Value *val,
 		Slapi_Value ***ivals, int ftype );
 static int int_compare(struct berval	*v1, struct berval	*v2);
 static int int_validate(struct berval *val);
+static void int_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 2252 */
 static char *names[] = { "INTEGER", "int", INTEGER_SYNTAX_OID, 0 };
@@ -147,6 +153,8 @@ int_init( Slapi_PBlock *pb )
 	    (void *) int_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) int_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) int_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= int_init %d\n", rc, 0, 0 );
@@ -157,8 +165,14 @@ static int
 int_filter_ava( Slapi_PBlock *pb, struct berval *bvfilter,
     Slapi_Value **bvals, int ftype, Slapi_Value **retVal )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_INT | SYNTAX_CES,
-                               ftype, retVal ) );
+	int filter_normalized = 0;
+	int syntax = SYNTAX_INT | SYNTAX_CES;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
+							   ftype, retVal ) );
 }
 
 static int
@@ -235,3 +249,14 @@ static int int_validate(
 exit:
         return(rc);
 }
+
+static void int_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_INT|SYNTAX_CES, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/nameoptuid.c

@@ -59,6 +59,12 @@ static int nameoptuid_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char
 		char *final, Slapi_Value ***ivals );
 static int nameoptuid_compare(struct berval	*v1, struct berval	*v2);
 static int nameoptuid_validate(struct berval *val);
+static void nameoptuid_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Name And Optional UID", "nameoptuid", NAMEANDOPTIONALUID_SYNTAX_OID, 0 };
@@ -138,6 +144,8 @@ nameoptuid_init( Slapi_PBlock *pb )
 	    (void *) nameoptuid_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) nameoptuid_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) nameoptuid_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= nameoptuid_init %d\n", rc, 0, 0 );
@@ -153,7 +161,13 @@ nameoptuid_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS | SYNTAX_DN,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS | SYNTAX_DN;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -272,3 +286,13 @@ exit:
 	return rc;
 }
 
+static void nameoptuid_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS | SYNTAX_DN, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/numericstring.c

@@ -59,6 +59,12 @@ static int numstr_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **an
 		char *final, Slapi_Value ***ivals );
 static int numstr_compare(struct berval	*v1, struct berval	*v2);
 static int numstr_validate(struct berval *val);
+static void numstr_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Numeric String", "numstr", NUMERICSTRING_SYNTAX_OID, 0 };
@@ -153,6 +159,8 @@ numstr_init( Slapi_PBlock *pb )
 	    (void *) numstr_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) numstr_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) numstr_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= numstr_init %d\n", rc, 0, 0 );
@@ -163,7 +171,13 @@ static int
 numstr_filter_ava( Slapi_PBlock *pb, struct berval *bvfilter,
 	Slapi_Value **bvals, int ftype, Slapi_Value **retVal )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_SI | SYNTAX_CES,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_SI | SYNTAX_CES;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
                                ftype, retVal ) );
 }
 
@@ -240,3 +254,14 @@ static int numstr_validate(
 exit:
 	return(rc);
 }
+
+static void numstr_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_SI|SYNTAX_CES, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/sicis.c

@@ -63,6 +63,12 @@ static int sicis_assertion2keys_ava( Slapi_PBlock *pb, Slapi_Value *val,
 static int sicis_assertion2keys_sub( Slapi_PBlock *pb, char *initial,
 		char **any, char *final, Slapi_Value ***ivals );
 static int sicis_compare(struct berval	*v1, struct berval	*v2);
+static void sicis_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 2252 */
 static char *names[] = { "SpaceInsensitiveString",
@@ -102,6 +108,8 @@ sicis_init( Slapi_PBlock *pb )
 	    (void *) SPACE_INSENSITIVE_STRING_SYNTAX_OID );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_COMPARE,
 	    (void *) sicis_compare );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) sicis_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= sicis_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -116,7 +124,13 @@ sicis_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_SI | SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_SI | SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -177,3 +191,14 @@ static int sicis_compare(
 {
 	return value_cmp(v1, v2, SYNTAX_SI|SYNTAX_CIS, 3 /* Normalise both values */);
 }
+
+static void sicis_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_SI|SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/string.c

@@ -62,7 +62,8 @@ string_filter_ava( struct berval *bvfilter, Slapi_Value **bvals, int syntax,
     int ftype, Slapi_Value **retVal )
 {
 	int	i, rc;
-	struct berval bvfilter_norm;
+	struct berval bvfilter_norm = {0, NULL};
+	struct berval *pbvfilter_norm = &bvfilter_norm;
 	char *alt = NULL;
 
 	if(retVal) {
@@ -72,19 +73,32 @@ string_filter_ava( struct berval *bvfilter, Slapi_Value **bvals, int syntax,
 		return( string_filter_approx( bvfilter, bvals, retVal ) );
 	}
 
-	bvfilter_norm.bv_val = slapi_ch_malloc( bvfilter->bv_len + 1 );
-	SAFEMEMCPY( bvfilter_norm.bv_val, bvfilter->bv_val, bvfilter->bv_len );
-	bvfilter_norm.bv_val[bvfilter->bv_len] = '\0';
-	/* 3rd arg: 1 - trim leading blanks */
-	value_normalize_ext( bvfilter_norm.bv_val, syntax, 1, &alt );
-	if (alt) {
-		slapi_ch_free_string(&bvfilter_norm.bv_val);
-		bvfilter_norm.bv_val = alt;
+	if (syntax & SYNTAX_NORM_FILT) {
+		pbvfilter_norm = bvfilter; /* already normalized */
+	} else {
+		bvfilter_norm.bv_val = slapi_ch_malloc( bvfilter->bv_len + 1 );
+		SAFEMEMCPY( bvfilter_norm.bv_val, bvfilter->bv_val, bvfilter->bv_len );
+		bvfilter_norm.bv_val[bvfilter->bv_len] = '\0';
+		/* 3rd arg: 1 - trim leading blanks */
+		value_normalize_ext( bvfilter_norm.bv_val, syntax, 1, &alt );
+		if (alt) {
+			slapi_ch_free_string(&bvfilter_norm.bv_val);
+			bvfilter_norm.bv_val = alt;
+			alt = NULL;
+		}
+		bvfilter_norm.bv_len = strlen(bvfilter_norm.bv_val);
 	}
-	bvfilter_norm.bv_len = strlen(bvfilter_norm.bv_val);
 
 	for ( i = 0; (bvals != NULL) && (bvals[i] != NULL); i++ ) {
-		rc = value_cmp( (struct berval*)slapi_value_get_berval(bvals[i]), &bvfilter_norm, syntax, 1/* Normalise the first value only */ );
+		int norm_val = 1; /* normalize the first value only */
+		/* if the NORMALIZED flag is set, skip normalizing */
+		if (slapi_value_get_flags(bvals[i]) & SLAPI_ATTR_FLAG_NORMALIZED) {
+			norm_val = 0;
+		}
+		/* note - do not return the normalized value in retVal - the
+		   caller will usually want the "raw" value, and can normalize it later
+		*/
+		rc = value_cmp( (struct berval*)slapi_value_get_berval(bvals[i]), pbvfilter_norm, syntax, norm_val );
                 switch ( ftype ) {
                 case LDAP_FILTER_GE:
                         if ( rc >= 0 ) {
@@ -205,7 +219,7 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
     Slapi_Value **bvals, int syntax )
 {
 	int		i, j, rc, size=0;
-	char		*p, *end, *realval, *tmpbuf, *bigpat = NULL;
+	char		*p, *end, *realval, *tmpbuf = NULL, *bigpat = NULL;
 	size_t		tmpbufsize;
 	char		pat[BUFSIZ];
 	char		buf[BUFSIZ];
@@ -218,6 +232,9 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
 	Slapi_Regex	*re = NULL;
 	const char  *re_result = NULL;
 	char *alt = NULL;
+	int filter_normalized = 0;
+	int free_re = 1;
+	struct subfilt *sf = NULL;
 
 	LDAPDebug( LDAP_DEBUG_FILTER, "=> string_filter_sub\n",
 	    0, 0, 0 );
@@ -234,54 +251,85 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
 	 */
 	time_up = ( timelimit==-1 ? -1 : optime + timelimit);
 
-	/*
-	 * construct a regular expression corresponding to the
-	 * filter and let regex do the work for each value
-	 * XXX should do this once and save it somewhere XXX
-	 */
-	pat[0] = '\0';
-	p = pat;
-	end = pat + sizeof(pat) - 2;	/* leave room for null */
-
-	if ( initial != NULL ) {
-		size = strlen( initial ) + 1; /* add 1 for "^" */
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_DATA, &sf );
+	if ( sf ) {
+		re = (Slapi_Regex *)sf->sf_private;
+		if ( re ) {
+			free_re = 0;
+		}
 	}
 
-	if ( any != NULL ) {
-		i = 0;
-		while ( any[i] ) {
-			size += strlen(any[i++]) + 2; /* add 2 for ".*" */
+	if (!re) {
+		/*
+		 * construct a regular expression corresponding to the
+		 * filter and let regex do the work for each value
+		 * XXX should do this once and save it somewhere XXX
+		 */
+		pat[0] = '\0';
+		p = pat;
+		end = pat + sizeof(pat) - 2;	/* leave room for null */
+
+		if ( initial != NULL ) {
+			size = strlen( initial ) + 1; /* add 1 for "^" */
 		}
-	}
 
-	if ( final != NULL ) {
-		 size += strlen( final ) + 3; /* add 3 for ".*" and "$" */
-	}
+		if ( any != NULL ) {
+			i = 0;
+			while ( any[i] ) {
+				size += strlen(any[i++]) + 2; /* add 2 for ".*" */
+			}
+		}
 
-	size *= 2; /* doubled in case all filter chars need escaping */
-	size++; /* add 1 for null */
+		if ( final != NULL ) {
+			size += strlen( final ) + 3; /* add 3 for ".*" and "$" */
+		}
 
-	if ( p + size > end ) {
-		bigpat = slapi_ch_malloc( size );
-		p = bigpat;
-	}
+		size *= 2; /* doubled in case all filter chars need escaping */
+		size++; /* add 1 for null */
 
-	if ( initial != NULL ) {
-		/* 3rd arg: 1 - trim leading blanks */
-		value_normalize_ext( initial, syntax, 1, &alt );
-		*p++ = '^';
-		if (alt) {
-			filter_strcpy_special_ext( p, alt, FILTER_STRCPY_ESCAPE_RECHARS );
-			slapi_ch_free_string(&alt);
-		} else {
-			filter_strcpy_special_ext( p, initial, FILTER_STRCPY_ESCAPE_RECHARS );
+		if ( p + size > end ) {
+			bigpat = slapi_ch_malloc( size );
+			p = bigpat;
 		}
-		p = strchr( p, '\0' );
-	}
-	if ( any != NULL ) {
-		for ( i = 0; any[i] != NULL; i++ ) {
+
+		if ( initial != NULL ) {
+			/* 3rd arg: 1 - trim leading blanks */
+			if (!filter_normalized) {
+				value_normalize_ext( initial, syntax, 1, &alt );
+			}
+			*p++ = '^';
+			if (alt) {
+				filter_strcpy_special_ext( p, alt, FILTER_STRCPY_ESCAPE_RECHARS );
+				slapi_ch_free_string(&alt);
+			} else {
+				filter_strcpy_special_ext( p, initial, FILTER_STRCPY_ESCAPE_RECHARS );
+			}
+			p = strchr( p, '\0' );
+		}
+		if ( any != NULL ) {
+			for ( i = 0; any[i] != NULL; i++ ) {
+				/* 3rd arg: 0 - DO NOT trim leading blanks */
+				if (!filter_normalized) {
+					value_normalize_ext( any[i], syntax, 0, &alt );
+				}
+				/* ".*" + value */
+				*p++ = '.';
+				*p++ = '*';
+				if (alt) {
+					filter_strcpy_special_ext( p, alt, FILTER_STRCPY_ESCAPE_RECHARS );
+					slapi_ch_free_string(&alt);
+				} else {
+					filter_strcpy_special_ext( p, any[i], FILTER_STRCPY_ESCAPE_RECHARS );
+				}
+				p = strchr( p, '\0' );
+			}
+		}
+		if ( final != NULL ) {
 			/* 3rd arg: 0 - DO NOT trim leading blanks */
-			value_normalize_ext( any[i], syntax, 0, &alt );
+			if (!filter_normalized) {
+				value_normalize_ext( final, syntax, 0, &alt );
+			}
 			/* ".*" + value */
 			*p++ = '.';
 			*p++ = '*';
@@ -289,38 +337,24 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
 				filter_strcpy_special_ext( p, alt, FILTER_STRCPY_ESCAPE_RECHARS );
 				slapi_ch_free_string(&alt);
 			} else {
-				filter_strcpy_special_ext( p, any[i], FILTER_STRCPY_ESCAPE_RECHARS );
+				filter_strcpy_special_ext( p, final, FILTER_STRCPY_ESCAPE_RECHARS );
 			}
-			p = strchr( p, '\0' );
-		}
-	}
-	if ( final != NULL ) {
-		/* 3rd arg: 0 - DO NOT trim leading blanks */
-		value_normalize_ext( final, syntax, 0, &alt );
-		/* ".*" + value */
-		*p++ = '.';
-		*p++ = '*';
-		if (alt) {
-			filter_strcpy_special_ext( p, alt, FILTER_STRCPY_ESCAPE_RECHARS );
-			slapi_ch_free_string(&alt);
+			strcat( p, "$" );
+		}
+
+		/* compile the regex */
+		p = (bigpat) ? bigpat : pat;
+		tmpbuf = NULL;
+		re = slapi_re_comp( p, &re_result );
+		if (NULL == re) {
+			LDAPDebug( LDAP_DEBUG_ANY, "re_comp (%s) failed (%s): %s\n",
+					   pat, p, re_result?re_result:"unknown" );
+			rc = LDAP_OPERATIONS_ERROR;
+			goto bailout;
 		} else {
-			filter_strcpy_special_ext( p, final, FILTER_STRCPY_ESCAPE_RECHARS );
+			LDAPDebug( LDAP_DEBUG_TRACE, "re_comp (%s)\n",
+					   escape_string( p, ebuf ), 0, 0 );
 		}
-		strcat( p, "$" );
-	}
-
-	/* compile the regex */
-	p = (bigpat) ? bigpat : pat;
-	tmpbuf = NULL;
-	re = slapi_re_comp( p, &re_result );
-	if (NULL == re) {
-		LDAPDebug( LDAP_DEBUG_ANY, "re_comp (%s) failed (%s): %s\n",
-		    pat, p, re_result?re_result:"unknown" );
-		rc = LDAP_OPERATIONS_ERROR;
-		goto bailout;
-	} else {
- 		LDAPDebug( LDAP_DEBUG_TRACE, "re_comp (%s)\n",
-				   escape_string( p, ebuf ), 0, 0 );
 	}
 
 	curtime = current_time();
@@ -353,8 +387,9 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
 			strncpy( realval, bvp->bv_val, tmpbufsize );
 		}
 		/* 3rd arg: 1 - trim leading blanks */
-		value_normalize_ext( realval, syntax, 1, &alt );
-
+		if (!(slapi_value_get_flags(bvals[j]) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			value_normalize_ext( realval, syntax, 1, &alt );
+		}
 		if (alt) {
 			tmprc = slapi_re_exec( re, alt, time_up );
 			slapi_ch_free_string(&alt);
@@ -373,7 +408,9 @@ string_filter_sub( Slapi_PBlock *pb, char *initial, char **any, char *final,
 		}
 	}
 bailout:
-	slapi_re_free(re);
+	if (free_re) {
+		slapi_re_free(re);
+	}
 	slapi_ch_free((void**)&tmpbuf );	/* NULL is fine */
 	slapi_ch_free((void**)&bigpat );	/* NULL is fine */
 
@@ -419,9 +456,12 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 			if (alt) {
 				slapi_ch_free_string(&c);
 		    	*nbvlp = slapi_value_new_string_passin(alt);
+				alt = NULL;
 			} else {
 		    	*nbvlp = slapi_value_new_string_passin(c);
 			}
+			/* new value is normalized */
+			slapi_value_set_flags(*nbvlp, slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
 		}
 		*ivals = nbvals;
 		break;
@@ -529,18 +569,23 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		bvdup= slapi_value_new(); 
 		for ( bvlp = bvals; bvlp && *bvlp; bvlp++ ) {
-			c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
 			/* 3rd arg: 1 - trim leading blanks */
-			value_normalize_ext( c, syntax, 1, &alt );
-			if (alt) {
-				slapi_ch_free_string(&c);
-				slapi_value_set_string_passin(bvdup, alt);
+			if (!(slapi_value_get_flags(*bvlp) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+				c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
+				value_normalize_ext( c, syntax, 1, &alt );
+				if (alt) {
+					slapi_ch_free_string(&c);
+					slapi_value_set_string_passin(bvdup, alt);
+					alt = NULL;
+				} else {
+					slapi_value_set_string_passin(bvdup, c);
+					c = NULL;
+				}
+				bvp = slapi_value_get_berval(bvdup);
 			} else {
-				slapi_value_set_string_passin(bvdup, c);
+				bvp = slapi_value_get_berval(*bvlp);
 			}
 
-			bvp = slapi_value_get_berval(bvdup);
-
 			/* leading */
 			if ( bvp->bv_len > substrlens[INDEX_SUBSTRBEGIN] - 2 ) {
 				buf[0] = '^';
@@ -549,6 +594,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRBEGIN]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
+				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
 				n++;
 			}
 
@@ -561,6 +607,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRMIDDLE]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
+				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
 				n++;
 			}
 
@@ -573,6 +620,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				buf[substrlens[INDEX_SUBSTREND] - 1] = '$';
 				buf[substrlens[INDEX_SUBSTREND]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
+				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
 				n++;
 			}
 		}
@@ -602,6 +650,7 @@ string_assertion2keys_ava(
 	char		*w, *c;
     Slapi_Value *tmpval=NULL;
     char *alt = NULL;
+    unsigned long flags = val ? slapi_value_get_flags(val) : 0;
 
     switch ( ftype ) {
     case LDAP_FILTER_EQUALITY_FAST: 
@@ -613,26 +662,34 @@ string_assertion2keys_ava(
         }
         memcpy(tmpval->bv.bv_val,slapi_value_get_string(val),len);
         tmpval->bv.bv_val[len]='\0';
-        /* 3rd arg: 1 - trim leading blanks */
-        value_normalize_ext(tmpval->bv.bv_val, syntax, 1, &alt );
-        if (alt) {
-            if (len >=  tmpval->bv.bv_len) {
-                slapi_ch_free_string(&tmpval->bv.bv_val);
+        if (!(flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
+            /* 3rd arg: 1 - trim leading blanks */
+            value_normalize_ext(tmpval->bv.bv_val, syntax, 1, &alt );
+            if (alt) {
+                if (len >=  tmpval->bv.bv_len) {
+                    slapi_ch_free_string(&tmpval->bv.bv_val);
+                }
+                tmpval->bv.bv_val = alt;
+                alt = NULL;
             }
-            tmpval->bv.bv_val = alt;
+            tmpval->bv.bv_len=strlen(tmpval->bv.bv_val);
         }
-        tmpval->bv.bv_len=strlen(tmpval->bv.bv_val);
+        slapi_value_set_flags(tmpval, flags|SLAPI_ATTR_FLAG_NORMALIZED);
         break;
 	case LDAP_FILTER_EQUALITY:
 		(*ivals) = (Slapi_Value **) slapi_ch_malloc( 2 * sizeof(Slapi_Value *) );
 		(*ivals)[0] = slapi_value_dup( val );
-		/* 3rd arg: 1 - trim leading blanks */
-		value_normalize_ext( (*ivals)[0]->bv.bv_val, syntax, 1, &alt );
-		if (alt) {
-			slapi_ch_free_string(&(*ivals)[0]->bv.bv_val);
-			(*ivals)[0]->bv.bv_val = alt;
+		if (!(flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			/* 3rd arg: 1 - trim leading blanks */
+			value_normalize_ext( (*ivals)[0]->bv.bv_val, syntax, 1, &alt );
+			if (alt) {
+				slapi_ch_free_string(&(*ivals)[0]->bv.bv_val);
+				(*ivals)[0]->bv.bv_val = alt;
+				(*ivals)[0]->bv.bv_len = strlen( (*ivals)[0]->bv.bv_val );
+				alt = NULL;
+			}
+			slapi_value_set_flags((*ivals)[0], flags|SLAPI_ATTR_FLAG_NORMALIZED);
 		}
-		(*ivals)[0]->bv.bv_len = strlen( (*ivals)[0]->bv.bv_val );
 		(*ivals)[1] = NULL;
 		break;
 

ldap/servers/plugins/syntaxes/syntax.h

@@ -54,6 +54,7 @@
 #define SYNTAX_DN		8	/* distinguished name: used with SYNTAX_CIS */
 #define SYNTAX_SI		16	/* space insensitive: used with SYNTAX_CIS */
 #define SYNTAX_INT		32	/* INTEGER */
+#define SYNTAX_NORM_FILT	64	/* filter already normalized */
 
 #define SUBBEGIN	3
 #define SUBMIDDLE	3

ldap/servers/plugins/syntaxes/tel.c

@@ -59,6 +59,12 @@ static int tel_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **any,
 		char *final, Slapi_Value ***ivals );
 static int tel_compare(struct berval	*v1, struct berval	*v2);
 static int tel_validate(struct berval *val);
+static void tel_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 2252 */
 static char *names[] = { "TelephoneNumber", "tel", TELEPHONE_SYNTAX_OID, 0 };
@@ -159,6 +165,8 @@ tel_init( Slapi_PBlock *pb )
 	    (void *) tel_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) tel_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) tel_normalize );
 
 	rc |= register_matching_rule_plugins();
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= tel_init %d\n", rc, 0, 0 );
@@ -174,7 +182,13 @@ tel_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_TEL | SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_TEL | SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -267,3 +281,14 @@ tel_validate(
 exit:
 	return rc;
 }
+
+static void tel_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_TEL|SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/teletex.c

@@ -60,6 +60,12 @@ static int teletex_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **a
 static int teletex_compare(struct berval	*v1, struct berval	*v2);
 static int teletex_validate(struct berval *val);
 static int ttx_param_validate(const char *start, const char *end);
+static void teletex_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Teletex Terminal Identifier", "teletextermid", TELETEXTERMID_SYNTAX_OID, 0 };
@@ -99,6 +105,8 @@ teletex_init( Slapi_PBlock *pb )
 	    (void *) teletex_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) teletex_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) teletex_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= teletex_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -113,7 +121,13 @@ teletex_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS,
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
 	    ftype, retVal ) );
 }
 
@@ -340,3 +354,14 @@ ttx_param_validate(
 exit:
 	return rc;
 }
+
+static void teletex_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/telex.c

@@ -59,6 +59,12 @@ static int telex_assertion2keys_sub( Slapi_PBlock *pb, char *initial, char **any
 		char *final, Slapi_Value ***ivals );
 static int telex_compare(struct berval	*v1, struct berval	*v2);
 static int telex_validate(struct berval *val);
+static void telex_normalize(
+	Slapi_PBlock *pb,
+	char    *s,
+	int     trim_spaces,
+	char    **alt
+);
 
 /* the first name is the official one from RFC 4517 */
 static char *names[] = { "Telex Number", "telexnumber", TELEXNUMBER_SYNTAX_OID, 0 };
@@ -98,6 +104,8 @@ telex_init( Slapi_PBlock *pb )
 	    (void *) telex_compare );
 	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_VALIDATE,
 	    (void *) telex_validate );
+	rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_NORMALIZE,
+	    (void *) telex_normalize );
 
 	LDAPDebug( LDAP_DEBUG_PLUGIN, "<= telex_init %d\n", rc, 0, 0 );
 	return( rc );
@@ -112,8 +120,14 @@ telex_filter_ava(
 	Slapi_Value **retVal
 )
 {
-	return( string_filter_ava( bvfilter, bvals, SYNTAX_CIS,
-	    ftype, retVal ) );
+	int filter_normalized = 0;
+	int syntax = SYNTAX_CIS;
+	slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	if (filter_normalized) {
+		syntax |= SYNTAX_NORM_FILT;
+	}
+	return( string_filter_ava( bvfilter, bvals, syntax,
+							   ftype, retVal ) );
 }
 
 
@@ -255,3 +269,13 @@ exit:
 	return rc;
 }
 
+static void telex_normalize(
+	Slapi_PBlock	*pb,
+	char	*s,
+	int		trim_spaces,
+	char	**alt
+)
+{
+	value_normalize_ext(s, SYNTAX_CIS, trim_spaces, alt);
+	return;
+}

ldap/servers/plugins/syntaxes/value.c

@@ -107,6 +107,10 @@ value_normalize_ext(
 	}
 	*alt = NULL;
 
+	if (NULL == s) {
+		return;
+	}
+
 	if ( ! (syntax & SYNTAX_CIS) && ! (syntax & SYNTAX_CES) ) {
 		return;
 	}

ldap/servers/slapd/attr.c

@@ -411,6 +411,7 @@ slapi_attr_value_find( const Slapi_Attr *a, const struct berval *v )
 
 	ava.ava_type = a->a_type;
 	ava.ava_value = *v;
+	ava.ava_private = NULL;
 	return(plugin_call_syntax_filter_ava( a, LDAP_FILTER_EQUALITY, &ava ));
 }
 
@@ -553,10 +554,12 @@ slapi_attr_value_cmp( const Slapi_Attr *a, const struct berval *v1, const struct
 	    cvals[0] = &tmpcval;
    		cvals[0]->v_csnset = NULL;
    		cvals[0]->bv = *v1;
+   		cvals[0]->v_flags = 0;
    		cvals[1] = NULL;
    		a2.a_present_values.va = cvals; /* JCM - PUKE */
    		ava.ava_type = a->a_type;
    		ava.ava_value = *v2;
+    	ava.ava_private = NULL;
 		retVal = plugin_call_syntax_filter_ava(&a2, LDAP_FILTER_EQUALITY, &ava);
 	}
 	return retVal;
@@ -673,6 +676,7 @@ attr_value_find_wsi(Slapi_Attr *a, const struct berval *bval, Slapi_Value **valu
 	 */
 	ava.ava_type = a->a_type;
 	ava.ava_value = *bval;
+	ava.ava_private = NULL;
 	retVal = plugin_call_syntax_filter_ava_sv(a, LDAP_FILTER_EQUALITY, &ava, value, 0 /* Present */);
 	
 	if(retVal==0)

ldap/servers/slapd/ava.c

@@ -67,6 +67,7 @@ get_ava(
 	}
 	ava->ava_type = slapi_attr_syntax_normalize(type);
 	slapi_ch_free_string( &type );
+    ava->ava_private = NULL;
 
 	return( 0 );
 }
@@ -97,6 +98,7 @@ rdn2ava(
 	strcpy_unescape_value( s, s );
 	ava->ava_value.bv_val = s;
 	ava->ava_value.bv_len = strlen( s );
+	ava->ava_private = NULL;
 
 	return( 0 );
 }

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -790,6 +790,7 @@ typedef struct _back_search_result_set
     Slapi_Entry*      sr_vlventry;          /* a special VLV Entry for when the ACL check fails */
     int               sr_flags;             /* Magic flags, defined below */
     int               sr_current_sizelimit; /* Current sizelimit */
+    Slapi_Filter*     sr_norm_filter;       /* search filter pre-normalized */
 } back_search_result_set;
 #define SR_FLAG_CAN_SKIP_FILTER_TEST 1 /* If set in sr_flags, means that we can safely skip the filter test */
 

ldap/servers/slapd/back-ldbm/filterindex.c

@@ -303,6 +303,7 @@ ava_candidates(
 
         tmp.bv = *bval;
         tmp.v_csnset=NULL;
+        tmp.v_flags = 0;
         fake.bv.bv_val=buf;
         fake.bv.bv_len=sizeof(buf);
         ptr[0]=&fake;

ldap/servers/slapd/back-ldbm/ldbm_attr.c

@@ -461,39 +461,6 @@ ldbm_compute_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi_
 	return -1; /* I see no ships */
 }
 
-/*
- * string_find(): case sensitive search for the substring str2 within str1.
- */
-static 
-char * string_find (
-        const char * str1,
-        const char * str2
-        )
-{
-        char *cp = (char *) str1;
-        char *s1, *s2;
-
-        if ( !*str2 )
-            return((char *)str1);
-
-        while (*cp)
-        {
-                s1 = cp;
-                s2 = (char *) str2;
-
-                while ( *s1 && *s2 && !(*s1-*s2) )
-                        s1++, s2++;
-
-                if (!*s2)
-                        return(cp);
-
-                cp++;
-        }
-
-        return(NULL);
-
-}
-
 /* What are we doing ?
 	The back-end can't search properly for the hasSubordinates and
 	numSubordinates attributes. The reason being that they're not
@@ -727,12 +694,10 @@ ldbm_compute_rewriter(Slapi_PBlock *pb)
 	slapi_pblock_get( pb, SLAPI_SEARCH_STRFILTER, &fstr );
 
 	if ( NULL != fstr ) {
-		char *lc_fstr = (char *)slapi_utf8StrToLower( (unsigned char *)fstr );
-
-		if (lc_fstr && string_find(lc_fstr,"subordinates")) {
+		if (PL_strcasestr(fstr, "subordinates")) {
 			Slapi_Filter	*f = NULL;
 			/* Look for special filters we want to leave alone */
-			if (0 == strcmp(lc_fstr, "(&(numsubordinates=*)(numsubordinates>=1))" )) {
+			if (0 == strcasecmp(fstr, "(&(numsubordinates=*)(numsubordinates>=1))" )) {
 				; /* Do nothing, this one works OK */
 			} else {
 				/* So let's grok the filter in detail and try to rewrite it */
@@ -744,8 +709,6 @@ ldbm_compute_rewriter(Slapi_PBlock *pb)
 				}
 			}
 		}
-
-		slapi_ch_free_string( &lc_fstr );
 	}
 	return rc; 
 }

ldap/servers/slapd/back-ldbm/ldbm_search.c

@@ -198,6 +198,100 @@ ldbm_back_search_cleanup(Slapi_PBlock *pb,
     return function_result;
 }
 
+static int
+ldbm_search_compile_filter(Slapi_Filter *f, void *arg)
+{
+    int rc = SLAPI_FILTER_SCAN_CONTINUE;
+    if (f->f_choice == LDAP_FILTER_SUBSTRINGS) {
+        char pat[BUFSIZ];
+        char *p, *end, *tmpbuf, *bigpat = NULL;
+        size_t size = 0;
+        Slapi_Regex *re = NULL;
+        const char *re_result = NULL;
+        int i = 0;
+        char ebuf[BUFSIZ];
+
+        PR_ASSERT(NULL == f->f_un.f_un_sub.sf_private);
+        /*
+         * construct a regular expression corresponding to the filter
+         */
+        pat[0] = '\0';
+        p = pat;
+        end = pat + sizeof(pat) - 2; /* leave room for null */
+
+        if (f->f_sub_initial != NULL) {
+            size = strlen(f->f_sub_initial) + 1; /* add 1 for "^" */
+        }
+
+        while (f->f_sub_any && f->f_sub_any[i]) {
+            size += strlen(f->f_sub_any[i++]) + 2; /* add 2 for ".*" */
+        }
+
+        if (f->f_sub_final != NULL) {
+            size += strlen(f->f_sub_final) + 3; /* add 3 for ".*" and "$" */
+        }
+
+        size *= 2; /* doubled in case all filter chars need escaping (regex special chars) */
+        size++; /* add 1 for null */
+
+        if (p + size > end) {
+            bigpat = slapi_ch_malloc(size);
+            p = bigpat;
+        }
+        if (f->f_sub_initial != NULL) {
+            *p++ = '^';
+            p = filter_strcpy_special_ext(p, f->f_sub_initial, FILTER_STRCPY_ESCAPE_RECHARS);
+        }
+        for (i = 0; f->f_sub_any && f->f_sub_any[i]; i++) {
+            /* ".*" + value */
+            *p++ = '.';
+            *p++ = '*';
+            p = filter_strcpy_special_ext(p, f->f_sub_any[i], FILTER_STRCPY_ESCAPE_RECHARS);
+        }
+        if (f->f_sub_final != NULL) {
+            /* ".*" + value */
+            *p++ = '.';
+            *p++ = '*';
+            p = filter_strcpy_special_ext(p, f->f_sub_final, FILTER_STRCPY_ESCAPE_RECHARS);
+            strcat(p, "$");
+        }
+
+        /* compile the regex */
+        p = bigpat ? bigpat : pat;
+        tmpbuf = NULL;
+        re = slapi_re_comp(p, &re_result);
+        if (NULL == re) {
+            LDAPDebug(LDAP_DEBUG_ANY, "ldbm_search_compile_filter: re_comp (%s) failed (%s): %s\n",
+                      pat, p, re_result?re_result:"unknown" );
+            rc = SLAPI_FILTER_SCAN_ERROR;
+        } else {
+            LDAPDebug(LDAP_DEBUG_TRACE, "ldbm_search_compile_filter: re_comp (%s)\n",
+                      escape_string(p, ebuf), 0, 0);
+            f->f_un.f_un_sub.sf_private = (void *)re;
+        }
+    } else if (f->f_choice == LDAP_FILTER_EQUALITY) {
+        /* store the flags in the ava_private - should be ok - points
+           to itself - no dangling references */
+        f->f_un.f_un_ava.ava_private = &f->f_flags;
+    }
+    return rc;
+}
+
+static int
+ldbm_search_free_compiled_filter(Slapi_Filter *f, void *arg)
+{
+    int rc = SLAPI_FILTER_SCAN_CONTINUE;
+    if ((f->f_choice == LDAP_FILTER_SUBSTRINGS) &&
+        (f->f_un.f_un_sub.sf_private)) {
+        slapi_re_free((Slapi_Regex *)f->f_un.f_un_sub.sf_private);
+        f->f_un.f_un_sub.sf_private = NULL;
+    } else if (f->f_choice == LDAP_FILTER_EQUALITY) {
+        /* clear the flags in the ava_private */
+        f->f_un.f_un_ava.ava_private = NULL;
+    }
+    return rc;
+}
+
 /*
  * Return values from ldbm_back_search are:
  *
@@ -774,6 +868,32 @@ ldbm_back_search( Slapi_PBlock *pb )
         }
     }
 
+    /* if we need to perform the filter test, pre-digest the filter to
+       speed up the filter test */
+    if ( !(sr->sr_flags & SR_FLAG_CAN_SKIP_FILTER_TEST) ||
+         li->li_filter_bypass_check ) {
+        int rc = 0, filt_errs = 0;
+        Slapi_Filter *filter= NULL;
+
+        slapi_pblock_get(pb, SLAPI_SEARCH_FILTER, &filter);
+        slapi_filter_free(sr->sr_norm_filter, 1);
+        sr->sr_norm_filter = slapi_filter_dup(filter);
+        /* step 1 - normalize all of the values used in the search filter */
+        slapi_filter_normalize(sr->sr_norm_filter, PR_TRUE /* normalize values too */);
+        /* step 2 - pre-compile the substr regex and the equality flags */
+        rc = slapi_filter_apply(sr->sr_norm_filter, ldbm_search_compile_filter,
+                                NULL, &filt_errs);
+        if (rc != SLAPI_FILTER_SCAN_NOMORE) {
+            LDAPDebug2Args(LDAP_DEBUG_ANY,
+                           "ERROR: could not pre-compile the search filter - error %d %d\n",
+                           rc, filt_errs);
+            if (rc == SLAPI_FILTER_SCAN_ERROR) {
+                tmp_err = LDAP_OPERATIONS_ERROR;
+                tmp_desc = "Could not compile regex for filter matching";
+            }
+        }
+    }
+
     /* Fix for bugid #394184, SD, 05 Jul 00 */
     /* tmp_err == -1: no error */
     return ldbm_back_search_cleanup(pb, li, sort_control, tmp_err, tmp_desc,
@@ -1234,6 +1354,12 @@ ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension )
         goto bail;
     }
 
+    if (sr->sr_norm_filter) {
+        int val = 1;
+        slapi_pblock_set( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &val );
+        filter = sr->sr_norm_filter;
+    }
+
     if (NULL == basesdn) {
         slapi_send_ldap_result( pb, LDAP_INVALID_DN_SYNTAX, NULL,
                                "Null target DN", 0, NULL );
@@ -1602,6 +1728,7 @@ new_search_result_set(IDList *idl, int vlv, int lookthroughlimit)
 static void
 delete_search_result_set( back_search_result_set **sr )
 {
+    int rc = 0, filt_errs = 0;
     if ( NULL == sr || NULL == *sr)
     {
         return;
@@ -1610,6 +1737,14 @@ delete_search_result_set( back_search_result_set **sr )
     {
         idl_free( (*sr)->sr_candidates );
     }
+    rc = slapi_filter_apply((*sr)->sr_norm_filter, ldbm_search_free_compiled_filter,
+                            NULL, &filt_errs);
+    if (rc != SLAPI_FILTER_SCAN_NOMORE) {
+        LDAPDebug2Args(LDAP_DEBUG_ANY,
+                       "ERROR: could not free the pre-compiled regexes in the search filter - error %d %d\n",
+                       rc, filt_errs);
+    }
+    slapi_filter_free((*sr)->sr_norm_filter, 1);
     memset( *sr, 0, sizeof( back_search_result_set ) );
     slapi_ch_free( (void**)sr );
 }

ldap/servers/slapd/filter.c

@@ -689,6 +689,7 @@ slapi_filter_dup(Slapi_Filter *f)
 
 	out->f_choice = f->f_choice;
 	out->f_hash = f->f_hash;
+	out->f_flags = f->f_flags;
 
 	LDAPDebug( LDAP_DEBUG_FILTER, "slapi_filter_dup type 0x%lX\n", f->f_choice, 0, 0 );
 	switch ( f->f_choice ) {
@@ -719,9 +720,6 @@ slapi_filter_dup(Slapi_Filter *f)
 	case LDAP_FILTER_OR:
 	case LDAP_FILTER_NOT:
 		outl = &out->f_list;
-
-/*		out->f_list = slapi_filter_dup(f->f_list);
-*/
 		for (fl = f->f_list; fl != NULL; fl = fl->f_next) {
 			(*outl) = slapi_filter_dup( fl );
 			(*outl)->f_next = 0;
@@ -733,18 +731,15 @@ slapi_filter_dup(Slapi_Filter *f)
 		break;
 
 	case LDAP_FILTER_EXTENDED:
-		/* something needs to be done here, but Im not sure how to do it
-		slapi_ch_free((void**)&f->f_mr_oid);
-		slapi_ch_free((void**)&f->f_mr_type);
-		slapi_ch_free((void **)&f->f_mr_value.bv_val );
-		if (f->f_mr.mrf_destroy != NULL) {
-		    Slapi_PBlock pb;
-		    pblock_init (&pb);
-		    if ( ! slapi_pblock_set (&pb, SLAPI_PLUGIN_OBJECT, f->f_mr.mrf_object)) {
-			f->f_mr.mrf_destroy (&pb);
-		    }
+		out->f_mr_oid = slapi_ch_strdup(f->f_mr_oid);
+		out->f_mr_type = slapi_ch_strdup(f->f_mr_type);
+		out->f_mr_value.bv_val = slapi_ch_strdup(f->f_mr_value.bv_val);
+		out->f_mr_value.bv_len = f->f_mr_value.bv_len;
+		out->f_mr_dnAttrs = f->f_mr_dnAttrs;
+		if (f->f_mr.mrf_match) {
+			int rc = plugin_mr_filter_create(&out->f_mr);
+			LDAPDebug1Arg( LDAP_DEBUG_FILTER, "slapi_filter_dup plugin_mr_filter_create returned %d\n", rc );
 		}
-		*/
 		break;
 
 	default:
@@ -1043,34 +1038,92 @@ slapi_filter_get_subfilt(
 }
 
 static void
-filter_normalize_ava( struct ava *ava, int ftype )
+filter_normalize_ava( struct slapi_filter *f, PRBool norm_values )
 {
-    char	*tmp;
+    char *tmp;
+    struct ava *ava;
 
-    if ( ava == NULL ) {
-	return;
+    if ( f == NULL ) {
+        return;
     }
+
+    ava = &f->f_ava;
     tmp = ava->ava_type;
     ava->ava_type = slapi_attr_syntax_normalize(tmp);
     slapi_ch_free((void**)&tmp );
-    /* value will be normalized later */
+    f->f_flags |= SLAPI_FILTER_NORMALIZED_TYPE;
+    if (norm_values) {
+        char *newval = NULL;
+        /* NOTE: assumes ava->ava_value.bv_val is NULL terminated - get_ava/ber_scanf 'o'
+           will NULL terminate the string by default */
+        slapi_attr_value_normalize(NULL, NULL, ava->ava_type, ava->ava_value.bv_val, 1, &newval);
+        if (newval && (newval != ava->ava_value.bv_val)) {
+            slapi_ch_free_string(&ava->ava_value.bv_val);
+            ava->ava_value.bv_val = newval;
+            ava->ava_value.bv_len = strlen(newval);
+        }
+        f->f_flags |= SLAPI_FILTER_NORMALIZED_VALUE;
+    }
 }
 
+static void
+filter_normalize_subfilt( struct slapi_filter *f, PRBool norm_values )
+{
+	struct subfilt *sf;
+
+	if ( f == NULL ) {
+	    return;
+	}
+
+	sf = &f->f_sub;
+	char *tmp = sf->sf_type;
+	sf->sf_type = slapi_attr_syntax_normalize(tmp);
+	slapi_ch_free((void**)&tmp );
+	f->f_flags |= SLAPI_FILTER_NORMALIZED_TYPE;
+	if (norm_values) {
+		char *newval = NULL;
+		Slapi_Attr attr;
+		int ii;
+
+		slapi_attr_init(&attr, sf->sf_type);
+		slapi_attr_value_normalize(NULL, &attr, NULL, sf->sf_initial, 1, &newval);
+		if (newval && (newval != sf->sf_initial)) {
+			slapi_ch_free_string(&sf->sf_initial);
+			sf->sf_initial = newval;
+		}
+		for (ii = 0; sf->sf_any && sf->sf_any[ii]; ++ii) {
+			newval = NULL;
+			/* do not trim spaces of sf_any values - see string_filter_sub() */
+			slapi_attr_value_normalize(NULL, &attr, NULL, sf->sf_any[ii], 0, &newval);
+			if (newval && (newval != sf->sf_any[ii])) {
+				slapi_ch_free_string(&sf->sf_any[ii]);
+				sf->sf_any[ii] = newval;
+			}
+		}
+		newval = NULL;
+		/* do not trim spaces of sf_final values - see string_filter_sub() */
+		slapi_attr_value_normalize(NULL, &attr, NULL, sf->sf_final, 0, &newval);
+		if (newval && (newval != sf->sf_final)) {
+			slapi_ch_free_string(&sf->sf_final);
+			sf->sf_final = newval;
+		}
+		attr_done(&attr);
+		f->f_flags |= SLAPI_FILTER_NORMALIZED_VALUE;
+	}
+}
 
-void filter_normalize( struct slapi_filter *f );
+void filter_normalize_ext( struct slapi_filter *f, PRBool norm_values );
 
 static void
-filter_normalize_list( struct slapi_filter *flist )
+filter_normalize_list( struct slapi_filter *flist, PRBool norm_values )
 {
     struct slapi_filter *f;
 
     for ( f = flist; f != NULL; f = f->f_next ) {
-	filter_normalize( f );
+	filter_normalize_ext( f, norm_values );
     }
 }
 
-
-
 /*
  * Normalize all values and types in a filter.  This isn't necessary
  * when we've read the slapi_filter off the wire, but if we've hand-constructed
@@ -1080,7 +1133,7 @@ filter_normalize_list( struct slapi_filter *flist )
  * normalized.
  */
 void
-filter_normalize( struct slapi_filter *f )
+filter_normalize_ext( struct slapi_filter *f, PRBool norm_values )
 {
     char	*tmp;
 
@@ -1093,38 +1146,49 @@ filter_normalize( struct slapi_filter *f )
     case LDAP_FILTER_LE:
     case LDAP_FILTER_APPROX:
     case LDAP_FILTER_EQUALITY:
-	filter_normalize_ava( &f->f_ava, f->f_choice );
+	filter_normalize_ava( f, norm_values );
 	break;
     case LDAP_FILTER_SUBSTRINGS:
-	tmp = f->f_sub_type;
-	f->f_sub_type = slapi_attr_syntax_normalize(tmp);
-	slapi_ch_free((void**)&tmp );
-	/* value will be normalized later */
+	filter_normalize_subfilt( f, norm_values );
 	break;
     case LDAP_FILTER_PRESENT:
 	tmp = f->f_type;
 	f->f_type = slapi_attr_syntax_normalize(tmp);
 	slapi_ch_free((void**)&tmp );
+	f->f_flags |= SLAPI_FILTER_NORMALIZED_TYPE;
 	break;
     case LDAP_FILTER_EXTENDED:
 	tmp = f->f_mr_type;
 	f->f_mr_type = slapi_attr_syntax_normalize(tmp);
 	slapi_ch_free((void**)&tmp );
+	f->f_flags |= SLAPI_FILTER_NORMALIZED_TYPE;
 	break;
     case LDAP_FILTER_AND:
-	filter_normalize_list( f->f_and );
+	filter_normalize_list( f->f_and, norm_values );
 	break;
     case LDAP_FILTER_OR:
-	filter_normalize_list( f->f_or );
+	filter_normalize_list( f->f_or, norm_values );
 	break;
     case LDAP_FILTER_NOT:
-	filter_normalize_list( f->f_not );
+	filter_normalize_list( f->f_not, norm_values );
 	break;
     default:
 	return;
     }
 }
-	
+
+void
+filter_normalize( struct slapi_filter *f )
+{
+    filter_normalize_ext(f, PR_FALSE);
+}
+
+void
+slapi_filter_normalize( struct slapi_filter *f, PRBool norm_values )
+{
+    filter_normalize_ext(f, norm_values);
+}
+
 void
 filter_print( struct slapi_filter *f )
 {

ldap/servers/slapd/filtercmp.c

@@ -91,6 +91,7 @@ static Slapi_Value **get_normalized_value(const Slapi_Attr *sattr, struct ava *a
 
     sv.bv = ava->ava_value;
     sv.v_csnset = NULL;
+    sv.v_flags = 0;
     svlist[0] = &sv;
     svlist[1] = NULL;
     if ((slapi_attr_values2keys_sv(sattr, svlist, &keylist,

ldap/servers/slapd/filterentry.c

@@ -498,8 +498,9 @@ test_extensible_filter(
 				 */
 				struct ava a;
 				a.ava_type= mrf->mrf_type;
-        		a.ava_value.bv_len= mrf->mrf_value.bv_len;
+				a.ava_value.bv_len= mrf->mrf_value.bv_len;
 				a.ava_value.bv_val = mrf->mrf_value.bv_val;
+				a.ava_private = NULL;
 				rc= test_ava_filter( callers_pb, e, e->e_attrs, &a, LDAP_FILTER_EQUALITY, 0 /* Don't Verify Access */ , 0 /* don't just verify access */, access_check_done );
 				if(rc!=LDAP_SUCCESS && mrf->mrf_dnAttrs)
 				{
@@ -566,8 +567,9 @@ test_extensible_filter(
 				 */
 				struct ava a;
 				a.ava_type= mrf->mrf_type;
-        		a.ava_value.bv_len= mrf->mrf_value.bv_len;
+				a.ava_value.bv_len= mrf->mrf_value.bv_len;
 				a.ava_value.bv_val = mrf->mrf_value.bv_val;
+				a.ava_private = NULL;
 				rc= test_ava_filter( callers_pb, e, e->e_attrs, &a, LDAP_FILTER_EQUALITY, 0 /* Don't Verify Access */ , 0 /* don't just verify access */, access_check_done );
 				if(rc!=LDAP_SUCCESS && mrf->mrf_dnAttrs)
 				{
@@ -646,7 +648,7 @@ test_filter_list(
 	return( nomatch );
 }
 
-void
+char *
 filter_strcpy_special_ext( char *d, char *s, int flags )
 {
 	for ( ; *s; s++ ) {
@@ -677,12 +679,13 @@ filter_strcpy_special_ext( char *d, char *s, int flags )
 		*d++ = *s;
 	}
 	*d = '\0';
+	return d;
 }
 
-void
+char *
 filter_strcpy_special( char *d, char *s )
 {
-	filter_strcpy_special_ext(d, s, 0);
+	return filter_strcpy_special_ext(d, s, 0);
 }
 
 int test_substring_filter(

ldap/servers/slapd/pblock.c

@@ -1244,6 +1244,12 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value )
 		}
 		(*(IFP *)value) = pblock->pb_plugin->plg_syntax_validate;
 		break;
+	case SLAPI_PLUGIN_SYNTAX_NORMALIZE:
+		if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_SYNTAX ) {
+			return( -1 );
+		}
+		(*(VFPV *)value) = pblock->pb_plugin->plg_syntax_normalize;
+		break;
 
 	/* controls we know about */
 	case SLAPI_MANAGEDSAIT:
@@ -1905,6 +1911,14 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value )
 		(*(LDAPControl ***)value) = pblock->pb_search_ctrls;
 		break;
 		
+	case SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED:
+		(*(int *)value) = pblock->pb_syntax_filter_normalized;
+		break;
+		
+	case SLAPI_PLUGIN_SYNTAX_FILTER_DATA:
+		(*(void **)value) = pblock->pb_syntax_filter_data;
+		break;
+		
 	default:
 		LDAPDebug( LDAP_DEBUG_ANY,
 		    "Unknown parameter block argument %d\n", arg, 0, 0 );
@@ -2717,6 +2731,12 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value )
 		}
 		pblock->pb_plugin->plg_syntax_validate = (IFP) value;
 		break;
+	case SLAPI_PLUGIN_SYNTAX_NORMALIZE:
+		if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_SYNTAX ) {
+			return( -1 );
+		}
+		pblock->pb_plugin->plg_syntax_normalize = (VFPV) value;
+		break;
 	case SLAPI_ENTRY_PRE_OP:
 		pblock->pb_pre_op_entry = (Slapi_Entry *) value;
 		break;
@@ -3399,6 +3419,14 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value )
 		pblock->pb_search_ctrls = (LDAPControl **) value;
 		break;
 
+	case SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED:
+		pblock->pb_syntax_filter_normalized = *((int *)value);
+		break;
+
+	case SLAPI_PLUGIN_SYNTAX_FILTER_DATA:
+		pblock->pb_syntax_filter_data = (void *)value;
+		break;
+
 	default:
 		LDAPDebug( LDAP_DEBUG_ANY,
 		    "Unknown parameter block argument %d\n", arg, 0, 0 );

ldap/servers/slapd/plugin_syntax.c

@@ -143,6 +143,13 @@ plugin_call_syntax_filter_ava_sv(
 
 	pblock_init( &pipb );
 	slapi_pblock_set( &pipb, SLAPI_PLUGIN, (void *) a->a_plugin );
+	if (ava->ava_private) {
+		int filter_normalized = 0;
+		int f_flags = 0;
+		f_flags = *(int *)ava->ava_private;
+		filter_normalized = f_flags | SLAPI_FILTER_NORMALIZED_VALUE;
+		slapi_pblock_set( &pipb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	}
 
 	rc = -1;	/* does not match by default */
 	switch ( ftype ) {
@@ -230,6 +237,7 @@ plugin_call_syntax_filter_sub_sv(
 	Slapi_PBlock	pipb;
 	int		rc;
 	IFP sub_fn = NULL;
+	int filter_normalized = 0;
 
 	LDAPDebug( LDAP_DEBUG_FILTER,
 	    "=> plugin_call_syntax_filter_sub_sv\n", 0, 0, 0 );
@@ -242,6 +250,12 @@ plugin_call_syntax_filter_sub_sv(
 	}
 
 	pblock_init( &pipb );
+	if (pb) {
+		slapi_pblock_get( pb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+		slapi_pblock_set( &pipb, SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED, &filter_normalized );
+	}
+	slapi_pblock_set( &pipb, SLAPI_PLUGIN_SYNTAX_FILTER_DATA, fsub );
+
 	/* use the substr matching rule plugin if available, otherwise, use
 	   the syntax plugin */
 	if (a->a_mr_sub_plugin) {
@@ -913,3 +927,29 @@ slapi_attr_assertion2keys_sub( /* JCM SLOW FUNCTION */
 	valuearray_free(&svout);
 	return rc;
 }
+
+void
+slapi_attr_value_normalize(
+	Slapi_PBlock *pb,
+	const Slapi_Attr *sattr, /* if sattr is NULL, type must be attr type name */
+	const char *type,
+	char *val,
+	int trim_spaces,
+	char **retval
+)
+{
+	Slapi_Attr myattr;
+	VFPV norm_fn = NULL;
+
+	if (!sattr) {
+		sattr = slapi_attr_init(&myattr, type);
+	}
+	norm_fn = sattr->a_plugin->plg_syntax_normalize;
+	if (norm_fn) {
+		(*norm_fn)(pb, val, trim_spaces, retval);
+	}
+	if (sattr == &myattr) {
+		attr_done(&myattr);
+	}
+	return;
+}

ldap/servers/slapd/proto-slap.h

@@ -606,9 +606,9 @@ void set_hash_filters(int i);
 /*
  * filterentry.c
  */
-void filter_strcpy_special( char *d, char *s );
+char *filter_strcpy_special( char *d, char *s );
 #define FILTER_STRCPY_ESCAPE_RECHARS 0x01
-void filter_strcpy_special_ext( char *d, char *s, int flags );
+char *filter_strcpy_special_ext( char *d, char *s, int flags );
 
 
 /*

ldap/servers/slapd/slap.h

@@ -214,6 +214,7 @@ typedef struct symbol_t {
 typedef void	(*VFP)(void *);
 typedef void	(*VFPP)(void **);
 typedef void	(*VFP0)(void);
+typedef void	(*VFPV)(); /* takes undefined arguments */
 #define LDAPI_INTERNAL	1
 #include "slapi-private.h"
 #include "pw.h"
@@ -1067,6 +1068,7 @@ struct slapdplugin {
 			char	*plg_un_syntax_oid;
 			IFP	plg_un_syntax_compare;
 			IFP	plg_un_syntax_validate;
+			VFPV plg_un_syntax_normalize;
 		} plg_un_syntax;
 #define plg_syntax_filter_ava		plg_un.plg_un_syntax.plg_un_syntax_filter_ava
 #define plg_syntax_filter_sub		plg_un.plg_un_syntax.plg_un_syntax_filter_sub
@@ -1078,6 +1080,7 @@ struct slapdplugin {
 #define plg_syntax_oid			plg_un.plg_un_syntax.plg_un_syntax_oid
 #define plg_syntax_compare		plg_un.plg_un_syntax.plg_un_syntax_compare
 #define plg_syntax_validate		plg_un.plg_un_syntax.plg_un_syntax_validate
+#define plg_syntax_normalize	plg_un.plg_un_syntax.plg_un_syntax_normalize
 
 		struct plg_un_acl_struct {
 			IFP	plg_un_acl_init;
@@ -1629,6 +1632,8 @@ typedef struct slapi_pblock {
 	LDAPControl	**pb_search_ctrls; /* for search operations, allows plugins to provide
 									  controls to pass for each entry or referral returned */
 	IFP		pb_mr_index_sv_fn; /* values and keys are Slapi_Value ** */
+	int		pb_syntax_filter_normalized; /* the syntax filter types/values are already normalized */
+	void		*pb_syntax_filter_data; /* extra data to pass to a syntax plugin function */
 } slapi_pblock;
 
 /* index if substrlens */

ldap/servers/slapd/slapi-plugin.h

@@ -4776,6 +4776,15 @@ int slapi_filter_compare(struct slapi_filter *f1, struct slapi_filter *f2);
 Slapi_Filter *slapi_filter_dup(Slapi_Filter *f);
 int slapi_filter_changetype(Slapi_Filter *f, const char *newtype);
 
+/**
+ * Normalize in-place the given filter.  Normalizes the attribute types always.
+ * If norm_values is true, will also normalize the values.
+ *
+ * \param f the filter to normalize
+ * \param norm_values if true, normalize the values in addition to the type names
+ */
+void slapi_filter_normalize(Slapi_Filter *f, PRBool norm_values);
+
 /**
  * Check whether a given attribute type is defined in schema or not
  *
@@ -5259,6 +5268,30 @@ int slapi_attr_assertion2keys_ava_sv( const Slapi_Attr *sattr, Slapi_Value *val,
 int slapi_attr_assertion2keys_sub_sv( const Slapi_Attr *sattr, char *initial,
 	char **any, char *final, Slapi_Value ***ivals );
 
+/**
+ * Normalize the given value using the syntax associated with the
+ * given attribute.  It will attempt to normalize the value in place.
+ * If it cannot, it will return the normalized value in retval.  If
+ * trim_spaces is true, whitepace characters will be trimmed from the
+ * ends of the string.  If sattr is NULL, the type will be used to look
+ * up the attribute syntax.  If sattr is not NULL, type is ignored.  If
+ * retval is set, the caller is responsible for freeing it.
+ *
+ * \param pb Slapi_PBlock to use
+ * \param sattr attribute to get the syntax from 
+ * \param type attribute to get the syntax from if sattr is NULL
+ * \param val value to normalize in place - must be NULL terminated
+ * \param trim_spaces trim whitespace from ends of string
+ * \param retval if value could not be normalized in place, this is the malloc'd memory containg the new value - caller must free
+ */
+void slapi_attr_value_normalize(
+	Slapi_PBlock *pb,
+	const Slapi_Attr *sattr, /* if sattr is NULL, type must be attr type name */
+	const char *type,
+	char *val,
+	int trim_spaces,
+	char **retval
+);
 
 /*
  * internal operation and plugin callback routines
@@ -6272,6 +6305,9 @@ typedef struct slapi_plugindesc {
 #define SLAPI_SYNTAX_SUBSTRLENS			709
 #define SLAPI_MR_SUBSTRLENS			SLAPI_SYNTAX_SUBSTRLENS /* alias */
 #define SLAPI_PLUGIN_SYNTAX_VALIDATE		710
+#define SLAPI_PLUGIN_SYNTAX_NORMALIZE		711
+#define SLAPI_PLUGIN_SYNTAX_FILTER_NORMALIZED	712
+#define SLAPI_PLUGIN_SYNTAX_FILTER_DATA		713
 
 /* ACL plugin functions and arguments */
 #define SLAPI_PLUGIN_ACL_INIT			730

ldap/servers/slapd/slapi-private.h

@@ -70,6 +70,8 @@ extern "C" {
 #define SLAPI_FILTER_TOMBSTONE 2
 #define SLAPI_FILTER_RUV 4
 #define SLAPI_ENTRY_LDAPSUBENTRY 2
+#define SLAPI_FILTER_NORMALIZED_TYPE 8
+#define SLAPI_FILTER_NORMALIZED_VALUE 16
 
 /*
     Optimized filter path. For example the following code was lifted from int.c (syntaxes plugin):

ldap/servers/slapd/utf8compare.c

@@ -870,7 +870,7 @@ slapi_utf8StrToLower(unsigned char *s)
     while ((np = (unsigned char *)ldap_utf8next((char *)p)) <= tail) {
         switch(sz = np - p) {
         case 1:
-            sprintf((char *)lp, "%c", tolower(*p));
+            *lp = tolower(*p);
             break;
         case 2:
             if (*p < UL2S || *p > UL2E) {	/* out of range */
@@ -1871,7 +1871,7 @@ slapi_utf8StrToUpper(unsigned char *s)
     while ((np = (unsigned char *)ldap_utf8next((char *)p)) <= tail) {
         switch(sz = np - p) {
         case 1:	/* ASCII */
-            sprintf((char *)up, "%c", toupper(*p));
+            *up = toupper(*p);
             break;
         case 2:	/* 2 bytes */
             if (*p < LU2S || *p > LU2E) {	/* out of range */

ldap/servers/slapd/value.c

@@ -159,6 +159,7 @@ slapi_value_dup(const Slapi_Value *v)
 {
 	Slapi_Value *newvalue= value_new(&v->bv,CSN_TYPE_UNKNOWN,NULL);
 	newvalue->v_csnset= csnset_dup(v->v_csnset);
+	newvalue->v_flags = v->v_flags;
 	return newvalue;
 
 }