Resolves: #199923

Summary: subtree search fails to find items under a db containing special characters (Comment#16)
Description: When dn contains rdn which includes '\\', it was escaped twice to generate a key for entrydn and caused mismatch in forming ancestorid index.  It ends up the subtree search fail.

ldap/servers/plugins/syntaxes/string.c

@@ -319,8 +319,10 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		for ( i = 0; i < numbvals; i++ )
 		{
-                    c = slapi_ch_strdup(slapi_value_get_string(bvals[i]));
-                    value_normalize( c, syntax, 1 /* trim leading blanks */ );
+			c = slapi_ch_strdup(slapi_value_get_string(bvals[i]));
+			/* if the NORMALIZED flag is set, skip normalizing */
+			if (!(slapi_value_get_flags(bvals[i]) & SLAPI_ATTR_FLAG_NORMALIZED))
+				value_normalize( c, syntax, 1 /* trim leading blanks */ );
 		    nbvals[i] = slapi_value_new_string_passin(c);
 		}
 		nbvals[i] = NULL;

ldap/servers/slapd/attrlist.c

@@ -289,3 +289,26 @@ int attrlist_replace(Slapi_Attr **alist, const char *type, struct berval **vals)
     return rc;
 }
 
+/*
+ * attrlist_replace_with_flags - replace the attribute value(s) with this value(s)
+ *
+ * Returns
+ * LDAP_SUCCESS - OK (including the attr not found)
+ * LDAP_OPERATIONS_ERROR - Existing duplicates in attribute.
+ */
+int attrlist_replace_with_flags(Slapi_Attr **alist, const char *type, struct berval **vals, int flags)
+{
+    Slapi_Attr **a = NULL;
+    Slapi_Value **values = NULL;
+    int rc = LDAP_SUCCESS;
+
+    if (vals == NULL || vals[0] == NULL) {
+        (void)attrlist_delete(alist, type);
+    } else {
+        attrlist_find_or_create(alist, type, &a);
+        valuearray_init_bervalarray_with_flags(vals, &values, flags);
+        rc = attr_replace(*a, values);
+    }
+    return rc;
+}
+

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -638,6 +638,7 @@ typedef struct _back_search_result_set
 #define BE_INDEX_TOMBSTONE	8   /* Index entry as a tombstone */
 #define BE_INDEX_DONT_ENCRYPT	16   /* Disable any encryption if this flag is set */
 #define BE_INDEX_EQUALITY	32  /* (w/DEL) remove the equality index */
+#define BE_INDEX_NORMALIZED SLAPI_ATTR_FLAG_NORMALIZED /* value already normalized */
 
 /* Name of attribute type used for binder-based look through limit */
 #define LDBM_LOOKTHROUGHLIMIT_AT	"nsLookThroughLimit"

ldap/servers/slapd/back-ldbm/import-threads.c

@@ -933,7 +933,7 @@ static int foreman_do_entrydn(ImportJob *job, FifoItem *fi)
 
     /* insert into the entrydn index */
     bv.bv_val = (void*)backentry_get_ndn(fi->entry);   /* jcm - Had to cast away const */
-    bv.bv_len = strlen(backentry_get_ndn(fi->entry));
+    bv.bv_len = strlen(bv.bv_val);
 
     /* We need to check here whether the DN is already present in
      * the entrydn index. If it is then the input ldif 
@@ -960,7 +960,7 @@ static int foreman_do_entrydn(ImportJob *job, FifoItem *fi)
     if ((ret = index_addordel_string(be, "entrydn", 
                                      bv.bv_val,
                                      fi->entry->ep_id,
-                                     BE_INDEX_ADD, NULL)) != 0) {
+                                     BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL)) != 0) {
         import_log_notice(job, "Error writing entrydn index "
                           "(error %d: %s)",
                           ret, dblayer_strerror(ret));

ldap/servers/slapd/back-ldbm/index.c

@@ -381,10 +381,10 @@ index_addordel_entry(
     /* if we are adding a tombstone entry (see ldbm_add.c) */
     if ((flags & BE_INDEX_TOMBSTONE) && (flags & BE_INDEX_ADD))
     {
-	Slapi_DN parent;
-	Slapi_DN *sdn = slapi_entry_get_sdn(e->ep_entry);
-	slapi_sdn_init(&parent);
-	slapi_sdn_get_parent(sdn, &parent);
+        Slapi_DN parent;
+        Slapi_DN *sdn = slapi_entry_get_sdn(e->ep_entry);
+        slapi_sdn_init(&parent);
+        slapi_sdn_get_parent(sdn, &parent);
         /*
          * Just index the "nstombstone" attribute value from the objectclass
          * attribute, and the nsuniqueid attribute value, and the entrydn value of the deleted entry.
@@ -404,7 +404,7 @@ index_addordel_entry(
             ldbm_nasty(errmsg, 1020, result);
             return( result );
         }
-	slapi_sdn_done(&parent);
+        slapi_sdn_done(&parent);
     }
     else
     {
@@ -414,8 +414,11 @@ index_addordel_entry(
               rc = slapi_entry_next_attr( e->ep_entry, attr, &attr ) ) {
             slapi_attr_get_type( attr, &type );
             svals = attr_get_present_values(attr);
-            result = index_addordel_values_sv( be, type, svals, NULL, e->ep_id,
-                                               flags, txn );
+            if ( 0 == strcmp( type, "entrydn" )) {
+                slapi_values_set_flags(svals, SLAPI_ATTR_FLAG_NORMALIZED);
+            }
+            result = index_addordel_values_sv( be, type, svals, NULL,
+                                               e->ep_id, flags, txn );
             if ( result != 0 ) {
                 ldbm_nasty(errmsg, 1030, result);
                 return( result );
@@ -423,13 +426,13 @@ index_addordel_entry(
         }
 
         /* update ancestorid index . . . */
-		/* . . . only if we are not deleting a tombstone entry - tombstone entries are not in the ancestor id index - see bug 603279 */
-	if (!((flags & BE_INDEX_TOMBSTONE) && (flags & BE_INDEX_DEL))) {
-	        result = ldbm_ancestorid_index_entry(be, e, flags, txn);
-		if ( result != 0 ) {
-			return( result );
-		}
-	}
+        /* . . . only if we are not deleting a tombstone entry - tombstone entries are not in the ancestor id index - see bug 603279 */
+        if (!((flags & BE_INDEX_TOMBSTONE) && (flags & BE_INDEX_DEL))) {
+                result = ldbm_ancestorid_index_entry(be, e, flags, txn);
+            if ( result != 0 ) {
+                return( result );
+            }
+        }
     }
     
     LDAPDebug( LDAP_DEBUG_TRACE, "<= index_%s_entry%s %d\n",
@@ -1643,6 +1646,8 @@ index_addordel_string(backend *be, const char *type, const char *s, ID id, int f
     sv.bv.bv_val= (void*)s;
     svp[0] = &sv;
     svp[1] = NULL;
+    if (flags & BE_INDEX_NORMALIZED)
+        slapi_value_set_flags(&sv, BE_INDEX_NORMALIZED);
     return index_addordel_values_ext_sv(be,type,svp,NULL,id,flags,txn,NULL,NULL);
 }
 

ldap/servers/slapd/back-ldbm/ldbm_add.c

@@ -195,7 +195,7 @@ ldbm_back_add( Slapi_PBlock *pb )
 		   slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_PARENT_ENTRY))
 		{
 			done_with_pblock_entry(pb,SLAPI_ADD_PARENT_ENTRY); /* Could be through this multiple times */
-			addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn);
+			addr.dn = (char*)slapi_sdn_get_dn (&parentsdn); /* get_copy_of_entry assumes the DN is not normalized */
 			addr.uniqueid = operation->o_params.p.p_add.parentuniqueid;
 			ldap_result_code= get_copy_of_entry(pb, &addr, &txn, SLAPI_ADD_PARENT_ENTRY, !is_replicated_operation);
 			/* need to set parentsdn or parentuniqueid if either is not set? */
@@ -234,7 +234,7 @@ ldbm_back_add( Slapi_PBlock *pb )
 	 */
     if(have_parent_address(&parentsdn, operation->o_params.p.p_add.parentuniqueid))
 	{
-		addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn);
+		addr.dn = (char*)slapi_sdn_get_dn (&parentsdn);
 		addr.uniqueid = operation->o_params.p.p_add.parentuniqueid;
 		parententry = find_entry2modify_only(pb,be,&addr,&txn);
 		if (parententry && parententry->ep_entry) {
@@ -898,7 +898,7 @@ add_update_entrydn_operational_attributes(struct backentry *ep)
     bvp[1] = NULL;
     bv.bv_val = (void*)backentry_get_ndn(ep);
     bv.bv_len = strlen( bv.bv_val );
-    entry_replace_values( ep->ep_entry, "entrydn", bvp );
+    entry_replace_values_with_flags( ep->ep_entry, "entrydn", bvp, SLAPI_ATTR_FLAG_NORMALIZED );
 }
 
 /*

ldap/servers/slapd/back-ldbm/ldbm_delete.c

@@ -250,7 +250,7 @@ ldbm_back_delete( Slapi_PBlock *pb )
     		struct backentry *parent = NULL;
 			entry_address parent_addr;
 
-			parent_addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn);
+			parent_addr.dn = (char*)slapi_sdn_get_dn (&parentsdn);
 			parent_addr.uniqueid = NULL;
     		parent = find_entry2modify_only(pb,be,&parent_addr,&txn);
     		if (NULL != parent) {

ldap/servers/slapd/back-ldbm/ldbm_modrdn.c

@@ -271,7 +271,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
 	}
 
 	/* Fetch and lock the parent of the entry that is moving */
-	oldparent_addr.dn = (char*)slapi_sdn_get_ndn (&dn_parentdn);
+	oldparent_addr.dn = (char*)slapi_sdn_get_dn (&dn_parentdn);
 	oldparent_addr.uniqueid = NULL;    		
 	parententry = find_entry2modify_only( pb, be, &oldparent_addr, NULL );
    	modify_init(&parent_modify_context,parententry);
@@ -360,10 +360,17 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
 		if(!(slapi_dn_isbesuffix(pb,slapi_sdn_get_ndn(&dn_olddn)) && isroot))
 		{
 			/* Here means that we didn't find the parent */
-			ldap_result_matcheddn = slapi_ch_strdup((char *) slapi_entry_get_dn(parententry->ep_entry));
+			if (parententry && parententry->ep_entry)
+			{
+				ldap_result_matcheddn = slapi_ch_strdup((char *) slapi_entry_get_dn(parententry->ep_entry));
+			}
+			else
+			{
+				ldap_result_matcheddn = "NULL";
+			}
 			ldap_result_code= LDAP_NO_SUCH_OBJECT;
 			LDAPDebug( LDAP_DEBUG_TRACE, "Parent does not exist matched %s, parentdn = %s\n", 
-				ldap_result_matcheddn == NULL ? "NULL" : ldap_result_matcheddn, slapi_sdn_get_ndn(&dn_parentdn), 0 );
+				ldap_result_matcheddn, slapi_sdn_get_ndn(&dn_parentdn), 0 );
 			goto error_return;
 		}
 	}
@@ -818,7 +825,8 @@ common_return:
     moddn_unlock_and_return_entries(be,&e,&existingentry);
     slapi_ch_free((void**)&child_entries);
     slapi_ch_free((void**)&child_entry_copies);
-    slapi_ch_free((void**)&ldap_result_matcheddn);
+    if (ldap_result_matcheddn && 0 != strcmp(ldap_result_matcheddn, "NULL"))
+        slapi_ch_free((void**)&ldap_result_matcheddn);
 	idl_free(children);
     slapi_sdn_done(&dn_olddn);
     slapi_sdn_done(&dn_newdn);

ldap/servers/slapd/entry.c

@@ -2795,6 +2795,17 @@ entry_replace_values(
     return attrlist_replace( &e->e_attrs, type, vals );
 }
 
+int
+entry_replace_values_with_flags(
+    Slapi_Entry		*e,
+    const char		*type,
+    struct berval	**vals,
+    int flags
+)
+{
+    return attrlist_replace_with_flags( &e->e_attrs, type, vals, flags );
+}
+
 int
 slapi_entry_flag_is_set( const Slapi_Entry *e, unsigned char flag )
 {

ldap/servers/slapd/proto-slap.h

@@ -81,6 +81,7 @@ void attrlist_add(Slapi_Attr **attrs, Slapi_Attr *a);
 int attrlist_count_subtypes(Slapi_Attr *a, const char *type);
 Slapi_Attr *attrlist_find_ex( Slapi_Attr *a, const char *type, int *type_name_disposition, char** actual_type_name, void **hint );
 int attrlist_replace(Slapi_Attr **alist, const char *type, struct berval **vals);
+int attrlist_replace_with_flags(Slapi_Attr **alist, const char *type, struct berval **vals, int flags);
 
 /*
  * attrsyntax.c
@@ -121,6 +122,7 @@ size_t value_size(const Slapi_Value *v);
  * valueset.c
  */
 int valuearray_init_bervalarray(struct berval **bvals, Slapi_Value ***cvals);
+int valuearray_init_bervalarray_with_flags(struct berval **bvals, Slapi_Value ***cvals, unsigned long flags);
 int valuearray_get_bervalarray(Slapi_Value **cvals, struct berval ***bvals); /* JCM SLOW FUNCTION */
 void valuearray_free(Slapi_Value ***va);
 Slapi_Value *valuearray_remove_value(const Slapi_Attr *a, Slapi_Value **va, const Slapi_Value *v);
@@ -642,6 +644,7 @@ void do_modrdn( Slapi_PBlock *pb );
  * modutil.c
  */
 int entry_replace_values( Slapi_Entry *e, const char *type, struct berval **vals );
+int entry_replace_values_with_flags( Slapi_Entry *e, const char *type, struct berval **vals, int flags );
 int entry_apply_mods( Slapi_Entry *e, LDAPMod **mods );
 int entry_apply_mod( Slapi_Entry *e, const LDAPMod *mod );
 void freepmods( LDAPMod **pmods );

ldap/servers/slapd/slap.h

@@ -379,6 +379,7 @@ struct slapi_value
 {
     struct berval bv;
     CSNSet *v_csnset;
+    unsigned long v_flags;
 };
 
 /*

ldap/servers/slapd/slapi-plugin.h

@@ -61,6 +61,7 @@ extern "C" {
 #define SLAPI_ATTR_FLAG_OBSOLETE	0x0040	/* an outdated definition */
 #define SLAPI_ATTR_FLAG_COLLECTIVE	0x0080	/* collective (not supported) */
 #define SLAPI_ATTR_FLAG_NOUSERMOD	0x0100	/* can't be modified over LDAP */
+#define SLAPI_ATTR_FLAG_NORMALIZED	0x0200	/* the attr value is normalized */
 
 /* operation flags */
 #define SLAPI_OP_FLAG_NEVER_CHAIN	0x00800 /* Do not chain the operation */	
@@ -429,6 +430,9 @@ Slapi_Value *slapi_value_init_berval(Slapi_Value *v, struct berval *bval);
 Slapi_Value *slapi_value_init_string(Slapi_Value *v,const char *s);
 Slapi_Value *slapi_value_init_string_passin(Slapi_Value *v, char *s);
 Slapi_Value *slapi_value_dup(const Slapi_Value *v);
+void slapi_value_set_flags(Slapi_Value *v, unsigned long flags);
+void slapi_values_set_flags(Slapi_Value **vs, unsigned long flags);
+unsigned long slapi_value_get_flags(Slapi_Value *v);
 void slapi_value_free(Slapi_Value **value);
 const struct berval *slapi_value_get_berval( const Slapi_Value *value );
 Slapi_Value *slapi_value_set_berval( Slapi_Value *value, const struct berval *bval );

ldap/servers/slapd/value.c

@@ -190,6 +190,30 @@ value_init(Slapi_Value *v, const struct berval *bval,CSNType t,const CSN *csn)
     return v;
 }
 
+void
+slapi_value_set_flags(Slapi_Value *v, unsigned long flags)
+{
+	PR_ASSERT(v!=NULL);
+	v->v_flags = flags;
+}
+
+void
+slapi_values_set_flags(Slapi_Value **vs, unsigned long flags)
+{
+	PR_ASSERT(vs!=NULL);
+	Slapi_Value **v;
+	for (v = vs; v && *v; v++) {
+		slapi_value_set_flags(*v, flags);
+	}
+}
+
+unsigned long
+slapi_value_get_flags(Slapi_Value *v)
+{
+	PR_ASSERT(v!=NULL);
+	return v->v_flags;
+}
+
 void 
 slapi_value_free(Slapi_Value **v)
 {

ldap/servers/slapd/valueset.c

@@ -247,6 +247,36 @@ valuearray_init_bervalarray(struct berval **bvals, Slapi_Value ***cvals)
     return n;
 }
 
+/*
+ * JCM SLOW FUNCTION
+ *
+ * WARNING: Use only if you absolutley need to...
+ * This function mostly exists to map from the old slapi berval
+ * based interface to the new Slapi_Value based interfaces.
+ */
+int
+valuearray_init_bervalarray_with_flags(struct berval **bvals, Slapi_Value ***cvals, unsigned long flags)
+{
+    int n;
+    for(n=0; bvals != NULL && bvals[n] != NULL; n++);
+    if(n==0)
+    {
+        *cvals = NULL;
+    }
+    else
+    {
+        int i;
+        *cvals = (Slapi_Value **) slapi_ch_malloc((n + 1) * sizeof(Slapi_Value *));
+        for(i=0;i<n;i++)
+        {
+            (*cvals)[i] = slapi_value_new_berval(bvals[i]);
+            slapi_value_set_flags((*cvals)[i], flags);
+        }
+        (*cvals)[i] = NULL;
+    }
+    return n;
+}
+
 /*
  * JCM SLOW FUNCTION
  *