Bug 745259 - Incorrect entryUSN index under high load in

             replicated environment

https://bugzilla.redhat.com/show_bug.cgi?id=745259

Bug Description: When replication conflicts occur in replacing
entryusn, dangling entryusn keys pointing the same entryid are
generated in the entryusn index file.  There is no way to clean
them up unless reindexing the entryusn.

Fix Description: When replication conflicts occur in the replace
op and new entry still contains the old value, then the old value
won't be removed from the index file.  Also, if the new value is
not added to the entry, the new value won't be added to the index
file.

The failure is informed to entryusn by by setting LDAP_MOD_IGNORE
in the mod.  Entryusn plugin bepostop_modify uses the info to
determine to increment the entryusn or not.

ldap/servers/plugins/usn/usn.c

@@ -60,6 +60,7 @@ static int usn_bepreop_delete(Slapi_PBlock *pb);
 static int usn_bepreop_modify(Slapi_PBlock *pb);
 static int usn_bepostop(Slapi_PBlock *pb);
 static int usn_bepostop_delete (Slapi_PBlock *pb);
+static int usn_bepostop_modify (Slapi_PBlock *pb);
 static int usn_start(Slapi_PBlock *pb);
 static int usn_close(Slapi_PBlock *pb);
 static int usn_get_attr(Slapi_PBlock *pb, const char* type, void *value);
@@ -180,7 +181,7 @@ usn_bepostop_init(Slapi_PBlock *pb)
         slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN,
                                  (void *)usn_bepostop_delete) != 0 ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN,
-                                 (void *)usn_bepostop) != 0 ||
+                                 (void *)usn_bepostop_modify) != 0 ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN,
                                  (void *)usn_bepostop) != 0) {
         slapi_log_error(SLAPI_LOG_FATAL, USN_PLUGIN_SUBSYSTEM,
@@ -517,6 +518,53 @@ bail:
     return rc;
 }
 
+/* count up the counter */
+static int
+usn_bepostop_modify (Slapi_PBlock *pb)
+{
+    int rc = -1;
+    Slapi_Backend *be = NULL;
+    LDAPMod **mods = NULL;
+    int i;
+
+    slapi_log_error(SLAPI_LOG_TRACE, USN_PLUGIN_SUBSYSTEM,
+                    "--> usn_bepostop_mod\n");
+
+    /* if op is not successful, don't increment the counter */
+    slapi_pblock_get(pb, SLAPI_RESULT_CODE, &rc);
+    if (LDAP_SUCCESS != rc) {
+        goto bail;
+    }
+
+    slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods);
+    for (i = 0; mods && mods[i]; i++) {
+        if (0 == strcasecmp(mods[i]->mod_type, SLAPI_ATTR_ENTRYUSN)) {
+            if (mods[i]->mod_op & LDAP_MOD_IGNORE) {
+    slapi_log_error(SLAPI_LOG_TRACE, USN_PLUGIN_SUBSYSTEM,
+                    "usn_bepostop_mod: MOD_IGNORE detected\n");
+                goto bail; /* conflict occurred.
+                              skip incrementing the counter. */
+            } else {
+                break;
+            }
+        }
+    }
+
+    slapi_pblock_get(pb, SLAPI_BACKEND, &be);
+    if (NULL == be) {
+        rc = LDAP_PARAM_ERROR;    
+        goto bail;
+    }
+
+    if (be->be_usn_counter) {
+        slapi_counter_increment(be->be_usn_counter);
+    }
+bail:
+    slapi_log_error(SLAPI_LOG_TRACE, USN_PLUGIN_SUBSYSTEM,
+                    "<-- usn_bepostop_mod\n");
+    return rc;
+}
+
 /* count up the counter */
 /* if the op is delete and the op was not successful, remove preventryusn */
 static int

ldap/servers/slapd/back-ldbm/index.c

@@ -485,7 +485,7 @@ index_addordel_entry(
 int 
 index_add_mods(
     backend *be,
-    const LDAPMod **mods,
+    LDAPMod **mods,
     struct backentry 	*olde,
     struct backentry 	*newe,
     back_txn *txn
@@ -580,10 +580,11 @@ index_add_mods(
                             flags |= BE_INDEX_EQUALITY;
                         }
                     } else {
-                        /* Remove duplicate value from deleted value array */
                         Slapi_Value *rval = valuearray_remove_value(curr_attr, deleted_valueArray, deleted_valueArray[j]);
                         slapi_value_free( &rval );
                         j--;
+                        /* indicates there was some conflict */
+                        mods[i]->mod_op |= LDAP_MOD_IGNORE;
                     }
                 }
             } else {
@@ -604,16 +605,38 @@ index_add_mods(
             if ( mods_valueArray == NULL ) {
                 rc = 0;
             } else {
-                rc = index_addordel_values_sv( be,
+                /* Verify if the value is in newe.
+                 * If it is in, we will add the attr value to the index file. */
+                slapi_entry_attr_find( newe->ep_entry, 
+                                       mods[i]->mod_type, &curr_attr );
+                
+                for (j = 0; mods_valueArray[j] != NULL; j++) {
+                    /* mods_valueArray[j] is in curr_attr ==> return 0 */
+                    if (slapi_attr_value_find(curr_attr,
+                                slapi_value_get_berval(mods_valueArray[j]))) {
+                        /* The value is NOT in newe, remove it. */
+                        Slapi_Value *rval = valuearray_remove_value(curr_attr,
+                                                            mods_valueArray,
+                                                            mods_valueArray[j]);
+                        slapi_value_free( &rval );
+                        /* indicates there was some conflict */
+                        mods[i]->mod_op |= LDAP_MOD_IGNORE;
+                    }
+                }
+                if (mods_valueArray) {
+                    rc = index_addordel_values_sv( be,
                                             mods[i]->mod_type, 
                                             mods_valueArray, NULL,
                                             id, BE_INDEX_ADD, txn );
+                } else {
+                    rc = 0;
+                }
             }
             break;
 
         case LDAP_MOD_DELETE:
             if ( (mods[i]->mod_bvalues == NULL) ||
-					 (mods[i]->mod_bvalues[0] == NULL) ) {
+                 (mods[i]->mod_bvalues[0] == NULL) ) {
                 rc = 0;
                 flags = BE_INDEX_DEL;
 
@@ -713,7 +736,7 @@ index_add_mods(
             }
             rc = 0;
             break;
-        }
+        } /* switch ( mods[i]->mod_op & ~LDAP_MOD_BVALUES ) */
 
         /* free memory */
         slapi_ch_free((void **)&tmp);
@@ -724,7 +747,7 @@ index_add_mods(
             ldbm_nasty(errmsg, 1040, rc);
             return( rc );
         }
-    }
+    } /* for ( i = 0; mods[i] != NULL; i++ ) */
 
     return( 0 );
 }

ldap/servers/slapd/back-ldbm/ldbm_modify.c

@@ -159,7 +159,7 @@ int modify_update_all(backend *be, Slapi_PBlock *pb,
 		}
 		goto error;
 	}
-	retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(mc->smods), mc->old_entry, mc->new_entry, txn );
+	retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(mc->smods), mc->old_entry, mc->new_entry, txn );
 	if ( 0 != retval ) {
 		if (DB_LOCK_DEADLOCK != retval)
 		{
@@ -456,7 +456,7 @@ ldbm_back_modify( Slapi_PBlock *pb )
 			goto error_return;
 		}
 		ec_in_cache = 1;
-		retval = index_add_mods( be, (const LDAPMod**)mods, e, ec, &txn );
+		retval = index_add_mods( be, mods, e, ec, &txn );
 		if (DB_LOCK_DEADLOCK == retval)
 		{
 			/* Abort and re-try */

ldap/servers/slapd/back-ldbm/ldbm_modrdn.c

@@ -1432,7 +1432,7 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods1), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods1), e, ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
@@ -1455,7 +1455,7 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods2), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods2), e, ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
@@ -1472,7 +1472,7 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods3), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods3), e, ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */

ldap/servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -278,7 +278,7 @@ int idl_new_compare_dups(
  * index.c
  */
 int index_addordel_entry( backend *be, struct backentry *e, int flags, back_txn *txn );
-int index_add_mods( backend *be, const LDAPMod**mods, struct backentry *olde, struct backentry *newe, back_txn *txn );
+int index_add_mods( backend *be, /*const*/LDAPMod**mods, struct backentry *olde, struct backentry *newe, back_txn *txn );
 int index_addordel_string(backend *be, const char *type, const char *s, ID id, int flags, back_txn *txn);
 int index_addordel_values_sv( backend *be, const char *type, Slapi_Value **vals, Slapi_Value **evals, ID id, int flags, back_txn *txn );
 int index_addordel_values_ext_sv( backend *be, const char *type, Slapi_Value **vals, Slapi_Value **evals, ID id, int flags, back_txn *txn,int *idl_disposition, void *buffer_handle );

ldap/servers/slapd/entrywsi.c

@@ -719,11 +719,15 @@ entry_apply_mod_wsi(Slapi_Entry *e, const LDAPMod *mod, const CSN *csn, int urp)
 		retVal = entry_replace_present_values_wsi( e, mod->mod_type, mod->mod_bvalues, csn, urp );
 		break;
 	}
-	for ( i = 0; mod->mod_bvalues != NULL && mod->mod_bvalues[i] != NULL; i++ )
-	{
-		LDAPDebug( LDAP_DEBUG_ARGS, "   %s: %s\n", mod->mod_type, mod->mod_bvalues[i]->bv_val, 0 );
+	if ( LDAPDebugLevelIsSet( LDAP_DEBUG_ARGS )) {
+		for ( i = 0;
+		      mod->mod_bvalues != NULL && mod->mod_bvalues[i] != NULL;
+		      i++ ) {
+			LDAPDebug( LDAP_DEBUG_ARGS, "   %s: %s\n",
+			           mod->mod_type, mod->mod_bvalues[i]->bv_val, 0 );
+		}
+		LDAPDebug( LDAP_DEBUG_ARGS, "   -\n", 0, 0, 0 );
 	}
-	LDAPDebug( LDAP_DEBUG_ARGS, "   -\n", 0, 0, 0 );
 
 	return retVal;
 }

ldap/servers/slapd/slapi-private.h

@@ -440,8 +440,10 @@ unsigned long operation_get_type(Slapi_Operation *op);
  * #define LDAP_MOD_ADD            0x00
  * #define LDAP_MOD_DELETE         0x01
  * #define LDAP_MOD_REPLACE        0x02
+ * #define LDAP_MOD_INCREMENT      0x03 -- Openldap extension
+ * #define LDAP_MOD_BVALUES        0x80
  */
-#define LDAP_MOD_IGNORE 0x09
+#define LDAP_MOD_IGNORE 0x100
 
 
 /* dl.c */