Ticket #360 - ldapmodify returns Operations error

https://fedorahosted.org/389/ticket/360
Resolves: Ticket #360
Bug Description: ldapmodify returns Operations error
Reviewed by: mreynolds (Thanks!)
Branch: master
Fix Description:
1) Fix handling of DB_LOCK_DEADLOCK conditions.  When a database operation
returns DB_LOCK_DEADLOCK, all cursors must be closed, and the transaction
aborted and retried.  If not in a transaction, the operation may be retried
immediately.  This fix adds this logic to many places in the db code where
it was lacking.
2) Fix resetting of the data when an operation has to be retried.  When
a transaction has to be retried, we must reset the data to the same state
it was before any of the operations in the transaction were attempted.  This
includes the entry cache state, which was lacking in a number of ways.  One
major problem with resetting the cache is that cache_add_tentative adds an
entry to the dncache, but not the idcache, in order to reserve a space in
the cache, and to prevent other entries with the same DN from being added
while the operation is in progress.  There was no good way to remove this
entry.  In the case of modrdn, removing the tentative entry would also
remove the real entry since they share the same entryid.  This fix also
makes sure that in error conditions, temporary entries are removed from
the cache and properly freed, and real entries are "rolled back" into the
cache.
3) Added a transaction stress thread.  This thread can simulate various types
of read and write locking that can cause conflicts and trigger regular
database operations to return DB_LOCK_DEADLOCK.  The usual culprit is read
locks which are held on pages that are searched outside of a transaction.
The stress thread can lock, via read cursors, all of the pages in all of
the indexes in the database, and hold these pages for some set period of
time, then loop and do it again.
4) It is quite easy to get the database in a deadlock situation, where a
update operation is waiting for a read lock to be released in order to
write to a page, while a search operation is waiting for a write lock to
be released in order to read lock the page.  If we are going to allow
concurrent searches during update operations, without making search requests
transacted, we need to have some way to detect these deadlocks.  The fastest
way to do it is to use the DB_TXN_NOWAIT flag when starting transactions.
This tells bdb to return immediately with a DB_LOCK_DEADLOCK if the
transaction cannot proceed due to a locked page (e.g. a search request
has read locked a page).  Alternately, we could have transactions wait
for some specified period of time, but if we think that this type of thread
contention is going to be rare, it makes sense to return immediately, if
our deadlock handling is fast and robust.
5) Fixed some memory leaks
6) The txn_test thread needs to know when the backend databases are
available - had to add a backend flag BE_STATE_STOPPING so that
the txn_thread knows when the databases are available
7) If the op was retried RETRY_TIMES times, return LDAP_BUSY instead of
OPERATIONS_ERROR - the problem really is that the server is busy with
other transactions, and the operation could go through if the client
were to retry.
8) Renaming an entry without changing the dn e.g. changing the case does
not cache the entry, so handle that
9) Added a delay when a deadlock is encountered in modrdn - same as the
other add/mod/del cases
Platforms tested: RHEL6 x86_64, Fedora 17
Flag Day: yes
Doc impact: yes

ldap/servers/plugins/replication/repl5_plugins.c

@@ -997,6 +997,11 @@ write_changelog_and_ruv (Slapi_PBlock *pb)
 		return 0;
 	}
 
+	slapi_pblock_get(pb, SLAPI_RESULT_CODE, &rc);
+	if (rc) { /* op failed - just return */
+		return 0;
+	}
+
 	/* we only log changes for operations applied to a replica */
 	repl_obj = replica_get_replica_for_op (pb);
 	if (repl_obj == NULL)

ldap/servers/slapd/add.c

@@ -665,6 +665,7 @@ static void op_shared_add (Slapi_PBlock *pb)
 		int	rc;
 		Slapi_Entry	*ec;
 		Slapi_DN *add_target_sdn = NULL;
+		Slapi_Entry *save_e = NULL;
 
 		slapi_pblock_set(pb, SLAPI_PLUGIN, be->be_database);
 		set_db_default_result_handlers(pb);
@@ -678,6 +679,8 @@ static void op_shared_add (Slapi_PBlock *pb)
 		if (be->be_add != NULL)
 		{
 			rc = (*be->be_add)(pb);
+			/* backend may change this if errors and not consumed */
+			slapi_pblock_get(pb, SLAPI_ADD_ENTRY, &save_e);
 			slapi_pblock_set(pb, SLAPI_ADD_ENTRY, ec);
 			if (rc == 0)
 			{
@@ -703,6 +706,10 @@ static void op_shared_add (Slapi_PBlock *pb)
 			}
 			else
 			{
+				/* restore e so we can free it below */
+				if (save_e) {
+					e = save_e;
+				}
 				if (rc == SLAPI_FAIL_DISKFULL)
 				{
 					operation_out_of_disk_space();

ldap/servers/slapd/back-ldbm/cache.c

@@ -294,7 +294,7 @@ dump_hash(Hashtable *ht)
             continue;
         }
         do {
-            PR_snprintf(ep_id, 16, "%u", ((struct backcommon *)e)->ep_id);
+            PR_snprintf(ep_id, 16, "%u-%u", ((struct backcommon *)e)->ep_id, ((struct backcommon *)e)->ep_refcnt);
             len = strlen(ep_id);
             if (ids_size < len + 1) {
                 LDAPDebug1Arg(LDAP_DEBUG_ANY, "%s\n", ep_ids);
@@ -857,7 +857,7 @@ entrycache_remove_int(struct cache *cache, struct backentry *e)
     const char *uuid;
 #endif
 
-    LOG("=> entrycache_remove_int (%s)\n", backentry_get_ndn(e), 0, 0);
+    LOG("=> entrycache_remove_int (%s) (%u) (%u)\n", backentry_get_ndn(e), e->ep_id, e->ep_refcnt);
     if (e->ep_state & ENTRY_STATE_NOTINCACHE)
     {
         return ret;
@@ -876,13 +876,22 @@ entrycache_remove_int(struct cache *cache, struct backentry *e)
     {
         LOG("remove %s from dn hash failed\n", ndn, 0, 0);
     }
-    if (remove_hash(cache->c_idtable, &(e->ep_id), sizeof(ID)))
+    /* if entry was added tentatively, it will be in the dntable
+       but not in the idtable - we cannot just remove it from
+       the idtable - in the case of modrdn, this will remove
+       the _real_ entry from the idtable, leading to a cache
+       imbalance
+    */
+    if (!(e->ep_state & ENTRY_STATE_CREATING))
     {
-       ret = 0;
-    }
-    else
-    {
-        LOG("remove %d from id hash failed\n", e->ep_id, 0, 0);
+        if (remove_hash(cache->c_idtable, &(e->ep_id), sizeof(ID)))
+        {
+            ret = 0;
+        }
+        else
+        {
+            LOG("remove %d from id hash failed\n", e->ep_id, 0, 0);
+        }
     }
 #ifdef UUIDCACHE_ON 
     uuid = slapi_entry_get_uniqueid(e->ep_entry);
@@ -907,6 +916,11 @@ entrycache_remove_int(struct cache *cache, struct backentry *e)
 
     /* mark for deletion (will be erased when refcount drops to zero) */
     e->ep_state |= ENTRY_STATE_DELETED;
+#if 0
+    if (slapi_is_loglevel_set(SLAPI_LOG_CACHE)) {
+        dump_hash(cache->c_idtable);
+    }
+#endif
     LOG("<= entrycache_remove_int: %d\n", ret, 0, 0);
     return ret;
 }
@@ -999,14 +1013,23 @@ static int entrycache_replace(struct cache *cache, struct backentry *olde,
      * cache tables, operation error 
      */
     if ( (olde->ep_state & ENTRY_STATE_NOTINCACHE) == 0 ) {
-
-        found = remove_hash(cache->c_dntable, (void *)oldndn, strlen(oldndn));
-        found &= remove_hash(cache->c_idtable, &(olde->ep_id), sizeof(ID));
+        int found_in_dn = remove_hash(cache->c_dntable, (void *)oldndn, strlen(oldndn));
+        int found_in_id = remove_hash(cache->c_idtable, &(olde->ep_id), sizeof(ID));
+#ifdef UUIDCACHE_ON
+        int found_in_uuid = remove_hash(cache->c_uuidtable, (void *)olduuid, strlen(olduuid));
+#endif
+        found = found_in_dn && found_in_id;
 #ifdef UUIDCACHE_ON
-        found &= remove_hash(cache->c_uuidtable, (void *)olduuid, strlen(olduuid));
+        found = found && found_in_uuid;
 #endif
         if (!found) {
-            LOG("entry cache replace: cache index tables out of sync\n", 0, 0, 0);
+#ifdef UUIDCACHE_ON
+            LOG("entry cache replace: cache index tables out of sync - found dn [%d] id [%d] uuid [%d]\n",
+                found_in_dn, found_in_id, found_in_uuid);
+#else
+            LOG("entry cache replace: cache index tables out of sync - found dn [%d] id [%d]\n",
+                found_in_dn, found_in_id, 0);
+#endif
             PR_Unlock(cache->c_mutex);
             return 1;
         }
@@ -1472,7 +1495,9 @@ int cache_lock_entry(struct cache *cache, struct backentry *e)
 void cache_unlock_entry(struct cache *cache, struct backentry *e)
 {
     LOG("=> cache_unlock_entry\n", 0, 0, 0);
-    PR_ExitMonitor(e->ep_mutexp);
+    if (PR_ExitMonitor(e->ep_mutexp)) {
+        LOG("=> cache_unlock_entry - monitor was not entered!!!\n", 0, 0, 0);
+    }
 }
 
 /* DN cache */

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -214,6 +214,7 @@ static int dblayer_start_deadlock_thread(struct ldbminfo *li);
 static int dblayer_start_checkpoint_thread(struct ldbminfo *li);
 static int dblayer_start_trickle_thread(struct ldbminfo *li);
 static int dblayer_start_perf_thread(struct ldbminfo *li);
+static int dblayer_start_txn_test_thread(struct ldbminfo *li);
 static int trans_batch_count=1;
 static int trans_batch_limit=0;
 static PRBool log_flush_thread=PR_FALSE;
@@ -226,6 +227,14 @@ static void dblayer_pop_pvt_txn();
 #define MEGABYTE (1024 * 1024)
 #define GIGABYTE (1024 * MEGABYTE)
 
+/* env. vars. you can set to stress txn handling */
+#define TXN_TESTING "TXN_TESTING" /* enables the txn test thread */
+#define TXN_TEST_HOLD_MSEC "TXN_TEST_HOLD_MSEC" /* time to hold open the db cursors */
+#define TXN_TEST_LOOP_MSEC "TXN_TEST_LOOP_MSEC" /* time to wait before looping again */
+#define TXN_TEST_USE_TXN "TXN_TEST_USE_TXN" /* use transactions or not */
+#define TXN_TEST_USE_RMW "TXN_TEST_USE_RMW" /* use DB_RMW for c_get flags or not */
+#define TXN_TEST_INDEXES "TXN_TEST_INDEXES" /* list of indexes to use - comma delimited - id2entry,entryrdn,etc. */
+#define TXN_TEST_VERBOSE "TXN_TEST_VERBOSE" /* be wordy */
 
 /* This function compares two index keys.  It is assumed
    that the values are already normalized, since they should have
@@ -1755,6 +1764,9 @@ dblayer_start(struct ldbminfo *li, int dbmode)
 
             /* Now open the performance counters stuff */
             perfctrs_init(li,&(priv->perf_private));
+            if (getenv(TXN_TESTING)) {
+                dblayer_start_txn_test_thread(li);
+            }
         }
         if (return_value != 0) {
             if (return_value == ENOMEM) {
@@ -2597,7 +2609,10 @@ int dblayer_instance_close(backend *be)
     if (NULL == inst)
         return -1;
 
-    if (getenv("USE_VALGRIND")) {
+    if (!inst->import_env) {
+        be->be_state = BE_STATE_STOPPING;
+    }
+    if (getenv("USE_VALGRIND") || slapi_is_loglevel_set(SLAPI_LOG_CACHE)) {
         /* 
          * if any string is set to an environment variable USE_VALGRIND,
          * when running a memory leak checking tool (e.g., valgrind),
@@ -3422,7 +3437,7 @@ int dblayer_txn_begin_ext(struct ldbminfo *li, back_txnid parent_txn, back_txn *
         return_value = TXN_BEGIN(pEnv->dblayer_DB_ENV,
                                  (DB_TXN*)parent_txn,
                                  &new_txn.back_txn_txn,
-                                 0);
+                                 DB_TXN_NOWAIT);
         if (0 != return_value) 
         {
             if(use_lock) slapi_rwlock_unlock(priv->dblayer_env->dblayer_env_lock);
@@ -3725,6 +3740,407 @@ dblayer_start_deadlock_thread(struct ldbminfo *li)
     return return_value;
 }
 
+static const u_int32_t default_flags = DB_NEXT;
+
+/* this is the loop delay - how long after we release the db pages
+   until we acquire them again */
+#define TXN_TEST_LOOP_WAIT(msecs) do {                             \
+    if (msecs) {                                                   \
+        DS_Sleep(PR_MillisecondsToInterval(slapi_rand() % msecs)); \
+    }                                                              \
+} while (0)
+
+/* this is how long we hold the pages open until we close the cursors */
+#define TXN_TEST_PAGE_HOLD(msecs) do {                             \
+    if (msecs) {                                                   \
+        DS_Sleep(PR_MillisecondsToInterval(slapi_rand() % msecs)); \
+    }                                                              \
+} while (0)
+
+typedef struct txn_test_iter {
+    DB *db;
+    DBC *cur;
+    size_t cnt;
+    const char *attr;
+    u_int32_t flags;
+    backend *be;
+} txn_test_iter;
+
+typedef struct txn_test_cfg {
+    PRUint32 hold_msec;
+    PRUint32 loop_msec;
+    u_int32_t flags;
+    int use_txn;
+    char **indexes;
+    int verbose;
+} txn_test_cfg;
+
+static txn_test_iter *
+new_txn_test_iter(DB *db, const char *attr, backend *be, u_int32_t flags)
+{
+    txn_test_iter *tti = (txn_test_iter *)slapi_ch_malloc(sizeof(txn_test_iter));
+    tti->db = db;
+    tti->cur = NULL;
+    tti->cnt = 0;
+    tti->attr = attr;
+    tti->flags = default_flags|flags;
+    tti->be = be;
+    return tti;
+}
+
+static void
+init_txn_test_iter(txn_test_iter *tti)
+{
+    if (tti->cur) {
+        if (tti->cur->dbp && (tti->cur->dbp->open_flags == 0x58585858)) {
+            /* already closed? */
+        } else if (tti->be && (tti->be->be_state != BE_STATE_STARTED)) {
+            /* already closed? */
+        } else {
+            tti->cur->c_close(tti->cur);
+        }
+        tti->cur = NULL;
+    }
+    tti->cnt = 0;
+    tti->flags = default_flags;
+}
+
+static void
+free_txn_test_iter(txn_test_iter *tti)
+{
+    init_txn_test_iter(tti);
+    slapi_ch_free((void **)&tti);
+}
+
+static void
+free_ttilist(txn_test_iter ***ttilist, size_t *tticnt)
+{
+    if (!ttilist || !*ttilist || !**ttilist) {
+        return;
+    }
+    while (*tticnt > 0) {
+        (*tticnt)--;
+        free_txn_test_iter((*ttilist)[*tticnt]);
+    }
+    slapi_ch_free((void *)ttilist);
+}
+
+static void
+init_ttilist(txn_test_iter **ttilist, size_t tticnt)
+{
+    if (!ttilist || !*ttilist) {
+        return;
+    }
+    while (tticnt > 0) {
+        tticnt--;
+        init_txn_test_iter(ttilist[tticnt]);
+    }
+}
+
+static void
+print_ttilist(txn_test_iter **ttilist, size_t tticnt)
+{
+    while (tticnt > 0) {
+        tticnt--;
+        LDAPDebug2Args(LDAP_DEBUG_ANY,
+                       "txn_test_threadmain: attr [%s] cnt [%lu]\n",
+                       ttilist[tticnt]->attr, ttilist[tticnt]->cnt);
+    }
+}
+
+#define TXN_TEST_IDX_OK_IF_NULL "nscpEntryDN"
+
+static void
+txn_test_init_cfg(txn_test_cfg *cfg)
+{
+    static char *indexlist = "aci,entryrdn,numsubordinates,uid,ancestorid,objectclass,uniquemember,cn,parentid,nsuniqueid,sn,id2entry," TXN_TEST_IDX_OK_IF_NULL;
+    char *indexlist_copy = NULL;
+
+    cfg->hold_msec = getenv(TXN_TEST_HOLD_MSEC) ? atoi(getenv(TXN_TEST_HOLD_MSEC)) : 200;
+    cfg->loop_msec = getenv(TXN_TEST_LOOP_MSEC) ? atoi(getenv(TXN_TEST_LOOP_MSEC)) : 10;
+    cfg->flags = getenv(TXN_TEST_USE_RMW) ? DB_RMW : 0;
+    cfg->use_txn = getenv(TXN_TEST_USE_TXN) ? 1 : 0;
+    if (getenv(TXN_TEST_INDEXES)) {
+        indexlist_copy = slapi_ch_strdup(getenv(TXN_TEST_INDEXES));
+    } else {
+        indexlist_copy = slapi_ch_strdup(indexlist);
+    }
+    cfg->indexes = slapi_str2charray(indexlist_copy, ",");
+    slapi_ch_free_string(&indexlist_copy);
+    cfg->verbose = getenv(TXN_TEST_VERBOSE) ? 1 : 0;
+
+    slapi_log_error(SLAPI_LOG_FATAL, "txn_test_threadmain",
+                    "Config hold_msec [%d] loop_msec [%d] rmw [%d] txn [%d] indexes [%s]\n",
+                    cfg->hold_msec, cfg->loop_msec, cfg->flags, cfg->use_txn,
+                    getenv(TXN_TEST_INDEXES) ? getenv(TXN_TEST_INDEXES) : indexlist);
+}
+
+static int txn_test_threadmain(void *param)
+{
+    dblayer_private *priv = NULL;
+    struct ldbminfo *li = NULL;
+    Object *inst_obj;
+    int rc = 0;
+    txn_test_iter **ttilist = NULL;
+    size_t tticnt = 0;
+    DB_TXN *txn = NULL;
+    txn_test_cfg cfg;
+    size_t counter = 0;
+    char keybuf[8192];
+    char databuf[8192];
+
+    PR_ASSERT(NULL != param);
+    li = (struct ldbminfo*)param;
+
+    priv = (dblayer_private*)li->li_dblayer_private;
+    PR_ASSERT(NULL != priv);
+
+    INCR_THREAD_COUNT(priv);
+
+    if (!priv->dblayer_enable_transactions) {
+        goto end;
+    }
+
+    txn_test_init_cfg(&cfg);
+
+wait_for_init:
+    free_ttilist(&ttilist, &tticnt);
+    DS_Sleep(PR_MillisecondsToInterval(1000));
+    if (priv->dblayer_stop_threads) {
+        goto end;
+    }
+    
+    for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
+         inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
+        char **idx = NULL;
+        ldbm_instance *inst = (ldbm_instance *)object_get_data(inst_obj);
+        backend *be = inst->inst_be;
+
+        if (be->be_state != BE_STATE_STARTED) {
+            object_release(inst_obj);
+            goto wait_for_init;
+        }
+
+        for (idx = cfg.indexes; idx && *idx; ++idx) {
+            DB *db = NULL;
+            if (be->be_state != BE_STATE_STARTED) {
+                object_release(inst_obj);
+                goto wait_for_init;
+            }
+
+            if (!strcmp(*idx, "id2entry")) {
+                dblayer_get_id2entry(be, &db);
+                if (db == NULL) {
+                    object_release(inst_obj);
+                    goto wait_for_init;
+                }
+            } else {
+                struct attrinfo *ai = NULL;
+                ainfo_get(be, *idx, &ai);
+                if (NULL == ai) {
+                    object_release(inst_obj);
+                    goto wait_for_init;
+                }
+                dblayer_get_index_file(be, ai, &db, 0);
+                if (NULL == db) {
+                    if (strcasecmp(*idx, TXN_TEST_IDX_OK_IF_NULL)) {
+                        object_release(inst_obj);
+                        goto wait_for_init;
+                    }
+                }
+            }
+            if (db) {
+                ttilist = (txn_test_iter **)slapi_ch_realloc((char *)ttilist, sizeof(txn_test_iter *) * (tticnt + 1));
+                ttilist[tticnt++] = new_txn_test_iter(db, *idx, be, cfg.flags);
+            }
+        }
+    }
+
+    while (!priv->dblayer_stop_threads) {
+retry_txn:
+        init_ttilist(ttilist, tticnt);
+        if (txn) {
+            TXN_ABORT(txn);
+            txn = NULL;
+        }
+        if (cfg.use_txn) {
+            rc = TXN_BEGIN(priv->dblayer_env->dblayer_DB_ENV, NULL, &txn, 0);
+            if (rc || !txn) {
+                LDAPDebug2Args(LDAP_DEBUG_ANY,
+                               "txn_test_threadmain failed to create a new transaction, err=%d (%s)\n",
+                               rc, dblayer_strerror(rc));
+            }
+        } else {
+            rc = 0;
+        }
+        if (!rc) {
+            DBT key;
+            DBT data;
+            size_t ii;
+            size_t donecnt = 0;
+            size_t cnt = 0;
+
+            /* phase 1 - open a cursor to each db */
+            if (cfg.verbose) {
+                LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                              "txn_test_threadmain: starting [%lu] indexes\n", tticnt);
+            }
+            for (ii = 0; ii < tticnt; ++ii) {
+                txn_test_iter *tti = ttilist[ii];
+
+retry_cursor:
+                if (priv->dblayer_stop_threads) {
+                    goto end;
+                }
+                if (tti->be->be_state != BE_STATE_STARTED) {
+                    if (txn) {
+                        TXN_ABORT(txn);
+                        txn = NULL;
+                    }
+                    goto wait_for_init;
+                }
+                if (tti->db->open_flags == 0xdbdbdbdb) {
+                    if (txn) {
+                        TXN_ABORT(txn);
+                        txn = NULL;
+                    }
+                    goto wait_for_init;
+                }
+                rc = tti->db->cursor(tti->db, txn, &tti->cur, 0);
+                if (DB_LOCK_DEADLOCK == rc) {
+                    if (cfg.verbose) {
+                        LDAPDebug0Args(LDAP_DEBUG_ANY,
+                                       "txn_test_threadmain cursor create deadlock - retry\n");
+                    }
+                    if (cfg.use_txn) {
+                        goto retry_txn;
+                    } else {
+                        goto retry_cursor;
+                    }
+                } else if (rc) {
+                    LDAPDebug2Args(LDAP_DEBUG_ANY,
+                                   "txn_test_threadmain failed to create a new cursor, err=%d (%s)\n",
+                                   rc, dblayer_strerror(rc));
+                }
+            }
+
+            memset(&key, 0, sizeof(key));
+            key.flags = DB_DBT_USERMEM;
+            key.data = keybuf;
+            key.ulen = sizeof(keybuf);
+            memset(&data, 0, sizeof(data));
+            data.flags = DB_DBT_USERMEM;
+            data.data = databuf;
+            data.ulen = sizeof(databuf);
+            /* phase 2 - iterate over each cursor at the same time until
+               1) get error
+               2) get deadlock
+               3) all cursors are exhausted
+            */
+            while (donecnt < tticnt) {
+                for (ii = 0; ii < tticnt; ++ii) {
+                    txn_test_iter *tti = ttilist[ii];
+                    if (tti->cur) {
+retry_get:
+                        if (priv->dblayer_stop_threads) {
+                            goto end;
+                        }
+                        if (tti->be->be_state != BE_STATE_STARTED) {
+                            if (txn) {
+                                TXN_ABORT(txn);
+                                txn = NULL;
+                            }
+                            goto wait_for_init;
+                        }
+                        if (tti->db->open_flags == 0xdbdbdbdb) {
+                            if (txn) {
+                                TXN_ABORT(txn);
+                                txn = NULL;
+                            }
+                            goto wait_for_init;
+                        }
+                        rc = tti->cur->c_get(tti->cur, &key, &data, tti->flags);
+                        if (DB_LOCK_DEADLOCK == rc) {
+                            if (cfg.verbose) {
+                                LDAPDebug0Args(LDAP_DEBUG_ANY,
+                                               "txn_test_threadmain cursor get deadlock - retry\n");
+                            }
+                            if (cfg.use_txn) {
+                                goto retry_txn;
+                            } else {
+                                goto retry_get;
+                            }
+                        } else if (DB_NOTFOUND == rc) {
+                            donecnt++; /* ran out of this one */
+                            tti->flags = DB_FIRST|cfg.flags; /* start over until all indexes are done */
+                        } else if (rc) {
+                            if ((DB_BUFFER_SMALL != rc) || cfg.verbose) {
+                                LDAPDebug2Args(LDAP_DEBUG_ANY,
+                                               "txn_test_threadmain failed to read a cursor, err=%d (%s)\n",
+                                               rc, dblayer_strerror(rc));
+                            }
+                            tti->cur->c_close(tti->cur);
+                            tti->cur = NULL;
+                            donecnt++;
+                        } else {
+                            tti->cnt++;
+                            tti->flags = default_flags|cfg.flags;
+                            cnt++;
+                        }
+                    }
+                }
+            }
+            TXN_TEST_PAGE_HOLD(cfg.hold_msec);
+            /*print_ttilist(ttilist, tticnt);*/
+            init_ttilist(ttilist, tticnt);
+            if (cfg.verbose) {
+                LDAPDebug2Args(LDAP_DEBUG_ANY,
+                               "txn_test_threadmain: finished [%lu] indexes [%lu] records\n", tticnt, cnt);
+            }
+            TXN_TEST_LOOP_WAIT(cfg.loop_msec);
+        } else {
+            TXN_TEST_LOOP_WAIT(cfg.loop_msec);
+        }
+        counter++;
+        if (!(counter % 40)) {
+            /* some operations get completely stuck - so every once in a while,
+               pause to allow those ops to go through */
+            DS_Sleep(PR_SecondsToInterval(1));
+        }
+    }
+
+end:
+    slapi_ch_array_free(cfg.indexes);
+    free_ttilist(&ttilist, &tticnt);
+    if (txn) {
+        TXN_ABORT(txn);
+    }        
+    DECR_THREAD_COUNT(priv);
+    return 0;
+}
+
+/*
+ * create a thread for transaction deadlock testing
+ */
+static int
+dblayer_start_txn_test_thread(struct ldbminfo *li)
+{
+    int return_value = 0;
+    if (NULL == PR_CreateThread (PR_USER_THREAD,
+                                 (VFP) (void *) txn_test_threadmain, li,
+                                 PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD,
+                                 PR_UNJOINABLE_THREAD,
+                                 SLAPD_DEFAULT_THREAD_STACKSIZE) )
+    {
+        PRErrorCode prerr = PR_GetError();
+        LDAPDebug(LDAP_DEBUG_ANY, "failed to create txn test thread, "
+                  SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n",
+                  prerr, slapd_pr_strerror(prerr), 0);
+        return_value = -1;
+    }
+    return return_value;
+}
+
 /* deadlock thread main function */
 
 static int deadlock_threadmain(void *param)

ldap/servers/slapd/back-ldbm/index.c

@@ -412,7 +412,7 @@ index_addordel_entry(
         }
         result = index_addordel_string(be, SLAPI_ATTR_NSCP_ENTRYDN, slapi_sdn_get_ndn(&parent), e->ep_id, flags, txn);
         if ( result != 0 ) {
-            ldbm_nasty(errmsg, 1020, result);
+            ldbm_nasty(errmsg, 1021, result);
             return( result );
         }
         slapi_sdn_done(&parent);
@@ -423,6 +423,7 @@ index_addordel_entry(
              */ 
             result = entryrdn_index_entry(be, e, flags, txn);
             if ( result != 0 ) {
+                ldbm_nasty(errmsg, 1023, result);
                 return( result );
             }
             /* To maintain tombstonenumsubordinates,
@@ -433,7 +434,7 @@ index_addordel_entry(
                 result = index_addordel_values_sv(be, LDBM_PARENTID_STR, svals, NULL,
                                                   e->ep_id, flags, txn);
                 if ( result != 0 ) {
-                    ldbm_nasty(errmsg, 1020, result);
+                    ldbm_nasty(errmsg, 1022, result);
                     return( result );
                 }
             }
@@ -481,6 +482,7 @@ index_addordel_entry(
         if (entryrdn_get_switch()) { /* subtree-rename: on */
             result = entryrdn_index_entry(be, e, flags, txn);
             if ( result != 0 ) {
+                ldbm_nasty(errmsg, 1031, result);
                 return( result );
             }
         }
@@ -608,13 +610,18 @@ index_add_mods(
             /* We need to first remove the old values from the 
              * index, if any. */
             if (deleted_valueArray) {
-                index_addordel_values_sv( be, mods[i]->mod_type,
-                                          deleted_valueArray, evals, id, 
-                                          flags, txn );
+                rc = index_addordel_values_sv( be, mods[i]->mod_type,
+                                               deleted_valueArray, evals, id, 
+                                               flags, txn );
+                if (rc) {
+                    ldbm_nasty(errmsg, 1041, rc);
+                    goto error;
+                }
             }
 
             /* Free valuearray */
             slapi_valueset_free(mod_vals);
+            mod_vals = NULL;
         case LDAP_MOD_ADD:
             if ( mods_valueArray == NULL ) {
                 rc = 0;
@@ -643,9 +650,13 @@ index_add_mods(
                 }
                 if (mods_valueArray) {
                     rc = index_addordel_values_sv( be,
-                                            mods[i]->mod_type, 
-                                            mods_valueArray, NULL,
-                                            id, BE_INDEX_ADD, txn );
+                                                   mods[i]->mod_type, 
+                                                   mods_valueArray, NULL,
+                                                   id, BE_INDEX_ADD, txn );
+                    if (rc) {
+                        ldbm_nasty(errmsg, 1042, rc);
+                        goto error;
+                    }
                 } else {
                     rc = 0;
                 }
@@ -702,12 +713,17 @@ index_add_mods(
 
                 /* Update the index, if necessary */
                 if (deleted_valueArray) {
-                    index_addordel_values_sv( be, mods[i]->mod_type,
-                                              deleted_valueArray, evals, id, 
-                                              flags, txn );
+                    rc = index_addordel_values_sv( be, mods[i]->mod_type,
+                                                   deleted_valueArray, evals, id, 
+                                                   flags, txn );
+                    if (rc) {
+                        ldbm_nasty(errmsg, 1043, rc);
+                        goto error;
+                    }
                 }
 
                 slapi_valueset_free(mod_vals);
+                mod_vals = NULL;
             } else {
 
                 /* determine if the presence key should be
@@ -753,15 +769,25 @@ index_add_mods(
                 rc = index_addordel_values_sv( be, basetype,
                                                mods_valueArray,
                                                evals, id, flags, txn );
+                if (rc) {
+                    ldbm_nasty(errmsg, 1044, rc);
+                    goto error;
+                }
             }
             rc = 0;
             break;
         } /* switch ( mods[i]->mod_op & ~LDAP_MOD_BVALUES ) */
 
+error:
         /* free memory */
         slapi_ch_free((void **)&tmp);
+        tmp = NULL;
         valuearray_free(&mods_valueArray);
+        mods_valueArray = NULL;
         slapi_valueset_free(all_vals);
+        all_vals = NULL;
+        slapi_valueset_free(mod_vals);
+        mod_vals = NULL;
 
         if ( rc != 0 ) {
             ldbm_nasty(errmsg, 1040, rc);
@@ -996,6 +1022,9 @@ index_read_ext_allids(
 	  idl = idl_fetch_ext( be, db, &key, db_txn, ai, err, allidslimit );
 	  if(*err == DB_LOCK_DEADLOCK) {
 	    ldbm_nasty("index read retrying transaction", 1045, *err);
+#ifdef FIX_TXN_DEADLOCKS
+#error can only retry here if txn == NULL - otherwise, have to abort and retry txn
+#endif
 	    continue;
 	  } else {
 	    break;
@@ -1124,6 +1153,9 @@ retry:
 			cursor->c_close(cursor);
 			cursor = NULL;
 			key->data = saved_key;
+#ifdef FIX_TXN_DEADLOCKS
+#error if txn != NULL, have to abort and retry the transaction, not just the cursor
+#endif
 			goto retry;
 		} else
 		{
@@ -1146,6 +1178,9 @@ retry:
 		cursor->c_close(cursor);
 		cursor = NULL;
 		key->data = saved_key;
+#ifdef FIX_TXN_DEADLOCKS
+#error if txn != NULL, have to abort and retry the transaction, not just the cursor
+#endif
 		goto retry;
 	}
 error:
@@ -1493,6 +1528,9 @@ index_range_read_ext(
           tmp = idl_fetch_ext( be, db, &cur_key, NULL, ai, err, allidslimit );
           if(*err == DB_LOCK_DEADLOCK) {
             ldbm_nasty("index_range_read retrying transaction", 1090, *err);
+#ifdef FIX_TXN_DEADLOCKS
+#error if txn != NULL, have to abort and retry the transaction, not just the fetch
+#endif
             continue;
           } else {
             break;
@@ -1641,7 +1679,7 @@ addordel_values(
 
 		if ( rc != 0)
 		{
-                        ldbm_nasty(errmsg, 1090, rc);
+                        ldbm_nasty(errmsg, 1096, rc);
 		}
 		index_free_prefix (prefix);
 		if (NULL != key.dptr && prefix != key.dptr)

ldap/servers/slapd/back-ldbm/ldbm_add.c

@@ -934,7 +934,7 @@ ldbm_back_add( Slapi_PBlock *pb )
 	if (retry_count == RETRY_TIMES) {
 		/* Failed */
 		LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in add\n", 0, 0, 0 );
-   		ldap_result_code= LDAP_OPERATIONS_ERROR;
+   		ldap_result_code= LDAP_BUSY;
 		goto error_return;
 	}
 

ldap/servers/slapd/back-ldbm/ldbm_delete.c

@@ -60,7 +60,8 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	struct ldbminfo	*li = NULL;
 	struct backentry *e = NULL;
 	struct backentry *tombstone = NULL;
-	struct backentry *original_entry = NULL;
+	Slapi_Entry *original_entry = NULL;
+	struct backentry *original_tombstone = NULL;
 	char *dn = NULL;
 	back_txn txn;
 	back_txnid parent_txn;
@@ -90,7 +91,6 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	int delete_tombstone_entry = 0;	/* We must remove the given tombstone entry from the DB	*/
 	int create_tombstone_entry = 0;	/* We perform a "regular" LDAP delete but since we use	*/
 									/* replication, we must create a new tombstone entry	*/
-	int e_in_cache = 0;
 	int tombstone_in_cache = 0;
 	entry_address *addr;
 	int addordel_flags = 0; /* passed to index_addordel */
@@ -98,6 +98,7 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	Slapi_Entry *orig_entry = NULL;
 	Slapi_DN parentsdn;
 	int opreturn = 0;
+	int free_delete_existing_entry = 0;
 
 	slapi_pblock_get( pb, SLAPI_BACKEND, &be);
 	slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li );
@@ -186,7 +187,6 @@ ldbm_back_delete( Slapi_PBlock *pb )
 		/* retval is -1 */
 		goto error_return; /* error result sent by find_entry2modify() */
 	}
-	e_in_cache = 1;
 
 	if ( slapi_entry_has_children( e->ep_entry ) )
 	{
@@ -206,6 +206,7 @@ ldbm_back_delete( Slapi_PBlock *pb )
 		 */
 		ldap_result_code= get_copy_of_entry(pb, addr, &txn,
 						SLAPI_DELETE_EXISTING_ENTRY, !is_replicated_operation);
+		free_delete_existing_entry = 1;
 		if(ldap_result_code==LDAP_OPERATIONS_ERROR ||
 		   ldap_result_code==LDAP_INVALID_DN_SYNTAX)
 		{
@@ -428,6 +429,11 @@ ldbm_back_delete( Slapi_PBlock *pb )
 		slapi_value_free(&tomb_value);
 		/* XXXggood above used to be: slapi_entry_add_string(tombstone->ep_entry, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE); */
 		/* JCMREPL - Add a description of what's going on? */
+
+		if ( (original_tombstone = backentry_dup( tombstone )) == NULL ) {
+			ldap_result_code= LDAP_OPERATIONS_ERROR;
+			goto error_return;
+		}
 	}
 
 	if (!is_ruv && !is_fixup_operation && !delete_tombstone_entry) {
@@ -443,7 +449,7 @@ ldbm_back_delete( Slapi_PBlock *pb )
 		}
 	}
 
-	if ( (original_entry = backentry_dup( e )) == NULL ) {
+	if ( (original_entry = slapi_entry_dup( e->ep_entry )) == NULL ) {
 		ldap_result_code= LDAP_OPERATIONS_ERROR;
 		goto error_return;
 	}
@@ -456,26 +462,45 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	txn.back_txn_txn = NULL; /* ready to create the child transaction */
 	for (retry_count = 0; retry_count < RETRY_TIMES; retry_count++) {
 		if (txn.back_txn_txn && (txn.back_txn_txn != parent_txn)) {
+			Slapi_Entry *ent = NULL;
+
 			dblayer_txn_abort(li,&txn);
 			slapi_pblock_set(pb, SLAPI_TXN, parent_txn);
-			if (e_in_cache) {
-				/* entry 'e' is in the entry cache.  Since we reset 'e' to
-				 * the original_entry, remove it from the cache.  */
-				CACHE_REMOVE(&inst->inst_cache, e);
-				cache_unlock_entry(&inst->inst_cache, e);
-				CACHE_RETURN(&inst->inst_cache, &e);
-				/* As we are about to delete it, 
-				 * we don't put the entry back to cache */
-				e_in_cache = 0; 
-			} else {
-				backentry_free(&e);
+
+			/* reset original entry */
+			slapi_pblock_get( pb, SLAPI_DELETE_EXISTING_ENTRY, &ent );
+			if (ent && (ent != original_entry) && free_delete_existing_entry) {
+				slapi_entry_free(ent);
+				slapi_pblock_set( pb, SLAPI_DELETE_EXISTING_ENTRY, NULL );
 			}
-			slapi_pblock_set( pb, SLAPI_DELETE_EXISTING_ENTRY, original_entry->ep_entry );
-			e = original_entry;
-			if ( (original_entry = backentry_dup( e )) == NULL ) {
+			slapi_entry_free(e->ep_entry);
+			e->ep_entry = original_entry;
+			if ( (original_entry = slapi_entry_dup( e->ep_entry )) == NULL ) {
 				ldap_result_code= LDAP_OPERATIONS_ERROR;
 				goto error_return;
 			}
+			slapi_pblock_set( pb, SLAPI_DELETE_EXISTING_ENTRY, original_entry );
+			free_delete_existing_entry = 0; /* owned by original_entry now */
+			if (nscpEntrySDN) {
+				nscpEntrySDN = slapi_entry_get_sdn(original_entry);
+			}
+
+			/* reset tombstone entry */
+			if (original_tombstone) {
+				if (tombstone_in_cache) {
+					CACHE_REMOVE(&inst->inst_cache, tombstone);
+					CACHE_RETURN(&inst->inst_cache, &tombstone);
+					tombstone_in_cache = 0; 
+				} else {
+					backentry_free(&tombstone);
+				}
+				tombstone = original_tombstone;
+				if ( (original_tombstone = backentry_dup( tombstone )) == NULL ) {
+					ldap_result_code= LDAP_OPERATIONS_ERROR;
+					goto error_return;
+				}
+			}
+
 			/* We're re-trying */
 			LDAPDebug0Args(LDAP_DEBUG_BACKLDBM,
 			               "Delete Retrying Transaction\n");
@@ -562,6 +587,14 @@ ldbm_back_delete( Slapi_PBlock *pb )
 			 * tentatively for now, then cache_add again when the original
 			 * entry is removed from the cache.
 			 */
+			if (cache_add_tentative( &inst->inst_cache, tombstone, NULL) == 0) {
+				tombstone_in_cache = 1;
+			} else if (!(tombstone->ep_state & ENTRY_STATE_NOTINCACHE)) {
+			    LDAPDebug1Arg(LDAP_DEBUG_CACHE,
+			                  "id2entry_add tombstone (%s) is in cache\n",
+			                  slapi_entry_get_dn(tombstone->ep_entry));
+			    tombstone_in_cache = 1;
+			}
 			retval = id2entry_add( be, tombstone, &txn );
 			if (DB_LOCK_DEADLOCK == retval) {
 				LDAPDebug( LDAP_DEBUG_ARGS, "delete 1 DB_LOCK_DEADLOCK\n", 0, 0, 0 );
@@ -576,14 +609,6 @@ ldbm_back_delete( Slapi_PBlock *pb )
 							  LDAP_OPERATIONS_ERROR, retry_count);
 				goto error_return;
 			}
-			if (cache_add_tentative( &inst->inst_cache, tombstone, NULL) == 0) {
-				tombstone_in_cache = 1;
-			} else if (!(tombstone->ep_state & ENTRY_STATE_NOTINCACHE)) {
-			    LDAPDebug1Arg(LDAP_DEBUG_CACHE,
-			                  "id2entry_add tombstone (%s) is in cache\n",
-			                  slapi_entry_get_dn(tombstone->ep_entry));
-			    tombstone_in_cache = 1;
-			}
 		}
 		else
 		{
@@ -966,7 +991,7 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	if (retry_count == RETRY_TIMES) {
 		/* Failed */
 		LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in delete\n", 0, 0, 0 );
-		ldap_result_code= LDAP_OPERATIONS_ERROR;
+		ldap_result_code= LDAP_BUSY;
 		retval = -1;
 		goto error_return;
 	}
@@ -1007,18 +1032,13 @@ ldbm_back_delete( Slapi_PBlock *pb )
 	}
 
 	/* delete from cache and clean up */
-	if (e_in_cache) {
+	if (e) {
 		CACHE_REMOVE(&inst->inst_cache, e);
 		cache_unlock_entry(&inst->inst_cache, e);
 		CACHE_RETURN(&inst->inst_cache, &e);
+		e = NULL;
 	}
-	if (tombstone_in_cache) {
-		if (CACHE_ADD( &inst->inst_cache, tombstone, NULL ) == 0) {
-			tombstone_in_cache = 1;
-		} else {
-			tombstone_in_cache = 0;
-		}
-	}
+
 	if (parent_found)
 	{
 		 /* Replace the old parent entry with the newly modified one */
@@ -1042,6 +1062,9 @@ error_return:
 	if (tombstone_in_cache)
 	{
 		CACHE_REMOVE( &inst->inst_cache, tombstone );
+		CACHE_RETURN( &inst->inst_cache, &tombstone );
+		tombstone = NULL;
+		tombstone_in_cache = 0;
 	}
 	else
 	{
@@ -1108,6 +1131,8 @@ common_return:
 	if (tombstone_in_cache)
 	{
 		CACHE_RETURN( &inst->inst_cache, &tombstone );
+		tombstone = NULL;
+		tombstone_in_cache = 0;
 	}
 	else
 	{
@@ -1126,7 +1151,7 @@ common_return:
 
 	/* Need to return to cache after post op plugins are called */
 	if (retval) { /* error case */
-		if (e && e_in_cache) {
+		if (e) {
 			cache_unlock_entry( &inst->inst_cache, e );
 			CACHE_RETURN( &inst->inst_cache, &e );
 		}
@@ -1156,8 +1181,13 @@ diskfull_return:
 		 */
 		slapi_pblock_set(pb, SLAPI_URP_NAMING_COLLISION_DN, slapi_ch_strdup (dn));
 	}
-	done_with_pblock_entry(pb, SLAPI_DELETE_EXISTING_ENTRY);
-	backentry_free(&original_entry);
+	if (free_delete_existing_entry) {
+		done_with_pblock_entry(pb, SLAPI_DELETE_EXISTING_ENTRY);
+	} else { /* owned by original_entry */
+		slapi_pblock_set(pb, SLAPI_DELETE_EXISTING_ENTRY, NULL);
+	}
+	slapi_entry_free(original_entry);
+	backentry_free(&original_tombstone);
 	slapi_ch_free((void**)&errbuf);
 	slapi_sdn_done(&sdn);
 	slapi_ch_free_string(&e_uniqueid);

ldap/servers/slapd/back-ldbm/ldbm_modify.c

@@ -460,25 +460,17 @@ ldbm_back_modify( Slapi_PBlock *pb )
 				/* New entry 'ec' is in the entry cache.
 				 * Remove it from the cache once. */
 				CACHE_REMOVE(&inst->inst_cache, ec);
-				cache_unlock_entry(&inst->inst_cache, e);
-				CACHE_RETURN(&inst->inst_cache, ec);
+				CACHE_RETURN(&inst->inst_cache, &ec);
 			} else {
 				backentry_free(&ec);
 			}
+			ec_in_cache = 0; /* added to cache by id2entry - have to remove to try again */
 			slapi_pblock_set( pb, SLAPI_MODIFY_EXISTING_ENTRY, original_entry->ep_entry );
 			ec = original_entry;
 			if ( (original_entry = backentry_dup( e )) == NULL ) {
 				ldap_result_code= LDAP_OPERATIONS_ERROR;
 				goto error_return;
 			}
-			/* Put new entry 'ec' into the entry cache. */
-			if (ec_in_cache && CACHE_ADD(&inst->inst_cache, ec, NULL) < 0) {
-				LDAPDebug1Arg(LDAP_DEBUG_ANY, 
-				              "ldbm_back_modify: adding %s to cache failed\n",
-				              slapi_entry_get_dn_const(ec->ep_entry));
-				ldap_result_code = LDAP_OPERATIONS_ERROR;
-				goto error_return;
-			}
 			LDAPDebug0Args(LDAP_DEBUG_BACKLDBM,
 			               "Modify Retrying Transaction\n");
 #ifndef LDBM_NO_BACKOFF_DELAY
@@ -603,7 +595,7 @@ ldbm_back_modify( Slapi_PBlock *pb )
 	}
 	if (retry_count == RETRY_TIMES) {
 		LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in modify\n", 0, 0, 0 );
-	   	ldap_result_code= LDAP_OPERATIONS_ERROR;
+	   	ldap_result_code= LDAP_BUSY;
 		goto error_return;
 	}
 
@@ -629,6 +621,7 @@ ldbm_back_modify( Slapi_PBlock *pb )
 	ec->ep_entry->e_virtual_watermark = 0;
 
 	/* we must return both e (which has been deleted) and new entry ec */
+	/* cache_replace removes e from the caches */
 	cache_unlock_entry( &inst->inst_cache, e );
 	CACHE_RETURN( &inst->inst_cache, &e );
 	/* 

ldap/servers/slapd/back-ldbm/ldbm_modrdn.c

@@ -46,11 +46,11 @@
 #include "back-ldbm.h"
 
 static const char *moddn_get_newdn(Slapi_PBlock *pb, Slapi_DN *dn_olddn, Slapi_DN *dn_newrdn, Slapi_DN *dn_newsuperiordn);
-static void moddn_unlock_and_return_entries(backend *be,struct backentry **targetentry, struct backentry **existingentry);
+static void moddn_unlock_and_return_entry(backend *be,struct backentry **targetentry);
 static int moddn_newrdn_mods(Slapi_PBlock *pb, const char *olddn, struct backentry *ec, Slapi_Mods *smods_wsi, int is_repl_op);
 static IDList *moddn_get_children(back_txn *ptxn, Slapi_PBlock *pb, backend *be, struct backentry *parententry, Slapi_DN *parentdn, struct backentry ***child_entries, struct backdn ***child_dns);
 static int moddn_rename_children(back_txn *ptxn, Slapi_PBlock *pb, backend *be, IDList *children, Slapi_DN *dn_parentdn, Slapi_DN *dn_newsuperiordn, struct backentry *child_entries[]);
-static int modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry *ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3);
+static int modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry **ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3, int *e_in_cache, int *ec_in_cache);
 static void mods_remove_nsuniqueid(Slapi_Mods *smods);
 
 #define MOD_SET_ERROR(rc, error, count)                                        \
@@ -68,6 +68,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     struct backentry *e= NULL;
     struct backentry *ec= NULL;
     int ec_in_cache= 0;
+    int e_in_cache= 0;
     back_txn txn;
     back_txnid parent_txn;
     int retval = -1;
@@ -81,7 +82,6 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     char *ldap_result_matcheddn= NULL;
     struct backentry *parententry= NULL;
     struct backentry *newparententry= NULL;
-    struct backentry *existingentry= NULL;
     struct backentry *original_entry = NULL;
     struct backentry *original_parent = NULL;
     struct backentry *original_newparent = NULL;
@@ -120,6 +120,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     const char *newdn = NULL;
     char *newrdn = NULL;
     int opreturn = 0;
+    int free_modrdn_existing_entry = 0;
 
     /* sdn & parentsdn need to be initialized before "goto *_return" */
     slapi_sdn_init(&dn_newdn);
@@ -254,6 +255,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
             {
                 goto error_return;
             }
+            free_modrdn_existing_entry = 1; /* need to free it */
         }
 
         /* <old superior> */
@@ -349,6 +351,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
         ldap_result_code= -1;
         goto error_return; /* error result sent by find_entry2modify() */
     }
+    e_in_cache = 1; /* e is in the cache and locked */
     /* Check that an entry with the same DN doesn't already exist. */
     {
         Slapi_Entry *entry;
@@ -518,17 +521,21 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     }
 
     /* create it in the cache - prevents others from creating it */
-    if (( cache_add_tentative( &inst->inst_cache, ec, NULL ) != 0 ) &&
+    if (( cache_add_tentative( &inst->inst_cache, ec, NULL ) != 0 ) ) {
+        ec_in_cache = 0; /* not in cache */
         /* allow modrdn even if the src dn and dest dn are identical */
-        ( 0 != slapi_sdn_compare((const Slapi_DN *)&dn_newdn,
-                                 (const Slapi_DN *)sdn)) )
-    {
-        /* somebody must've created it between dn2entry() and here */
-        /* JCMREPL - Hmm... we can't permit this to happen...? */
-        ldap_result_code= LDAP_ALREADY_EXISTS;
-        goto error_return;
+        if ( 0 != slapi_sdn_compare((const Slapi_DN *)&dn_newdn,
+                                    (const Slapi_DN *)sdn) ) {
+            /* somebody must've created it between dn2entry() and here */
+            /* JCMREPL - Hmm... we can't permit this to happen...? */
+            ldap_result_code= LDAP_ALREADY_EXISTS;
+            goto error_return;
+        }
+        /* so if the old dn is the same as the new dn, the entry will not be cached
+           until it is replaced with cache_replace */
+    } else {
+        ec_in_cache = 1;
     }
-    ec_in_cache= 1;
 
     /* Build the list of modifications required to the existing entry */
     {
@@ -725,15 +732,6 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
         ldap_result_code= LDAP_OPERATIONS_ERROR;
         goto error_return;
     }
-    if ( (original_parent = backentry_dup( parententry )) == NULL ) {
-        ldap_result_code= LDAP_OPERATIONS_ERROR;
-        goto error_return;
-    }
-    if ( newparententry &&
-         ((original_newparent = backentry_dup( newparententry )) == NULL) ) {
-        ldap_result_code= LDAP_OPERATIONS_ERROR;
-        goto error_return;
-    }
     slapi_pblock_get(pb, SLAPI_MODRDN_TARGET_ENTRY, &target_entry);
     if ( (original_targetentry = slapi_entry_dup(target_entry)) == NULL ) {
         ldap_result_code= LDAP_OPERATIONS_ERROR;
@@ -754,6 +752,8 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     {
         if (txn.back_txn_txn && (txn.back_txn_txn != parent_txn))
         {
+            Slapi_Entry *ent = NULL;
+
             dblayer_txn_abort(li,&txn);
             /* txn is no longer valid - reset slapi_txn to the parent */
             slapi_pblock_set(pb, SLAPI_TXN, parent_txn);
@@ -761,6 +761,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
             slapi_pblock_get(pb, SLAPI_MODRDN_NEWRDN, &newrdn);
             slapi_ch_free_string(&newrdn);
             slapi_pblock_set(pb, SLAPI_MODRDN_NEWRDN, original_newrdn);
+            slapi_sdn_set_normdn_byref(&dn_newrdn, original_newrdn);
             original_newrdn = slapi_ch_strdup(original_newrdn);
 
             slapi_pblock_get(pb, SLAPI_MODRDN_NEWSUPERIOR_SDN, &dn_newsuperiordn);
@@ -769,27 +770,51 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
             orig_dn_newsuperiordn = slapi_sdn_dup(orig_dn_newsuperiordn);
             if (ec_in_cache) {
                 /* New entry 'ec' is in the entry cache.
-                 * Remove it from teh cache once. */
+                 * Remove it from the cache . */
                 CACHE_REMOVE(&inst->inst_cache, ec);
-                cache_unlock_entry(&inst->inst_cache, e);
-                CACHE_RETURN(&inst->inst_cache, ec);
+                CACHE_RETURN(&inst->inst_cache, &ec);
+#ifdef DEBUG_CACHE
+                PR_ASSERT(ec == NULL);
+#endif
+                ec_in_cache = 0;
             } else {
                 backentry_free(&ec);
             }
-            slapi_pblock_set( pb, SLAPI_MODRDN_EXISTING_ENTRY, original_entry->ep_entry );
+            /* make sure the original entry is back in the cache if it was removed */
+            if (!e_in_cache) {
+                CACHE_ADD(&inst->inst_cache, e, NULL);
+                e_in_cache = 1;
+            }
+            slapi_pblock_get( pb, SLAPI_MODRDN_EXISTING_ENTRY, &ent );
+            if (ent && (ent != original_entry->ep_entry)) {
+                slapi_entry_free(ent);
+                slapi_pblock_set( pb, SLAPI_MODRDN_EXISTING_ENTRY, NULL );
+            }
             ec = original_entry;
             if ( (original_entry = backentry_dup( ec )) == NULL ) {
                 ldap_result_code= LDAP_OPERATIONS_ERROR;
                 goto error_return;
             }
-            if (ec_in_cache && 
+            slapi_pblock_set( pb, SLAPI_MODRDN_EXISTING_ENTRY, original_entry->ep_entry );
+            free_modrdn_existing_entry = 0; /* owned by original_entry now */
+            if (!ec_in_cache) {
                 /* Put the resetted entry 'ec' into the cache again. */
-                (cache_add_tentative( &inst->inst_cache, ec, NULL ) != 0)) {
-                LDAPDebug1Arg(LDAP_DEBUG_ANY, 
-                              "ldbm_back_modrdn: adding %s to cache failed\n",
-                              slapi_entry_get_dn_const(ec->ep_entry));
-                ldap_result_code = LDAP_OPERATIONS_ERROR;
-                goto error_return;
+                if (cache_add_tentative( &inst->inst_cache, ec, NULL ) != 0) {
+                    ec_in_cache = 0; /* not in cache */
+                    /* allow modrdn even if the src dn and dest dn are identical */
+                    if ( 0 != slapi_sdn_compare((const Slapi_DN *)&dn_newdn,
+                                                (const Slapi_DN *)sdn) ) {
+                        LDAPDebug1Arg(LDAP_DEBUG_ANY, 
+                                      "ldbm_back_modrdn: adding %s to cache failed\n",
+                                      slapi_entry_get_dn_const(ec->ep_entry));
+                        ldap_result_code = LDAP_OPERATIONS_ERROR;
+                        goto error_return;
+                    }
+                    /* so if the old dn is the same as the new dn, the entry will not be cached
+                       until it is replaced with cache_replace */
+                } else {
+                    ec_in_cache = 1;
+                }
             }
 
             slapi_pblock_get(pb, SLAPI_MODRDN_TARGET_ENTRY, &target_entry );
@@ -800,28 +825,16 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
                 goto error_return;
             }
 
-            backentry_free(&parententry);
-            slapi_pblock_set( pb, SLAPI_MODRDN_PARENT_ENTRY, original_parent->ep_entry );
-            parententry = original_parent;
-            if ( (original_entry = backentry_dup( parententry )) == NULL ) {
-                ldap_result_code= LDAP_OPERATIONS_ERROR;
-                goto error_return;
-            }
-
-            backentry_free(&newparententry);
-            if (original_newparent) {
-                slapi_pblock_set( pb, SLAPI_MODRDN_NEWPARENT_ENTRY,
-                                  original_newparent->ep_entry );
-            }
-            newparententry = original_entry;
-            if ( newparententry &&
-                 ((original_entry = backentry_dup(newparententry)) == NULL) ) {
-                ldap_result_code= LDAP_OPERATIONS_ERROR;
-                goto error_return;
-            }
             /* We're re-trying */
             LDAPDebug0Args(LDAP_DEBUG_BACKLDBM,
                            "Modrdn Retrying Transaction\n");
+#ifndef LDBM_NO_BACKOFF_DELAY
+            {
+            PRIntervalTime interval;
+            interval = PR_MillisecondsToInterval(slapi_rand() % 100);
+            DS_Sleep(interval);
+            }
+#endif
         }
         retval = dblayer_txn_begin(li,parent_txn,&txn);
         if (0 != retval) {
@@ -852,7 +865,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
         /*
          * Update the indexes for the entry.
          */
-        retval = modrdn_rename_entry_update_indexes(&txn, pb, li, e, ec, &smods_generated, &smods_generated_wsi, &smods_operation_wsi);
+        retval = modrdn_rename_entry_update_indexes(&txn, pb, li, e, &ec, &smods_generated, &smods_generated_wsi, &smods_operation_wsi, &e_in_cache, &ec_in_cache);
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
@@ -979,11 +992,14 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
         {
             Slapi_RDN newsrdn;
             slapi_rdn_init_sdn(&newsrdn, (const Slapi_DN *)&dn_newdn);
-            rc = entryrdn_rename_subtree(be, (const Slapi_DN *)sdn, &newsrdn,
-                                         (const Slapi_DN *)dn_newsuperiordn,
-                                         e->ep_id, &txn);
+            retval = entryrdn_rename_subtree(be, (const Slapi_DN *)sdn, &newsrdn,
+                                             (const Slapi_DN *)dn_newsuperiordn,
+                                             e->ep_id, &txn);
             slapi_rdn_done(&newsrdn);
-            if (rc) {
+            if (retval != 0) {
+                if (retval == DB_LOCK_DEADLOCK) continue;
+                if (retval == DB_RUNRECOVERY || LDBM_OS_ERR_IS_DISKFULL(retval))
+                    disk_full = 1;
                 MOD_SET_ERROR(ldap_result_code, 
                               LDAP_OPERATIONS_ERROR, retry_count);
                 goto error_return;
@@ -1035,7 +1051,7 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     {
         /* Failed */
         LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in modrdn\n", 0, 0, 0 );
-        ldap_result_code= LDAP_OPERATIONS_ERROR;
+        ldap_result_code= LDAP_BUSY;
         goto error_return;
     }
 
@@ -1128,12 +1144,6 @@ ldbm_back_modrdn( Slapi_PBlock *pb )
     }
 
     retval= 0;
-#if 0 /* this new entry in the cache can be used for future; don't remove it */
-    /* remove from cache so that memory can be freed by cache_return */
-    if (ec_in_cache) {
-        CACHE_REMOVE(&inst->inst_cache, ec);
-    }
-#endif
     goto common_return;
 
 error_return:
@@ -1151,13 +1161,6 @@ error_return:
         CACHE_REMOVE(&inst->inst_dncache, bdn);
         CACHE_RETURN(&inst->inst_dncache, &bdn);
     }
-    if( ec!=NULL ) {
-        if (ec_in_cache) {
-            CACHE_REMOVE(&inst->inst_cache, ec);
-        } else {
-            backentry_free( &ec );
-        }
-    }
     if(children)
     {
         int i = 0;
@@ -1245,13 +1248,6 @@ error_return:
 
 common_return:
 
-    /* Free up the resource we don't need any more */
-    if(ec_in_cache) {
-        CACHE_RETURN( &inst->inst_cache, &ec );
-    }
-
-    moddn_unlock_and_return_entries(be,&e,&existingentry);
-
     /* result code could be used in the bepost plugin functions. */
     slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code);
     /*
@@ -1259,6 +1255,32 @@ common_return:
      */
     plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN);
 
+    /* Free up the resource we don't need any more */
+    if (ec) {
+        /* remove the new entry from the cache if the op failed -
+           otherwise, leave it in */
+        if (ec_in_cache && retval) {
+            CACHE_REMOVE( &inst->inst_cache, ec );
+        }
+        if (ec_in_cache) {
+            CACHE_RETURN( &inst->inst_cache, &ec );
+        } else {
+            backentry_free( &ec );
+        }
+        ec = NULL;
+        ec_in_cache = 0;
+    }
+
+    /* put e back in the cache if the modrdn failed */
+    if (e) {
+        if (!e_in_cache && retval) {
+            CACHE_ADD(&inst->inst_cache, e, NULL);
+            e_in_cache = 1;
+        }
+    }
+
+    moddn_unlock_and_return_entry(be,&e);
+
     if (ruv_c_init) {
         modify_term(&ruv_c, be);
     }
@@ -1281,7 +1303,11 @@ common_return:
     slapi_sdn_done(&dn_parentdn);
     modify_term(&parent_modify_context,be);
     modify_term(&newparent_modify_context,be);
-    done_with_pblock_entry(pb,SLAPI_MODRDN_EXISTING_ENTRY);
+    if (free_modrdn_existing_entry) {
+        done_with_pblock_entry(pb,SLAPI_MODRDN_EXISTING_ENTRY);
+    } else { /* owned by original_entry */
+        slapi_pblock_set(pb, SLAPI_MODRDN_EXISTING_ENTRY, NULL);
+    }
     done_with_pblock_entry(pb,SLAPI_MODRDN_PARENT_ENTRY);
     done_with_pblock_entry(pb,SLAPI_MODRDN_NEWPARENT_ENTRY);
     done_with_pblock_entry(pb,SLAPI_MODRDN_TARGET_ENTRY);
@@ -1354,10 +1380,9 @@ moddn_get_newdn(Slapi_PBlock *pb, Slapi_DN *dn_olddn, Slapi_DN *dn_newrdn, Slapi
  * Return the entries to the cache.
  */
 static void
-moddn_unlock_and_return_entries(
+moddn_unlock_and_return_entry(
     backend *be,
-    struct backentry **targetentry, 
-    struct backentry **existingentry)
+    struct backentry **targetentry)
     {
         ldbm_instance *inst = (ldbm_instance *) be->be_instance_info;
 
@@ -1367,10 +1392,6 @@ moddn_unlock_and_return_entries(
             CACHE_RETURN( &inst->inst_cache, targetentry );
             *targetentry= NULL;
         }
-        if ( *existingentry!=NULL ) {
-            CACHE_RETURN( &inst->inst_cache, existingentry );
-            *existingentry= NULL;
-        }
     }
 
 
@@ -1540,8 +1561,8 @@ moddn_newrdn_mods(Slapi_PBlock *pb, const char *olddn, struct backentry *ec, Sla
     }
     else
     {
-    LDAPDebug( LDAP_DEBUG_TRACE, "moddn_newrdn_mods failed: could not parse new rdn %s\n",
-           newrdn, 0, 0);
+        LDAPDebug( LDAP_DEBUG_TRACE, "moddn_newrdn_mods failed: could not parse new rdn %s\n",
+                   newrdn, 0, 0);
         return LDAP_OPERATIONS_ERROR;
     }
     
@@ -1568,7 +1589,7 @@ mods_remove_nsuniqueid(Slapi_Mods *smods)
  * mods contains the list of attribute change made.
  */
 static int
-modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry *ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3)
+modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry **ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3, int *e_in_cache, int *ec_in_cache)
 {
     backend *be;
     ldbm_instance *inst;
@@ -1576,36 +1597,41 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
     char *msg;
     Slapi_Operation *operation;
     int is_ruv = 0;                 /* True if the current entry is RUV */
+    int orig_ec_in_cache = 0;
 
     slapi_pblock_get( pb, SLAPI_BACKEND, &be );
     slapi_pblock_get( pb, SLAPI_OPERATION, &operation );
     is_ruv = operation_is_flag_set(operation, OP_FLAG_REPL_RUV);
     inst = (ldbm_instance *) be->be_instance_info;
 
+    orig_ec_in_cache = *ec_in_cache;
     /*
      * Update the ID to Entry index. 
      * Note that id2entry_add replaces the entry, so the Entry ID stays the same.
      */
-    retval = id2entry_add( be, ec, ptxn );
+    retval = id2entry_add( be, *ec, ptxn );
     if (DB_LOCK_DEADLOCK == retval)
     {
         /* Retry txn */
+        LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes: id2entry_add deadlock\n" );
         goto error_return;
     }
     if (retval != 0)
     {
-        LDAPDebug( LDAP_DEBUG_ANY, "id2entry_add failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 );
+        LDAPDebug( LDAP_DEBUG_ANY, "modrdn_rename_entry_update_indexes: id2entry_add failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 );
         goto error_return;
     }
+    *ec_in_cache = 1; /* id2entry_add adds to cache if not already in */
     if(smods1!=NULL && slapi_mods_get_num_mods(smods1)>0)
     {
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods1), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods1), e, *ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
+            LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes: index_add_mods1 deadlock\n" );
             goto error_return;
         }
         if (retval != 0)
@@ -1625,10 +1651,11 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods2), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods2), e, *ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
+            LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes: index_add_mods2 deadlock\n" );
             goto error_return;
         }
         if (retval != 0)
@@ -1642,10 +1669,11 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
         /*
          * update the indexes: lastmod, rdn, etc.
          */
-        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods3), e, ec, ptxn );
+        retval = index_add_mods( be, slapi_mods_get_ldapmods_byref(smods3), e, *ec, ptxn );
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Retry txn */
+            LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes: index_add_mods3 deadlock\n" );
             goto error_return;
         }
         if (retval != 0)
@@ -1661,10 +1689,11 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
      */
     if (!is_ruv)
     {
-        retval= vlv_update_all_indexes(ptxn, be, pb, e, ec);
+        retval= vlv_update_all_indexes(ptxn, be, pb, e, *ec);
         if (DB_LOCK_DEADLOCK == retval)
         {
             /* Abort and re-try */
+            LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes: vlv_update_all_indexes deadlock\n" );
             goto error_return;
         }
         if (retval != 0)
@@ -1673,9 +1702,18 @@ modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbm
             goto error_return;
         }
     }
-    if (cache_replace( &inst->inst_cache, e, ec ) != 0 ) {
+    if (cache_replace( &inst->inst_cache, e, *ec ) != 0 ) {
+        LDAPDebug0Args( LDAP_DEBUG_BACKLDBM, "modrdn_rename_entry_update_indexes cache_replace failed\n");
         retval= -1;
         goto error_return;
+    } else {
+        *e_in_cache = 0; /* e un-cached */
+    }
+    if (orig_ec_in_cache) {
+        /* ec was already added to the cache via cache_add_tentative (to reserve its spot in the cache)
+           and/or id2entry_add - so it already had one refcount - cache_replace adds another refcount -
+           drop the extra ref added by cache_replace */
+        CACHE_RETURN( &inst->inst_cache, ec );
     }
 error_return:
     return retval;
@@ -1687,7 +1725,7 @@ moddn_rename_child_entry(
     Slapi_PBlock *pb, 
     struct ldbminfo *li, 
     struct backentry *e, 
-    struct backentry *ec, 
+    struct backentry **ec, 
     int parentdncomps, 
     char **newsuperiordns, 
     int newsuperiordncomps,
@@ -1711,8 +1749,10 @@ moddn_rename_child_entry(
     int olddncomps= 0;
     int need= 1; /* For the '\0' */
     int i;
+    int e_in_cache = 1;
+    int ec_in_cache = 0;
 
-    olddn = slapi_entry_get_dn(ec->ep_entry);
+    olddn = slapi_entry_get_dn((*ec)->ep_entry);
     if (NULL == olddn) {
         return retval;
     }
@@ -1747,9 +1787,9 @@ moddn_rename_child_entry(
         }
     }
     slapi_ldap_value_free( olddns );
-    slapi_entry_set_dn( ec->ep_entry, newdn );
+    slapi_entry_set_dn( (*ec)->ep_entry, newdn );
     /* add the entrydn operational attributes */
-    add_update_entrydn_operational_attributes (ec);
+    add_update_entrydn_operational_attributes (*ec);
 
     /*
      * Update the DN CSN of the entry.
@@ -1767,13 +1807,14 @@ moddn_rename_child_entry(
         slapi_mods_add( &smods, LDAP_MOD_DELETE, LDBM_ENTRYDN_STR, 
                     strlen( backentry_get_ndn(e) ), backentry_get_ndn(e) );
         slapi_mods_add( &smods, LDAP_MOD_REPLACE, LDBM_ENTRYDN_STR, 
-                    strlen( backentry_get_ndn(ec) ), backentry_get_ndn(ec) );
+                    strlen( backentry_get_ndn(*ec) ), backentry_get_ndn(*ec) );
         smodsp = &smods;
         /*
          * Update all the indexes.
          */
         retval = modrdn_rename_entry_update_indexes(ptxn, pb, li, e, ec,
-                                                    smodsp, NULL, NULL);
+                                                    smodsp, NULL, NULL,
+                                                    &e_in_cache, &ec_in_cache);
         /* JCMREPL - Should the children get updated modifiersname and lastmodifiedtime? */
         slapi_mods_done(&smods);
     }
@@ -1850,7 +1891,7 @@ moddn_rename_children(
     opcsn = operation_get_csn (operation);
     for (i=0,retval=0; retval == 0 && child_entries[i] && child_entry_copies[i]; i++) {
         retval = moddn_rename_child_entry(ptxn, pb, li, child_entries[i],
-                                         child_entry_copies[i], parentdncomps,
+                                         &child_entry_copies[i], parentdncomps,
                                          newsuperiordns, newsuperiordncomps,
                                          opcsn );
     }

ldap/servers/slapd/back-ldbm/misc.c

@@ -53,7 +53,7 @@ void ldbm_nasty(const char* str, int c, int err)
     char buffer[200];
     if (err == DB_LOCK_DEADLOCK) {
         PR_snprintf(buffer,200,"%s WARNING %d",str,c);
-        LDAPDebug(LDAP_DEBUG_TRACE,"%s, err=%d %s\n",
+        LDAPDebug(LDAP_DEBUG_BACKLDBM,"%s, err=%d %s\n",
                   buffer,err,(msg = dblayer_strerror( err )) ? msg : "");
    } else if (err == DB_RUNRECOVERY) {
         LDAPDebug2Args(LDAP_DEBUG_ANY, "FATAL ERROR at %s (%d); "

ldap/servers/slapd/back-ldbm/seq.c

@@ -240,6 +240,9 @@ ldbm_back_seq( Slapi_PBlock *pb )
 				  idl = idl_fetch( be, db, &key, NULL, ai, &err );
 				  if(err == DB_LOCK_DEADLOCK) {
 				    ldbm_nasty("ldbm_back_seq deadlock retry", 1600, err);
+#ifdef FIX_TXN_DEADLOCKS
+#error if txn != NULL, have to retry the entire transaction
+#endif
 				    continue;
 				  } else {
 				    break;

ldap/servers/slapd/slap.h

@@ -1242,7 +1242,8 @@ enum
 	BE_STATE_STOPPED = 1,	/* backend is initialized but not started */
 	BE_STATE_STARTED,		/* backend is started */
 	BE_STATE_CLEANED,		/* backend was cleaned up */
-	BE_STATE_DELETED		/* backend is removed */
+	BE_STATE_DELETED,		/* backend is removed */
+	BE_STATE_STOPPING       /* told to stop but not yet stopped */
 };
 
 struct conn;