Bug 572018 - Upgrading from 1.2.5 to 1.2.6.a2 deletes userRoot

https://bugzilla.redhat.com/show_bug.cgi?id=572018

Description: The upgrade tool dn2rdn shares the reindex code, in
which a cursor is set to each entry in id2entry.db# converting
dn to rdn and put it into the btree with db->put one by one.
It turned out the strategy does not work for the case the btree
structure is modified.  In that case, the btree could get corrupted.
This patch creates a new id2entry_tmp.db# from the scratch.  Once
all the entries are successfully converted, the old db is removed
and id2entry_tmp.db# is renamed to id2entry.db#.

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -845,4 +845,7 @@ typedef struct _back_search_result_set
  * starting with '.' to reduce the risk to match an ordinary backend name */
 #define CHANGELOG_BACKUPDIR ".repl_changelog_backup"
 
+/* For dblayer_get_aux_id2entry_ext */
+#define DBLAYER_AUX_ID2ENTRY_TMP 0x1
+
 #endif /* _back_ldbm_h_ */

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -2138,9 +2138,8 @@ int dblayer_instance_start(backend *be, int mode)
 #if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 3300
         return_value = dbp->set_malloc(dbp, (void *)slapi_ch_malloc);
         if (0 != return_value) {
-            LDAPDebug(LDAP_DEBUG_ANY,
-                      "dbp->set_malloc failed %d\n",
-                      return_value, 0, 0);
+            LDAPDebug1Arg(LDAP_DEBUG_ANY, "dbp->set_malloc failed %d\n",
+                          return_value);
 
             goto out;
         }
@@ -2163,6 +2162,26 @@ int dblayer_instance_start(backend *be, int mode)
             if (0 != return_value)
                 goto out;
             dbp = inst->inst_id2entry;
+            return_value = dbp->set_pagesize(dbp,
+                (priv->dblayer_page_size == 0) ? DBLAYER_PAGESIZE :
+                priv->dblayer_page_size);
+            if (0 != return_value) {
+                LDAPDebug(LDAP_DEBUG_ANY,
+                      "dbp->set_pagesize(%lu or %lu) failed %d\n",
+                      priv->dblayer_page_size, DBLAYER_PAGESIZE,
+                      return_value);
+                goto out;
+            }
+
+#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 3300
+            return_value = dbp->set_malloc(dbp, (void *)slapi_ch_malloc);
+            if (0 != return_value) {
+                LDAPDebug1Arg(LDAP_DEBUG_ANY, "dbp->set_malloc failed %d\n",
+                              return_value);
+
+                goto out;
+            }
+#endif
             slapi_ch_free_string(&abs_id2entry_file);
         }
         DB_OPEN(mypEnv->dblayer_openflags,
@@ -2273,8 +2292,29 @@ int dblayer_release_id2entry(backend *be, DB *pDB)
  * - create a dedicated db env and db handler for id2entry.
  * - introduced for upgradedb not to share the env and db handler with
  *   other index files to support multiple passes and merge.
+ * - Argument path is for returning the full path for the id2entry.db#,
+ *   if the memory to store the address of the full path is given.  The
+ *   caller is supposed to release the full path.
  */
-int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv)
+int
+dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv, char **path)
+{
+    return dblayer_get_aux_id2entry_ext(be, ppDB, ppEnv, path, 0);
+}
+
+/*
+ * flags:
+ * DBLAYER_AUX_ID2ENTRY_TMP -- create id2entry_tmp.db#
+ *
+ * - if non-NULL *ppEnv is given, env is already open.
+ *   Just open an id2entry[_tmp].db#.
+ * - Argument path is for returning the full path for the id2entry[_tmp].db#,
+ *   if the memory to store the address of the full path is given.  The
+ *   caller is supposed to release the full path.
+ */
+int
+dblayer_get_aux_id2entry_ext(backend *be, DB **ppDB, DB_ENV **ppEnv, 
+                             char **path, int flags)
 {
     ldbm_instance *inst;
     struct dblayer_private_env *mypEnv = NULL;
@@ -2284,7 +2324,8 @@ int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv)
     dblayer_private *opriv = NULL;
     dblayer_private *priv = NULL;
     char *subname = NULL;
-    int envflags;
+    int envflags = 0;
+    int dbflags = 0;
     size_t cachesize;
     PRFileInfo prfinfo;
     PRStatus prst;
@@ -2295,8 +2336,11 @@ int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv)
 
     PR_ASSERT(NULL != be);
     
+    if ((NULL == ppEnv) || (NULL == ppDB)) {
+        LDAPDebug0Args(LDAP_DEBUG_ANY, "No memory for DB_ENV or DB handle\n");
+        goto done;
+    }
     *ppDB = NULL;
-    *ppEnv = NULL;
     inst = (ldbm_instance *) be->be_instance_info;
     if (NULL == inst)
     {
@@ -2352,87 +2396,102 @@ int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv)
         ldbm_delete_dirs(priv->dblayer_home_directory);
     }
     rval = mkdir_p(priv->dblayer_home_directory, 0700);
-    if (0 != rval)
+    if (rval)
     {
-        LDAPDebug(LDAP_DEBUG_ANY,
-            "can't create env dir: persistent id2entry is not available\n", 0, 0, 0);
+        LDAPDebug0Args(LDAP_DEBUG_ANY,
+               "can't create env dir: persistent id2entry is not available\n");
         goto done;
     }
 
-    /* use our own env */
-    rval = dblayer_make_env(&mypEnv, li);
-    if (rval != 0) {
-        LDAPDebug(LDAP_DEBUG_ANY,
-            "Unable to create new DB_ENV for import/export! %d\n", rval, 0, 0);
-        goto err;
+    /* use our own env if not passed */
+    if (!*ppEnv) {
+        rval = dblayer_make_env(&mypEnv, li);
+        if (rval) {
+            LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                  "Unable to create new DB_ENV for import/export! %d\n", rval);
+            goto err;
+        }
     }
 
     envflags = DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE;
-    cachesize = 1048576; /* 1M */
+    cachesize = 10485760; /* 10M */
 
-    mypEnv->dblayer_DB_ENV->set_cachesize(mypEnv->dblayer_DB_ENV,
+    if (!*ppEnv) {
+        mypEnv->dblayer_DB_ENV->set_cachesize(mypEnv->dblayer_DB_ENV,
                 0, cachesize, priv->dblayer_ncache);
 
-    /* probably want to change this -- but for now, create the 
-     * mpool files in the instance directory.
-     */
-    mypEnv->dblayer_openflags = envflags;
-    data_directories[0] = inst->inst_parent_dir_name;
-    dblayer_set_data_dir(priv, mypEnv, data_directories);
-    rval = (mypEnv->dblayer_DB_ENV->open)(mypEnv->dblayer_DB_ENV,
+        /* probably want to change this -- but for now, create the 
+         * mpool files in the instance directory.
+         */
+        mypEnv->dblayer_openflags = envflags;
+        data_directories[0] = inst->inst_parent_dir_name;
+        dblayer_set_data_dir(priv, mypEnv, data_directories);
+        rval = (mypEnv->dblayer_DB_ENV->open)(mypEnv->dblayer_DB_ENV,
             priv->dblayer_home_directory, envflags, priv->dblayer_file_mode);
-    if (rval != 0)
-    {
-        LDAPDebug(LDAP_DEBUG_ANY,
-            "Unable to open new DB_ENV for upgradedb/reindex %d\n", rval, 0, 0);
-        goto err;
+        if (rval) {
+            LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                  "Unable to open new DB_ENV for upgradedb/reindex %d\n", rval);
+            goto err;
+        }
+        *ppEnv = mypEnv->dblayer_DB_ENV;
     }
-    *ppEnv = mypEnv->dblayer_DB_ENV;
-
-    rval = db_create(&dbp, mypEnv->dblayer_DB_ENV, 0);
-    if (0 != rval) {
-        LDAPDebug(LDAP_DEBUG_ANY,
-                  "Unable to create id2entry db handler! %d\n", rval, 0, 0);
+    rval = db_create(&dbp, *ppEnv, 0);
+    if (rval) {
+        LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                      "Unable to create id2entry db handler! %d\n", rval);
         goto err;
     }
 
     rval = dbp->set_pagesize(dbp, (priv->dblayer_page_size == 0) ?
                         DBLAYER_PAGESIZE : priv->dblayer_page_size);
-    if (0 != rval)
-    {
+    if (rval) {
         LDAPDebug(LDAP_DEBUG_ANY,
-            "dbp->set_pagesize(%lu or %lu) failed %d\n",
-            priv->dblayer_page_size, DBLAYER_PAGESIZE, rval);
+                  "dbp->set_pagesize(%lu or %lu) failed %d\n",
+                  priv->dblayer_page_size, DBLAYER_PAGESIZE, rval);
         goto err;
     }
 
-    id2entry_file = slapi_ch_smprintf("%s/%s",
-        inst->inst_dir_name, ID2ENTRY LDBM_FILENAME_SUFFIX);
+    if (flags & DBLAYER_AUX_ID2ENTRY_TMP) {
+        id2entry_file = slapi_ch_smprintf("%s/%s_tmp%s",
+            inst->inst_dir_name, ID2ENTRY, LDBM_FILENAME_SUFFIX);
+        dbflags = DB_CREATE;
+    } else {
+        id2entry_file = slapi_ch_smprintf("%s/%s",
+            inst->inst_dir_name, ID2ENTRY LDBM_FILENAME_SUFFIX);
+    }
 
     PR_ASSERT(dblayer_inst_exists(inst, NULL));
-    DB_OPEN(mypEnv->dblayer_openflags,
-            dbp, NULL/* txnid */, id2entry_file, subname, DB_BTREE,
-            0, priv->dblayer_file_mode, rval);
-    if (0 != rval)
-    {
+    DB_OPEN(envflags, dbp, NULL/* txnid */, id2entry_file, subname, DB_BTREE,
+            dbflags, priv->dblayer_file_mode, rval);
+    if (rval) {
         LDAPDebug(LDAP_DEBUG_ANY,
                   "dbp->open(\"%s\") failed: %s (%d)\n",
                   id2entry_file, dblayer_strerror(rval), rval);
-        if (strstr(dblayer_strerror(rval), "Permission denied"))
-        {
-            LDAPDebug(LDAP_DEBUG_ANY,
-                "Instance directory %s may not be writable\n", inst_dirp, 0, 0);
+        if (strstr(dblayer_strerror(rval), "Permission denied")) {
+            LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                  "Instance directory %s may not be writable\n", inst_dirp);
         }
         goto err;
     }
     *ppDB = dbp;
+    rval = 0; /* to make it sure ... */
     goto done;
 err:
-    if (*ppEnv)
+    if (*ppEnv) {
        (*ppEnv)->close(*ppEnv, 0);
-    if (priv->dblayer_home_directory)
+       *ppEnv = NULL;
+    }
+    if (id2entry_file) {
+        slapi_ch_free_string(&id2entry_file);
+    }
+    if (priv->dblayer_home_directory) {
         ldbm_delete_dirs(priv->dblayer_home_directory);
+    }
 done:
+    if ((0 == rval) && path) { /* only when successfull */
+        *path = slapi_ch_smprintf("%s/%s",
+                                  inst->inst_parent_dir_name, id2entry_file);
+    }
     slapi_ch_free_string(&id2entry_file);
     if (priv) {
         slapi_ch_free_string(&priv->dblayer_home_directory);

ldap/servers/slapd/back-ldbm/import-threads.c

@@ -909,7 +909,10 @@ index_producer(void *param)
 
     /* vars for Berkeley DB */
     DB_ENV *env = NULL;
+    char *id2entry = NULL;
+    char *tmpid2entry = NULL;
     DB *db = NULL;
+    DB *tmp_db = NULL;
     DBC *dbc = NULL;
     DBT key = {0};
     DBT data = {0};
@@ -935,11 +938,20 @@ index_producer(void *param)
     info->state = RUNNING;
 
     /* open id2entry with dedicated db env and db handler */
-    if ( dblayer_get_aux_id2entry( be, &db, &env ) != 0  || db == NULL ||
-         env == NULL) {
-        LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 );
+    if ( dblayer_get_aux_id2entry( be, &db, &env, &id2entry ) != 0  ||
+         db == NULL || env == NULL) {
+        LDAPDebug0Args(LDAP_DEBUG_ANY, "Could not open id2entry\n" );
         goto error;
     }
+    if (job->flags & FLAG_DN2RDN) {
+        /* open new id2entry for the rdn format entries */
+        if ( dblayer_get_aux_id2entry_ext( be, &tmp_db, &env, &tmpid2entry,
+                                           DBLAYER_AUX_ID2ENTRY_TMP ) != 0  ||
+             tmp_db == NULL || env == NULL) {
+            LDAPDebug0Args(LDAP_DEBUG_ANY, "Could not open new id2entry\n");
+            goto error;
+        }
+    }
 
     /* get a cursor to we can walk over the table */
     db_rval = db->cursor(db, NULL, &dbc, 0);
@@ -1013,8 +1025,8 @@ index_producer(void *param)
                     data.dptr = slapi_entry2str_with_options(e, &len, options);
                     data.dsize = len + 1;
 
-                    /* store it  */
-                    rc = db->put( db, NULL, &key, &data, 0);
+                    /* store it in the new id2entry db file */
+                    rc = tmp_db->put( tmp_db, NULL, &key, &data, 0);
                     if (rc) {
                         LDAPDebug2Args( LDAP_DEBUG_TRACE,
                                    "index_producer: converting an entry "
@@ -1125,8 +1137,34 @@ index_producer(void *param)
         }
     }
 
-    dbc->c_close(dbc);
-    dblayer_release_aux_id2entry( be, db, env );
+    dbc->c_close(dbc); dbc = NULL;
+    db->close(db, 0);
+    if (job->flags & FLAG_DN2RDN) {
+        /* remove id2entry, then rename tmp to id2entry */
+        tmp_db->close(tmp_db, 0);
+        rc = db_create(&db, env, 0);
+        if (rc) {
+            LDAPDebug2Args(LDAP_DEBUG_ANY,
+                           "Creating db handle to rename %s to %s failed.\n",
+                           tmpid2entry, id2entry);
+            goto bail;
+        }
+        rc = db->remove(db, id2entry, NULL, 0);
+        if (rc) {
+            LDAPDebug1Arg(LDAP_DEBUG_ANY, "Removing %s failed.\n", id2entry);
+            goto bail;
+        }
+        db_create(&db, env, 0);
+        rc = db->rename(db, tmpid2entry, NULL, id2entry, 0);
+        if (rc) {
+            LDAPDebug2Args(LDAP_DEBUG_ANY, "Renaming %s to %s failed.\n",
+                           tmpid2entry, id2entry);
+            goto bail;
+        }
+    }
+    dblayer_release_aux_id2entry(be, NULL, env);
+    slapi_ch_free_string(&id2entry);
+    slapi_ch_free_string(&tmpid2entry);
     info->state = FINISHED;
     return;
 
@@ -1134,7 +1172,29 @@ error:
     if (dbc) {
         dbc->c_close(dbc);
     }
+    if (job->flags & FLAG_DN2RDN) {
+        if (tmp_db) {
+            tmp_db->close(tmp_db, 0);
+            rc = db_create(&tmp_db, env, 0);
+            if (rc) {
+                LDAPDebug1Arg(LDAP_DEBUG_ANY,
+                              "Creating db handle to remove %s s failed.\n",
+                              tmpid2entry);
+                goto bail;
+            }
+            /* remove tmp */
+            rc = tmp_db->remove(tmp_db, tmpid2entry, NULL, 0);
+            if (rc) {
+                LDAPDebug1Arg(LDAP_DEBUG_ANY, "Removing %s failed.\n",
+                              tmpid2entry);
+                goto bail;
+            }
+        }
+    }
+bail:
     dblayer_release_aux_id2entry( be, db, env );
+    slapi_ch_free_string(&id2entry);
+    slapi_ch_free_string(&tmpid2entry);
     info->state = ABORTED;
 }
 
@@ -1255,7 +1315,7 @@ upgradedn_producer(void *param)
     info->state = RUNNING;
 
     /* open id2entry with dedicated db env and db handler */
-    if ( dblayer_get_aux_id2entry( be, &db, &env ) != 0  || db == NULL ||
+    if ( dblayer_get_aux_id2entry( be, &db, &env, NULL ) != 0  || db == NULL ||
          env == NULL) {
         LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 );
         goto error;

ldap/servers/slapd/back-ldbm/import.c

@@ -1349,7 +1349,7 @@ int import_main_offline(void *arg)
     /* Now do the numsubordinates attribute */
     import_log_notice(job, "Indexing complete.  Post-processing...");
     /* [610066] reindexed db cannot be used in the following backup/restore */
-    if ( !(job->flags & FLAG_REINDEXING) &&
+    if ( (!(job->flags & FLAG_REINDEXING) || (job->flags & FLAG_DN2RDN)) &&
          (ret = update_subordinatecounts(be, job->mothers, job->encrypt, NULL))
          != 0 ) {
         import_log_notice(job, "Failed to update numsubordinates attributes");

ldap/servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -111,7 +111,8 @@ int dblayer_erase_index_file(backend *be, struct attrinfo *a, int no_force_chkpt
 int dblayer_erase_index_file_nolock(backend *be, struct attrinfo *a, int no_force_chkpt); 
 int dblayer_get_id2entry(backend *be, DB **ppDB);
 int dblayer_release_id2entry(backend *be, DB *pDB);
-int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv);
+int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv, char **path);
+int dblayer_get_aux_id2entry_ext(backend *be, DB **ppDB, DB_ENV **ppEnv, char **path, int flags);
 int dblayer_release_aux_id2entry(backend *be, DB *pDB, DB_ENV *pEnv);
 int dblayer_txn_init(struct ldbminfo *li, back_txn *txn);
 int dblayer_txn_begin(struct ldbminfo *li,back_txnid parent_txn, back_txn *txn);