[155905] mrclone failure: race condition between force_checkpoint and dblayer_close

1) Let dblayer_force_checkpoint check if the db env is still good or not.
   If it's already released, it just returns.
2) In _dblayer_delete_instance_dir, it actually does not need to invoke db
   threads.  Set DBLAYER_NO_DBTHREADS_MODE when calling dblayer_start.

ldap/servers/slapd/back-ldbm/archive.c

@@ -322,7 +322,7 @@ int ldbm_back_ldbm2archive( Slapi_PBlock *pb )
 
     /* start the database code up, do not attempt to perform recovery */
     if (run_from_cmdline &&
-        0 != dblayer_start(li,DBLAYER_ARCHIVE_MODE|DBLAYER_CMDLINE_MODE)) {
+        0 != dblayer_start(li,DBLAYER_ARCHIVE_MODE|DBLAYER_NO_DBTHREADS_MODE)) {
         LDAPDebug(LDAP_DEBUG_ANY, "db2archive: Failed to init database\n",
                   0, 0, 0);
         if (task) {
@@ -336,7 +336,7 @@ int ldbm_back_ldbm2archive( Slapi_PBlock *pb )
 
     /* close the database down again */
     if (run_from_cmdline &&
-        0 != dblayer_close(li,DBLAYER_ARCHIVE_MODE|DBLAYER_CMDLINE_MODE)) {
+        0 != dblayer_close(li,DBLAYER_ARCHIVE_MODE|DBLAYER_NO_DBTHREADS_MODE)) {
         LDAPDebug(LDAP_DEBUG_ANY, "db2archive: Failed to close database\n",
                   0, 0, 0);
         if (task) {

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -214,7 +214,7 @@ extern int ldbm_warn_if_no_db;
 #define DBLAYER_INDEX_MODE                  0x80
 #define DBLAYER_CLEAN_RECOVER_MODE          0x100
 
-#define DBLAYER_CMDLINE_MODE                0x1000
+#define DBLAYER_NO_DBTHREADS_MODE           0x1000
 
 #define DBLAYER_RESTORE_MASK                (DBLAYER_RESTORE_MODE|DBLAYER_RESTORE_NO_RECOVERY_MODE)
 

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -1388,12 +1388,12 @@ int dblayer_start(struct ldbminfo *li, int dbmode)
         case DBLAYER_RESTORE_MODE:
             open_flags |= DB_RECOVER_FATAL;
             open_flags &= ~DB_RECOVER;    /* shouldn't set both */
-            if (!(dbmode & DBLAYER_CMDLINE_MODE))
+            if (!(dbmode & DBLAYER_NO_DBTHREADS_MODE))
                 dbmode = DBLAYER_NORMAL_MODE; /* to restart helper threads */
             break;
         case DBLAYER_RESTORE_NO_RECOVERY_MODE:
             open_flags &= ~(DB_RECOVER | DB_RECOVER_FATAL);
-            if (!(dbmode & DBLAYER_CMDLINE_MODE))
+            if (!(dbmode & DBLAYER_NO_DBTHREADS_MODE))
                 dbmode = DBLAYER_NORMAL_MODE; /* to restart helper threads */
         }
     }
@@ -1547,7 +1547,9 @@ int dblayer_start(struct ldbminfo *li, int dbmode)
 
 
         /* Now attempt to start up the checkpoint and deadlock threads */
-        if ( (DBLAYER_NORMAL_MODE & dbmode ) && (0 == return_value)) {
+        /* note: need to be '==', not '&' to omit DBLAYER_NO_DBTHREADS_MODE */
+        if ( (DBLAYER_NORMAL_MODE == dbmode ) && 
+             (0 == return_value)) {
             /* update the dbversion file */
             dbversion_write(li, region_dir, NULL);
 
@@ -3934,16 +3936,13 @@ static int dblayer_force_checkpoint(struct ldbminfo *li)
   dblayer_private *priv = (dblayer_private *)li->li_dblayer_private;
   struct dblayer_private_env *pEnv;
   
-  if (NULL == priv){
-  /* already terminated.  nothing to do */  
+  if (NULL == priv || NULL == priv->dblayer_env){
+    /* already terminated.  nothing to do */  
     return -1;
   }
    
   pEnv= priv->dblayer_env;
   
-  
-  PR_ASSERT(pEnv != NULL);
-  
   if (priv->dblayer_enable_transactions) {
     
     LDAPDebug(LDAP_DEBUG_TRACE, "Checkpointing database ...\n", 0, 0, 0);
@@ -4008,7 +4007,8 @@ static int _dblayer_delete_instance_dir(ldbm_instance *inst, int startdb)
     {
 		if (startdb)
 		{
-			rval = dblayer_start(li, DBLAYER_NORMAL_MODE);
+			/* close immediately; no need to run db threads */
+			rval = dblayer_start(li, DBLAYER_NORMAL_MODE|DBLAYER_NO_DBTHREADS_MODE);
 			if (rval)
 			{
 				LDAPDebug(LDAP_DEBUG_ANY, "_dblayer_delete_instance_dir: dblayer_start failed! %s (%d)\n",
@@ -5402,7 +5402,8 @@ int dblayer_restore(struct ldbminfo *li, char *src_dir, Slapi_Task *task, char *
 
     if (li->li_flags & TASK_RUNNING_FROM_COMMANDLINE)
     {
-        dbmode |= DBLAYER_CMDLINE_MODE;
+        /* command line mode; no need to run db threads */
+        dbmode |= DBLAYER_NO_DBTHREADS_MODE;
     }
     else /* on-line mode */
     {