Ticket #48005 - ns-slapd crash in shutdown phase

Description: There was a small window that long running tasks access its
own task object after it's aready released by main thread in the shutdown
period.  This patch adds refcounter to such threads and make destructor
wait until the counter becomes 0.  Plus, the shutdown check is added to
their task callbacks.

Following tasks are updated by this patch:
  slapd/task.c:                         "fixup tombstones"
  posix-winsync/posix-winsync-config.c: "memberuid task"
  replication/repl5_replica_config.c:   "cleanallruv"
  replication/repl5_replica_config.c:   "abort cleanallruv"
  syntaxes/validate_task.c:             "syntax validate"
  automember/automember.c:              "automember rebuild membership"
  automember/automember.c:              "automember export updates"
  automember/automember.c:              "automember map updates"
  linkedattrs/linked_attrs.c:           "fixup linked attributes"
  memberof/memberof.c:                  "memberof task"
  schema_reload/schema_reload.c:        "schema reload task"
  usn/usn_cleanup.c:                    "USN tombstone cleanup task"

Following tasks are already covered:
  slapd/task.c: "import"
  slapd/task.c: "index"
  slapd/task.c: "upgradedb"

Following tasks are processed in an ordinary worker thread; no need to change
  slapd/task.c: "sysconfig reload"
  slapd/task.c: "export"
  slapd/task.c: "backup"
  slapd/task.c: "restore"

https://fedorahosted.org/389/ticket/48005

Reviewed by [email protected] (Thank you, Mark!!)

ldap/servers/plugins/automember/automember.c

@@ -119,9 +119,9 @@ static int automember_task_add_map_entries(Slapi_PBlock *pb, Slapi_Entry *e, Sla
 void automember_rebuild_task_thread(void *arg);
 void automember_export_task_thread(void *arg);
 void automember_map_task_thread(void *arg);
-void automember_task_destructor(Slapi_Task *task);
-void automember_task_export_destructor(Slapi_Task *task);
-void automember_task_map_destructor(Slapi_Task *task);
+static void automember_task_destructor(Slapi_Task *task);
+static void automember_task_export_destructor(Slapi_Task *task);
+static void automember_task_map_destructor(Slapi_Task *task);
 
 #define DEFAULT_FILE_MODE PR_IRUSR | PR_IWUSR
 
@@ -1962,11 +1962,15 @@ fetch_attr(Slapi_Entry *e, const char *attrname, const char *default_val)
     return slapi_value_get_string(val);
 }
 
-void
+static void
 automember_task_destructor(Slapi_Task *task)
 {
     if (task) {
         task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
         if (mydata) {
             slapi_ch_free_string(&mydata->bind_dn);
             slapi_sdn_free(&mydata->base_dn);
@@ -1976,11 +1980,15 @@ automember_task_destructor(Slapi_Task *task)
     }
 }
 
-void
+static void
 automember_task_export_destructor(Slapi_Task *task)
 {
     if (task) {
         task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
         if (mydata) {
             slapi_ch_free_string(&mydata->ldif_out);
             slapi_ch_free_string(&mydata->bind_dn);
@@ -1991,11 +1999,15 @@ automember_task_export_destructor(Slapi_Task *task)
     }
 }
 
-void
+static void
 automember_task_map_destructor(Slapi_Task *task)
 {
     if (task) {
         task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
         if (mydata) {
             slapi_ch_free_string(&mydata->ldif_out);
             slapi_ch_free_string(&mydata->ldif_in);
@@ -2114,7 +2126,8 @@ out:
  *  Search using the basedn, filter, and scope provided from the task data.
  *  Then loop of each entry, and apply the membership if applicable.
  */
-void automember_rebuild_task_thread(void *arg){
+void automember_rebuild_task_thread(void *arg)
+{
     Slapi_Task *task = (Slapi_Task *)arg;
     struct configEntry *config = NULL;
     Slapi_PBlock *search_pb = NULL, *fixup_pb = NULL;
@@ -2124,6 +2137,12 @@ void automember_rebuild_task_thread(void *arg){
     int result = 0;
     int i = 0;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_rebuild_task_thread --> refcount incremented.\n" );
     /*
      *  Fetch our task data from the task
      */
@@ -2192,7 +2211,8 @@ void automember_rebuild_task_thread(void *arg){
                 if (slapi_dn_issuffix(slapi_entry_get_dn(entries[i]), config->scope) &&
                     (slapi_filter_test_simple(entries[i], config->filter) == 0))
                 {
-                    if(automember_update_membership(config, entries[i], NULL)){
+                    if (slapi_is_shutting_down() ||
+                        automember_update_membership(config, entries[i], NULL)) {
                         result = SLAPI_PLUGIN_FAILURE;
                         automember_config_unlock();
                         goto out;
@@ -2226,6 +2246,9 @@ out:
     }
     slapi_task_inc_progress(task);
     slapi_task_finish(task, result);
+    slapi_task_dec_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_rebuild_task_thread <-- refcount decremented.\n" );
 }
 
 /*
@@ -2328,7 +2351,8 @@ out:
     return rv;
 }
 
-void automember_export_task_thread(void *arg){
+void automember_export_task_thread(void *arg)
+{
     Slapi_Task *task = (Slapi_Task *)arg;
     Slapi_PBlock *search_pb = NULL;
     Slapi_Entry **entries = NULL;
@@ -2340,6 +2364,13 @@ void automember_export_task_thread(void *arg){
     int i = 0;
     int rc = 0;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_export_task_thread --> refcount incremented.\n" );
+
     td = (task_data *)slapi_task_get_data(task);
     slapi_task_begin(task, 1);
     slapi_task_log_notice(task, "Automember export task starting.  Exporting changes to (%s)", td->ldif_out);
@@ -2394,7 +2425,8 @@ void automember_export_task_thread(void *arg){
                 if (slapi_dn_issuffix(slapi_sdn_get_dn(td->base_dn), config->scope) &&
                     (slapi_filter_test_simple(entries[i], config->filter) == 0))
                 { 
-                    if(automember_update_membership(config, entries[i], ldif_fd)){
+                    if (slapi_is_shutting_down() ||
+                        automember_update_membership(config, entries[i], ldif_fd)) {
                         result = SLAPI_DSE_CALLBACK_ERROR;
                         automember_config_unlock();
                         goto out;
@@ -2423,6 +2455,9 @@ out:
     }
     slapi_task_inc_progress(task);
     slapi_task_finish(task, result);
+    slapi_task_dec_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_export_task_thread <-- refcount decremented.\n" );
 }
 
 /*
@@ -2507,7 +2542,8 @@ out:
  *  Read in the text entries from ldif_in, and convert them to slapi_entries.
  *  Then, write to ldif_out what the updates would be if these entries were added
  */
-void automember_map_task_thread(void *arg){
+void automember_map_task_thread(void *arg)
+{
     Slapi_Task *task = (Slapi_Task *)arg;
     Slapi_Entry *e = NULL;
     int result = SLAPI_DSE_CALLBACK_OK;
@@ -2527,6 +2563,12 @@ void automember_map_task_thread(void *arg){
 #endif
     int rc = 0;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_map_task_thread --> refcount incremented.\n" );
     td = (task_data *)slapi_task_get_data(task);
     slapi_task_begin(task, 1);
     slapi_task_log_notice(task, "Automember map task starting...  Reading entries from (%s)"
@@ -2586,7 +2628,8 @@ void automember_map_task_thread(void *arg){
                     if (slapi_dn_issuffix(slapi_entry_get_dn_const(e), config->scope) &&
                         (slapi_filter_test_simple(e, config->filter) == 0))
                     {
-                        if(automember_update_membership(config, e, ldif_fd_out)){
+                        if (slapi_is_shutting_down() ||
+                            automember_update_membership(config, e, ldif_fd_out)) {
                             result = SLAPI_DSE_CALLBACK_ERROR;
                             slapi_entry_free(e);
                             slapi_ch_free_string(&entrystr);
@@ -2620,6 +2663,9 @@ out:
     }
     slapi_task_inc_progress(task);
     slapi_task_finish(task, result);
+    slapi_task_dec_refcount(task);
+    slapi_log_error( SLAPI_LOG_PLUGIN, AUTOMEMBER_PLUGIN_SUBSYSTEM,
+                     "automember_map_task_thread <-- refcount decremented.\n" );
 }
 
 /*

ldap/servers/plugins/linkedattrs/fixup_task.c

@@ -119,6 +119,10 @@ linked_attrs_fixup_task_destructor(Slapi_Task *task)
 {
 	if (task) {
 		task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
 		if (mydata) {
 			slapi_ch_free_string(&mydata->linkdn);
 			slapi_ch_free_string(&mydata->bind_dn);
@@ -137,6 +141,12 @@ linked_attrs_fixup_task_thread(void *arg)
 	int found_config = 0;
 	int rc = 0;
 
+	if (!task) {
+		return; /* no task */
+	}
+	slapi_task_inc_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, LINK_PLUGIN_SUBSYSTEM,
+	                "linked_attrs_fixup_task_thread --> refcount incremented.\n" );
 	/* Fetch our task data from the task */
 	td = (task_data *)slapi_task_get_data(task);
 
@@ -154,8 +164,8 @@ linked_attrs_fixup_task_thread(void *arg)
     linked_attrs_read_lock();
     main_config = linked_attrs_get_config();
     if (!PR_CLIST_IS_EMPTY(main_config)) {
-       struct configEntry *config_entry = NULL;
-       PRCList *list = PR_LIST_HEAD(main_config);
+        struct configEntry *config_entry = NULL;
+        PRCList *list = PR_LIST_HEAD(main_config);
 
         while (list != main_config) {
             config_entry = (struct configEntry *) list;
@@ -204,6 +214,10 @@ linked_attrs_fixup_task_thread(void *arg)
 
 	/* this will queue the destruction of the task */
 	slapi_task_finish(task, rc);
+
+	slapi_task_dec_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, LINK_PLUGIN_SUBSYSTEM,
+	                "linked_attrs_fixup_task_thread <-- refcount decremented.\n");
 }
 
 static void 
@@ -269,7 +283,7 @@ linked_attrs_fixup_links(struct configEntry *config)
             if(rc == 0){
                 slapi_back_transaction_commit(fixup_pb);
             } else {
-            	slapi_back_transaction_abort(fixup_pb);
+                slapi_back_transaction_abort(fixup_pb);
             }
             slapi_pblock_destroy(fixup_pb);
         }
@@ -352,11 +366,20 @@ linked_attrs_remove_backlinks_callback(Slapi_Entry *e, void *callback_data)
     int rc = 0;
     Slapi_DN *sdn = slapi_entry_get_sdn(e);
     char *type = (char *)callback_data;
-    Slapi_PBlock *pb = slapi_pblock_new();
+    Slapi_PBlock *pb = NULL;
     char *val[1];
     LDAPMod mod;
     LDAPMod *mods[2];
 
+    /* 
+     * If the server is ordered to shutdown, stop the fixup and return an error.
+     */
+    if (slapi_is_shutting_down()) {
+        rc = -1;
+        goto bail;
+    }
+
+    pb = slapi_pblock_new();
     /* Remove all values of the passed in type. */
     val[0] = 0;
 
@@ -377,7 +400,7 @@ linked_attrs_remove_backlinks_callback(Slapi_Entry *e, void *callback_data)
     slapi_modify_internal_pb(pb);
 
     slapi_pblock_destroy(pb);
-
+bail:
     return rc;
 }
 
@@ -394,6 +417,13 @@ linked_attrs_add_backlinks_callback(Slapi_Entry *e, void *callback_data)
     LDAPMod mod;
     LDAPMod *mods[2];
 
+    /* 
+     * If the server is ordered to shutdown, stop the fixup and return an error.
+     */
+    if (slapi_is_shutting_down()) {
+        rc = -1;
+        goto done;
+    }
     /* Setup the modify operation.  Only the target will
      * change, so we only need to do this once. */
     val[0] = linkdn;

ldap/servers/plugins/memberof/memberof.c

@@ -2638,6 +2638,12 @@ void memberof_fixup_task_thread(void *arg)
 	int rc = 0;
 	Slapi_PBlock *fixup_pb = NULL;
 
+	if (!task) {
+		return; /* no task */
+	}
+	slapi_task_inc_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM,
+	                "memberof_fixup_task_thread --> refcount incremented.\n" );
 	/* Fetch our task data from the task */
 	td = (task_data *)slapi_task_get_data(task);
 
@@ -2707,6 +2713,9 @@ void memberof_fixup_task_thread(void *arg)
 
 	/* this will queue the destruction of the task */
 	slapi_task_finish(task, rc);
+	slapi_task_dec_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM,
+	                "memberof_fixup_task_thread <-- refcount decremented.\n");
 }
 
 /* extract a single value from the entry (as a string) -- if it's not in the
@@ -2798,8 +2807,14 @@ out:
 void
 memberof_task_destructor(Slapi_Task *task)
 {
+	slapi_log_error( SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM,
+		"memberof_task_destructor -->\n" );
 	if (task) {
 		task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
 		if (mydata) {
 			slapi_ch_free_string(&mydata->dn);
 			slapi_ch_free_string(&mydata->bind_dn);
@@ -2808,6 +2823,8 @@ memberof_task_destructor(Slapi_Task *task)
 			slapi_ch_free((void **)&mydata);
 		}
 	}
+	slapi_log_error( SLAPI_LOG_PLUGIN, MEMBEROF_PLUGIN_SUBSYSTEM,
+		"memberof_task_destructor <--\n" );
 }
 
 int memberof_fix_memberof(MemberOfConfig *config, char *dn, char *filter_str)
@@ -2846,6 +2863,14 @@ int memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data)
 	memberof_del_dn_data del_data = {0, config->memberof_attr};
 	Slapi_ValueSet *groups = 0;
 
+	/* 
+	 * If the server is ordered to shutdown, stop the fixup and return an error.
+	 */
+	if (slapi_is_shutting_down()) {
+		rc = -1;
+		goto bail;
+	}
+	/* get a list of all of the groups this user belongs to */
 	groups = memberof_get_groups(config, sdn);
 
 	/* If we found some groups, replace the existing memberOf attribute
@@ -2893,6 +2918,6 @@ int memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data)
 	}
 
 	slapi_valueset_free(groups);
-	
+bail:
 	return rc;
 }

ldap/servers/plugins/memberof/memberof.h

@@ -82,7 +82,7 @@ typedef struct memberofconfig {
 	char *memberof_attr;
 	int allBackends;
 	Slapi_DN *entryScope;
-        Slapi_DN *entryScopeExcludeSubtree;
+	Slapi_DN *entryScopeExcludeSubtree;
 	Slapi_Filter *group_filter;
 	Slapi_Attr **group_slapiattrs;
 	int skip_nested;

ldap/servers/plugins/posix-winsync/posix-group-task.c

@@ -165,6 +165,10 @@ posix_group_task_destructor(Slapi_Task *task)
 {
     if (task) {
         task_data *mydata = (task_data *) slapi_task_get_data(task);
+        while (slapi_task_get_refcount(task) > 0) {
+            /* Yield to wait for the fixup task finishes. */
+            DS_Sleep (PR_MillisecondsToInterval(100));
+        }
         if (mydata) {
             slapi_ch_free_string(&mydata->dn);
             slapi_ch_free_string(&mydata->filter_str);
@@ -172,6 +176,8 @@ posix_group_task_destructor(Slapi_Task *task)
             slapi_ch_free((void **) &mydata);
         }
     }
+    slapi_log_error(SLAPI_LOG_PLUGIN, POSIX_WINSYNC_PLUGIN_NAME,
+                    "posix_group_task_destructor <--\n");
 }
 
 #if 0 /* NOT USED */
@@ -245,17 +251,28 @@ posix_group_fix_memberuid_callback(Slapi_Entry *e, void *callback_data)
                     "_fix_memberuid ==>\n");
     cb_data *the_cb_data = (cb_data *) callback_data;
 
-    int rc;
+    int rc = 0;
     Slapi_Attr *muid_attr = NULL;
     Slapi_Value *v = NULL;
 
-    Slapi_Mods *smods = slapi_mods_new();
-
-    char *dn = slapi_entry_get_dn(e);
-    Slapi_DN *sdn = slapi_entry_get_sdn(e);
+    Slapi_Mods *smods = NULL;
+    char *dn = NULL;
+    Slapi_DN *sdn = NULL;
     LDAPMod **mods = NULL;
     int is_posix_group = 0;
 
+    /* 
+     * If the server is ordered to shutdown, stop the fixup and return an error.
+     */
+    if (slapi_is_shutting_down()) {
+        rc = -1;
+        goto bail;
+    }
+
+    smods = slapi_mods_new();
+    dn = slapi_entry_get_dn(e);
+    sdn = slapi_entry_get_sdn(e);
+
     if (hasObjectClass(e, "posixGroup")) {
         is_posix_group = 1;
     }
@@ -441,7 +458,7 @@ posix_group_fix_memberuid_callback(Slapi_Entry *e, void *callback_data)
         slapi_pblock_destroy(mod_pb);
     }
     slapi_mods_free(&smods);
-
+bail:
     slapi_log_error(SLAPI_LOG_PLUGIN, POSIX_WINSYNC_PLUGIN_NAME,
                     "_fix_memberuid <==\n");
     /*
@@ -450,7 +467,7 @@ posix_group_fix_memberuid_callback(Slapi_Entry *e, void *callback_data)
      * uniqueMember attribute.  But "not found" error shoud not
      * be returned, which stops the further fixup task.
      */
-    return 0;
+    return rc;
 }
 
 static void
@@ -463,6 +480,12 @@ posix_group_fixup_task_thread(void *arg)
     task_data *td = NULL;
     int rc = 0;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, POSIX_WINSYNC_PLUGIN_NAME,
+                    "posix_group_fixup_task_thread --> refcount incremented.\n" );
     /* Fetch our task data from the task */
     td = (task_data *) slapi_task_get_data(task);
 
@@ -491,4 +514,7 @@ posix_group_fixup_task_thread(void *arg)
 
     slapi_log_error(SLAPI_LOG_PLUGIN, POSIX_WINSYNC_PLUGIN_NAME,
                     "_task_thread <==\n");
+    slapi_task_dec_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, POSIX_WINSYNC_PLUGIN_NAME,
+                    "posix_group_fixup_task_thread <-- refcount decremented.\n");
 }

ldap/servers/plugins/replication/repl5_replica_config.c

@@ -109,6 +109,8 @@ static CSN* replica_cleanallruv_find_maxcsn(Replica *replica, ReplicaId rid, cha
 static int replica_cleanallruv_get_replica_maxcsn(Repl_Agmt *agmt, char *rid_text, char *basedn, CSN **csn);
 static void preset_cleaned_rid(ReplicaId rid);
 static multimaster_mtnode_extension * _replica_config_get_mtnode_ext (const Slapi_Entry *e);
+static void replica_cleanall_ruv_destructor(Slapi_Task *task);
+static void replica_cleanall_ruv_abort_destructor(Slapi_Task *task);
 
 /*
  * Note: internal add/modify/delete operations should not be run while
@@ -1509,6 +1511,10 @@ replica_cleanall_ruv_task(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
         rc = SLAPI_DSE_CALLBACK_ERROR;
         goto out;
     }
+
+    /* register our destructor for waiting the task is done */
+    slapi_task_set_destructor_fn(task, replica_cleanall_ruv_destructor);
+
     /*
      *  Get our task settings
      */
@@ -1752,6 +1758,13 @@ replica_cleanallruv_thread(void *arg)
     int aborted = 0;
     int rc = 0;
 
+    if (!data) {
+        return; /* no data */
+    }
+    if (data->task) {
+        slapi_task_inc_refcount(data->task);
+        slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name, "replica_cleanallruv_thread --> refcount incremented.\n");
+    }
     /*
      *  Initialize our settings
      */
@@ -1974,6 +1987,8 @@ done:
     }
     if(data->task){
         slapi_task_finish(data->task, rc);
+        slapi_task_dec_refcount(data->task);
+        slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name, "replica_cleanallruv_thread <-- refcount decremented.\n");
     }
     if(data->payload){
         ber_bvfree(data->payload);
@@ -1989,6 +2004,36 @@ done:
     slapi_ch_free((void **)&data);
 }
 
+static void
+replica_cleanall_ruv_destructor(Slapi_Task *task)
+{
+	slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
+		"replica_cleanall_ruv_destructor -->\n" );
+	if (task) {
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
+	}
+	slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
+		"replica_cleanall_ruv_destructor <--\n" );
+}
+
+static void
+replica_cleanall_ruv_abort_destructor(Slapi_Task *task)
+{
+	slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
+		"replica_cleanall_ruv_abort_destructor -->\n" );
+	if (task) {
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
+	}
+	slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
+		"replica_cleanall_ruv_abort_destructor <--\n" );
+}
+
 /*
  *  Loop over the agmts, and check if they are in the last phase of cleaning, meaning they have
  *  released cleanallruv data from the config
@@ -2775,6 +2820,10 @@ replica_cleanall_ruv_abort(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter
 
     /* allocate new task now */
     task = slapi_new_task(slapi_entry_get_ndn(e));
+
+    /* register our destructor for waiting the task is done */
+    slapi_task_set_destructor_fn(task, replica_cleanall_ruv_abort_destructor);
+
     /*
      *  Get our task settings
      */
@@ -2921,6 +2970,13 @@ replica_abort_task_thread(void *arg)
     int release_it = 0;
     int count = 0, rc = 0;
 
+    if (!data) {
+        return; /* no data */
+    }
+    if (data->task) {
+        slapi_task_inc_refcount(data->task);
+        slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name, "replica_abort_task_thread --> refcount incremented.\n");
+    }
     cleanruv_log(data->task, ABORT_CLEANALLRUV_ID, "Aborting task for rid(%d)...",data->rid);
 
     /*
@@ -3028,6 +3084,8 @@ done:
 
     if(data->task){
         slapi_task_finish(data->task, agmt_not_notified);
+        slapi_task_dec_refcount(data->task);
+        slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name, "replica_abort_task_thread <-- refcount incremented.\n");
     }
     if(data->repl_obj && release_it)
         object_release(data->repl_obj);

ldap/servers/plugins/schema_reload/schema_reload.c

@@ -86,6 +86,7 @@ static int schemareload_add(Slapi_PBlock *pb, Slapi_Entry *e,
                     void *arg);
 static int schemareload_start(Slapi_PBlock *pb);
 static int schemareload_close(Slapi_PBlock *pb);
+static void schemareload_destructor(Slapi_Task *task);
 
 /* 
  * Init function
@@ -159,6 +160,12 @@ schemareload_thread(void *arg)
     int total_work = 2;
     task_data *td = NULL;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, "schemareload",
+                    "schemareload_thread --> refcount incremented.\n" );
     /* Fetch our task data from the task */
     td = (task_data *)slapi_task_get_data(task);
 
@@ -174,7 +181,11 @@ schemareload_thread(void *arg)
     rv = slapi_validate_schema_files(td->schemadir);
     slapi_task_inc_progress(task);
 
-    if (LDAP_SUCCESS == rv) {
+    if (slapi_is_shutting_down()) {
+        slapi_task_log_notice(task, "Server is shuttoing down; Schema validation aborted.");
+        slapi_task_log_status(task, "Server is shuttoing down; Schema validation aborted.");
+        slapi_log_error(SLAPI_LOG_FATAL, "schemareload", "Server is shuttoing down; Schema validation aborted.");
+    } else if (LDAP_SUCCESS == rv) {
         slapi_task_log_notice(task, "Schema validation passed.");
         slapi_task_log_status(task, "Schema validation passed.");
         slapi_log_error(SLAPI_LOG_FATAL, "schemareload", "Schema validation passed.\n");
@@ -192,16 +203,18 @@ schemareload_thread(void *arg)
             slapi_task_log_status(task, "Schema reload task failed.");
             slapi_log_error(SLAPI_LOG_FATAL, "schemareload", "Schema reload task failed.\n");
         }
-        PR_Unlock(schemareload_lock);
     } else {
         slapi_task_log_notice(task, "Schema validation failed.");
         slapi_task_log_status(task, "Schema validation failed.");
         slapi_log_error(SLAPI_LOG_FATAL, "schemareload", "Schema validation failed.\n");
-        PR_Unlock(schemareload_lock);
     }
+    PR_Unlock(schemareload_lock);
 
     /* this will queue the destruction of the task */
     slapi_task_finish(task, rv);
+    slapi_task_dec_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, "schemareload",
+                    "schemareload_thread <-- refcount decremented.\n");
 }
 
 /* extract a single value from the entry (as a string) -- if it's not in the
@@ -226,6 +239,10 @@ schemareload_destructor(Slapi_Task *task)
 {
     if (task) {
         task_data *mydata = (task_data *)slapi_task_get_data(task);
+        while (slapi_task_get_refcount(task) > 0) {
+            /* Yield to wait for the fixup task finishes. */
+            DS_Sleep (PR_MillisecondsToInterval(100));
+        }
         if (mydata) {
             slapi_ch_free_string(&mydata->schemadir);
             slapi_ch_free_string(&mydata->bind_dn);

ldap/servers/plugins/syntaxes/validate_task.c

@@ -179,6 +179,10 @@ syntax_validate_task_destructor(Slapi_Task *task)
 {
 	if (task) {
 		task_data *mydata = (task_data *)slapi_task_get_data(task);
+		while (slapi_task_get_refcount(task) > 0) {
+			/* Yield to wait for the fixup task finishes. */
+			DS_Sleep (PR_MillisecondsToInterval(100));
+		}
 		if (mydata) {
 			slapi_ch_free_string(&mydata->dn);
 			slapi_ch_free_string(&mydata->filter_str);
@@ -197,6 +201,12 @@ syntax_validate_task_thread(void *arg)
 	task_data *td = NULL;
 	Slapi_PBlock *search_pb = slapi_pblock_new();
 
+	if (!task) {
+		return; /* no task */
+	}
+	slapi_task_inc_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, SYNTAX_PLUGIN_SUBSYSTEM,
+	                "syntax_validate_task_thread --> refcount incremented.\n" );
 	/* Fetch our task data from the task */
 	td = (task_data *)slapi_task_get_data(task);
 
@@ -231,16 +241,26 @@ syntax_validate_task_thread(void *arg)
 
 	/* this will queue the destruction of the task */
 	slapi_task_finish(task, rc);
+	slapi_task_dec_refcount(task);
+	slapi_log_error(SLAPI_LOG_PLUGIN, SYNTAX_PLUGIN_SUBSYSTEM,
+	                "syntax_validate_task_thread <-- refcount decremented.\n"); 
 }
 
 static int
 syntax_validate_task_callback(Slapi_Entry *e, void *callback_data)
 {
-        int rc = 0;
-        char *dn = slapi_entry_get_dn(e);
+	int rc = 0;
+	char *dn = slapi_entry_get_dn(e);
 	task_data *td = (task_data *)callback_data;
 	Slapi_PBlock *pb = NULL;
 
+	/* 
+	 * If the server is ordered to shutdown, stop the fixup and return an error.
+	 */
+	if (slapi_is_shutting_down()) {
+		rc = -1;
+		goto bail;
+	}
 	/* Override the syntax checking config to force syntax checking. */
 	if (slapi_entry_syntax_check(NULL, e, 1) != 0) {
 		char *error_text = NULL;
@@ -261,7 +281,7 @@ syntax_validate_task_callback(Slapi_Entry *e, void *callback_data)
 		/* Keep a tally of the number of invalid entries found. */
 		slapi_counter_increment(td->invalid_entries);
 	}
-
+bail:
 	return rc;
 }
 

ldap/servers/plugins/usn/usn_cleanup.c

@@ -49,6 +49,8 @@ struct usn_cleanup_data {
 
 static int usn_cleanup_add(Slapi_PBlock *pb, Slapi_Entry *e,
             Slapi_Entry *eAfter, int *returncode, char *returntext, void *arg);
+static void usn_cleanup_task_destructor(Slapi_Task *task);
+
 
 int
 usn_cleanup_start(Slapi_PBlock *pb)
@@ -83,8 +85,14 @@ usn_cleanup_thread(void *arg)
     Slapi_PBlock *delete_pb = NULL;
     char *filter = "objectclass=nsTombstone";
 
+    if (!task) {
+        return; /* no task */
+    }
     slapi_log_error(SLAPI_LOG_TRACE, USN_PLUGIN_SUBSYSTEM,
                     "--> usn_cleanup_thread\n");
+    slapi_task_inc_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, USN_PLUGIN_SUBSYSTEM,
+                    "usn_cleanup_thread --> refcount incremented.\n" );
 
     if (NULL == usn_get_identity()) { /* plugin is not initialized */
         slapi_task_log_notice(task, "USN plugin is not initialized\n");
@@ -195,14 +203,12 @@ bail:
     if (cleanup_data->maxusn_to_delete) {
         slapi_ch_free_string(&filter);
     }
-    slapi_ch_free_string(&cleanup_data->maxusn_to_delete);
-    slapi_ch_free_string(&cleanup_data->suffix);
-    slapi_ch_free_string(&cleanup_data->bind_dn);
-    slapi_ch_free((void **)&cleanup_data);
 
     /* this will queue the destruction of the task */
     slapi_task_finish(task, rv);
-
+    slapi_task_dec_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, USN_PLUGIN_SUBSYSTEM,
+                    "usn_cleanup_thread <-- refcount decremented.\n");
     slapi_log_error(SLAPI_LOG_TRACE, USN_PLUGIN_SUBSYSTEM,
                     "<-- usn_cleanup_thread\n");
 }
@@ -283,7 +289,7 @@ usn_cleanup_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
     backend = slapi_entry_attr_get_charptr(e, "backend");
     maxusn = slapi_entry_attr_get_charptr(e, "maxusn_to_delete");
 
-    if (NULL == suffix && NULL == backend) {
+    if (!suffix && !backend) {
         slapi_log_error(SLAPI_LOG_FATAL, USN_PLUGIN_SUBSYSTEM,
             "USN tombstone cleanup: Both suffix and backend are missing.\n");
         *returncode = LDAP_PARAM_ERROR;
@@ -292,7 +298,7 @@ usn_cleanup_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
     }
 
     /* suffix is not given, but backend is; get the suffix */
-    if (NULL == suffix && NULL != backend) {
+    if (!suffix && backend) {
         be = slapi_be_select_by_instance_name(backend);
         be_suffix = slapi_be_getsuffix(be, 0);
         if (be_suffix) {
@@ -317,12 +323,6 @@ usn_cleanup_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
         goto bail;
     }
 
-    cleanup_data =
-      (struct usn_cleanup_data *)slapi_ch_malloc(sizeof(struct usn_cleanup_data));
-    cleanup_data->suffix = slapi_ch_strdup(suffix);
-    cleanup_data->maxusn_to_delete = slapi_ch_strdup(maxusn);
-    cleanup_data->bind_dn = slapi_ch_strdup(bind_dn);
-
     /* allocate new task now */
     task = slapi_plugin_new_task(slapi_entry_get_ndn(e), arg);
     if (task == NULL) {
@@ -330,11 +330,21 @@ usn_cleanup_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
             "USN tombstone cleanup: unable to allocate new task.\n");
         *returncode = LDAP_OPERATIONS_ERROR;
         rv = SLAPI_DSE_CALLBACK_ERROR;
-        slapi_ch_free((void**)&cleanup_data);
         goto bail;
     }
 
+    /* register our destructor for cleaning up our private data */
+    slapi_task_set_destructor_fn(task, usn_cleanup_task_destructor);
+
     /* Stash our argument in the task for use by the task thread */
+    cleanup_data =
+      (struct usn_cleanup_data *)slapi_ch_malloc(sizeof(struct usn_cleanup_data));
+    cleanup_data->suffix = suffix;
+    suffix = NULL; /* don't free in this function */
+    cleanup_data->maxusn_to_delete = maxusn;
+    maxusn = NULL; /* don't free in this function */
+    cleanup_data->bind_dn = bind_dn;
+    bind_dn = NULL; /* don't free in this function */
     slapi_task_set_data(task, cleanup_data);
 
     /* start the USN tombstone cleanup task as a separate thread */
@@ -361,3 +371,23 @@ bail:
     return rv;
 }
 
+static void
+usn_cleanup_task_destructor(Slapi_Task *task)
+{
+    slapi_log_error(SLAPI_LOG_PLUGIN, USN_PLUGIN_SUBSYSTEM, "usn_cleanup_task_destructor -->\n");
+    if (task) {
+        struct usn_cleanup_data *mydata = (struct usn_cleanup_data *)slapi_task_get_data(task);
+        while (slapi_task_get_refcount(task) > 0) {
+            /* Yield to wait for the fixup task finishes. */
+            DS_Sleep (PR_MillisecondsToInterval(100));
+        }
+        if (mydata) {
+            slapi_ch_free_string(&mydata->suffix);
+            slapi_ch_free_string(&mydata->maxusn_to_delete);
+            slapi_ch_free_string(&mydata->bind_dn);
+            /* Need to cast to avoid a compiler warning */
+            slapi_ch_free((void **)&mydata);
+        }
+    }
+    slapi_log_error(SLAPI_LOG_PLUGIN, USN_PLUGIN_SUBSYSTEM, "usn_cleanup_task_destructor <--\n");
+}

ldap/servers/slapd/slapi-plugin.h

@@ -7967,6 +7967,15 @@ void slapi_plugin_op_finished(void *arg);
 #define RDN_IS_CONFLICT  0x2
 int slapi_is_special_rdn(const char *rdn, int flag);
 
+/**
+ * Sleeps for PRIntervalTime ticks defined in NSPR library
+ *
+ * \param PRIntervalTime ticks
+ *
+ * \return Nothing
+ */
+void    DS_Sleep(PRIntervalTime ticks);
+
 #ifdef __cplusplus
 }
 #endif

ldap/servers/slapd/slapi-private.h

@@ -1245,8 +1245,6 @@ void bervalarray_add_berval_fast(struct berval ***vals, const struct berval *add
 
 /***** End of items added for the replication plugin. ***********************/
 
-void    DS_Sleep(PRIntervalTime ticks);
-
 /* macro to specify the behavior of upgradedb & upgradednformat */
 #define SLAPI_UPGRADEDB_FORCE    0x1  /* reindex all (no check w/ idl switch) */
 #define SLAPI_UPGRADEDB_SKIPINIT 0x2  /* call upgradedb as part of other op */

ldap/servers/slapd/task.c

@@ -113,6 +113,8 @@ static const char *fetch_attr(Slapi_Entry *e, const char *attrname,
 static Slapi_Entry *get_internal_entry(Slapi_PBlock *pb, char *dn);
 static void modify_internal_entry(char *dn, LDAPMod **mods);
 
+static void fixup_tombstone_task_destructor(Slapi_Task *task);
+
 /***********************************
  * Public Functions
  ***********************************/ 
@@ -2218,6 +2220,12 @@ task_fixup_tombstone_thread(void *arg)
     int fixup_count = 0;
     int rc, i, j;
 
+    if (!task) {
+        return; /* no task */
+    }
+    slapi_task_inc_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, TASK_TOMBSTONE_FIXUP,
+                    "fixup_tombstone_task_thread --> refcount incremented.\n" );
     slapi_task_begin(task, 1);
     slapi_task_log_notice(task, "Beginning tombstone fixup task...\n");
     slapi_log_error(SLAPI_LOG_REPL, TASK_TOMBSTONE_FIXUP,
@@ -2233,8 +2241,14 @@ task_fixup_tombstone_thread(void *arg)
 
     /* Okay check the specified backends only */
     for(i = 0; base && base[i]; i++){
-        Slapi_PBlock *search_pb = slapi_pblock_new();
+        Slapi_PBlock *search_pb = NULL;
+
+        if (slapi_is_shutting_down()) {
+            rc = -1;
+            goto bail;
+        }
 
+        search_pb = slapi_pblock_new();
         /* find entries that need fixing... */
         slapi_search_internal_set_pb(search_pb, base[i], LDAP_SCOPE_SUBTREE,
                 filter, NULL, 0, NULL, NULL, plugin_get_default_component_id(), 0);
@@ -2247,8 +2261,7 @@ task_fixup_tombstone_thread(void *arg)
             slapi_log_error(SLAPI_LOG_REPL, TASK_TOMBSTONE_FIXUP,
                     "Failed to search backend for tombstones, error %d\n", rc);
             slapi_pblock_destroy(search_pb);
-            slapi_task_finish(task, rc);
-            return;
+            goto bail;
         }
 
         slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries);
@@ -2281,9 +2294,11 @@ task_fixup_tombstone_thread(void *arg)
     slapi_log_error(SLAPI_LOG_REPL, TASK_TOMBSTONE_FIXUP, "%s %d tombstones.\n",
                     task_data->stripcsn ? "Stripped" : "Fixed", fixup_count);
     slapi_task_inc_progress(task);
+bail:
     slapi_task_finish(task, rc);
-    slapi_ch_array_free(base);
-    slapi_ch_free((void **)&task_data);
+    slapi_task_dec_refcount(task);
+    slapi_log_error(SLAPI_LOG_PLUGIN, TASK_TOMBSTONE_FIXUP,
+                    "fixup_tombstone_task_thread <-- refcount decremented.\n" );
 }
 
 
@@ -2387,6 +2402,8 @@ task_fixup_tombstones_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *eAfter,
     }
 
     task = slapi_new_task(slapi_entry_get_ndn(e));
+    /* register our destructor for cleaning up our private data */
+    slapi_task_set_destructor_fn(task, fixup_tombstone_task_destructor);
     task_data = (struct task_tombstone_data *)slapi_ch_calloc(1, sizeof(struct task_tombstone_data));
     task_data->base = base;
     task_data->task = task;
@@ -2422,6 +2439,26 @@ done:
     return SLAPI_DSE_CALLBACK_OK;
 }
 
+static void
+fixup_tombstone_task_destructor(Slapi_Task *task)
+{
+    slapi_log_error(SLAPI_LOG_PLUGIN, TASK_TOMBSTONE_FIXUP,
+                    "fixup_tombstone_task_destructor -->\n" );
+    if (task) {
+        struct task_tombstone_data *mydata = (struct task_tombstone_data *)slapi_task_get_data(task);
+        while (slapi_task_get_refcount(task) > 0) {
+            /* Yield to wait for the fixup task finishes. */
+            DS_Sleep (PR_MillisecondsToInterval(100));
+        }
+        if (mydata) {
+            slapi_ch_array_free(mydata->base);
+            slapi_ch_free((void **)&mydata);
+        }
+    }
+    slapi_log_error(SLAPI_LOG_PLUGIN, TASK_TOMBSTONE_FIXUP,
+                    "fixup_tombstone_task_destructor <--\n" );
+}
+
 /* cleanup old tasks that may still be in the DSE from a previous session
  * (this can happen if the server crashes [no matter how unlikely we like
  * to think that is].)