Ticket #315 - ns-slapd exits/crashes if /var fills up

Bug Description:  Once /var fills up the DS will crash.

Fix Description:  Created a new feature to monitor the disk space used by DS.
                  Once the available disk space gets critical we shutdown
                  the process.

                  For complete details see "Disk_Monitoring" text file

https://fedorahosted.org/389/ticket/315

Reviewed by:

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -6613,6 +6613,24 @@ ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info)
         }
         break;
     }
+    case BACK_INFO_DIRECTORY:
+    {
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        if (li) {
+            *(char **)info = li->li_directory;
+            rc = 0;
+        }
+        break;
+    }
+    case BACK_INFO_LOG_DIRECTORY:
+    {
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        if (li) {
+            *(char **)info = ldbm_config_db_logdirectory_get_ext((void *)li);
+            rc = 0;
+        }
+        break;
+    }
     default:
         break;
     }

ldap/servers/slapd/back-ldbm/ldbm_config.c

@@ -469,7 +469,7 @@ static int ldbm_config_dbncache_set(void *arg, void *value, char *errorbuf, int
     return retval;
 }
 
-static void *ldbm_config_db_logdirectory_get(void *arg) 
+void *ldbm_config_db_logdirectory_get(void *arg)
 {
     struct ldbminfo *li = (struct ldbminfo *) arg;
 
@@ -483,7 +483,17 @@ static void *ldbm_config_db_logdirectory_get(void *arg)
         return (void *) slapi_ch_strdup(li->li_dblayer_private->dblayer_log_directory);
     else
         return (void *) slapi_ch_strdup(li->li_new_directory);
+}
+
+/* Does not return a copy of the string - used by disk space monitoring feature */
+void *ldbm_config_db_logdirectory_get_ext(void *arg)
+{
+    struct ldbminfo *li = (struct ldbminfo *) arg;
 
+    if (strlen(li->li_dblayer_private->dblayer_log_directory) > 0)
+        return (void *)li->li_dblayer_private->dblayer_log_directory;
+    else
+        return (void *)li->li_new_directory;
 }
 
 static int ldbm_config_db_logdirectory_set(void *arg, void *value, char *errorbuf, int phase, int apply) 

ldap/servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -615,6 +615,7 @@ int dbversion_exists(struct ldbminfo *li, const char *directory);
 int ldbm_config_load_dse_info(struct ldbminfo *li);
 void ldbm_config_setup_default(struct ldbminfo *li);
 void ldbm_config_internal_set(struct ldbminfo *li, char *attrname, char *value);
+void *ldbm_config_db_logdirectory_get_ext(void *arg);
 void ldbm_instance_config_internal_set(ldbm_instance *inst, char *attrname, char *value);
 void ldbm_instance_config_setup_default(ldbm_instance *inst);
 int ldbm_instance_postadd_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg);

ldap/servers/slapd/daemon.c

@@ -59,6 +59,7 @@
 #include <sys/time.h>
 #include <sys/wait.h>
 #include <pthread.h>
+#include <mntent.h>
 #endif
 #include <time.h>
 #include <signal.h>
@@ -81,15 +82,17 @@
 /* for some reason, linux tty stuff defines CTIME */
 #ifdef LINUX
 #undef CTIME
+#include <sys/statfs.h>
+#else
+#include <sys/statvfs.h>
+#include <sys/mnttab.h>
 #endif
 #include "slap.h"
 #include "slapi-plugin.h"
-
 #include "snmp_collator.h"
 #include <private/pprio.h>
-
 #include <ssl.h>
-
+#include <stdio.h>
 #include "fe.h"
 
 #if defined(ENABLE_LDAPI)
@@ -126,9 +129,12 @@ PRFileDesc*		signalpipe[2];
 static int writesignalpipe = SLAPD_INVALID_SOCKET;
 static int readsignalpipe = SLAPD_INVALID_SOCKET;
 
-#define FDS_SIGNAL_PIPE 0
-
+static PRThread *disk_thread_p = NULL;
+static PRCondVar *diskmon_cvar = NULL;
+static PRLock *diskmon_mutex = NULL;
+void disk_monitoring_stop();
 
+#define FDS_SIGNAL_PIPE 0
 
 static int get_configured_connection_table_size();
 #ifdef RESOLVER_NEEDS_LOW_FILE_DESCRIPTORS
@@ -155,7 +161,6 @@ HANDLE  hServDoneEvent = NULL;
 
 static int createsignalpipe( void );
 
-
 #if defined( _WIN32 )
 /* Set an event to hook the NT Service termination */
 void *slapd_service_exit_wait()
@@ -470,6 +475,444 @@ time_thread(void *nothing)
     return(NULL);
 }
 
+/*
+ *  Return a copy of the mount point for the specified directory
+ */
+#ifdef SOLARIS
+char *
+disk_mon_get_mount_point(char *dir)
+{
+    struct mnttab *mnt;
+    struct stat s;
+    dev_t dev_id;
+    FILE *fp;
+
+    fp = fopen("/etc/mnttab", "r");
+
+    if (fp == NULL || stat(dir, &s) != 0) {
+        return NULL;
+    }
+
+    dev_id = s.st_dev;
+
+    while((mnt = getmntent(fp))){
+        if (stat(mnt->mnt_mountp, &s) != 0) {
+            continue;
+        }
+        if (s.st_dev == dev_id) {
+            return (slapi_ch_strdup(mnt->mnt_mountp));
+        }
+    }
+
+    return NULL;
+}
+#elif HPUX
+char *
+disk_mon_get_mount_point(char *dir)
+{
+    struct mntent *mnt;
+    struct stat s;
+    dev_t dev_id;
+    FILE *fp;
+
+    if ((fp = setmntent("/etc/mnttab", "r")) == NULL) {
+        return NULL;
+    }
+
+    if (stat(dir, &s) != 0) {
+        return NULL;
+    }
+
+    dev_id = s.st_dev;
+
+    while((mnt = getmntent(fp))){
+        if (stat(mnt->mnt_dir, &s) != 0) {
+            continue;
+        }
+        if (s.st_dev == dev_id) {
+            endmntent(fp);
+            return (slapi_ch_strdup(mnt->mnt_dir));
+        }
+    }
+    endmntent(fp);
+
+    return NULL;
+}
+#else /* Linux */
+char *
+disk_mon_get_mount_point(char *dir)
+{
+    struct mntent *mnt;
+    struct stat s;
+    dev_t dev_id;
+    FILE *fp;
+
+    if (stat(dir, &s) != 0) {
+        return NULL;
+    }
+
+    dev_id = s.st_dev;
+    if ((fp = setmntent("/proc/mounts", "r")) == NULL) {
+        return NULL;
+    }
+    while((mnt = getmntent(fp))){
+        if (stat(mnt->mnt_dir, &s) != 0) {
+            continue;
+        }
+        if (s.st_dev == dev_id) {
+            endmntent(fp);
+            return (slapi_ch_strdup(mnt->mnt_dir));
+        }
+    }
+    endmntent(fp);
+
+    return NULL;
+}
+#endif
+
+/*
+ *  Get the mount point of the directory, and add it to the
+ *  list.  Skip duplicate mount points.
+ */
+void
+disk_mon_add_dir(char ***list, char *directory)
+{
+    char *dir = disk_mon_get_mount_point(directory);
+
+    if(dir == NULL)
+        return;
+
+    if(!charray_inlist(*list,dir)){
+        slapi_ch_array_add(list, dir);
+    } else {
+        slapi_ch_free((void **)&dir);
+    }
+}
+
+/*
+ *  We gather all the log, txn log, config, and db directories
+ */
+void
+disk_mon_get_dirs(char ***list, int logs_critical){
+    slapdFrontendConfig_t *config = getFrontendConfig();
+    Slapi_Backend *be = NULL;
+    char *cookie = NULL;
+    char *dir = NULL;
+
+    if(logs_critical){
+        slapi_rwlock_rdlock(config->cfg_rwlock);
+        disk_mon_add_dir(list, config->accesslog);
+        disk_mon_add_dir(list, config->errorlog);
+        disk_mon_add_dir(list, config->auditlog);
+        slapi_rwlock_unlock(config->cfg_rwlock);
+    }
+
+    /* Add /var just to be safe */
+#ifdef LOCALSTATEDIR
+    disk_mon_add_dir(list, LOCALSTATEDIR);
+#else
+    disk_mon_add_dir(list, "/var");
+#endif
+
+    /* config and backend directories */
+    slapi_rwlock_rdlock(config->cfg_rwlock);
+    disk_mon_add_dir(list, config->configdir);
+    slapi_rwlock_unlock(config->cfg_rwlock);
+
+    be = slapi_get_first_backend (&cookie);
+    while (be) {
+        if(slapi_back_get_info(be, BACK_INFO_DIRECTORY, (void **)&dir) == LDAP_SUCCESS){  /* db directory */
+        	disk_mon_add_dir(list, dir);
+        }
+        if(slapi_back_get_info(be, BACK_INFO_LOG_DIRECTORY, (void **)&dir) == LDAP_SUCCESS){  /*  txn log dir */
+        	disk_mon_add_dir(list, dir);
+        }
+        be = (backend *)slapi_get_next_backend (cookie);
+    }
+}
+
+/*
+ *  This function checks the list of directories to see if any are below the
+ *  threshold.  We return the the directory/free disk space of the most critical
+ *  directory.
+ */
+char *
+disk_mon_check_diskspace(char **dirs, PRInt64 threshold, PRInt64 *disk_space)
+{
+#ifdef LINUX
+    struct statfs buf;
+#else
+    struct statvfs buf;
+#endif
+    PRInt64 worst_disk_space = threshold;
+    PRInt64 freeBytes = 0;
+    PRInt64 blockSize = 0;
+    char *worst_dir = NULL;
+    int hit_threshold = 0;
+    int i = 0;
+
+    for(i = 0; dirs && dirs[i]; i++){
+#ifndef LINUX
+        if (statvfs(dirs[i], &buf) != -1)
+#else
+        if (statfs(dirs[i], &buf) != -1)
+#endif
+        {
+            LL_UI2L(freeBytes, buf.f_bavail);
+            LL_UI2L(blockSize, buf.f_bsize);
+            LL_MUL(freeBytes, freeBytes, blockSize);
+
+            if(LL_UCMP(freeBytes, <, threshold)){
+                hit_threshold = 1;
+                if(LL_UCMP(freeBytes, <, worst_disk_space)){
+                    worst_disk_space = freeBytes;
+                    worst_dir = dirs[i];
+                }
+            }
+        }
+    }
+
+    if(hit_threshold){
+        *disk_space = worst_disk_space;
+        return worst_dir;
+    } else {
+        *disk_space = 0;
+        return NULL;
+    }
+}
+
+#define LOGGING_OFF 0
+#define LOGGING_ON 1
+/*
+ *  Disk Space Monitoring Thread
+ *
+ *  We need to monitor the free disk space of critical disks.
+ *
+ *  If we get below the free disk space threshold, start taking measures
+ *  to avoid additional disk space consumption by stopping verbose logging,
+ *  access/audit logging, and deleting rotated logs.
+ *
+ *  If this is not enough, then we need to shut slapd down to avoid
+ *  possibly corrupting the db.
+ *
+ *  Future - it would be nice to be able to email an alert.
+ */
+void
+disk_monitoring_thread(void *nothing)
+{
+    char errorbuf[BUFSIZ];
+    char **dirs = NULL;
+    char *dirstr = NULL;
+    PRInt64 previous_mark = 0;
+    PRInt64 disk_space = 0;
+    PRInt64 threshold = 0;
+    time_t start = 0;
+    time_t now = 0;
+    int deleted_rotated_logs = 0;
+    int logging_critical = 0;
+    int preserve_logging = 0;
+    int passed_threshold = 0;
+    int verbose_logging = 0;
+    int using_accesslog = 0;
+    int using_auditlog = 0;
+    int logs_disabled = 0;
+    int grace_period = 0;
+    int first_pass = 1;
+    int halfway = 0;
+    int ok_now = 0;
+
+    while(!g_get_shutdown()) {
+        if(!first_pass){
+            PR_Lock(diskmon_mutex);
+            PR_WaitCondVar(diskmon_cvar, PR_SecondsToInterval(10));
+            PR_Unlock(diskmon_mutex);
+            /*
+             *  We need to subtract from disk_space to account for the
+             *  logging we just did, it doesn't hurt if we subtract a
+             *  little more than necessary.
+             */
+            previous_mark = disk_space - 512;
+            ok_now = 0;
+        } else {
+            first_pass = 0;
+        }
+        /*
+         *  Get the config settings, as they could have changed
+         */
+        logging_critical = config_get_disk_logging_critical();
+        preserve_logging = config_get_disk_preserve_logging();
+        grace_period = 60 * config_get_disk_grace_period(); /* convert it to seconds */
+        verbose_logging = config_get_errorlog_level();
+        threshold = config_get_disk_threshold();
+        halfway = threshold / 2;
+
+        if(config_get_auditlog_logging_enabled()){
+            using_auditlog = 1;
+        }
+        if(config_get_accesslog_logging_enabled()){
+            using_accesslog = 1;
+        }
+        /*
+         *  Check the disk space.  Always refresh the list, as backends can be added
+         */
+        slapi_ch_array_free(dirs);
+        dirs = NULL;
+        disk_mon_get_dirs(&dirs, logging_critical);
+        dirstr = disk_mon_check_diskspace(dirs, threshold, &disk_space);
+        if(dirstr == NULL){
+            /*
+             *  Good, none of our disks are within the threshold,
+             *  reset the logging if we turned it off
+             */
+            if(passed_threshold){
+            	if(logs_disabled){
+            		LDAPDebug(LDAP_DEBUG_ANY, "Disk space is now within acceptable levels.  "
+                        "Restoring the log settings.\n",0,0,0);
+                    if(using_accesslog){
+                        config_set_accesslog_enabled(LOGGING_ON);
+                    }
+                    if(using_auditlog){
+                        config_set_auditlog_enabled(LOGGING_ON);
+                    }
+                } else {
+                	LDAPDebug(LDAP_DEBUG_ANY, "Disk space is now within acceptable levels.\n",0,0,0);
+                }
+            	deleted_rotated_logs = 0;
+            	passed_threshold = 0;
+            	previous_mark = 0;
+            	logs_disabled = 0;
+            }
+            continue;
+        } else {
+            passed_threshold = 1;
+        }
+        /*
+         *  Check if we are already critical
+         */
+        if(disk_space < 4096){ /* 4 k */
+            LDAPDebug(LDAP_DEBUG_ANY, "Disk space is critically low on disk (%s), remaining space: %d Kb.  "
+                "Signaling slapd for shutdown...\n", dirstr , (disk_space / 1024), 0);
+            g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+            return;
+        }
+        /*
+         *  If we are low, see if we are using verbose error logging, and turn it off
+         */
+        if(verbose_logging){
+            LDAPDebug(LDAP_DEBUG_ANY, "Disk space is low on disk (%s), remaining space: %d Kb, "
+                "setting error loglevel to zero.\n", dirstr, (disk_space / 1024), 0);
+            config_set_errorlog_level(CONFIG_LOGLEVEL_ATTRIBUTE, 0, errorbuf, CONFIG_APPLY);
+            continue;
+        }
+        /*
+         *  If we are low, there's no verbose logging, logs are not critical, then disable the
+         *  access/audit logs, log another error, and continue.
+         */
+        if(!logs_disabled && (!preserve_logging || !logging_critical)){
+            if(disk_space < previous_mark){
+                LDAPDebug(LDAP_DEBUG_ANY, "Disk space is too low on disk (%s), remaining space: %d Kb, "
+                    "disabling access and audit logging.\n", dirstr, (disk_space / 1024), 0);
+                config_set_accesslog_enabled(LOGGING_OFF);
+                config_set_auditlog_enabled(LOGGING_OFF);
+                logs_disabled = 1;
+            }
+            continue;
+        }
+        /*
+         *  If we are low, we turned off verbose logging, logs are not critical, and we disabled
+         *  access/audit logging, then delete the rotated logs, log another error, and continue.
+         */
+        if(!deleted_rotated_logs && (!preserve_logging || !logging_critical)){
+            if(disk_space < previous_mark){
+                LDAPDebug(LDAP_DEBUG_ANY, "Disk space is too low on disk (%s), remaining space: %d Kb, "
+                    "deleting rotated logs.\n", dirstr, (disk_space / 1024), 0);
+                log__delete_rotated_logs();
+                deleted_rotated_logs = 1;
+            }
+            continue;
+        }
+        /*
+         *  Ok, we've done what we can, log a message if we continue to lose available disk space
+         */
+        if(disk_space < previous_mark){
+            LDAPDebug(LDAP_DEBUG_ANY, "Disk space is too low on disk (%s), remaining space: %d Kb\n",
+                dirstr, (disk_space / 1024), 0);
+        }
+        /*
+         *
+         *  If we are below the halfway mark, and we did everything else,
+         *  go into shutdown mode. If the disk space doesn't get critical,
+         *  wait for the grace period before shutting down.  This gives an
+         *  admin the chance to clean things up.
+         *
+         */
+        if(disk_space < halfway){
+            LDAPDebug(LDAP_DEBUG_ANY, "Disk space on (%s) is too far below the threshold(%d bytes).  "
+                "Waiting %d minutes for disk space to be cleaned up before shutting slapd down...\n",
+                dirstr, threshold, (grace_period / 60));
+            time(&start);
+            now = start;
+            while( (now - start) < grace_period ){
+                if(g_get_shutdown()){
+                    return;
+                }
+                /*
+                 *  Sleep for a little bit, but we don't want to run out of disk space
+                 *  while sleeping for the entire grace period
+                 */
+                DS_Sleep(PR_SecondsToInterval(1));
+                /*
+                 *  Now check disk space again in hopes some space was freed up
+                 */
+                dirstr = disk_mon_check_diskspace(dirs, threshold, &disk_space);
+                if(!dirstr){
+                    /*
+                     *  Excellent, we are back to acceptable levels, reset everything...
+                     */
+                    LDAPDebug(LDAP_DEBUG_ANY, "Available disk space is now acceptable (%d bytes).  Aborting"
+                                              " shutdown, and restoring the log settings.\n",disk_space,0,0);
+                    if(!preserve_logging && using_accesslog){
+                        config_set_accesslog_enabled(LOGGING_ON);
+                    }
+                    if(!preserve_logging && using_auditlog){
+                        config_set_auditlog_enabled(LOGGING_ON);
+                    }
+                    deleted_rotated_logs = 0;
+                    passed_threshold = 0;
+                    logs_disabled = 0;
+                    previous_mark = 0;
+                    ok_now = 1;
+                    start = 0;
+                    now = 0;
+                    break;
+                } else if(disk_space < 4096){ /* 4 k */
+                    /*
+                     *  Disk space is critical, log an error, and shut it down now!
+                     */
+                    LDAPDebug(LDAP_DEBUG_ANY, "Disk space is critically low on disk (%s), remaining space: %d Kb."
+                        "  Signaling slapd for shutdown...\n", dirstr, (disk_space / 1024), 0);
+                    g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+                    return;
+                }
+                time(&now);
+            }
+
+            if(ok_now){
+                /*
+                 *  Disk space is acceptable, resume normal processing
+                 */
+                continue;
+            }
+            /*
+             *  If disk space was freed up we would of detected in the above while loop.  So shut it down.
+             */
+            LDAPDebug(LDAP_DEBUG_ANY, "Disk space is still too low (%d Kb).  Signaling slapd for shutdown...\n",
+                (disk_space / 1024), 0, 0);
+            g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+            return;
+        }
+    }
+}
 
 void slapd_daemon( daemon_ports_t *ports )
 {
@@ -563,7 +1006,44 @@ void slapd_daemon( daemon_ports_t *ports )
 		g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
 	}
 
-	/* We are now ready to accept imcoming connections */
+    /*
+     *  If we are monitoring disk space, then create the mutex, the cvar,
+     *  and the monitoring thread.
+     */
+    if( config_get_disk_monitoring() ){
+        if ( ( diskmon_mutex = PR_NewLock() ) == NULL ) {
+            slapi_log_error(SLAPI_LOG_FATAL, NULL,
+                "Cannot create new lock for disk space monitoring. "
+                SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n",
+                PR_GetError(), slapd_pr_strerror( PR_GetError() ));
+            g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+        }
+        if ( diskmon_mutex ){
+            if(( diskmon_cvar = PR_NewCondVar( diskmon_mutex )) == NULL ) {
+                slapi_log_error(SLAPI_LOG_FATAL, NULL,
+                    "Cannot create new condition variable for disk space monitoring. "
+                    SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n",
+                    PR_GetError(), slapd_pr_strerror( PR_GetError() ));
+                g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+            }
+        }
+        if( diskmon_mutex && diskmon_cvar ){
+            disk_thread_p = PR_CreateThread(PR_SYSTEM_THREAD,
+                (VFP) (void *) disk_monitoring_thread, NULL,
+                PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD,
+                PR_JOINABLE_THREAD,
+                SLAPD_DEFAULT_THREAD_STACKSIZE);
+            if ( NULL == disk_thread_p ) {
+                PRErrorCode errorCode = PR_GetError();
+                LDAPDebug(LDAP_DEBUG_ANY, "Unable to create disk monitoring thread - Shutting Down ("
+                    SLAPI_COMPONENT_NAME_NSPR " error %d - %s)\n",
+                    errorCode, slapd_pr_strerror(errorCode), 0);
+                g_set_shutdown( SLAPI_SHUTDOWN_EXIT );
+            }
+        }
+    }
+
+	/* We are now ready to accept incoming connections */
 #if defined( XP_WIN32 )
 	if ( n_tcps != SLAPD_INVALID_SOCKET
 				&& listen( n_tcps, DAEMON_LISTEN_SIZE ) == -1 ) {
@@ -807,6 +1287,7 @@ void slapd_daemon( daemon_ports_t *ports )
 	be_flushall();
 	op_thread_cleanup();
 	housekeeping_stop(); /* Run this after op_thread_cleanup() logged sth */
+	disk_monitoring_stop(disk_thread_p);
 
 #ifndef _WIN32
 	threads = g_get_active_threadcnt();
@@ -3128,10 +3609,10 @@ void configure_ns_socket( int * ns )
 	        on = 0;
 		setsockopt( *ns, IPPROTO_TCP, TCP_NODELAY, (char * ) &on, sizeof(on) );
 	} /* else (!enable_nagle) */
-		 
-	
+
+
 	return;
-	       
+
 }
 
 
@@ -3158,3 +3639,13 @@ get_loopback_by_addr( void )
 	    AF_INET, &hp, hbuf, sizeof(hbuf), &herrno );
 }
 #endif /* RESOLVER_NEEDS_LOW_FILE_DESCRIPTORS */
+
+void
+disk_monitoring_stop()
+{
+	if ( disk_thread_p ) {
+		PR_Lock( diskmon_mutex );
+		PR_NotifyCondVar( diskmon_cvar );
+		PR_Unlock( diskmon_mutex );
+	}
+}

ldap/servers/slapd/libglobs.c

@@ -662,6 +662,26 @@ static struct config_get_and_set {
 		NULL, 0,
 		(void**)&global_slapdFrontendConfig.default_naming_context,
 		CONFIG_STRING, (ConfigGetFunc)config_get_default_naming_context},
+	{CONFIG_DISK_MONITORING, config_set_disk_monitoring,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.disk_monitoring, CONFIG_ON_OFF,
+		(ConfigGetFunc)config_get_disk_monitoring},
+	{CONFIG_DISK_THRESHOLD, config_set_disk_threshold,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.disk_threshold, CONFIG_INT,
+		(ConfigGetFunc)config_get_disk_threshold},
+	{CONFIG_DISK_GRACE_PERIOD, config_set_disk_grace_period,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.disk_grace_period,
+		CONFIG_INT, (ConfigGetFunc)config_get_disk_grace_period},
+	{CONFIG_DISK_PRESERVE_LOGGING, config_set_disk_logging_critical,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.disk_logging_critical,
+		CONFIG_ON_OFF, (ConfigGetFunc)config_get_disk_logging_critical},
+	{CONFIG_DISK_PRESERVE_LOGGING, config_set_disk_preserve_logging,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.disk_preserve_logging,
+		CONFIG_ON_OFF, (ConfigGetFunc)config_get_disk_preserve_logging},
 #ifdef MEMPOOL_EXPERIMENTAL
 	,{CONFIG_MEMPOOL_SWITCH_ATTRIBUTE, config_set_mempool_switch,
 		NULL, 0,
@@ -1050,6 +1070,12 @@ FrontendConfig_init () {
   cfg->allowed_to_delete_attrs = slapi_ch_strdup("nsslapd-listenhost nsslapd-securelistenhost nsslapd-defaultnamingcontext");
   cfg->default_naming_context = NULL; /* store normalized dn */
 
+  cfg->disk_monitoring = LDAP_OFF;
+  cfg->disk_threshold = 2097152;  /* 2 mb */
+  cfg->disk_grace_period = 60; /* 1 hour */
+  cfg->disk_preserve_logging = LDAP_OFF;
+  cfg->disk_logging_critical = LDAP_OFF;
+
 #ifdef MEMPOOL_EXPERIMENTAL
   cfg->mempool_switch = LDAP_ON;
   cfg->mempool_maxfreelist = 1024;
@@ -1159,7 +1185,98 @@ config_value_is_null( const char *attrname, const char *value, char *errorbuf,
 	return 0;
 }
 
+int
+config_set_disk_monitoring( const char *attrname, char *value, char *errorbuf, int apply )
+{
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal = LDAP_SUCCESS;
 
+    retVal = config_set_onoff ( attrname, value, &(slapdFrontendConfig->disk_monitoring),
+                                errorbuf, apply);
+    return retVal;
+}
+
+int
+config_set_disk_threshold( const char *attrname, char *value, char *errorbuf, int apply )
+{
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal = LDAP_SUCCESS;
+    long threshold = 0;
+    char *endp = NULL;
+
+    if ( config_value_is_null( attrname, value, errorbuf, 0 )) {
+        return LDAP_OPERATIONS_ERROR;
+    }
+
+    threshold = strtol(value, &endp, 10);
+
+    if ( *endp != '\0' || threshold < 2048 ) {
+        PR_snprintf ( errorbuf, SLAPI_DSE_RETURNTEXT_SIZE, "%s: \"%s\" is invalid, threshold must be greater than 2048 and less then %ld",
+            attrname, value, LONG_MAX );
+        retVal = LDAP_OPERATIONS_ERROR;
+        return retVal;
+    }
+
+    if (apply) {
+        CFG_LOCK_WRITE(slapdFrontendConfig);
+        slapdFrontendConfig->disk_threshold = threshold;
+        CFG_UNLOCK_WRITE(slapdFrontendConfig);
+    }
+
+    return retVal;
+}
+
+int
+config_set_disk_preserve_logging( const char *attrname, char *value, char *errorbuf, int apply )
+{
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal = LDAP_SUCCESS;
+
+    retVal = config_set_onoff ( attrname, value, &(slapdFrontendConfig->disk_preserve_logging),
+                                errorbuf, apply);
+    return retVal;
+}
+
+int
+config_set_disk_logging_critical( const char *attrname, char *value, char *errorbuf, int apply )
+{
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal = LDAP_SUCCESS;
+
+    retVal = config_set_onoff ( attrname, value, &(slapdFrontendConfig->disk_logging_critical),
+                                errorbuf, apply);
+    return retVal;
+}
+
+int
+config_set_disk_grace_period( const char *attrname, char *value, char *errorbuf, int apply )
+{
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal = LDAP_SUCCESS;
+    int period = 0;
+    char *endp = NULL;
+
+    if ( config_value_is_null( attrname, value, errorbuf, 0 )) {
+        return LDAP_OPERATIONS_ERROR;
+    }
+
+    period = strtol(value, &endp, 10);
+
+    if ( *endp != '\0' || period < 1 ) {
+        PR_snprintf ( errorbuf, SLAPI_DSE_RETURNTEXT_SIZE, "%s: \"%s\" is invalid, grace period must be at least 1 minute",
+                      attrname, value);
+        retVal = LDAP_OPERATIONS_ERROR;
+        return retVal;
+    }
+
+    if (apply) {
+        CFG_LOCK_WRITE(slapdFrontendConfig);
+        slapdFrontendConfig->disk_grace_period = period;
+        CFG_UNLOCK_WRITE(slapdFrontendConfig);
+    }
+
+    return retVal;
+}
 
 int 
 config_set_port( const char *attrname, char *port, char *errorbuf, int apply ) {
@@ -3552,6 +3669,66 @@ config_get_port(){
 
 }
 
+int
+config_get_disk_monitoring(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal;
+
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->disk_monitoring;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
+int
+config_get_disk_preserve_logging(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal;
+
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->disk_preserve_logging;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
+int
+config_get_disk_logging_critical(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal;
+
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->disk_logging_critical;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
+int
+config_get_disk_grace_period(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal;
+
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->disk_grace_period;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
+long
+config_get_disk_threshold(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    long retVal;
+
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->disk_threshold;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
 char *
 config_get_ldapi_filename(){
   char *retVal;
@@ -4008,8 +4185,6 @@ config_get_pw_maxfailure() {
 
 }
 
-
-
 int
 config_get_pw_inhistory() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4022,9 +4197,6 @@ config_get_pw_inhistory() {
   return retVal; 
 }
 
-
-
-
 long
 config_get_pw_lockduration() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4038,7 +4210,6 @@ config_get_pw_lockduration() {
 
 }
 
-
 long
 config_get_pw_resetfailurecount() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4088,7 +4259,6 @@ config_get_pw_unlock() {
   return retVal; 
 }
 
-
 int
 config_get_pw_lockout(){
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4101,7 +4271,6 @@ config_get_pw_lockout(){
   return retVal;
 }
 
-
 int
 config_get_pw_gracelimit() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4115,7 +4284,6 @@ config_get_pw_gracelimit() {
 
 }
 
-
 int
 config_get_lastmod(){
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4128,7 +4296,6 @@ config_get_lastmod(){
   return retVal; 
 }
 
-
 int
 config_get_enquote_sup_oc(){
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4141,7 +4308,6 @@ config_get_enquote_sup_oc(){
   return retVal; 
 }
 
-
 int
 config_get_nagle() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4150,8 +4316,8 @@ config_get_nagle() {
   CFG_LOCK_READ(slapdFrontendConfig);
   retVal = slapdFrontendConfig->nagle;
   CFG_UNLOCK_READ(slapdFrontendConfig);
-return retVal; }
-
+  return retVal;
+}
 
 int
 config_get_accesscontrol() {
@@ -4197,7 +4363,7 @@ config_get_security() {
   CFG_UNLOCK_READ(slapdFrontendConfig);
 
   return retVal;
- }
+}
 			 
 int
 slapi_config_get_readonly() {
@@ -4209,8 +4375,7 @@ slapi_config_get_readonly() {
   CFG_UNLOCK_READ(slapdFrontendConfig);
 
   return retVal;
- }
-
+}
 
 int
 config_get_schemacheck() {
@@ -4222,7 +4387,7 @@ config_get_schemacheck() {
   CFG_UNLOCK_READ(slapdFrontendConfig);
   
   return retVal;
- }
+}
 
 int
 config_get_syntaxcheck() {
@@ -4270,7 +4435,7 @@ config_get_ds4_compatible_schema() {
   CFG_UNLOCK_READ(slapdFrontendConfig);
   
   return retVal;
- }
+}
 
 int
 config_get_schema_ignore_trailing_spaces() {
@@ -4282,7 +4447,7 @@ config_get_schema_ignore_trailing_spaces() {
   CFG_UNLOCK_READ(slapdFrontendConfig);
   
   return retVal;
- }
+}
 
 char *
 config_get_rootdn() {
@@ -4325,7 +4490,6 @@ config_get_rootpwstoragescheme() {
   return retVal; 
 }
 
-
 #ifndef _WIN32
 
 char *
@@ -4406,8 +4570,6 @@ config_get_reservedescriptors(){
   return retVal; 
 }
 
-
-
 int
 config_get_ioblocktimeout(){
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4418,8 +4580,7 @@ config_get_ioblocktimeout(){
   CFG_UNLOCK_READ(slapdFrontendConfig);	
 
   return retVal;
- }
-
+}
 
 int
 config_get_idletimeout(){
@@ -4574,7 +4735,6 @@ config_get_pw_minage(){
   return retVal; 
 }
 
-
 long
 config_get_pw_warning() {
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4587,7 +4747,6 @@ config_get_pw_warning() {
   return retVal;
 }
 
-
 int
 config_get_errorlog_level(){
   slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4600,7 +4759,6 @@ config_get_errorlog_level(){
   return retVal; 
 }
 
-
 /*  return integer -- don't worry about locking similar to config_check_referral_mode 
     below */
 
@@ -4627,6 +4785,16 @@ config_get_auditlog_logging_enabled(){
   return retVal;
 }
 
+int
+config_get_accesslog_logging_enabled(){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    int retVal;
+
+    retVal = slapdFrontendConfig->accesslog_logging_enabled;
+
+    return retVal;
+}
+
 char *config_get_referral_mode(void)
 {
     slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
@@ -4649,8 +4817,7 @@ config_get_conntablesize(void){
   CFG_UNLOCK_READ(slapdFrontendConfig);	
 
   return retVal;
- }
-
+}
 
 /* return yes/no without actually copying the referral url
    we don't worry about another thread changing this value
@@ -4661,7 +4828,6 @@ int config_check_referral_mode(void)
     return(slapdFrontendConfig->refer_mode & REFER_MODE_ON);
 }
 
-
 int
 config_get_outbound_ldap_io_timeout(void)
 {
@@ -4674,7 +4840,6 @@ config_get_outbound_ldap_io_timeout(void)
 	return retVal; 
 }
 
-
 int
 config_get_unauth_binds_switch(void)
 {
@@ -4687,7 +4852,6 @@ config_get_unauth_binds_switch(void)
 	return retVal;
 }
 
-
 int
 config_get_require_secure_binds(void)
 {
@@ -6281,3 +6445,32 @@ config_allowed_to_delete_attrs(const char *attr_type)
 	return rc;
 }
 
+void
+config_set_accesslog_enabled(int value){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    char errorbuf[BUFSIZ];
+
+    CFG_LOCK_WRITE(slapdFrontendConfig);
+    slapdFrontendConfig->accesslog_logging_enabled = value;
+    if(value){
+        log_set_logging(CONFIG_ACCESSLOG_LOGGING_ENABLED_ATTRIBUTE, "on", SLAPD_ACCESS_LOG, errorbuf, CONFIG_APPLY);
+    } else {
+        log_set_logging(CONFIG_ACCESSLOG_LOGGING_ENABLED_ATTRIBUTE, "off", SLAPD_ACCESS_LOG, errorbuf, CONFIG_APPLY);
+    }
+    CFG_UNLOCK_WRITE(slapdFrontendConfig);
+}
+
+void
+config_set_auditlog_enabled(int value){
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    char errorbuf[BUFSIZ];
+
+    CFG_LOCK_WRITE(slapdFrontendConfig);
+    slapdFrontendConfig->auditlog_logging_enabled = value;
+    if(value){
+        log_set_logging(CONFIG_AUDITLOG_LOGGING_ENABLED_ATTRIBUTE, "on", SLAPD_AUDIT_LOG, errorbuf, CONFIG_APPLY);
+    } else {
+        log_set_logging(CONFIG_AUDITLOG_LOGGING_ENABLED_ATTRIBUTE, "off", SLAPD_AUDIT_LOG, errorbuf, CONFIG_APPLY);
+    }
+    CFG_UNLOCK_WRITE(slapdFrontendConfig);
+}

ldap/servers/slapd/log.c

@@ -2532,6 +2532,69 @@ delete_logfile:
 	return 1;
 }
 
+/*
+ *  This function is used by the disk monitoring thread (daemon.c)
+ *
+ *  When we get close to running out of disk space we delete the rotated logs
+ *  as a last resort to help keep the server up and running.
+ */
+void
+log__delete_rotated_logs()
+{
+	struct logfileinfo *logp = NULL;
+	char buffer[BUFSIZ];
+	char tbuf[TBUFSIZE];
+
+	/*
+	 *  Access Log
+	 */
+	logp = loginfo.log_access_logchain;
+	while (logp) {
+		tbuf[0] = buffer[0] = '\0';
+		log_convert_time (logp->l_ctime, tbuf, 1);
+		PR_snprintf (buffer, sizeof(buffer), "%s.%s", loginfo.log_access_file, tbuf);
+
+		LDAPDebug(LDAP_DEBUG_ANY,"Deleted Rotated Log: %s\n",buffer,0,0);  /* MARK */
+
+		if (PR_Delete(buffer) != PR_SUCCESS) {
+			logp = logp->l_next;
+			continue;
+		}
+		loginfo.log_numof_access_logs--;
+		logp = logp->l_next;
+	}
+	/*
+	 *  Audit Log
+	 */
+	logp = loginfo.log_audit_logchain;
+	while (logp) {
+		tbuf[0] = buffer[0] = '\0';
+		log_convert_time (logp->l_ctime, tbuf, 1);
+		PR_snprintf (buffer, sizeof(buffer), "%s.%s", loginfo.log_audit_file, tbuf);
+		if (PR_Delete(buffer) != PR_SUCCESS) {
+			logp = logp->l_next;
+			continue;
+		}
+		loginfo.log_numof_audit_logs--;
+		logp = logp->l_next;
+	}
+	/*
+	 *  Error log
+	 */
+	logp = loginfo.log_error_logchain;
+	while (logp) {
+		tbuf[0] = buffer[0] = '\0';
+		log_convert_time (logp->l_ctime, tbuf, 1);
+		PR_snprintf (buffer, sizeof(buffer), "%s.%s", loginfo.log_error_file, tbuf);
+		if (PR_Delete(buffer) != PR_SUCCESS) {
+			logp = logp->l_next;
+			continue;
+		}
+		loginfo.log_numof_error_logs--;
+		logp = logp->l_next;
+	}
+}
+
 #define ERRORSLOG 1
 #define ACCESSLOG 2
 #define AUDITLOG  3
@@ -3776,7 +3839,7 @@ log__open_errorlogfile(int logfile_state, int locked)
 	while (logp) {
 		log_convert_time (logp->l_ctime, tbuf, 1 /*short */);
 		PR_snprintf(buffer, sizeof(buffer), "LOGINFO:%s%s.%s (%lu) (%" 
-			NSPRI64 "d)\n", PREVLOGFILE, loginfo.log_error_file, tbuf, 
+			NSPRI64 "d)\n", PREVLOGFILE, loginfo.log_error_file, tbuf,
 			logp->l_ctime, logp->l_size);
 		LOG_WRITE(fpinfo, buffer, strlen(buffer), 0);
 		logp = logp->l_next;

ldap/servers/slapd/proto-slap.h

@@ -380,6 +380,11 @@ int config_set_entryusn_global( const char *attrname, char *value, char *errorbu
 int config_set_allowed_to_delete_attrs( const char *attrname, char *value, char *errorbuf, int apply );
 int config_set_entryusn_import_init( const char *attrname, char *value, char *errorbuf, int apply );
 int config_set_default_naming_context( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_disk_monitoring( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_disk_threshold( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_disk_grace_period( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_disk_preserve_logging( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_disk_logging_critical( const char *attrname, char *value, char *errorbuf, int apply );
 
 #if !defined(_WIN32) && !defined(AIX)
 int config_set_maxdescriptors( const char *attrname, char *value, char *errorbuf, int apply );
@@ -526,8 +531,15 @@ int config_get_entryusn_global(void);
 char *config_get_allowed_to_delete_attrs(void);
 char *config_get_entryusn_import_init(void);
 char *config_get_default_naming_context(void);
-
 int config_allowed_to_delete_attrs(const char *attr_type);
+void config_set_accesslog_enabled(int value);
+void config_set_auditlog_enabled(int value);
+int config_get_accesslog_logging_enabled();
+int config_get_disk_monitoring();
+long config_get_disk_threshold();
+int config_get_disk_grace_period();
+int config_get_disk_preserve_logging();
+int config_get_disk_logging_critical();
 
 int is_abspath(const char *);
 char* rel2abspath( char * );
@@ -737,7 +749,7 @@ int check_log_max_size(
  
 
 void g_set_accesslog_level(int val);
-
+void log__delete_rotated_logs();
 
 /*
  * util.c

ldap/servers/slapd/slap.h

@@ -1989,6 +1989,11 @@ typedef struct _slapdEntryPoints {
 #define CONFIG_ENTRYUSN_IMPORT_INITVAL	"nsslapd-entryusn-import-initval"
 #define CONFIG_ALLOWED_TO_DELETE_ATTRIBUTE	"nsslapd-allowed-to-delete-attrs"
 #define CONFIG_DEFAULT_NAMING_CONTEXT "nsslapd-defaultnamingcontext"
+#define CONFIG_DISK_MONITORING "nsslapd-disk-monitoring"
+#define CONFIG_DISK_THRESHOLD "nsslapd-disk-monitoring-threshold"
+#define CONFIG_DISK_GRACE_PERIOD "nsslapd-disk-monitoring-grace-period"
+#define CONFIG_DISK_PRESERVE_LOGGING "nsslapd-disk-monitoring-preserve-logging"
+#define CONFIG_DISK_LOGGING_CRITICAL "nsslapd-disk-monitoring-logging-critical"
 
 #ifdef MEMPOOL_EXPERIMENTAL
 #define CONFIG_MEMPOOL_SWITCH_ATTRIBUTE "nsslapd-mempool"
@@ -2214,6 +2219,13 @@ typedef struct _slapdFrontendConfig {
   char *entryusn_import_init;   /* Entry USN: determine the initital value of import */
   int pagedsizelimit;
   char *default_naming_context; /* Default naming context (normalized) */
+
+  /* disk monitoring */
+  int disk_monitoring;
+  int disk_threshold;
+  int disk_grace_period;
+  int disk_preserve_logging;
+  int disk_logging_critical;
 } slapdFrontendConfig_t;
 
 /* possible values for slapdFrontendConfig_t.schemareplace */

ldap/servers/slapd/slapi-plugin.h

@@ -6903,7 +6903,9 @@ enum
     BACK_INFO_DBENV_OPENFLAGS,     /* Get the dbenv openflags */
     BACK_INFO_CRYPT_INIT,          /* Ctrl: clcrypt_init */
     BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */
-    BACK_INFO_CRYPT_DECRYPT_VALUE  /* Ctrl: clcrypt_decrypt_value */
+    BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */
+    BACK_INFO_DIRECTORY,           /* Get the directory path */
+    BACK_INFO_LOG_DIRECTORY        /* Get the txn log directory */
 };
 
 struct _back_info_crypt_init {