Ticket 49021 - Automatic thread tuning

Bug Description:  We currently always set 30 threads on systems no matter their
size. We should tune the threads more appropriately out of the box.

Fix Description:  Add a thread autotuning. We start at 16 threads, and ramp up,
reducing the rate off addition as we grow to a cap of 512 threads.

https://fedorahosted.org/389/ticket/49021

Author: wibrown

Review by: nhosoi, mreynolds (Thanks)

ldap/servers/slapd/libglobs.c

@@ -1457,7 +1457,7 @@ FrontendConfig_init(void) {
     init_require_secure_binds = cfg->require_secure_binds = LDAP_OFF;
     cfg->allow_anon_access = SLAPD_DEFAULT_ALLOW_ANON_ACCESS;
     init_slapi_counters = cfg->slapi_counters = LDAP_ON;
-    cfg->threadnumber = SLAPD_DEFAULT_MAX_THREADS;
+    cfg->threadnumber = util_get_hardware_threads();
     cfg->maxthreadsperconn = SLAPD_DEFAULT_MAX_THREADS_PER_CONN;
     cfg->reservedescriptors = SLAPD_DEFAULT_RESERVE_FDS;
     cfg->idletimeout = SLAPD_DEFAULT_IDLE_TIMEOUT;
@@ -3867,34 +3867,38 @@ config_set_encryptionalias( const char *attrname, char *value, char *errorbuf, i
   return retVal;
 }
 
-int 
+int
 config_set_threadnumber( const char *attrname, char *value, char *errorbuf, int apply ) {
-  int retVal = LDAP_SUCCESS;
-  long threadnum = 0;
-  char *endp = NULL;
+    int retVal = LDAP_SUCCESS;
+    long threadnum = 0;
+    char *endp = NULL;
 
-  slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
 
-  if ( config_value_is_null( attrname, value, errorbuf, 0 )) {
-	return LDAP_OPERATIONS_ERROR;
-  }
+    if ( config_value_is_null( attrname, value, errorbuf, 0 )) {
+        return LDAP_OPERATIONS_ERROR;
+    }
 
-  errno = 0;
-  threadnum = strtol(value, &endp, 10);
-  
-  if ( *endp != '\0' || errno == ERANGE || threadnum < 1 || threadnum > 65535 ) {
-	slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE,
-	      "%s: invalid value \"%s\", maximum thread number must range from 1 to 65535", attrname, value);
-	retVal = LDAP_OPERATIONS_ERROR;
-  }
-  
-  if (apply) {
-	CFG_LOCK_WRITE(slapdFrontendConfig);
-	/*	max_threads = threadnum; */
-	slapdFrontendConfig->threadnumber = threadnum;
-	CFG_UNLOCK_WRITE(slapdFrontendConfig);
-  }
-  return retVal;
+    errno = 0;
+    threadnum = strtol(value, &endp, 10);
+
+    /* Means we want to re-run the hardware detection. */
+    if (threadnum == -1) {
+        threadnum = util_get_hardware_threads();
+    }
+
+    if ( *endp != '\0' || errno == ERANGE || threadnum < 1 || threadnum > 65535 ) {
+        slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE,
+          "%s: invalid value \"%s\", maximum thread number must range from 1 to 65535", attrname, value);
+        retVal = LDAP_OPERATIONS_ERROR;
+    }
+    if (apply) {
+        CFG_LOCK_WRITE(slapdFrontendConfig);
+        /*  max_threads = threadnum; */
+        slapdFrontendConfig->threadnumber = threadnum;
+        CFG_UNLOCK_WRITE(slapdFrontendConfig);
+    }
+    return retVal;
 }
 
 int 
@@ -5497,16 +5501,25 @@ config_get_encryptionalias(void) {
   return retVal; 
 }
 
-int
+long
 config_get_threadnumber(void) {
-  slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
-  int retVal;
-  
-  CFG_LOCK_READ(slapdFrontendConfig);
-  retVal = slapdFrontendConfig->threadnumber;
-  CFG_UNLOCK_READ(slapdFrontendConfig);
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    long retVal;
 
-  return retVal;
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapdFrontendConfig->threadnumber;
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    if (retVal == -1) {
+        retVal = util_get_hardware_threads();
+    }
+
+    /* We *still* can't detect hardware threads. Okay, return 30 :( */
+    if (retVal == -1) {
+        retVal = 30;
+    }
+
+    return retVal;
 }
 
 int

ldap/servers/slapd/proto-slap.h

@@ -472,7 +472,7 @@ char *config_get_rootpwstoragescheme(void);
 char *config_get_localuser(void);
 char *config_get_workingdir(void);
 char *config_get_encryptionalias(void);
-int config_get_threadnumber(void);
+long config_get_threadnumber(void);
 int config_get_maxthreadsperconn(void);
 int config_get_maxdescriptors(void);
 int config_get_reservedescriptors(void);

ldap/servers/slapd/slap.h

@@ -257,8 +257,8 @@ typedef void	(*VFPV)(); /* takes undefined arguments */
 #define SLAPD_DEFAULT_OUTBOUND_LDAP_IO_TIMEOUT_STR "300000"
 #define SLAPD_DEFAULT_RESERVE_FDS           64
 #define SLAPD_DEFAULT_RESERVE_FDS_STR       "64"
-#define SLAPD_DEFAULT_MAX_THREADS           30      /* connection pool threads */
-#define SLAPD_DEFAULT_MAX_THREADS_STR       "30"
+#define SLAPD_DEFAULT_MAX_THREADS           -1      /* connection pool threads */
+#define SLAPD_DEFAULT_MAX_THREADS_STR       "-1"
 #define SLAPD_DEFAULT_MAX_THREADS_PER_CONN  5       /* allowed per connection */
 #define SLAPD_DEFAULT_MAX_THREADS_PER_CONN_STR "5"
 #define SLAPD_DEFAULT_MAX_BERSIZE_STR       "0"
@@ -2405,7 +2405,7 @@ typedef struct _slapdFrontendConfig {
   char *SNMPorganization;
   char *SNMPlocation;
   char *SNMPcontact;
-  int threadnumber;
+  long threadnumber;
   int timelimit;
   char *accesslog;
   struct berval **defaultreferral;

ldap/servers/slapd/slapi-private.h

@@ -1382,6 +1382,13 @@ int util_info_sys_pages(size_t *pagesize, size_t *pages, size_t *procpages, size
  */
 int util_is_cachesize_sane(size_t *cachesize);
 
+/**
+ * Retrieve the number of threads the server should run with based on this hardware.
+ *
+ * \return -1 if the hardware detection failed. Any positive value is threads to use.
+ */
+long util_get_hardware_threads(void);
+
 /**
  * Write an error message to the given error buffer.
  *

ldap/servers/slapd/util.c

@@ -1829,6 +1829,54 @@ out:
     return issane;
 }
 
+long
+util_get_hardware_threads(void) {
+#ifdef LINUX
+    long hw_threads = sysconf(_SC_NPROCESSORS_ONLN);
+    long threads = 0;
+    slapi_log_err(SLAPI_LOG_TRACE, "util_get_hardware_threads", "Detected %lu hardware threads\n", threads);
+    /*
+     * Now we determine the number to run with based on threads. Initially, for
+     * low processor counts we ramp up quickly, we plateau a little, then, we
+     * at high numbers start to plateau and increase slowly.
+     * Should be
+     * 1 -> 16
+     * 2 -> 16
+     * 4 -> 24
+     * 8 -> 32
+     * 16 -> 48
+     * 32 -> 64
+     * 64 -> 96
+     * 128 -> 192
+     * 256 -> 384
+     * 512 -> 512
+     * 1024 -> 512
+     * 2048 -> 512
+     */
+
+    if (hw_threads >= 0 && hw_threads < 4) {
+        threads = 16;
+    } else if (hw_threads >= 4 && hw_threads < 32) {
+        threads = 16 + (hw_threads * 2);
+    } else if (hw_threads >= 32 && hw_threads < 64) {
+        threads = (hw_threads * 2);
+    } else if (hw_threads >= 64 && hw_threads < 512) {
+        /* Same as *1.5 */
+        threads = (hw_threads * 2) - (hw_threads / 2);
+    } else {
+        /* Cap at 512 for now ... */
+        threads = 512;
+    }
+    slapi_log_err(SLAPI_LOG_INFO, "util_get_hardware_threads", "Automatically configuring %lu threads\n", threads);
+
+    return threads;
+#else
+    slapi_log_err(SLAPI_LOG_ERR, "util_get_hardware_threads", "ERROR: Cannot detect hardware threads on this platform. This is probably a bug!\n");
+    /* Can't detect threads on this platform! */
+    return -1;
+#endif
+}
+
 void
 slapi_create_errormsg(
     char        *errorbuf,