Ticket 49028 - Autosize database cache by default.

Bug Description:  We ship Directory Server with very conservative defaults.
These are often not fit for production, yet many people run with them and
complain about the performance of Directory Server.

Fix Description:  Instead of relying on the admin to tune their system, by
defaulting to the autotuning newly installed systems will get the "best possible"
experience. Please see:

http://www.port389.org/docs/389ds/design/autotuning.html

Additionally, we now clamp autotuned caches to 64mb boundaries, and default to
cachesizes of 0. If we detect 0, we run the autotune, if else we leave it alone.
This way existing installs won't see a change, but new deployments will
autoscale to the hardware. Existing users of autosizing who want the value to
change every startup will still have the same behaviour.

https://fedorahosted.org/389/ticket/49028

Author: wibrown

Review by: nhosoi (Thanks!)

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -160,8 +160,11 @@ typedef unsigned short u_int16_t;
 #define LDBM_VERSION_31          "Netscape-ldbm/3.1"
 #define LDBM_FILENAME_SUFFIX     LDBM_SUFFIX
 #define    DBVERSION_FILENAME    "DBVERSION"
-#define DEFAULT_CACHE_SIZE       (size_t)33554432
-#define DEFAULT_CACHE_SIZE_STR   "33554432"
+/* 0 here means to let the autotuning reset the value on first run */
+/* cache can't get any smaller than this (in bytes) */
+#define MINCACHESIZE             (size_t)512000
+#define DEFAULT_CACHE_SIZE       (size_t)0
+#define DEFAULT_CACHE_SIZE_STR   "0"
 #define DEFAULT_CACHE_ENTRIES    -1        /* no limit */
 #define DEFAULT_DNCACHE_SIZE     (size_t)16777216
 #define DEFAULT_DNCACHE_SIZE_STR "16777216"

ldap/servers/slapd/back-ldbm/cache.c

@@ -21,8 +21,6 @@
 /* #define CACHE_DEBUG * causes slowdown */
 #endif
 
-/* cache can't get any smaller than this (in bytes) */
-#define MINCACHESIZE       (size_t)512000
 
 /* don't let hash be smaller than this # of slots */
 #define MINHASHSIZE       1024
@@ -657,10 +655,13 @@ static void entrycache_set_max_size(struct cache *cache, size_t bytes)
     struct backentry *eflushtemp = NULL;
 
     if (bytes < MINCACHESIZE) {
-       bytes = MINCACHESIZE;
-       slapi_log_err(SLAPI_LOG_WARNING,
-                "entrycache_set_max_size", "Minimum cache size is %lu -- rounding up\n",
-                MINCACHESIZE);
+        /* During startup, this value can be 0 to indicate an autotune is about
+         * to happen. In that case, suppress this warning.
+         */
+        if (bytes > 0) {
+            slapi_log_err(SLAPI_LOG_WARNING, "entrycache_set_max_size", "Minimum cache size is %lu -- rounding up\n", MINCACHESIZE);
+        }
+        bytes = MINCACHESIZE;
     }
     cache_lock(cache);
     cache->c_maxsize = bytes;

ldap/servers/slapd/back-ldbm/ldbm_config.c

@@ -420,7 +420,11 @@ static int ldbm_config_dbcachesize_set(void *arg, void *value, char *errorbuf, i
         /* Stop the user configuring a stupidly small cache */
         /* min: 8KB (page size) * def thrd cnts (threadnumber==20). */
 #define DBDEFMINSIZ     500000
-        if (val < DBDEFMINSIZ) {
+        /* We allow a value of 0, because the autotuting in start.c will
+         * register that, and trigger the recalculation of the dbcachesize as
+         * needed on the next start up.
+         */
+        if (val < DBDEFMINSIZ && val > 0) {
             slapi_log_err(SLAPI_LOG_NOTICE,"ldbm_config_dbcachesize_set", "cache too small, increasing to %dK bytes\n",
                     DBDEFMINSIZ/1000);
             val = DBDEFMINSIZ;
@@ -435,14 +439,17 @@ static int ldbm_config_dbcachesize_set(void *arg, void *value, char *errorbuf, i
         }
         if (CONFIG_PHASE_RUNNING == phase) {
             li->li_new_dbcachesize = val;
-            slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_config_dbcachesize_set",
-            	"New db cache size will not take affect until the server is restarted\n");
+            if (val == 0) {
+                slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_config_dbcachesize_set", "cache size reset to 0, will be autosized on next startup.\n");
+            } else {
+                slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_config_dbcachesize_set", "New db cache size will not take affect until the server is restarted\n");
+            }
         } else {
             li->li_new_dbcachesize = val;
             li->li_dbcachesize = val;
         }
     }
-    
+
     return retval;
 }
 
@@ -1501,7 +1508,7 @@ static config_info ldbm_config[] = {
     {CONFIG_MODE, CONFIG_TYPE_INT_OCTAL, "0600", &ldbm_config_mode_get, &ldbm_config_mode_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_IDLISTSCANLIMIT, CONFIG_TYPE_INT, "4000", &ldbm_config_allidsthreshold_get, &ldbm_config_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_directory_get, &ldbm_config_directory_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE|CONFIG_FLAG_SKIP_DEFAULT_SETTING},
-    {CONFIG_DBCACHESIZE, CONFIG_TYPE_SIZE_T, DEFAULT_DBCACHE_SIZE_STR, &ldbm_config_dbcachesize_get, &ldbm_config_dbcachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
+    {CONFIG_DBCACHESIZE, CONFIG_TYPE_SIZE_T, DEFAULT_CACHE_SIZE_STR, &ldbm_config_dbcachesize_get, &ldbm_config_dbcachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_DBNCACHE, CONFIG_TYPE_INT, "0", &ldbm_config_dbncache_get, &ldbm_config_dbncache_set, CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_MAXPASSBEFOREMERGE, CONFIG_TYPE_INT, "100", &ldbm_config_maxpassbeforemerge_get, &ldbm_config_maxpassbeforemerge_set, 0},
     
@@ -1536,7 +1543,7 @@ static config_info ldbm_config[] = {
     {CONFIG_DB_HOME_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_db_home_directory_get, &ldbm_config_db_home_directory_set, 0},
     {CONFIG_IMPORT_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "-1", &ldbm_config_import_cache_autosize_get, &ldbm_config_import_cache_autosize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "0", &ldbm_config_cache_autosize_get, &ldbm_config_cache_autosize_set, 0},
-    {CONFIG_CACHE_AUTOSIZE_SPLIT, CONFIG_TYPE_INT, "50", &ldbm_config_cache_autosize_split_get, &ldbm_config_cache_autosize_split_set, 0},
+    {CONFIG_CACHE_AUTOSIZE_SPLIT, CONFIG_TYPE_INT, "40", &ldbm_config_cache_autosize_split_get, &ldbm_config_cache_autosize_split_set, 0},
     {CONFIG_IMPORT_CACHESIZE, CONFIG_TYPE_SIZE_T, "20000000", &ldbm_config_import_cachesize_get, &ldbm_config_import_cachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE},
     {CONFIG_IDL_SWITCH, CONFIG_TYPE_STRING, "new", &ldbm_config_idl_get_idl_new, &ldbm_config_idl_set_tune, CONFIG_FLAG_ALWAYS_SHOW},
     {CONFIG_IDL_UPDATE, CONFIG_TYPE_ONOFF, "on", &ldbm_config_idl_get_update, &ldbm_config_idl_set_update, 0},
@@ -1616,7 +1623,7 @@ ldbm_config_read_instance_entries(struct ldbminfo *li, const char *backend_type)
     return rc;
 }
 
-/* Reads in any config information held in the dse for the ldbm plugin.  
+/* Reads in any config information held in the dse for the ldbm plugin.
  * Creates dse entries used to configure the ldbm plugin and dblayer
  * if they don't already exist.  Registers dse callback functions to
  * maintain those dse entries.  Returns 0 on success.

ldap/servers/slapd/back-ldbm/start.c

@@ -17,6 +17,8 @@
 
 #include "back-ldbm.h"
 
+
+
 static int initialized = 0;
 
 int
@@ -25,6 +27,254 @@ ldbm_back_isinitialized()
     return initialized;
 }
 
+static int
+ldbm_back_start_autotune(struct ldbminfo *li) {
+    Object *inst_obj = NULL;
+    ldbm_instance *inst = NULL;
+    /* size_t is a platform unsigned int, IE uint64_t */
+    size_t total_cache_size = 0;
+    size_t pagesize = 0;
+    size_t pages = 0;
+    size_t procpages __attribute__((unused)) = 0;
+    size_t availpages = 0;
+    size_t cache_size_to_configure = 0;
+    size_t zone_pages = 0;
+    size_t db_pages = 0;
+    size_t entry_pages = 0;
+    size_t import_pages = 0;
+    size_t zone_size = 0;
+    size_t import_size = 0;
+    size_t cache_size = 0;
+    size_t db_size = 0;
+    /* For clamping the autotune value to a 64Mb boundary */
+    size_t clamp_pages = 0;
+    size_t clamp_div = 0;
+    size_t clamp_mod = 0;
+    /* Backend count */
+    size_t backend_count = 0;
+
+    int_fast32_t autosize_percentage = 0;
+    int_fast32_t autosize_db_percentage_split = 0;
+    int_fast32_t import_percentage = 0;
+    int32_t issane = 0;
+    char *msg = ""; /* This will be set by one of the two cache sizing paths below. */
+    char size_to_str[32];    /* big enough to hold %ld */
+
+    /* == Begin autotune == */
+
+    /*
+    * The process that we take here now defaults to autotune first, then override
+    * with manual values if so chosen.
+    *
+    * This means first off, we need to check for valid autosizing values.
+    * We then calculate what our system tuning would be. We clamp these to the
+    * nearest value. IE 487MB would be 510656512 bytes, so we clamp this to
+    * 536870912 bytes, aka 512MB. This is aligned to 64MB boundaries.
+    *
+    * Now that we have these values, we then check the values of dbcachesize
+    * and cachememsize. If they are 0, we set them to the auto-calculated value.
+    * If they are non-0, we skip the value.
+    *
+    * This way, we are really autotuning on "first run", and if the admin wants
+    * to up the values, they merely need to reset the value to 0, and let the
+    * server restart.
+    *
+    * wibrown 2017
+    */
+
+    /* sanity check the autosizing values,
+     no value or sum of values larger than 100.
+    */
+    backend_count = objset_size(li->li_instance_set);
+
+    /* If autosize == 0, set autosize_per to 10. */
+    if (li->li_cache_autosize <= 0) {
+        /* First, set our message. In the case autosize is 0, we calculate some
+         * sane defaults and populate these values, but it's only on first run.
+         */
+        msg = "This can be corrected by altering the values of nsslapd-dbcachesize, nsslapd-cachememsize and nsslapd-dncachememsize\n";
+        autosize_percentage = 10;
+    } else {
+        /* In this case we really are setting the values each start up, so
+         * change the msg.
+         */
+        msg = "This can be corrected by altering the values of nsslapd-cache-autosize, nsslapd-cache-autosize-split and nsslapd-dncachememsize\n";
+        autosize_percentage = li->li_cache_autosize;
+    }
+    /* Has to be less than 0, 0 means to disable I think */
+    if (li->li_import_cache_autosize < 0) {
+        import_percentage = 50;
+    } else {
+        import_percentage = li->li_import_cache_autosize;
+    }
+    /* This doesn't control the availability of the feature, so we can take the
+     * default from ldbm_config.c
+     */
+    autosize_db_percentage_split = li->li_cache_autosize_split;
+
+
+    /* Check the values are sane. */
+    if ((autosize_percentage > 100) || (import_percentage > 100) || (autosize_db_percentage_split > 100) ||
+            ((autosize_percentage > 0) && (import_percentage > 0) && (autosize_percentage + import_percentage > 100))) {
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "Cache autosizing: bad settings, value or sum of values can not larger than 100.\n");
+        return SLAPI_FAIL_GENERAL;
+    }
+
+    if (util_info_sys_pages(&pagesize, &pages, &procpages, &availpages) != 0) {
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "Unable to determine system page limits\n");
+        return SLAPI_FAIL_GENERAL;
+    }
+
+    if (pagesize == 0) {
+        /* If this happens, we are in a very bad state indeed... */
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "Unable to determine system page size\n");
+        return SLAPI_FAIL_GENERAL;
+    }
+
+    /* calculate the needed values */
+    zone_pages = (autosize_percentage * pages) / 100;
+    zone_size = zone_pages * pagesize;
+    /* This is how much we "might" use, lets check it's sane. */
+    /* In the case it is not, this will *reduce* the allocation */
+    issane = util_is_cachesize_sane(&zone_size);
+    if (!issane) {
+        slapi_log_err(SLAPI_LOG_WARNING, "ldbm_back_start", "Your autosized cache values have been reduced. Likely your nsslapd-cache-autosize percentage is too high.\n");
+        slapi_log_err(SLAPI_LOG_WARNING, "ldbm_back_start", "%s", msg);
+    }
+    /* It's valid, lets divide it up and set according to user prefs */
+    zone_pages = zone_size / pagesize;
+    db_pages = (autosize_db_percentage_split * zone_pages) / 100;
+
+    /* Cap the DB size at 512MB, as this doesn't help perf much more (lkrispen's advice) */
+    if ((db_pages * pagesize) > (512 * MEGABYTE)) {
+        db_pages = (512 * MEGABYTE) / pagesize;
+    }
+
+    /* Number of entry cache pages per backend. */
+    entry_pages = (zone_pages - db_pages) / backend_count;
+    /* Now, clamp this value to a 64mb boundary. */
+    /* How many pages are in 64mb? */
+    clamp_pages = (64 * MEGABYTE) / pagesize;
+    /* Now divide the entry pages by this, and also mod. If mod != 0, we need
+     * to add 1 to the diveded number. This should give us:
+     * 510 * 1024 * 1024 == 510MB
+     * 534773760 bytes
+     * 130560 pages at 4096 pages.
+     * 16384 pages for 64Mb
+     * 130560 / 16384 = 7
+     * 130560 % 16384 = 15872 which is != 0
+     * therfore 7 + 1, aka 8 * 16384 = 131072 pages = 536870912 bytes = 512MB.
+     */
+    clamp_div = entry_pages / clamp_pages;
+    clamp_mod = entry_pages % clamp_pages;
+    if (clamp_mod != 0) {
+        /* If we want to clamp down, remove this line. This would change the above from 510mb -> 448mb. */
+        clamp_div += 1;
+        entry_pages = clamp_div * clamp_pages;
+    }
+
+    slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "found %luk physical memory\n", pages*(pagesize/1024));
+    slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "found %luk avaliable\n", zone_pages*(pagesize/1024));
+
+    /* We've now calculated the autotuning values. Do we need to apply it?
+     * we use the logic of "if size is 0, or autosize is > 0. This way three
+     * options can happen.
+     *
+     * First, during first run, dbcache is 0, and autosize is 0. So we apply
+     * the autotuned value ONLY on first run.
+     * Second, once the admin sets a value, or autotuning set a value, it sticks.
+     * Third, if the admin really does want autosizing to take effect every
+     * start up, we disregard the defined value.
+     */
+
+    /* First, check the dbcache */
+    if (li->li_dbcachesize == 0 || li->li_cache_autosize > 0) {
+        slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache autosizing: db cache: %luk\n", db_pages*(pagesize/1024));
+        cache_size_to_configure = (unsigned long)(db_pages * pagesize);
+        if (cache_size_to_configure < (500 * MEGABYTE)) {
+            cache_size_to_configure = (unsigned long)((db_pages * pagesize) / 1.25);
+        }
+        /* Have to set this value through text. */
+        sprintf(size_to_str, "%lu", cache_size_to_configure);
+        ldbm_config_internal_set(li, CONFIG_DBCACHESIZE, size_to_str);
+    }
+    total_cache_size += li->li_dbcachesize;
+
+    /* For each backend */
+    /*   apply the appropriate cache size if 0 */
+    li->li_cache_autosize_ec = (unsigned long)entry_pages * pagesize;
+
+    for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
+            inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
+
+        inst = (ldbm_instance *)object_get_data(inst_obj);
+        cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
+
+        /* This is the point where we decide to apply or not.
+         * We have to check for the mincachesize as setting 0 resets
+         * to this value. This could cause an issue with a *tiny* install, but
+         * it's highly unlikely.
+         */
+        if (cache_size == 0 || cache_size == MINCACHESIZE || li->li_cache_autosize > 0) {
+            slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache autosizing: %s entry cache (%lu total): %luk\n", inst->inst_name, backend_count, entry_pages*(pagesize/1024));
+            cache_set_max_entries(&(inst->inst_cache), -1);
+            cache_set_max_size(&(inst->inst_cache), li->li_cache_autosize_ec, CACHE_TYPE_ENTRY);
+        }
+        /* Refresh this value now. */
+        cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
+        db_size = dblayer_get_id2entry_size(inst);
+        if (cache_size < db_size) {
+            slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start",
+                  "%s: entry cache size %lu B is "
+                  "less than db size %lu B; "
+                  "We recommend to increase the entry cache size "
+                  "nsslapd-cachememsize.\n",
+                  inst->inst_name, cache_size, db_size);
+        }
+        /* We need to get each instances dncache size to add to the total */
+        /* Else we can't properly check the cache allocations below */
+        /* Trac 48831 exists to allow this to be auto-sized too ... */
+        total_cache_size += (PRUint64)cache_get_max_size(&(inst->inst_dncache));
+        total_cache_size += cache_size;
+    }
+    /* autosizing importCache */
+    if (li->li_import_cache_autosize > 0) {
+        /* Use import percentage here, as it's been corrected for -1 behaviour */
+        import_pages = (import_percentage * pages) / 100;
+        import_size = import_pages * pagesize;
+        issane = util_is_cachesize_sane(&import_size);
+        if (!issane) {
+            slapi_log_err(SLAPI_LOG_WARNING, "ldbm_back_start", "Your autosized import cache values have been reduced. Likely your nsslapd-import-cache-autosize percentage is too high.\n");
+        }
+        /* We just accept the reduced allocation here. */
+        import_pages = import_size / pagesize;
+        slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache autosizing: import cache: %luk\n",
+        import_pages*(pagesize/1024));
+
+        sprintf(size_to_str, "%lu", (unsigned long)(import_pages * pagesize));
+        ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, size_to_str);
+    }
+
+    /* Finally, lets check that the total result is sane. */
+    slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "total cache size: %lu B; \n", total_cache_size);
+
+    issane = util_is_cachesize_sane(&total_cache_size);
+    if (!issane) {
+        /* Right, it's time to panic */
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "It is highly likely your memory configuration of all backends will EXCEED your systems memory.\n");
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "In a future release this WILL prevent server start up. You MUST alter your configuration.\n");
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "Total entry cache size: %lu B; dbcache size: %lu B; available memory size: %lu B; \n",
+                  (PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, availpages * pagesize
+        );
+        slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "%s\n", msg);
+        /* WB 2016 - This should be UNCOMMENTED in a future release */
+        /* return SLAPI_FAIL_GENERAL; */
+    }
+
+    /* == End autotune == */
+    return 0;
+}
+
 /*
  * Start the LDBM plugin, and all its instances.
  */
@@ -32,34 +282,9 @@ int
 ldbm_back_start( Slapi_PBlock *pb )
 {
   struct ldbminfo  *li;
-  char *home_dir;
-  int action;
-  int retval; 
-  int issane = 0;
-  PRUint64 total_cache_size = 0;
-  size_t pagesize = 0;
-  size_t pages = 0;
-  size_t procpages = 0;
-  size_t availpages = 0;
-  char *msg = ""; /* This will be set by one of the two cache sizing paths below. */
-
-  char s[32];    /* big enough to hold %ld */
-  unsigned long cache_size_to_configure = 0;
-  int zone_pages;
-  int db_pages;
-  int entry_pages;
-  int import_pages;
-  size_t zone_size;
-  size_t import_size;
-  size_t total_size;
-  Object *inst_obj;
-  ldbm_instance *inst;   
-  PRUint64 cache_size;
-  PRUint64 dncache_size;
-  PRUint64 db_size;
-#ifndef LINUX
-  PRUint64 memsize = pages * pagesize;
-#endif
+  char *home_dir = NULL;
+  int action = 0 ;
+  int retval = 0;
 
   slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_start", "ldbm backend starting\n");
 
@@ -124,163 +349,12 @@ ldbm_back_start( Slapi_PBlock *pb )
       ldbm_config_internal_set(li, CONFIG_DIRECTORY, "get default");
   }
 
-  /* sanity check the autosizing values,
-     no value or sum of values larger than 100.
-  */
-  if ((li->li_cache_autosize > 100) ||
-      (li->li_cache_autosize_split > 100) ||
-      (li->li_import_cache_autosize > 100) ||
-      ((li->li_cache_autosize > 0) && (li->li_import_cache_autosize > 0) &&
-      (li->li_cache_autosize + li->li_import_cache_autosize > 100))) {
-      slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Cache autosizing: bad settings, "
-        "value or sum of values can not larger than 100.\n");
-  } else {
-      if (util_info_sys_pages(&pagesize, &pages, &procpages, &availpages) != 0) {
-          slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Unable to determine system page limits\n");
-          return SLAPI_FAIL_GENERAL;
-      }
-      if (pagesize) {
-          if (li->li_cache_autosize == 0) {
-              /* First, set our message. */
-              msg = "This can be corrected by altering the values of nsslapd-dbcachesize, nsslapd-cachememsize and nsslapd-dncachememsize\n";
-
-              for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
-                   inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
-                  inst = (ldbm_instance *)object_get_data(inst_obj);
-                  cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
-                  db_size = dblayer_get_id2entry_size(inst);
-                  if (cache_size < db_size) {
-                      slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start",
-                              "%s: entry cache size %lu B is "
-                              "less than db size %lu B; "
-                              "We recommend to increase the entry cache size "
-                              "nsslapd-cachememsize.\n",
-                              inst->inst_name, cache_size, db_size);
-                  } else {
-                      slapi_log_err(SLAPI_LOG_BACKLDBM,
-                                "ldbm_back_start", "%s: entry cache size: %lu B; db size: %lu B\n",
-                                inst->inst_name, cache_size, db_size);
-                  }
-                  /* Get the dn_cachesize */
-                  dncache_size = (PRUint64)cache_get_max_size(&(inst->inst_dncache));
-                  total_cache_size += cache_size + dncache_size;
-                  slapi_log_err(SLAPI_LOG_BACKLDBM,
-                            "ldbm_back_start", "total cache size: %lu B; \n",
-                            total_cache_size);
-              }
-              slapi_log_err(SLAPI_LOG_BACKLDBM,
-                        "ldbm_back_start", "Total entry cache size: %lu B; "
-                        "dbcache size: %lu B; "
-                        "available memory size: %lu B;\n",
-#ifdef LINUX
-                        (PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, availpages * pagesize
-#else
-                        (PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, memsize
-#endif
-                );
-
-          /* autosizing dbCache and entryCache */
-          } else if (li->li_cache_autosize > 0) {
-              msg = "This can be corrected by altering the values of nsslapd-cache-autosize, nsslapd-cache-autosize-split and nsslapd-dncachememsize\n";
-              zone_pages = (li->li_cache_autosize * pages) / 100;
-              zone_size = zone_pages * pagesize;
-              /* This is how much we "might" use, lets check it's sane. */
-              /* In the case it is not, this will *reduce* the allocation */
-              issane = util_is_cachesize_sane(&zone_size);
-              if (!issane) {
-                  slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start",
-                		 "Your autosized cache values have been reduced. Likely your nsslapd-cache-autosize percentage is too high.\n");
-                  slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "%s", msg);
-              }
-              /* It's valid, lets divide it up and set according to user prefs */
-              zone_pages = zone_size / pagesize;
-              db_pages = (li->li_cache_autosize_split * zone_pages) / 100;
-              entry_pages = (zone_pages - db_pages) / objset_size(li->li_instance_set);
-              /* We update this for the is-sane check below. */
-              total_cache_size = (zone_pages - db_pages) * pagesize;
-
-              slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing. found %luk physical memory\n",
-                pages*(pagesize/1024));
-              slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing. found %luk avaliable\n",
-                zone_pages*(pagesize/1024));
-              slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing: db cache: %luk, "
-                "each entry cache (%d total): %luk\n",
-                db_pages*(pagesize/1024), objset_size(li->li_instance_set),
-                entry_pages*(pagesize/1024));
-    
-              /* libdb allocates 1.25x the amount we tell it to,
-               * but only for values < 500Meg 
-               * For the larger memory, the overhead is relatively small. */
-              cache_size_to_configure = (unsigned long)(db_pages * pagesize);
-              if (cache_size_to_configure < (500 * MEGABYTE)) {
-                  cache_size_to_configure = (unsigned long)((db_pages * pagesize) / 1.25);
-              }
-              sprintf(s, "%lu", cache_size_to_configure);
-              ldbm_config_internal_set(li, CONFIG_DBCACHESIZE, s);
-              li->li_cache_autosize_ec = (unsigned long)entry_pages * pagesize;
-    
-              for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
-                  inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
-                  inst = (ldbm_instance *)object_get_data(inst_obj);
-                  cache_set_max_entries(&(inst->inst_cache), -1);
-                  cache_set_max_size(&(inst->inst_cache),
-                                    li->li_cache_autosize_ec, CACHE_TYPE_ENTRY);
-                  /* We need to get each instances dncache size to add to the total */
-                  /* Else we can't properly check the cache allocations below */
-                  /* Trac 48831 exists to allow this to be auto-sized too ... */
-                  total_cache_size += (PRUint64)cache_get_max_size(&(inst->inst_dncache));
-              }
-          }    
-          /* autosizing importCache */
-          if (li->li_import_cache_autosize > 0) {
-              /* For some reason, -1 means 50 ... */
-              if (li->li_import_cache_autosize == -1) {
-                    li->li_import_cache_autosize = 50;
-              }
-              import_pages = (li->li_import_cache_autosize * pages) / 100;
-              import_size = import_pages * pagesize;
-              issane = util_is_cachesize_sane(&import_size);
-              if (!issane) {
-                  slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start",
-                          "Your autosized import cache values have been reduced. "
-                          "Likely your nsslapd-import-cache-autosize percentage is too high.\n");
-              }
-              /* We just accept the reduced allocation here. */
-              import_pages = import_size / pagesize;
-              slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache autosizing: import cache: %luk\n",
-                import_pages*(pagesize/1024));
-    
-              sprintf(s, "%lu", (unsigned long)(import_pages * pagesize));
-              ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s);
-          }
-      }
-  }
-
-  /* Finally, lets check that the total result is sane. */
-
-  total_size = total_cache_size + (PRUint64)li->li_dbcachesize;
-  issane = util_is_cachesize_sane(&total_size);
-  if (!issane) {
-    /* Right, it's time to panic */
-    slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start",
-        "It is highly likely your memory configuration of all backends will EXCEED your systems memory.\n");
-    slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start",
-        "In a future release this WILL prevent server start up. You MUST alter your configuration.\n");
-    slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start", "Total entry cache size: %lu B; "
-              "dbcache size: %lu B; available memory size: %lu B; \n",
-#ifdef LINUX
-              (PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, availpages * pagesize
-#else
-              (PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, memsize
-#endif
-    );
-    slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "%s\n", msg);
-    /* WB 2016 - This should be UNCOMMENTED in a future release */
-    /* return SLAPI_FAIL_GENERAL; */
+  retval = ldbm_back_start_autotune(li);
+  if (retval != 0) {
+      slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Failed to set database tuning on backends\n");
+      return SLAPI_FAIL_GENERAL;
   }
 
-
-
   retval = check_db_version(li, &action);
   if (0 != retval)
   {