590931 - rhds81 import - hardcoded pages_limit for nsslapd-import-cache-autosize

Fix Description:
1. Got rid of the old hardcoded limit 200MB.
2. Introduced the memory hard limit and soft limit.
   Standalone command line import ldif2db behaves as follows:
   If import cache autosize is enabled:
       nsslapd-import-cache-autosize: -1 or 1 ~ 99
       (if the value is greater than or equal to 100,
        it's reset to 50 with a warning.)
   the import cache size is calculated as
       nsslapd-import-cache-autosize * pages / 125
       (./125 instead of ./100 is for adjusting the BDB overhead.)
   If import cache is disabled:
       nsslapd-import-cache-autosize: 0
   get the nsslapd-import-cachesize.
   Calculate the memory size left after allocating the import cache size.
   If the size is less than the hard limit, it issues an error and quit.
   If the size is greater than the hard limit and less than the soft limit,
   it issues a warning, but continues the import task.

Note: this function is called only if the import is executed as a stand
alone command line (ldif2db).

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -869,7 +869,7 @@ void dblayer_sys_pages(size_t *pagesize, size_t *pages, size_t *procpages, size_
 #ifdef OS_solaris
     *pagesize = (int)sysconf(_SC_PAGESIZE);
     *pages = (int)sysconf(_SC_PHYS_PAGES);
-        *availpages = dblayer_getvirtualmemsize() / *pagesize;
+    *availpages = dblayer_getvirtualmemsize() / *pagesize;
     /* solaris has THE most annoying way to get this info */
     if (procpages) {
         struct prpsinfo psi;
@@ -1757,59 +1757,105 @@ int dblayer_start(struct ldbminfo *li, int dbmode)
     return 0;
 }
 
-void
-autosize_import_cache(struct ldbminfo *li)
+/*
+ * If import cache autosize is enabled:
+ *    nsslapd-import-cache-autosize: -1 or 1 ~ 99
+ * calculate the import cache size.
+ * If import cache is disabled:
+ *    nsslapd-import-cache-autosize: 0
+ * get the nsslapd-import-cachesize.
+ * Calculate the memory size left after allocating the import cache size.
+ * If the size is less than the hard limit, it issues an error and quit.
+ * If the size is greater than the hard limit and less than the soft limit,
+ * it issues a warning, but continues the import task.
+ *
+ * Note: this function is called only if the import is executed as a stand
+ * alone command line (ldif2db).
+ */
+int
+check_and_set_import_cache(struct ldbminfo *li)
 {
+    size_t import_pages = 0;
+    size_t pagesize, pages, procpages, availpages;
+    size_t soft_limit = 0;
+    size_t hard_limit = 0;
+    size_t page_delta = 0;
+    char s[64];   /* big enough to hold %ld */
+
+    dblayer_sys_pages(&pagesize, &pages, &procpages, &availpages);
+    if (0 == pagesize || 0 == pages) {
+        LDAPDebug2Args(LDAP_DEBUG_ANY, "check_and_set_import_cache: "
+                       "Failed to get pagesize: %ld or pages: %ld\n",
+                       pagesize, pages);
+        return ENOENT;
+    }
+    LDAPDebug(LDAP_DEBUG_ANY, "check_and_set_import_cache: "
+                  "pagesize: %ld, pages: %ld, procpages: %ld\n",
+                  pagesize, pages, procpages);
+
+    /* Soft limit: pages equivalent to 1GB (defined in dblayer.h) */
+    soft_limit = (DBLAYER_IMPORTCACHESIZE_SL*1024) / (pagesize/1024);
+    /* Hard limit: pages equivalent to 100MB (defined in dblayer.h) */
+    hard_limit = (DBLAYER_IMPORTCACHESIZE_HL*1024) / (pagesize/1024);
     /*
      * default behavior for ldif2db import cache,
      * nsslapd-import-cache-autosize==-1,
      * autosize 50% mem to import cache
      */
-    if (li->li_import_cache_autosize == -1) {
+    if (li->li_import_cache_autosize < 0) {
         li->li_import_cache_autosize = 50;
     }
 
     /* sanity check */
-    if (li->li_import_cache_autosize > 100) {
-        LDAPDebug(LDAP_DEBUG_ANY,
-            "cache autosizing: bad setting, "
-            "import cache autosizing value should not be larger than 100(%).\n"
-            "set: 100(%).\n", NULL, NULL, NULL);
-        li->li_import_cache_autosize = 100;
+    if (li->li_import_cache_autosize >= 100) {
+        LDAPDebug0Args(LDAP_DEBUG_ANY,
+            "check_and_set_import_cache: "
+            "import cache autosizing value "
+            "(nsslapd-import-cache-autosize) should not be "
+            "greater than or equal to 100(%). Reset to 50(%).\n");
+        li->li_import_cache_autosize = 50;
     }
 
-    /* autosizing importCache */
-    if (li->li_import_cache_autosize > 0) {
-        size_t pagesize, pages, procpages, availpages;
+    if (li->li_import_cache_autosize == 0) {
+        /* user specified importCache */
+        import_pages = li->li_import_cachesize / pagesize;
 
-        dblayer_sys_pages(&pagesize, &pages, &procpages, &availpages);
-        LDAPDebug(LDAP_DEBUG_ANY, "autosize_import_cache: "
-                  "pagesize: %d, pages: %d, procpages: %d\n",
-                  pagesize, pages, procpages);
-        if (pagesize) {
-            char s[32];   /* big enough to hold %ld */
-            int import_pages;
-            int pages_limit = (200 * 1024) / (pagesize/1024);
-            import_pages = (li->li_import_cache_autosize * pages) / 125;
-            /* We don't want to go wild with memory when auto-sizing, cap the 
-             * cache size at 200 Megs to try to avoid situations where we 
-             * attempt to allocate more memory than there is free page pool for, or 
-             * where there's some system limit on the size of process memory
-             */
-            if (import_pages > pages_limit) {
-                import_pages = pages_limit;
-            }
-            LDAPDebug(LDAP_DEBUG_ANY, "cache autosizing: import cache: %dk \n",
-                import_pages*(pagesize/1024), NULL, NULL);
-            LDAPDebug(LDAP_DEBUG_ANY,
-                          "li_import_cache_autosize: %d, import_pages: %d, pagesize: %d\n", 
-                          li->li_import_cache_autosize, import_pages,
-                          pagesize);
+    } else {
+        /* autosizing importCache */
+        /* ./125 instead of ./100 is for adjusting the BDB overhead. */
+        import_pages = (li->li_import_cache_autosize * pages) / 125;
+    }
 
-            sprintf(s, "%lu", (unsigned long)(import_pages * pagesize));
-            ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s);
-        }
+    page_delta = pages - import_pages;
+    if (page_delta < hard_limit) {
+        LDAPDebug(LDAP_DEBUG_ANY, 
+            "After allocating import cache %ldKB, "
+            "the available memory is %ldKB, "
+            "which is less than the hard limit %ldKB. "
+            "Please decrease the import cache size and rerun import.\n",
+            import_pages*(pagesize/1024), page_delta*(pagesize/1024),
+            hard_limit*(pagesize/1024));
+        return ENOMEM;
+    }
+    if (page_delta < soft_limit) {
+        LDAPDebug(LDAP_DEBUG_ANY, 
+            "WARNING: After allocating import cache %ldKB, "
+            "the available memory is %ldKB, "
+            "which is less than the soft limit %ldKB. "
+            "You may want to decrease the import cache size and "
+            "rerun import.\n",
+            import_pages*(pagesize/1024), page_delta*(pagesize/1024),
+            soft_limit*(pagesize/1024));
+    }
+
+    LDAPDebug1Arg(LDAP_DEBUG_ANY, "Import allocates %ldKB import cache.\n", 
+                  import_pages*(pagesize/1024));
+    if (li->li_import_cache_autosize > 0) { /* import cache autosizing */
+        /* set the calculated import cache size to the config */
+        sprintf(s, "%lu", (unsigned long)(import_pages * pagesize));
+        ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s);
     }
+    return 0;
 }
 
 /* mode is one of
@@ -2011,15 +2057,13 @@ int dblayer_instance_start(backend *be, int mode)
                 oflags |= DB_PRIVATE;
             }
             PR_Lock(li->li_config_mutex);
-            if ((li->li_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE) &&
-                (li->li_import_cache_autosize)) /* Autosizing importCache
-                                                 * Need to re-eval every time
-                                                 * to guarantee the memory is
-                                                 * really available 
-                                                 * (just for command line I/F)
-                                                 */
-            {
-                autosize_import_cache(li);
+            /* import cache checking and autosizing is available only
+             * for the command line */
+            if (li->li_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE) {
+                return_value = check_and_set_import_cache(li);
+                if (return_value) {
+                    goto out;
+                }
             }
             cachesize = li->li_import_cachesize;
             PR_Unlock(li->li_config_mutex);

ldap/servers/slapd/back-ldbm/dblayer.h

@@ -97,6 +97,14 @@
 #define	DB_REGION_NAME	25		/* DB: named regions, no backing file. */
 #endif
 
+/* Used in check_and_set_import_cache */
+/* After allocating the import cache, free memory must be left more than
+ * the hard limit to run import. */
+/* If the free memory size left is greater than hard limit and less than
+ * soft limit, the import utility issues a warning, but it runs */
+#define DBLAYER_IMPORTCACHESIZE_HL 100  /* import cache hard limit 100MB */
+#define DBLAYER_IMPORTCACHESIZE_SL 1024 /* import cache soft limit 1GB */
+
 struct dblayer_private_env {
 	DB_ENV	*dblayer_DB_ENV;
 	PRRWLock * dblayer_env_lock;

ldap/servers/slapd/back-ldbm/ldif2ldbm.c

@@ -632,7 +632,6 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb )
 
         li->li_flags |= SLAPI_TASK_RUNNING_FROM_COMMANDLINE;
         ldbm_config_load_dse_info(li);
-        autosize_import_cache(li);
     }
 
     /* Find the instance that the ldif2db will be done on. */
@@ -2484,7 +2483,9 @@ int ldbm_back_upgradedb(Slapi_PBlock *pb)
         {
             ldbm_config_load_dse_info(li);
         }
-        autosize_import_cache(li);
+        if (check_and_set_import_cache(li) < 0) {
+            return -1;
+        }
     }
     else
     {

ldap/servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -164,7 +164,7 @@ void dblayer_set_recovery_required(struct ldbminfo *li);
 char *dblayer_get_home_dir(struct ldbminfo *li, int *dbhome);
 char *dblayer_get_full_inst_dir(struct ldbminfo *li, ldbm_instance *inst,
                                 char *buf, int buflen);
-void autosize_import_cache(struct ldbminfo *li);
+int check_and_set_import_cache(struct ldbminfo *li);
 
 int dblayer_db_uses_locking(DB_ENV *db_env);
 int dblayer_db_uses_transactions(DB_ENV *db_env);