Bug 663484 - Entry usn plugin fails to properly tag entries on initialization

https://bugzilla.redhat.com/show_bug.cgi?id=663484

Description:
1) Introducing a config parameter nsslapd-entryusn-import-initval.
If the value is digit, e.g., 0, 10, 100 ..., the imported entries
will have the entryusn value.
If the value is not digit, e.g., "next", the imported entries
will have the last entryusn + 1 from the database existed before
the import was exected or initialized by the master.

2) Internal OID LDBM_ENTRYUSN_OID has been redifined in back-ldbm.h
to avoid a possible conflict.

3) If an attribute to modify is an operational attribute and
that is the only one modify operation (not one of the series
of modify operations), the server returns UNWILLING_TO_PERFORM.

In addition, this patch fixes a minor invalid access to a freed
memory in cl5DeleteDBSync.

ldap/servers/plugins/replication/cl5_api.c

@@ -659,18 +659,21 @@ int cl5DeleteDBSync (Object *replica)
 	rc = _cl5GetDBFile (replica, &obj);
 	if (rc == CL5_SUCCESS)
 	{
+        char *filename = NULL;
         file = (CL5DBFile*)object_get_data (obj);
         PR_ASSERT (file);
+        /* file->name is freed in _cl5DBDeleteFile */
+        filename = slapi_ch_strdup(file->name);
 
         _cl5DBDeleteFile (obj);
 
         /* wait until the file is gone */
-        while (PR_Access (file->name, PR_ACCESS_EXISTS) == PR_SUCCESS)
+        while (PR_Access (filename, PR_ACCESS_EXISTS) == PR_SUCCESS)
         {
             DS_Sleep (PR_MillisecondsToInterval(100));
         }
-
-	}
+        slapi_ch_free_string(&filename);
+    }
     else
     {
         Replica *r = (Replica*)object_get_data (replica);

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -805,8 +805,8 @@ typedef struct _back_search_result_set
 #define LDBM_DNCOMP_OID				"2.16.840.1.113730.3.1.603"
 #define LDBM_PARENTID_OID			"2.16.840.1.113730.3.1.604"
 #define LDBM_ENTRYID_OID			"2.16.840.1.113730.3.1.605"
-#define LDBM_ENTRYUSN_OID			"2.16.840.1.113730.3.1.606"
-#define LDBM_ENTRYRDN_OID			"2.16.840.1.113730.3.1.607"
+#define LDBM_ENTRYUSN_OID			"2.16.840.1.113730.3.1.2096"
+/* #define LDBM_ENTRYRDN_OID			"2.16.840.1.113730.3.1.2097" */
 
 #define LDBM_ANCESTORID_STR         "ancestorid"
 #define LDBM_ENTRYDN_STR            "entrydn"

ldap/servers/slapd/back-ldbm/import-threads.c

@@ -58,6 +58,7 @@
 
 static void import_wait_for_space_in_fifo(ImportJob *job, size_t new_esize);
 static int import_get_and_add_parent_rdns(ImportWorkerInfo *info, ldbm_instance *inst, DB *db, ID id, ID *total_id, Slapi_RDN *srdn, int *curr_entry);
+static int _get_import_entryusn(ImportJob *job, Slapi_Value **usn_value);
 
 static struct backentry *import_make_backentry(Slapi_Entry *e, ID id)
 {
@@ -393,6 +394,9 @@ import_producer(void *param)
     info->state = RUNNING;
     import_init_ldif(&c);
 
+    /* Get entryusn, if needed. */
+    _get_import_entryusn(job, &(job->usn_value));
+
     /* jumpstart by opening the first file */
     curr_file = 0;
     fd = -1;
@@ -667,24 +671,11 @@ import_producer(void *param)
             goto error;
         }
 
-        /* 
-         * Check if entryusn plugin is enabled.  
-         * If yes, add "entryusn: 0" to the entry 
-         * if it does not have the attr type .
-         */
-        if (plugin_enabled("USN", (void *)plugin_get_default_component_id())) {
-            if (slapi_entry_attr_find(ep->ep_entry, SLAPI_ATTR_ENTRYUSN,
-                                                      &attr)) { /* not found */
-                /* add entryusn: 0 to the entry */
-                Slapi_Value *usn_value = NULL;
-                struct berval usn_berval = {0};
-                usn_berval.bv_val = slapi_ch_smprintf("0");
-                usn_berval.bv_len = strlen(usn_berval.bv_val);
-                usn_value = slapi_value_new_berval(&usn_berval);
-                slapi_entry_add_value(ep->ep_entry,
-                                      SLAPI_ATTR_ENTRYUSN, usn_value);
-                slapi_value_free(&usn_value);
-            }
+        /* if usn_value is available AND the entry does not have it, */
+        if (job->usn_value && slapi_entry_attr_find(ep->ep_entry, 
+                                               SLAPI_ATTR_ENTRYUSN, &attr)) {
+            slapi_entry_add_value(ep->ep_entry, SLAPI_ATTR_ENTRYUSN,
+                                  job->usn_value);
         }
 
         /* Now we have this new entry, all decoded
@@ -731,8 +722,8 @@ import_producer(void *param)
                     "ending line %d of file \"%s\"",
                     escape_string(slapi_entry_get_dn(e), ebuf),
                     curr_lineno, curr_filename);
-            import_log_notice(job, "REASON: entry too large (%lu bytes) for "
-                    "the buffer size (%lu bytes)", newesize, job->fifo.bsize);
+            import_log_notice(job, "REASON: entry too large (%u bytes) for "
+                    "the buffer size (%u bytes)", newesize, job->fifo.bsize);
             backentry_free(&ep);
             job->skipped++;
             continue;
@@ -773,11 +764,13 @@ import_producer(void *param)
         }
     }
 
+    slapi_value_free(&(job->usn_value));
     import_free_ldif(&c);
     info->state = FINISHED;
     return;
 
 error:
+    slapi_value_free(&(job->usn_value));
     info->state = ABORTED;
 }
 
@@ -860,8 +853,8 @@ index_set_entry_to_fifo(ImportWorkerInfo *info, Slapi_Entry *e,
         char ebuf[BUFSIZ];
         import_log_notice(job, "WARNING: skipping entry \"%s\"",
                     escape_string(slapi_entry_get_dn(e), ebuf));
-        import_log_notice(job, "REASON: entry too large (%lu bytes) for "
-                    "the buffer size (%lu bytes)", newesize, job->fifo.bsize);
+        import_log_notice(job, "REASON: entry too large (%u bytes) for "
+                    "the buffer size (%u bytes)", newesize, job->fifo.bsize);
         backentry_free(&ep);
         job->skipped++;
         rc = 0; /* go to the next loop */
@@ -1735,8 +1728,8 @@ upgradedn_producer(void *param)
             char ebuf[BUFSIZ];
             import_log_notice(job, "WARNING: skipping entry \"%s\"",
                     escape_string(slapi_entry_get_dn(e), ebuf));
-            import_log_notice(job, "REASON: entry too large (%lu bytes) for "
-                    "the buffer size (%lu bytes)", newesize, job->fifo.bsize);
+            import_log_notice(job, "REASON: entry too large (%u bytes) for "
+                    "the buffer size (%u bytes)", newesize, job->fifo.bsize);
             backentry_free(&ep);
             job->skipped++;
             continue;
@@ -2823,6 +2816,13 @@ static int bulk_import_queue(ImportJob *job, Slapi_Entry *entry)
         pw_encodevals( (Slapi_Value **)va ); /* jcm - had to cast away const */
     }
 
+    /* if usn_value is available AND the entry does not have it, */
+    if (job->usn_value && slapi_entry_attr_find(ep->ep_entry, 
+                                               SLAPI_ATTR_ENTRYUSN, &attr)) {
+        slapi_entry_add_value(ep->ep_entry, SLAPI_ATTR_ENTRYUSN, 
+                              job->usn_value);
+    }
+
     /* Now we have this new entry, all decoded
      * Next thing we need to do is:
      * (1) see if the appropriate fifo location contains an
@@ -2871,8 +2871,8 @@ static int bulk_import_queue(ImportJob *job, Slapi_Entry *entry)
         char ebuf[BUFSIZ];
         import_log_notice(job, "WARNING: skipping entry \"%s\"",
                     escape_string(slapi_entry_get_dn(ep->ep_entry), ebuf));
-        import_log_notice(job, "REASON: entry too large (%lu bytes) for "
-                    "the import buffer size (%lu bytes).   Try increasing nsslapd-cachememsize.", newesize, job->fifo.bsize);
+        import_log_notice(job, "REASON: entry too large (%u bytes) for "
+                    "the import buffer size (%u bytes).   Try increasing nsslapd-cachememsize.", newesize, job->fifo.bsize);
         backentry_clear_entry(ep);      /* entry is released in the frontend on failure*/
         backentry_free( &ep );          /* release the backend wrapper, here */
         PR_Unlock(job->wire_lock);
@@ -2953,7 +2953,17 @@ int ldbm_back_wire_import(Slapi_PBlock *pb)
     slapi_pblock_get(pb, SLAPI_BULK_IMPORT_STATE, &state);
     if (state == SLAPI_BI_STATE_START) {
         /* starting a new import */
-        return bulk_import_start(pb);
+        int rc = bulk_import_start(pb);
+        if (!rc) {
+            /* job must be available since bulk_import_start was successful */
+            job =
+              (ImportJob *)slapi_get_object_extension(li->li_bulk_import_object,
+                                                     pb->pb_conn,
+                                                     li->li_bulk_import_handle);
+            /* Get entryusn, if needed. */
+            _get_import_entryusn(job, &(job->usn_value));
+        }
+        return rc;
     }
 
     PR_ASSERT(pb->pb_conn != NULL);
@@ -2982,6 +2992,7 @@ int ldbm_back_wire_import(Slapi_PBlock *pb)
     thread = job->main_thread;
 
     if (state == SLAPI_BI_STATE_DONE) {
+        slapi_value_free(&(job->usn_value));
         /* finished with an import */
         job->flags |= FLAG_PRODUCER_DONE;
         /* "job" struct may vanish at any moment after we set the DONE
@@ -3482,3 +3493,55 @@ bail:
         return rc;
     }
 }
+
+static int
+_get_import_entryusn(ImportJob *job, Slapi_Value **usn_value)
+{
+#define USN_COUNTER_BUF_LEN        64 /* enough size for 64 bit integers */
+    static char counter_buf[USN_COUNTER_BUF_LEN] = {0};
+    char *usn_init_str = NULL;
+    long long usn_init;
+    char *endptr = NULL;
+    struct berval usn_berval = {0};
+
+    if (NULL == usn_value) {
+        return 1;
+    }
+    *usn_value = NULL;
+    /* 
+     * Check if entryusn plugin is enabled.  
+     * If yes, get entryusn to set depending upon nsslapd-entryusn-import-init
+     */
+    if (!plugin_enabled("USN", (void *)plugin_get_default_component_id())) {
+        return 1;
+    }
+    /* get the import_init config param */
+    usn_init_str = config_get_entryusn_import_init();
+    if (usn_init_str) {
+        /* nsslapd-entryusn-import-init has a value */
+        usn_init = strtoll(usn_init_str, &endptr, 10);
+        if (errno || (0 == usn_init && endptr == usn_init_str)) {
+            ldbm_instance *inst = job->inst;
+            backend *be = inst->inst_be;
+            /* import_init value is not digit.
+             * Use the counter which stores the old DB's
+             * next entryusn. */
+            PR_snprintf(counter_buf, USN_COUNTER_BUF_LEN,
+                        "%" NSPRI64 "d",
+                        slapi_counter_get_value(be->be_usn_counter));
+        } else {
+            /* import_init value is digit.
+             * Initialize the entryusn values with the digit */
+            PR_snprintf(counter_buf, USN_COUNTER_BUF_LEN, "%s", usn_init_str);
+        }
+        slapi_ch_free_string(&usn_init_str);
+    } else {
+        /* nsslapd-entryusn-import-init is not defined */
+         /* Initialize to 0 by default */
+        PR_snprintf(counter_buf, USN_COUNTER_BUF_LEN, "0");
+    }
+    usn_berval.bv_val = counter_buf;
+    usn_berval.bv_len = strlen(usn_berval.bv_val);
+    *usn_value = slapi_value_new_berval(&usn_berval);
+    return 0;
+}

ldap/servers/slapd/back-ldbm/import.h

@@ -156,6 +156,7 @@ typedef struct {
     PRCondVar *wire_cv;         /* ... and ordering the startup */
     PRThread *main_thread;      /* for FRI: import_main() thread id */
 	int encrypt;
+	Slapi_Value *usn_value;     /* entryusn for import */
 } ImportJob;
 
 #define FLAG_INDEX_ATTRS	0x01	/* should we index the attributes? */

ldap/servers/slapd/back-ldbm/ldbm_usn.c

@@ -142,7 +142,8 @@ usn_get_last_usn(Slapi_Backend *be, PRUint64 *last_usn)
     if (0 != rc) {
         /* entryusn.db# is missing; it would be the first time. */
         slapi_log_error(SLAPI_LOG_FATAL, "usn_get_last_usn", 
-                        "failed to open the entryusn index: %d", rc);
+                        "WARNING: failed to open the entryusn index: %d; "
+                        "Creating it...\n", rc);
         goto bail;
     }
 

ldap/servers/slapd/back-ldbm/ldif2ldbm.c

@@ -682,6 +682,28 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb )
         /* from the command line, libdb needs to be started up */
         ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off");
 
+        /* If USN plugin is enabled, 
+         * initialize the USN counter to get the next USN */
+        if (plugin_enabled("USN", li->li_identity)) {
+            /* close immediately; no need to run db threads */
+            int rc = dblayer_start(li,
+                                 DBLAYER_NORMAL_MODE|DBLAYER_NO_DBTHREADS_MODE);
+            if (rc) {
+                LDAPDebug2Args(LDAP_DEBUG_ANY,
+                    "ldbm_back_ldif2ldbm: dblayer_start failed! %s (%d)\n",
+                    dblayer_strerror(rc), rc);
+                goto fail;
+            }
+            /* initialize the USN counter */
+            ldbm_usn_init(li);
+            rc = dblayer_close(li, DBLAYER_NORMAL_MODE);
+            if (rc) {
+                LDAPDebug2Args(LDAP_DEBUG_ANY,
+                    "ldbm_back_ldif2ldbm: dblayer_close failed! %s (%d)\n",
+                    dblayer_strerror(rc), rc);
+            }
+        }
+
         if (0 != (ret = dblayer_start(li, DBLAYER_IMPORT_MODE)) ) {
             if (LDBM_OS_ERR_IS_DISKFULL(ret)) {
                 LDAPDebug(LDAP_DEBUG_ANY, "ERROR: Failed to init database.  "

ldap/servers/slapd/libglobs.c

@@ -630,6 +630,10 @@ static struct config_get_and_set {
 		NULL, 0,
 		(void**)&global_slapdFrontendConfig.entryusn_global, CONFIG_ON_OFF,
 		(ConfigGetFunc)config_get_entryusn_global},
+	{CONFIG_ENTRYUSN_IMPORT_INITVAL, config_set_entryusn_import_init,
+		NULL, 0,
+		(void**)&global_slapdFrontendConfig.entryusn_import_init,
+		CONFIG_STRING, (ConfigGetFunc)config_get_entryusn_import_init},
 	{CONFIG_ALLOWED_TO_DELETE_ATTRIBUTE, config_set_allowed_to_delete_attrs,
 		NULL, 0,
 		(void**)&global_slapdFrontendConfig.allowed_to_delete_attrs,
@@ -1012,6 +1016,7 @@ FrontendConfig_init () {
   cfg->auditlog_exptimeunit = slapi_ch_strdup("month");
 
   cfg->entryusn_global = LDAP_OFF; 
+  cfg->entryusn_import_init = slapi_ch_strdup("0"); 
   cfg->allowed_to_delete_attrs = slapi_ch_strdup("nsslapd-listenhost nsslapd-securelistenhost");
 
 #ifdef MEMPOOL_EXPERIMENTAL
@@ -5584,6 +5589,35 @@ config_set_entryusn_global( const char *attrname, char *value,
     return retVal;
 }
 
+char *
+config_get_entryusn_import_init(void)
+{
+    char *retVal;
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+    CFG_LOCK_READ(slapdFrontendConfig);
+    retVal = slapi_ch_strdup(slapdFrontendConfig->entryusn_import_init);
+    CFG_UNLOCK_READ(slapdFrontendConfig);
+
+    return retVal;
+}
+
+
+int
+config_set_entryusn_import_init( const char *attrname, char *value,
+                                 char *errorbuf, int apply )
+{
+    int retVal = LDAP_SUCCESS;
+    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
+
+    if (apply) {
+        CFG_LOCK_WRITE(slapdFrontendConfig);
+        slapi_ch_free_string(&(slapdFrontendConfig->entryusn_import_init));
+        slapdFrontendConfig->entryusn_import_init = slapi_ch_strdup(value);
+        CFG_UNLOCK_WRITE(slapdFrontendConfig);
+    }
+    return retVal;
+}
+
 char *
 config_get_allowed_to_delete_attrs(void)
 {
@@ -5596,6 +5630,7 @@ config_get_allowed_to_delete_attrs(void)
     return retVal;
 }
 
+
 int
 config_set_allowed_to_delete_attrs( const char *attrname, char *value,
                                     char *errorbuf, int apply )

ldap/servers/slapd/modify.c

@@ -285,8 +285,6 @@ do_modify( Slapi_PBlock *pb )
 			slapi_ch_free((void **)&(mod->mod_type));
 			slapi_ch_free((void **)&mod);
             continue;
-			/* send_ldap_result( pb, LDAP_UNWILLING_TO_PERFORM, NULL, NULL, 0, NULL );
-			goto free_and_return; */
 		}
 		
 		/* check for password change */
@@ -299,6 +297,11 @@ do_modify( Slapi_PBlock *pb )
 		slapi_mods_add_ldapmod (&smods, mod);
 	}
 
+	if (ignored_some_mods && (0 == smods.num_elements)) {
+		send_ldap_result( pb, LDAP_UNWILLING_TO_PERFORM, NULL, NULL, 0, NULL );
+		goto free_and_return;
+	}
+
 	/* check for decoding error */
 	if ( (tag != LBER_END_OF_SEQORSET) && (len != -1) )
 	{

ldap/servers/slapd/proto-slap.h

@@ -373,6 +373,7 @@ int config_set_csnlogging(const char *attrname, char *value, char *errorbuf, int
 int config_set_force_sasl_external(const char *attrname, char *value, char *errorbuf, int apply );
 int config_set_entryusn_global( const char *attrname, char *value, char *errorbuf, int apply );
 int config_set_allowed_to_delete_attrs( const char *attrname, char *value, char *errorbuf, int apply );
+int config_set_entryusn_import_init( const char *attrname, char *value, char *errorbuf, int apply );
 
 
 #if !defined(_WIN32) && !defined(AIX)
@@ -514,6 +515,7 @@ int config_get_system_page_bits();
 int config_get_force_sasl_external();
 int config_get_entryusn_global(void);
 char *config_get_allowed_to_delete_attrs(void);
+char *config_get_entryusn_import_init(void);
 
 int is_abspath(const char *);
 char* rel2abspath( char * );

ldap/servers/slapd/slap.h

@@ -1913,6 +1913,7 @@ typedef struct _slapdEntryPoints {
 #define CONFIG_OUTBOUND_LDAP_IO_TIMEOUT_ATTRIBUTE "nsslapd-outbound-ldap-io-timeout"
 #define CONFIG_FORCE_SASL_EXTERNAL_ATTRIBUTE "nsslapd-force-sasl-external"
 #define CONFIG_ENTRYUSN_GLOBAL	"nsslapd-entryusn-global"
+#define CONFIG_ENTRYUSN_IMPORT_INITVAL	"nsslapd-entryusn-import-initval"
 #define CONFIG_ALLOWED_TO_DELETE_ATTRIBUTE	"nsslapd-allowed-to-delete-attrs"
 
 #ifdef MEMPOOL_EXPERIMENTAL
@@ -2131,7 +2132,8 @@ typedef struct _slapdFrontendConfig {
 #endif /* MEMPOOL_EXPERIMENTAL */
   int force_sasl_external;      /* force SIMPLE bind to be SASL/EXTERNAL if client cert credentials were supplied */
   int entryusn_global;          /* Entry USN: Use global counter */
-  char **allowed_to_delete_attrs;/* charray of config attrs allowed to delete */
+  char *allowed_to_delete_attrs;/* list of config attrs allowed to delete */
+  char *entryusn_import_init;   /* Entry USN: determine the initital value of import */
 } slapdFrontendConfig_t;
 
 /* possible values for slapdFrontendConfig_t.schemareplace */