Ticket - 49562 integrate changelog database to main database

Bug description:
	PHASE 2 of backend redesign:
	http://www.port389.org/docs/389ds/design/integrate-changelog-database-and-backend-database.html
        Mainly changelog managed its own access to the database and it uses a global
        config entry (cn=changelog5,cn=config) not related to the backend/replica.

Fix description:
	The fix is described in the design.
        Plus:
         - use-after-free (remove+add replica, set)
         - various leaks (triggered with CI tests fixup-tombstone, cascading)
	 - Plus some changes in the CI tests

https://pagure.io/389-ds-base/issue/49562

Reviewed by: Mark Reynolds, William Brown, Thierry Bordaz

dirsrvtests/tests/suites/password/regression_test.py

@@ -8,11 +8,11 @@
 import pytest
 import time
 from lib389._constants import PASSWORD, DN_DM, DEFAULT_SUFFIX
-from lib389._constants import SUFFIX, PASSWORD, DN_DM, DN_CONFIG, PLUGIN_RETRO_CHANGELOG, DEFAULT_SUFFIX, DEFAULT_CHANGELOG_DB
+from lib389._constants import SUFFIX, PASSWORD, DN_DM, DN_CONFIG, PLUGIN_RETRO_CHANGELOG, DEFAULT_SUFFIX, DEFAULT_CHANGELOG_DB, DEFAULT_BENAME
 from lib389 import Entry
 from lib389.topologies import topology_m1 as topo_master
 from lib389.idm.user import UserAccounts
-from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer
+from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer, ds_supports_new_changelog
 from lib389.topologies import topology_st as topo
 from lib389.idm.organizationalunit import OrganizationalUnits
 
@@ -43,13 +43,17 @@ def _check_unhashed_userpw(inst, user_dn, is_present=False):
     """Check if unhashed#user#password attribute is present or not in the changelog"""
     unhashed_pwd_attribute = 'unhashed#user#password'
 
-    changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB)
-    for dbfile in os.listdir(changelog_dbdir):
-        if dbfile.endswith('.db'):
-            changelog_dbfile = os.path.join(changelog_dbdir, dbfile)
-            log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile))
-    log.info('Running dbscan -f to check {} attr'.format(unhashed_pwd_attribute))
-    dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile)
+    if ds_supports_new_changelog():
+        dbscanOut = inst.dbscan(DEFAULT_BENAME, 'changelog')
+    else:
+        changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB)
+        for dbfile in os.listdir(changelog_dbdir):
+            if dbfile.endswith('.db'):
+                changelog_dbfile = os.path.join(changelog_dbdir, dbfile)
+                log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile))
+        log.info('Running dbscan -f to check {} attr'.format(unhashed_pwd_attribute))
+        dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile)
+
     for entry in dbscanOut.split(b'dbid: '):
         if ensure_bytes('operation: modify') in entry and ensure_bytes(user_dn) in entry and ensure_bytes('userPassword') in entry:
             if is_present:

dirsrvtests/tests/suites/replication/changelog_test.py

@@ -22,12 +22,16 @@ from lib389.plugins import RetroChangelogPlugin
 from lib389.dseldif import DSEldif
 from lib389.tasks import *
 from lib389.utils import *
+from lib389.utils import ldap, os, logging, ensure_bytes, ds_is_newer, ds_supports_new_changelog
 
 pytestmark = pytest.mark.tier1
 
 TEST_ENTRY_NAME = 'replusr'
 NEW_RDN_NAME = 'cl5usr'
-CHANGELOG = 'cn=changelog5,cn=config'
+if ds_supports_new_changelog():
+    CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
+else:
+    CHANGELOG = 'cn=changelog5,cn=config'
 RETROCHANGELOG = 'cn=Retro Changelog Plugin,cn=plugins,cn=config'
 MAXAGE = 'nsslapd-changelogmaxage'
 TRIMINTERVAL = 'nsslapd-changelogtrim-interval'
@@ -73,12 +77,17 @@ def _create_changelog_dump(topo):
     """Dump changelog using nss5task and check if ldap operations are logged"""
 
     log.info('Dump changelog using nss5task and check if ldap operations are logged')
-    changelog_dir = topo.ms['master1'].get_changelog_dir()
+    if ds_supports_new_changelog():
+        changelog_dir = topo.ms['master1'].get_ldif_dir()
+        changelog_end = '_cl.ldif'
+    else:
+        changelog_dir = topo.ms['master1'].get_changelog_dir()
+        changelog_end = '.ldif'
     replicas = Replicas(topo.ms["master1"])
     replica = replicas.get(DEFAULT_SUFFIX)
     log.info('Remove ldif files, if present in: {}'.format(changelog_dir))
     for files in os.listdir(changelog_dir):
-        if files.endswith('.ldif'):
+        if files.endswith(changelog_end):
             changelog_file = os.path.join(changelog_dir, files)
             try:
                 os.remove(changelog_file)
@@ -94,7 +103,7 @@ def _create_changelog_dump(topo):
 
     log.info('Check if changelog ldif file exist in: {}'.format(changelog_dir))
     for files in os.listdir(changelog_dir):
-        if files.endswith('.ldif'):
+        if files.endswith(changelog_end):
             changelog_ldif = os.path.join(changelog_dir, files)
             log.info('Changelog ldif file exist: {}'.format(changelog_ldif))
             return changelog_ldif
@@ -129,22 +138,23 @@ def get_ldap_error_msg(e, type):
 
 @pytest.fixture(scope="module")
 def changelog_init(topo):
-    """Initialize the test environment by changing log dir and
-    enabling cn=Retro Changelog Plugin,cn=plugins,cn=config
-     """
+    """ changlog dir is not configuarable, just
+    enable cn=Retro Changelog Plugin,cn=plugins,cn=config
+    """
     log.info('Testing Ticket 47669 - Test duration syntax in the changelogs')
 
     # bind as directory manager
     topo.ms["master1"].log.info("Bind as %s" % DN_DM)
     topo.ms["master1"].simple_bind_s(DN_DM, PASSWORD)
 
-    try:
-        changelogdir = os.path.join(os.path.dirname(topo.ms["master1"].dbdir), 'changelog')
-        topo.ms["master1"].modify_s(CHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-changelogdir',
-                                                                    ensure_bytes(changelogdir))])
-    except ldap.LDAPError as e:
-        log.error('Failed to modify ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc')))
-        assert False
+    if not ds_supports_new_changelog():
+        try:
+            changelogdir = os.path.join(os.path.dirname(topo.ms["master1"].dbdir), 'changelog')
+            topo.ms["master1"].modify_s(CHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-changelogdir',
+                                                                       ensure_bytes(changelogdir))])
+        except ldap.LDAPError as e:
+            log.error('Failed to modify ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc')))
+            assert False
 
     try:
         topo.ms["master1"].modify_s(RETROCHANGELOG, [(ldap.MOD_REPLACE, 'nsslapd-pluginEnabled', b'on')])
@@ -204,7 +214,10 @@ def remove_ldif_files_from_changelogdir(topo, extension):
     """
     Remove existing ldif files from changelog dir
     """
-    changelog_dir = topo.ms['master1'].get_changelog_dir()
+    if ds_supports_new_changelog():
+        changelog_dir = topo.ms['master1'].get_ldif_dir()
+    else:
+        changelog_dir = topo.ms['master1'].get_changelog_dir()
 
     log.info('Remove %s files, if present in: %s' % (extension, changelog_dir))
     for files in os.listdir(changelog_dir):
@@ -220,6 +233,7 @@ def remove_ldif_files_from_changelogdir(topo, extension):
 
                 
 @pytest.mark.xfail(ds_is_older('1.3.10.1', '1.4.3'), reason="bug bz1685059")
[email protected](reason="does not work for prefix builds")
 @pytest.mark.bz1685059
 @pytest.mark.ds50498
 @pytest.mark.bz1769296
@@ -351,7 +365,10 @@ def test_dsconf_dump_changelog_files_removed(topo):
         10. .ldif.done generated files are present in the changelog dir
      """
 
-    changelog_dir = topo.ms['master1'].get_changelog_dir()
+    if ds_supports_new_changelog():
+        changelog_dir = topo.ms['master1'].get_ldif_dir()
+    else:
+        changelog_dir = topo.ms['master1'].get_changelog_dir()
     instance = topo.ms['master1']
     instance_url = 'ldap://%s:%s' % (HOST_MASTER_1, PORT_MASTER_1)
 
@@ -466,7 +483,10 @@ def test_verify_changelog_online_backup(topo):
         log.fatal('test_changelog5: Online backup failed')
         assert False
 
-    backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB)
+    if ds_supports_new_changelog():
+        backup_checkdir = os.path.join(backup_dir, DEFAULT_BENAME, 'changelog.db')
+    else:
+        backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB)
     if os.path.exists(backup_checkdir):
         log.info('Database backup is created successfully')
     else:
@@ -524,7 +544,10 @@ def test_verify_changelog_offline_backup(topo):
         assert False
     topo.ms['master1'].start()
 
-    backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB)
+    if ds_supports_new_changelog():
+        backup_checkdir = os.path.join(backup_dir, DEFAULT_BENAME, 'changelog.db')
+    else:
+        backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup', DEFAULT_CHANGELOG_DB)
     if os.path.exists(backup_checkdir):
         log.info('Database backup is created successfully')
     else:
@@ -603,6 +626,7 @@ def test_ticket47669_changelog_triminterval(topo, changelog_init):
 
 
 @pytest.mark.ds47669
[email protected](ds_supports_new_changelog(), reason="changelog compaction is done by the backend itself, with id2entry as well, nsslapd-changelogcompactdb-interval is no longer supported")
 def test_changelog_compactdbinterval(topo, changelog_init):
     """Check nsslapd-changelog compactdbinterval values
 

dirsrvtests/tests/suites/replication/changelog_trimming_test.py

@@ -8,6 +8,7 @@ from lib389.properties import *
 from lib389.topologies import topology_m1 as topo
 from lib389.replica import Changelog5
 from lib389.idm.domain import Domain
+from lib389.utils import ensure_bytes, ds_supports_new_changelog
 
 pytestmark = pytest.mark.tier1
 
@@ -18,6 +19,10 @@ else:
     logging.getLogger(__name__).setLevel(logging.INFO)
 log = logging.getLogger(__name__)
 
+CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
+MAXAGE = 'nsslapd-changelogmaxage'
+MAXENTRIES = 'nsslapd-changelogmaxentries'
+TRIMINTERVAL = 'nsslapd-changelogtrim-interval'
 
 def do_mods(master, num):
     """Perform a num of mods on the default suffix
@@ -26,6 +31,16 @@ def do_mods(master, num):
     for i in range(num):
         domain.replace('description', 'change %s' % i)
 
+def set_value(master, attr, val):
+    """
+    Helper function to add/replace attr: val and check the added value
+    """
+    try:
+        master.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, attr, ensure_bytes(val))])
+    except ldap.LDAPError as e:
+        log.error('Failed to add ' + attr + ': ' + val + ' to ' + plugin + ': error {}'.format(get_ldap_error_msg(e,'desc')))
+        assert False
+
 @pytest.fixture(scope="module")
 def setup_max_entries(topo, request):
     """Configure logging and changelog max entries
@@ -34,9 +49,12 @@ def setup_max_entries(topo, request):
 
     master.config.loglevel((ErrorLog.REPLICA,), 'error')
 
-    cl = Changelog5(master)
-    cl.set_max_entries('2')
-    cl.set_trim_interval('300')
+    if ds_supports_new_changelog():
+        set_value(master, MAXENTRIES, '2')
+        set_value(master, TRIMINTERVAL, '300')
+    else:
+        cl = Changelog5(master)
+        cl.set_trim_interval('300')
 
 @pytest.fixture(scope="module")
 def setup_max_age(topo, request):
@@ -45,9 +63,13 @@ def setup_max_age(topo, request):
     master = topo.ms["master1"]
     master.config.loglevel((ErrorLog.REPLICA,), 'error')
 
-    cl = Changelog5(master)
-    cl.set_max_age('5')
-    cl.set_trim_interval('300')
+    if ds_supports_new_changelog():
+        set_value(master, MAXAGE, '5')
+        set_value(master, TRIMINTERVAL, '300')
+    else:
+        cl = Changelog5(master)
+        cl.set_max_age('5')
+        cl.set_trim_interval('300')
 
 def test_max_age(topo, setup_max_age):
     """Test changing the trimming interval works with max age
@@ -68,7 +90,8 @@ def test_max_age(topo, setup_max_age):
     log.info("Testing changelog triming interval with max age...")
 
     master = topo.ms["master1"]
-    cl = Changelog5(master)
+    if not ds_supports_new_changelog():
+        cl = Changelog5(master)
 
     # Do mods to build if cl entries
     do_mods(master, 10)
@@ -78,7 +101,10 @@ def test_max_age(topo, setup_max_age):
         log.fatal('Trimming event unexpectedly occurred')
         assert False
 
-    cl.set_trim_interval('5')
+    if ds_supports_new_changelog():
+        set_value(master, TRIMINTERVAL, '5')
+    else:
+        cl.set_trim_interval('5')
 
     time.sleep(6)  # Trimming should have occured
 
@@ -106,7 +132,8 @@ def test_max_entries(topo, setup_max_entries):
 
     log.info("Testing changelog triming interval with max entries...")
     master = topo.ms["master1"]
-    cl = Changelog5(master)
+    if not ds_supports_new_changelog():
+        cl = Changelog5(master)
 
     # reset errors log
     master.deleteErrorLogs()
@@ -118,7 +145,10 @@ def test_max_entries(topo, setup_max_entries):
         log.fatal('Trimming event unexpectedly occurred')
         assert False
 
-    cl.set_trim_interval('5')
+    if ds_supports_new_changelog():
+        set_value(master, TRIMINTERVAL, '5')
+    else:
+        cl.set_trim_interval('5')
 
     time.sleep(6)  # Trimming should have occured
 

dirsrvtests/tests/suites/replication/cleanallruv_test.py

@@ -82,6 +82,7 @@ def check_ruvs(msg, topology_m4, m4rid):
         clean = False
         replicas = Replicas(inst)
         replica = replicas.get(DEFAULT_SUFFIX)
+        log.info('check_ruvs for replica %s:%s (suffix:rid)' % (replica.get_suffix(), replica.get_rid()))
 
         count = 0
         while not clean and count < 20:
@@ -582,13 +583,15 @@ def test_stress_clean(topology_m4, m4rid):
     ldbm_config = LDBMConfig(topology_m4.ms["master4"])
 
     # Put all the masters under load
-    m1_add_users = AddUsers(topology_m4.ms["master1"], 2000)
+    # not too high load else it takes a long time to converge and
+    # the test result becomes instable
+    m1_add_users = AddUsers(topology_m4.ms["master1"], 500)
     m1_add_users.start()
-    m2_add_users = AddUsers(topology_m4.ms["master2"], 2000)
+    m2_add_users = AddUsers(topology_m4.ms["master2"], 500)
     m2_add_users.start()
-    m3_add_users = AddUsers(topology_m4.ms["master3"], 2000)
+    m3_add_users = AddUsers(topology_m4.ms["master3"], 500)
     m3_add_users.start()
-    m4_add_users = AddUsers(topology_m4.ms["master4"], 2000)
+    m4_add_users = AddUsers(topology_m4.ms["master4"], 500)
     m4_add_users.start()
 
     # Allow sometime to get replication flowing in all directions

dirsrvtests/tests/suites/replication/encryption_cl5_test.py

@@ -8,7 +8,8 @@
 #
 import logging
 import pytest
-from lib389.utils import ensure_bytes
+import pdb
+from lib389.utils import ensure_bytes, ds_supports_new_changelog
 from lib389.replica import ReplicationManager
 from lib389.dseldif import DSEldif
 from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES
@@ -45,26 +46,34 @@ def _enable_changelog_encryption(inst, encrypt_algorithm):
     dse_ldif = DSEldif(inst)
     log.info('Configuring changelog encryption:{} for: {}'.format(inst.serverid, encrypt_algorithm))
     inst.stop()
-    dse_ldif.replace(DN_CHANGELOG, 'nsslapd-encryptionalgorithm', encrypt_algorithm)
-    if dse_ldif.get(DN_CHANGELOG, 'nsSymmetricKey'):
-        dse_ldif.delete(DN_CHANGELOG, 'nsSymmetricKey')
+    if ds_supports_new_changelog():
+        changelog = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
+    else:
+        changelog = DN_CHANGELOG
+
+    dse_ldif.replace(changelog, 'nsslapd-encryptionalgorithm', encrypt_algorithm)
+    if dse_ldif.get(changelog, 'nsSymmetricKey'):
+        dse_ldif.delete(changelog, 'nsSymmetricKey')
     inst.start()
 
 
 def _check_unhashed_userpw_encrypted(inst, change_type, user_dn, user_pw, is_encrypted):
     """Check if unhashed#user#password attribute value is encrypted or not"""
 
-    changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB)
-    for dbfile in os.listdir(changelog_dbdir):
-        if dbfile.endswith('.db'):
-            changelog_dbfile = os.path.join(changelog_dbdir, dbfile)
-            log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile))
-    log.info('Running dbscan -f to check {} attr'.format(ATTRIBUTE))
-    dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile)
+    if ds_supports_new_changelog():
+        dbscanOut = inst.dbscan(DEFAULT_BENAME, 'changelog')
+    else:
+        changelog_dbdir = os.path.join(os.path.dirname(inst.dbdir), DEFAULT_CHANGELOG_DB)
+        for dbfile in os.listdir(changelog_dbdir):
+            if dbfile.endswith('.db'):
+                changelog_dbfile = os.path.join(changelog_dbdir, dbfile)
+                log.info('Changelog dbfile file exist: {}'.format(changelog_dbfile))
+        log.info('Running dbscan -f to check {} attr'.format(ATTRIBUTE))
+        dbscanOut = inst.dbscan(DEFAULT_CHANGELOG_DB, changelog_dbfile)
     count = 0
     for entry in dbscanOut.split(b'dbid: '):
         if ensure_bytes('operation: {}'.format(change_type)) in entry and\
-           ensure_bytes(ATTRIBUTE) in entry and ensure_bytes(user_dn) in entry:
+           ensure_bytes(ATTRIBUTE) in entry and ensure_bytes(user_dn.lower()) in entry.lower():
             count += 1
             user_pw_attr = ensure_bytes('{}: {}'.format(ATTRIBUTE, user_pw))
             if is_encrypted:
@@ -112,7 +121,16 @@ def test_algorithm_unhashed(topology_with_tls, encryption):
     _enable_changelog_encryption(m1, encryption)
 
     for inst1, inst2 in ((m1, m2), (m2, m1)):
-        user_props = TEST_USER_PROPERTIES.copy()
+        # need to create a user specific to the encryption
+        # else the two runs will hit the same user
+        user_props={
+                    'uid': 'testuser_%s' % encryption,
+                    'cn' : 'testuser_%s' % encryption,
+                    'sn' : 'user',
+                    'uidNumber' : '1000',
+                    'gidNumber' : '1000',
+                    'homeDirectory' : '/home/testuser_%s' % encryption
+                }
         user_props["userPassword"] = PASSWORD
         users = UserAccounts(inst1, DEFAULT_SUFFIX)
         tuser = users.create(properties=user_props)

dirsrvtests/tests/suites/replication/regression_test.py

@@ -29,6 +29,7 @@ pytestmark = pytest.mark.tier1
 NEW_SUFFIX_NAME = 'test_repl'
 NEW_SUFFIX = 'o={}'.format(NEW_SUFFIX_NAME)
 NEW_BACKEND = 'repl_base'
+CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
 MAXAGE_ATTR = 'nsslapd-changelogmaxage'
 MAXAGE_STR = '30'
 TRIMINTERVAL_STR = '5'
@@ -41,6 +42,16 @@ else:
     logging.getLogger(__name__).setLevel(logging.INFO)
 log = logging.getLogger(__name__)
 
[email protected](scope="module")
+def set_value(master, attr, val):
+    """
+    Helper function to add/replace attr: val and check the added value
+    """
+    try:
+        master.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, attr, ensure_bytes(val))])
+    except ldap.LDAPError as e:
+        log.error('Failed to add ' + attr + ': ' + val + ' to ' + plugin + ': error {}'.format(get_ldap_error_msg(e,'desc')))
+        assert False
 
 def find_start_location(file, no):
     log_pattern = re.compile("slapd_daemon - slapd started.")
@@ -675,13 +686,28 @@ def test_cleanallruv_repl(topo_m3):
     m1_m3 = M1.agreement.list(suffix=SUFFIX, consumer_host=M3.host, consumer_port=M3.port)
     m3_m1 = M3.agreement.list(suffix=SUFFIX, consumer_host=M1.host, consumer_port=M1.port)
 
-    log.info("Get the changelog enteries for M1 and M2")
-    changelog_m1 = Changelog5(M1)
-    changelog_m2 = Changelog5(M2)
-
     log.info("Modify nsslapd-changelogmaxage=30 and nsslapd-changelogtrim-interval=5 for M1 and M2")
-    changelog_m1.set_max_age(MAXAGE_STR)
-    changelog_m1.set_trim_interval(TRIMINTERVAL_STR)
+    if ds_supports_new_changelog():
+        CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
+
+        #set_value(M1, MAXAGE_ATTR, MAXAGE_STR)
+        try:
+            M1.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, MAXAGE_ATTR, ensure_bytes(MAXAGE_STR))])
+        except ldap.LDAPError as e:
+            log.error('Failed to add ' + MAXAGE_ATTR, + ': ' + MAXAGE_STR + ' to ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc')))
+            assert False
+
+        #set_value(M2, TRIMINTERVAL, TRIMINTERVAL_STR)
+        try:
+            M2.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, TRIMINTERVAL, ensure_bytes(TRIMINTERVAL_STR))])
+        except ldap.LDAPError as e:
+            log.error('Failed to add ' + TRIMINTERVAL, + ': ' + TRIMINTERVAL_STR + ' to ' + CHANGELOG + ': error {}'.format(get_ldap_error_msg(e,'desc')))
+            assert False
+    else:
+        log.info("Get the changelog enteries for M1 and M2")
+        changelog_m1 = Changelog5(M1)
+        changelog_m1.set_max_age(MAXAGE_STR)
+        changelog_m1.set_trim_interval(TRIMINTERVAL_STR)
 
     log.info("Add test users to 3 masters")
     users_m1 = UserAccounts(M1, DEFAULT_SUFFIX)

ldap/admin/src/scripts/DSCreate.pm.in

@@ -481,7 +481,7 @@ sub makeOtherConfigFiles {
         if ($!) {
             return ('error_copying_file', $src, $dest, $!);
         }
-        if (@errs = changeOwnerMode($inf, 4, $dest)) {
+        if (@errs = changeOwnerMode($inf, 6, $dest)) {
             return @errs;
         }
     }

ldap/admin/src/scripts/db2ldif.in

@@ -25,6 +25,7 @@ usage()
     echo "        -x                - Suffix to exclude"
     echo "        -a outputfile     - Name of the exported LDIF file"
     echo "        -r                - Include replication data"
+    echo "        -R                - Include changelog data"
     echo "        -E                - Decrypt attributes"
     echo "        -u                - Do not export the nsUniqueId attribute"
     echo "        -U                - Do not wrap long lines"
@@ -100,7 +101,7 @@ then
     exit 1
 fi
 
-while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qVc:" flag
+while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qRVc:" flag
 do
     case $flag in
         h) usage
@@ -119,6 +120,7 @@ do
         S) args=$args" -S";;
         v) args=$args" -v";;
         r) args=$args" -r";;
+        R) args=$args" -R";;        
         C) args=$args" -C";;
         u) args=$args" -u";;
         U) args=$args" -U";;

ldap/admin/src/scripts/ldif2db.in

@@ -30,6 +30,7 @@ usage()
     echo "        -G name           - Namespace id for name based uniqueid (-g deterministic)"
     echo "        -O                - Do not index the attributes"
     echo "        -E                - Encrypt attributes"
+    echo "        -R                - Import changelog data"
     echo "        -q                - Quiet mode - suppresses output"
     echo "        -V                - Verbose output"
     echo "        -v                - Display version"
@@ -54,7 +55,7 @@ handleopts()
     return 0
 }
 
-while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:EqV" flag
+while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:ERqV" flag
 do
     case $flag in
         h) usage
@@ -71,6 +72,7 @@ do
         t) args=$args" -t \"$OPTARG\"";;
         D) args=$args" -D \"$OPTARG\"";;
         E) args=$args" -E";;
+        R) args=$args" -R";;
         v) args=$args" -v";;
         N) args=$args" -N";;
         C) args=$args" -C";;

ldap/servers/plugins/replication/cl5.h

@@ -21,17 +21,18 @@
 typedef struct changelog5Config
 {
     char *dir;
-    /* These 2 parameters are needed for changelog trimming. Already present in 5.0 */
+    /* These 3 parameters are needed for changelog trimming. */
     char *maxAge;
     int maxEntries;
-    /* the changelog DB configuration parameters are defined as CL5DBConfig in cl5_api.h */
-    CL5DBConfig dbconfig;
-    char *symmetricKey;
-    long compactInterval;
     long trimInterval;
+    /* configuration of changelog encryption */
+    char *encryptionAlgorithm;
+    char *symmetricKey;
 } changelog5Config;
 
-/* initializes changelog*/
+/* upgrade changelog*/
+int changelog5_upgrade(void);
+/* initialize changelog*/
 int changelog5_init(void);
 /* cleanups changelog data */
 void changelog5_cleanup(void);
@@ -41,6 +42,11 @@ int changelog5_config_init(void);
 void changelog5_config_cleanup(void);
 /* reads changelog configuration */
 int changelog5_read_config(changelog5Config *config);
+/* transforms entry to internal config */
+void changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config);
+/* registeri/unregister functions to handle config changes */
+int changelog5_register_config_callbacks(const char *dn, Replica *replica);
+int changelog5_remove_config_callbacks(const char *dn);
 /* cleanups the content of the config structure */
 void changelog5_config_done(changelog5Config *config);
 /* frees the content and the config structure */

ldap/servers/plugins/replication/cl5_api.h

@@ -36,15 +36,6 @@
 
 /***** Data Structures *****/
 
-/* changelog configuration structure */
-typedef struct cl5dbconfig
-{
-    uint32_t pageSize;           /* page size in bytes */
-    PRInt32 fileMode;          /* file mode */
-    char *encryptionAlgorithm; /* nsslapd-encryptionalgorithm */
-    char *symmetricKey;
-} CL5DBConfig;
-
 /* changelog entry format */
 typedef struct cl5entry
 {
@@ -81,6 +72,9 @@ typedef struct cl5entry
 /* data structure that allows iteration through changelog */
 typedef struct cl5replayiterator CL5ReplayIterator;
 
+/* database information for th echangelog */
+typedef struct cl5DBFileHandle cldb_Handle;
+
 /* changelog state */
 typedef enum {
     CL5_STATE_NONE,    /* changelog has not been initialized */
@@ -137,9 +131,6 @@ void cl5Cleanup(void);
    Description:    opens changelog ; must be called after changelog is
                 initialized using cl5Init. It is thread safe and the second
                 call is ignored.
-   Parameters:  dir - changelog dir
-                config - db configuration parameters; currently not used
-                openMode - open mode
    Return:        CL5_SUCCESS if successful;
                 CL5_BAD_DATA if invalid directory is passed;
                 CL5_BAD_DBVERSION if dbversion file is missing or has unexpected data
@@ -147,7 +138,7 @@ void cl5Cleanup(void);
                 CL5_MEMORY_ERROR if memory allocation fails;
                 CL5_DB_ERROR if db initialization or open fails.
  */
-int cl5Open(const char *dir, const CL5DBConfig *config);
+int cl5Open(void);
 
 /* Name:        cl5Close
    Description:    closes changelog and cleanups changelog module; waits until
@@ -159,21 +150,11 @@ int cl5Open(const char *dir, const CL5DBConfig *config);
  */
 int cl5Close(void);
 
-/* Name:        cl5Delete
-   Description:    removes changelog
-   Parameters:  dir - changelog directory
-   Return:        CL5_SUCCESS if successful;
-                CL5_BAD_STATE if the changelog is not in closed state;
-                CL5_BAD_DATA if invalid directory supplied
-                CL5_SYSTEM_ERROR if NSPR call fails
- */
-int cl5Delete(const char *dir);
-
-/* Name:        cl5DeleteDBSync
-   Description: The same as cl5DeleteDB except the function does not return
-                until the file is removed.
+/* Name:        cldb_RemoveReplicaDB
+   Description: Clear the cldb information from the replica 
+                and delete the database file
 */
-int cl5DeleteDBSync(Replica *replica);
+int cldb_RemoveReplicaDB(Replica *replica);
 
 /* Name:        cl5GetUpperBoundRUV
    Description: retrieves vector that represent the upper bound of changes
@@ -202,7 +183,7 @@ int cl5GetUpperBoundRUV(Replica *r, RUV **ruv);
                 CL5_SYSTEM_ERROR if NSPR call fails;
                 CL5_MEMORY_ERROR if memory allocation fails.
  */
-int cl5ExportLDIF(const char *ldifFile, Replica **replicas);
+int cl5ExportLDIF(const char *ldifFile, Replica *replica);
 
 /* Name:        cl5ImportLDIF
    Description:    imports ldif file into changelog; changelog must be in the closed state
@@ -217,7 +198,7 @@ int cl5ExportLDIF(const char *ldifFile, Replica **replicas);
                 CL5_SYSTEM_ERROR if NSPR call fails;
                 CL5_MEMORY_ERROR if memory allocation fails.
  */
-int cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica **replicas);
+int cl5ImportLDIF(const char *clDir, const char *ldifFile, Replica *replica);
 
 /* Name:        cl5GetState
    Description:    returns database state
@@ -231,12 +212,11 @@ int cl5GetState(void);
    Description:    sets changelog trimming parameters
    Parameters:  maxEntries - maximum number of entries in the log;
                 maxAge - maximum entry age;
-                compactInterval - interval to compact changelog db;
                 trimInterval - interval for changelog trimming.
    Return:        CL5_SUCCESS if successful;
                 CL5_BAD_STATE if changelog has not been open
  */
-int cl5ConfigTrimming(int maxEntries, const char *maxAge, int compactInterval, int trimInterval);
+int cl5ConfigTrimming(Replica *replica, int maxEntries, const char *maxAge, int trimInterval);
 
 void cl5DestroyIterator(void *iterator);
 
@@ -248,7 +228,6 @@ void cl5DestroyIterator(void *iterator);
                    replica object since generation can change while operation
                    is in progress (if the data is reloaded). !!!
                 op - operation to write
-                local - this is a non-replicated operation
                 txn - the containing transaction
    Return:        CL5_SUCCESS if function is successful;
                 CL5_BAD_DATA if invalid op is passed;
@@ -256,7 +235,7 @@ void cl5DestroyIterator(void *iterator);
                 CL5_MEMORY_ERROR if memory allocation failed;
                 CL5_DB_ERROR if any other db error occurred;
  */
-int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local, void *txn);
+int cl5WriteOperationTxn(cldb_Handle *cldb, const slapi_operation_parameters *op, void *txn);
 
 /* Name:        cl5WriteOperation
    Description:    writes operation to changelog
@@ -266,14 +245,13 @@ int cl5WriteOperationTxn(const char *repl_name, const char *repl_gen, const slap
                    replica object since generation can change while operation
                    is in progress (if the data is reloaded). !!!
                 op - operation to write
-                local - this is a non-replicated operation
    Return:        CL5_SUCCESS if function is successful;
                 CL5_BAD_DATA if invalid op is passed;
                 CL5_BAD_STATE if db has not been initialized;
                 CL5_MEMORY_ERROR if memory allocation failed;
                 CL5_DB_ERROR if any other db error occurred;
  */
-int cl5WriteOperation(const char *repl_name, const char *repl_gen, const slapi_operation_parameters *op, PRBool local);
+int cl5WriteOperation(cldb_Handle *cldb, const slapi_operation_parameters *op);
 
 /* Name:        cl5CreateReplayIterator
    Description:    creates an iterator that allows to retrieve changes that should
@@ -321,29 +299,13 @@ int cl5GetNextOperationToReplay(CL5ReplayIterator *iterator,
  */
 void cl5DestroyReplayIterator(CL5ReplayIterator **iterator);
 
-/* Name:        cl5DeleteOnClose
-   Description:    marks changelog for deletion when it is closed
-   Parameters:  flag; if flag = 1 then delete else don't
-   Return:        none
- */
-
-void cl5DeleteOnClose(PRBool rm);
-
-/* Name:        cl5GetDir
-   Description:    returns changelog directory; must be freed by the caller;
-   Parameters:  none
+/* Name:        cl5GetLdifDir
+   Description:    returns the default ldif directory; must be freed by the caller;
+   Parameters:  backend used for export/import
    Return:        copy of the directory; caller needs to free the string
  */
 
-char *cl5GetDir(void);
-
-/* Name: cl5Exist
-   Description: checks if a changelog exists in the specified directory
-   Parameters: clDir - directory to check;
-   Return: 1 - if changelog exists; 0 - otherwise
- */
-
-PRBool cl5Exist(const char *clDir);
+char *cl5GetLdifDir(Slapi_Backend *be);
 
 /* Name: cl5GetOperationCount
    Description: returns number of entries in the changelog. The changelog must be
@@ -371,39 +333,24 @@ void cl5_operation_parameters_done(struct slapi_operation_parameters *sop);
 */
 
 int cl5CreateDirIfNeeded(const char *dir);
-int cl5DBData2Entry(const char *data, PRUint32 len, CL5Entry *entry);
+int cl5DBData2Entry(const char *data, PRUint32 len, CL5Entry *entry, void *clcrypt_handle);
 
 PRBool cl5HelperEntry(const char *csnstr, CSN *csn);
 CSN **cl5BuildCSNList(const RUV *consRuv, const RUV *supRuv);
 void cl5DestroyCSNList(CSN ***csns);
 
-int cl5_is_diskfull(void);
-int cl5_diskspace_is_available(void);
+int cl5Export(Slapi_PBlock *pb);
+int cl5Import(Slapi_PBlock *pb);
 
-/* Name: cl5DbDirIsEmpty
-   Description: See if the given cldb directory is empty or doesn't yet exist.
-   Parameters:    dir - Contains the name of the directory.
-   Return:        TRUE - directory does not exist or is empty, is NULL, or is
-                       an empty string
-                FALSE - otherwise
-*/
-int cl5DbDirIsEmpty(const char *dir);
-
-/* Name: cl5WriteRUV
-   Description: Write RUVs into changelog db's.  Called before backup.
-   Parameters:    none
-   Return:        TRUE
-*/
-int cl5WriteRUV(void);
+int cl5NotifyRUVChange(Replica *replica);
 
-/* Name: cl5DeleteRUV
-   Description: Read and delete RUVs from changelog db's.  Called after backup.
-   Parameters:    none
-   Return:        TRUE
-*/
-int cl5DeleteRUV(void);
-void cl5CleanRUV(ReplicaId rid);
+void cl5CleanRUV(ReplicaId rid, Replica *replica);
 void cl5NotifyCleanup(int rid);
 void trigger_cl_purging(cleanruv_purge_data *purge_data);
+int cldb_SetReplicaDB(Replica *replica, void *arg);
+int cldb_UnSetReplicaDB(Replica *replica, void *arg);
+int cldb_StartTrimming(Replica *replica);
+int cldb_StopTrimming(Replica *replica, void *arg);
+int cldb_StopThreads(Replica *replica, void *arg);
 
 #endif

ldap/servers/plugins/replication/cl5_clcache.c

@@ -129,7 +129,6 @@ struct clc_busy_list
 struct clc_pool
 {
     Slapi_RWLock *pl_lock;        /* cl writer and agreements */
-    DB_ENV **pl_dbenv;            /* pointer to DB_ENV for all the changelog files */
     CLC_Busy_List *pl_busy_lists; /* busy buffer lists, one list per changelog file */
     int pl_buffer_cnt_now;        /* total number of buffers */
     int pl_buffer_cnt_min;        /* free a newly returned buffer if _now > _min */
@@ -163,16 +162,12 @@ static void csn_dup_or_init_by_csn(CSN **csn1, CSN *csn2);
  * once and only once when process starts.
  */
 int
-clcache_init(DB_ENV **dbenv)
+clcache_init(void)
 {
     if (_pool) {
         return 0; /* already initialized */
     }
-    if (NULL == dbenv) {
-        return -1;
-    }
     _pool = (struct clc_pool *)slapi_ch_calloc(1, sizeof(struct clc_pool));
-    _pool->pl_dbenv = dbenv;
     _pool->pl_buffer_cnt_min = DEFAULT_CLC_BUFFER_COUNT_MIN;
     _pool->pl_buffer_cnt_max = DEFAULT_CLC_BUFFER_COUNT_MAX;
     _pool->pl_buffer_default_pages = DEFAULT_CLC_BUFFER_COUNT_MAX;
@@ -1140,7 +1135,6 @@ clcache_destroy()
             bl = next;
         }
         _pool->pl_busy_lists = NULL;
-        _pool->pl_dbenv = NULL;
         if (_pool->pl_lock) {
             slapi_rwlock_unlock(_pool->pl_lock);
             slapi_destroy_rwlock(_pool->pl_lock);

ldap/servers/plugins/replication/cl5_clcache.h

@@ -20,7 +20,7 @@
 
 typedef struct clc_buffer CLC_Buffer;
 
-int clcache_init(DB_ENV **dbenv);
+int clcache_init(void);
 void clcache_set_config(void);
 int clcache_get_buffer(CLC_Buffer **buf, DB *db, ReplicaId consumer_rid, const RUV *consumer_ruv, const RUV *local_ruv);
 int clcache_load_buffer(CLC_Buffer *buf, CSN **anchorCSN, int *continue_on_miss);

ldap/servers/plugins/replication/cl5_config.c

@@ -29,6 +29,10 @@
 #define CONFIG_BASE "cn=changelog5,cn=config" /*"cn=changelog,cn=supplier,cn=replication5.0,cn=replication,cn=config"*/
 #define CONFIG_FILTER "(objectclass=*)"
 
+/* the changelog config is now separate for each backend in "cn=changelog,<backend>,cn=ldbm database,cn=plugins,cn=config" */
+#define CL_CONFIG_BASE "cn=ldbm database,cn=plugins,cn=config"
+#define CL_CONFIG_FILTER "cn=changelog"
+
 static Slapi_RWLock *s_configLock; /* guarantees that only on thread at a time
                                 modifies changelog configuration */
 
@@ -37,12 +41,12 @@ static int changelog5_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *
 static int changelog5_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);
 static int changelog5_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);
 static int dont_allow_that(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *e, int *returncode, char *returntext, void *arg);
-
-static void changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config);
+static int cldb_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);
+static int cldb_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);
+static int cldb_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);
 static changelog5Config *changelog5_dup_config(changelog5Config *config);
+
 static void replace_bslash(char *dir);
-static int notify_replica(Replica *r, void *arg);
-static int _is_absolutepath(char *dir);
 
 int
 changelog5_config_init()
@@ -61,6 +65,7 @@ changelog5_config_init()
         return 1;
     }
 
+    /* callbacks to handle attempts to modify the old cn=changelog5 config */
     slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE,
                                    CONFIG_FILTER, changelog5_config_add, NULL);
     slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE,
@@ -70,6 +75,7 @@ changelog5_config_init()
     slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, CONFIG_BASE, LDAP_SCOPE_BASE,
                                    CONFIG_FILTER, changelog5_config_delete, NULL);
 
+
     return 0;
 }
 
@@ -111,11 +117,11 @@ changelog5_read_config(changelog5Config *config)
             changelog5_extract_config(entries[0], config);
         } else {
             memset(config, 0, sizeof(*config));
-            rc = LDAP_SUCCESS;
+            rc = LDAP_NO_SUCH_OBJECT;
         }
     } else {
         memset(config, 0, sizeof(*config));
-        rc = LDAP_SUCCESS;
+        rc = LDAP_NO_SUCH_OBJECT;
     }
 
     slapi_free_search_results_internal(pb);
@@ -124,6 +130,21 @@ changelog5_read_config(changelog5Config *config)
     return rc;
 }
 
+static changelog5Config *
+changelog5_dup_config(changelog5Config *config)
+{
+    changelog5Config *dup = (changelog5Config *)slapi_ch_calloc(1, sizeof(changelog5Config));
+
+    if (config->maxAge)
+        dup->maxAge = slapi_ch_strdup(config->maxAge);
+
+    dup->maxEntries = config->maxEntries;
+    dup->trimInterval = config->trimInterval;
+
+    return dup;
+}
+
+
 void
 changelog5_config_done(changelog5Config *config)
 {
@@ -132,8 +153,7 @@ changelog5_config_done(changelog5Config *config)
         slapi_ch_free_string(&config->maxAge);
         slapi_ch_free_string(&config->dir);
         slapi_ch_free_string(&config->symmetricKey);
-        slapi_ch_free_string(&config->dbconfig.encryptionAlgorithm);
-        slapi_ch_free_string(&config->dbconfig.symmetricKey);
+        slapi_ch_free_string(&config->encryptionAlgorithm);
     }
 }
 
@@ -152,95 +172,15 @@ changelog5_config_add(Slapi_PBlock *pb __attribute__((unused)),
                       char *returntext,
                       void *arg __attribute__((unused)))
 {
-    int rc;
-    changelog5Config config;
-
-    *returncode = LDAP_SUCCESS;
-
-    slapi_rwlock_wrlock(s_configLock);
-
-    /* we already have a configured changelog - don't need to do anything
-       since add operation will fail */
-    if (cl5GetState() == CL5_STATE_OPEN) {
-        *returncode = 1;
-        if (returntext) {
-            strcpy(returntext, "attempt to add changelog when it already exists");
-        }
-
-        slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl,
-                      "changelog5_config_add - Changelog already exist; "
-                      "request ignored\n");
-        goto done;
-    }
-
-    changelog5_extract_config(e, &config);
-    if (config.dir == NULL) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "NULL changelog directory");
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_add - NULL changelog directory\n");
-        goto done;
-    }
-
-    if (!cl5DbDirIsEmpty(config.dir)) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
-                        "The changelog directory [%s] already exists and is not empty.  "
-                        "Please choose a directory that does not exist or is empty.\n",
-                        config.dir);
-        }
-
-        goto done;
-    }
-
-    /* start the changelog */
-    rc = cl5Open(config.dir, &config.dbconfig);
-    if (rc != CL5_SUCCESS) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to start changelog; error - %d", rc);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_add - Failed to start changelog\n");
-        goto done;
-    }
-
-    /* set trimming parameters */
-    rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval);
-    if (rc != CL5_SUCCESS) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to configure changelog trimming; error - %d", rc);
-        }
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_add - Failed to configure changelog trimming\n");
-        goto done;
+    /* we no longer support a separate changelog configuration */
+    slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl,
+                  "changelog5_config_add - Separate changelog no longer supported; "
+                  "use cn=changelog,<backend> instead\n");
+ 
+    if (returntext) {
+        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Changelog configuration is part of the backend configuration");
     }
-
-    /* notify all the replicas that the changelog is configured
-       so that the can log dummy changes if necessary. */
-    replica_enumerate_replicas(notify_replica, NULL);
-
-#ifdef TEST_CL5
-    testChangelog(TEST_ITERATION);
-#endif
-
-done:;
-    slapi_rwlock_unlock(s_configLock);
-    changelog5_config_done(&config);
-    if (*returncode == LDAP_SUCCESS) {
-        if (returntext) {
-            returntext[0] = '\0';
-        }
-
-        return SLAPI_DSE_CALLBACK_OK;
-    }
-
+    *returncode = LDAP_UNWILLING_TO_PERFORM;
     return SLAPI_DSE_CALLBACK_ERROR;
 }
 
@@ -252,60 +192,65 @@ changelog5_config_modify(Slapi_PBlock *pb,
                          char *returntext,
                          void *arg __attribute__((unused)))
 {
-    int rc = 0;
-    LDAPMod **mods;
-    int i;
-    changelog5Config config;
-    changelog5Config *originalConfig = NULL;
-    char *currentDir = NULL;
-
+    /* we no longer support a separate changelog configuration */
+    /* the entry does not exist and the client will be notified
+     */
+    slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl,
+                  "changelog5_config_modify - Separate changelog no longer supported; "
+                  "request ignored\n");
+ 
     *returncode = LDAP_SUCCESS;
+    return SLAPI_DSE_CALLBACK_OK;
+}
 
-    /* changelog must be open before its parameters can be modified */
-    if (cl5GetState() != CL5_STATE_OPEN) {
-        if (returntext) {
-            strcpy(returntext, "changelog is not configured");
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_modify - Changelog is not configured\n");
-        return SLAPI_DSE_CALLBACK_ERROR;
-    }
+static int
+changelog5_config_delete(Slapi_PBlock *pb __attribute__((unused)),
+                         Slapi_Entry *e __attribute__((unused)),
+                         Slapi_Entry *entryAfter __attribute__((unused)),
+                         int *returncode,
+                         char *returntext,
+                         void *arg __attribute__((unused)))
+{
+    /* we no longer support a separate changelog configuration */
+    /* the entry does not exist and the client will be notified
+     */
+    slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl,
+                  "changelog5_config_delete - Separate changelog no longer supported; "
+                  "request ignored\n");
+ 
+    *returncode = LDAP_SUCCESS;
+    return SLAPI_DSE_CALLBACK_OK;
+}
 
-    slapi_rwlock_wrlock(s_configLock);
+static int
+cldb_config_add(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg)
+{
+    return SLAPI_DSE_CALLBACK_OK;
+}
 
-    /* changelog must be open before its parameters can be modified */
-    if (cl5GetState() != CL5_STATE_OPEN) {
-        *returncode = 1;
-        if (returntext) {
-            strcpy(returntext, "changelog is not configured");
-        }
+static int
+cldb_config_modify(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg)
+{
+    int rc = 0;
+    LDAPMod **mods;
+    *returncode = LDAP_SUCCESS;
+    changelog5Config config;
+    changelog5Config *originalConfig = NULL;
+    Replica *replica = (Replica *)arg;
 
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_modify - Changelog is not configured\n");
-        goto done;
-    }
 
-    /*
-     * Extract all the original configuration: This is needed to ensure that the configuration
-     * is trully reloaded. This was not needed before 091401 because the changelog configuration
-     * was always hardcoded (NULL was being passed to cl5Open). Now we need to ensure we pass to
-     * cl5Open the proper configuration...
-     */
     changelog5_extract_config(e, &config);
     originalConfig = changelog5_dup_config(&config);
 
     /* Reset all the attributes that have been potentially modified by the current MODIFY operation */
-    slapi_ch_free_string(&config.dir);
-    config.dir = NULL;
     config.maxEntries = CL5_NUM_IGNORE;
-    config.compactInterval = CL5_NUM_IGNORE;
     slapi_ch_free_string(&config.maxAge);
     config.maxAge = slapi_ch_strdup(CL5_STR_IGNORE);
     config.trimInterval = CL5_NUM_IGNORE;
 
+
     slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods);
-    for (i = 0; mods && mods[i] != NULL; i++) {
+    for (size_t i = 0; mods && mods[i] != NULL; i++) {
         if (mods[i]->mod_op & LDAP_MOD_DELETE) {
             /* We don't support deleting changelog attributes */
         } else if (mods[i]->mod_values == NULL) {
@@ -328,19 +273,7 @@ changelog5_config_modify(Slapi_PBlock *pb,
                 }
 
                 /* replace existing value */
-                if (strcasecmp(config_attr, CONFIG_CHANGELOG_DIR_ATTRIBUTE) == 0) {
-                    if (config_attr_value && config_attr_value[0] != '\0') {
-                        slapi_ch_free_string(&config.dir);
-                        config.dir = slapi_ch_strdup(config_attr_value);
-                        replace_bslash(config.dir);
-                    } else {
-                        *returncode = 1;
-                        if (returntext) {
-                            strcpy(returntext, "null changelog directory");
-                        }
-                        goto done;
-                    }
-                } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE) == 0) {
+                if (strcasecmp(config_attr, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE) == 0) {
                     if (config_attr_value && config_attr_value[0] != '\0') {
                         config.maxEntries = atoi(config_attr_value);
                     } else {
@@ -361,20 +294,6 @@ changelog5_config_modify(Slapi_PBlock *pb,
                         *returncode = LDAP_UNWILLING_TO_PERFORM;
                         goto done;
                     }
-                } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE) == 0) {
-                    if (slapi_is_duration_valid(config_attr_value)) {
-                        config.compactInterval = (long)slapi_parse_duration(config_attr_value);
-                    } else {
-                        if (returntext) {
-                            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
-                                        "%s: invalid value \"%s\", %s must range from 0 to %lld or digit[sSmMhHdD]",
-                                        CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE, config_attr_value,
-                                        CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE,
-                                        (long long int)LONG_MAX);
-                        }
-                        *returncode = LDAP_UNWILLING_TO_PERFORM;
-                        goto done;
-                    }
                 } else if (strcasecmp(config_attr, CONFIG_CHANGELOG_TRIM_ATTRIBUTE) == 0) {
                     if (slapi_is_duration_valid(config_attr_value)) {
                         config.trimInterval = (long)slapi_parse_duration(config_attr_value);
@@ -410,8 +329,6 @@ changelog5_config_modify(Slapi_PBlock *pb,
      * except config.dir */
     if (config.maxEntries == CL5_NUM_IGNORE)
         config.maxEntries = originalConfig->maxEntries;
-    if (config.compactInterval == CL5_NUM_IGNORE)
-        config.compactInterval = originalConfig->compactInterval;
     if (config.trimInterval == CL5_NUM_IGNORE)
         config.trimInterval = originalConfig->trimInterval;
     if (strcmp(config.maxAge, CL5_STR_IGNORE) == 0) {
@@ -420,106 +337,11 @@ changelog5_config_modify(Slapi_PBlock *pb,
             config.maxAge = slapi_ch_strdup(originalConfig->maxAge);
     }
 
-    /* attempt to change chagelog dir */
-    if (config.dir) {
-        currentDir = cl5GetDir();
-        if (currentDir == NULL) {
-            /* something is wrong: we should never be here */
-            *returncode = 1;
-            if (returntext) {
-                strcpy(returntext, "internal failure");
-            }
-
-            goto done;
-        }
-
-        if (strcmp(currentDir, config.dir) != 0) {
-            if (!cl5DbDirIsEmpty(config.dir)) {
-                *returncode = 1;
-                if (returntext) {
-                    PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
-                                "The changelog directory [%s] already exists and is not empty.  "
-                                "Please choose a directory that does not exist or is empty.\n",
-                                config.dir);
-                }
-
-                goto done;
-            }
-
-            if (!_is_absolutepath(config.dir) || (CL5_SUCCESS != cl5CreateDirIfNeeded(config.dir))) {
-                *returncode = 1;
-                if (returntext) {
-                    PL_strncpyz(returntext, "invalid changelog directory or insufficient access", SLAPI_DSE_RETURNTEXT_SIZE);
-                }
-
-                goto done;
-            }
-
-            /* changelog directory changed - need to remove the
-               previous changelog and create new one */
-
-            slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name_cl,
-                          "changelog5_config_modify - Changelog directory changed; "
-                          "old dir - %s, new dir - %s; recreating changelog.\n",
-                          currentDir, config.dir);
-
-            rc = cl5Close();
-            if (rc != CL5_SUCCESS) {
-                *returncode = 1;
-                if (returntext) {
-                    PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to close changelog; error - %d", rc);
-                }
-
-                slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                              "changelog5_config_modify - Failed to close changelog\n");
-                goto done;
-            } else {
-                slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl,
-                              "changelog5_config_modif - Closed the changelog\n");
-            }
-
-            rc = cl5Delete(currentDir);
-            if (rc != CL5_SUCCESS) {
-                *returncode = 1;
-                if (returntext) {
-                    PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to remove changelog; error - %d", rc);
-                }
-
-                slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                              "changelog5_config_modify - Failed to remove changelog\n");
-                goto done;
-            } else {
-                slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl,
-                              "changelog5_config_modify - Deleted the changelog at %s\n", currentDir);
-            }
-
-            rc = cl5Open(config.dir, &config.dbconfig);
-            if (rc != CL5_SUCCESS) {
-                *returncode = 1;
-                if (returntext) {
-                    PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Failed to restart changelog; error - %d", rc);
-                }
-
-                slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                              "changelog5_config_modify - Failed to restart changelog\n");
-                /* before finishing, let's try to do some error recovery */
-                if (CL5_SUCCESS != cl5Open(currentDir, &config.dbconfig)) {
-                    slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                                  "changelog5_config_modify - Failed to restore previous changelog\n");
-                }
-                goto done;
-            } else {
-                slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name_cl,
-                              "changelog5_config_modify - Opened the changelog at %s\n", config.dir);
-            }
-        }
-    }
-
     /* one of the changelog parameters is modified */
     if (config.maxEntries != CL5_NUM_IGNORE ||
         config.trimInterval != CL5_NUM_IGNORE ||
         strcmp(config.maxAge, CL5_STR_IGNORE) != 0) {
-        rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval);
+        rc = cl5ConfigTrimming(replica, config.maxEntries, config.maxAge, config.trimInterval);
         if (rc != CL5_SUCCESS) {
             *returncode = 1;
             if (returntext) {
@@ -538,9 +360,6 @@ done:;
     changelog5_config_done(&config);
     changelog5_config_free(&originalConfig);
 
-    /* slapi_ch_free accepts NULL pointer */
-    slapi_ch_free((void **)&currentDir);
-
     if (*returncode == LDAP_SUCCESS) {
 
         if (returntext) {
@@ -554,100 +373,12 @@ done:;
 }
 
 static int
-changelog5_config_delete(Slapi_PBlock *pb __attribute__((unused)),
-                         Slapi_Entry *e __attribute__((unused)),
-                         Slapi_Entry *entryAfter __attribute__((unused)),
-                         int *returncode,
-                         char *returntext,
-                         void *arg __attribute__((unused)))
+cldb_config_delete(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg)
 {
-    int rc;
-    char *currentDir = NULL;
-    *returncode = LDAP_SUCCESS;
-
-    /* changelog must be open before it can be deleted */
-    if (cl5GetState() != CL5_STATE_OPEN) {
-        *returncode = 1;
-        if (returntext) {
-            PL_strncpyz(returntext, "changelog is not configured", SLAPI_DSE_RETURNTEXT_SIZE);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_delete - Chagelog is not configured\n");
-        return SLAPI_DSE_CALLBACK_ERROR;
-    }
-
-    slapi_rwlock_wrlock(s_configLock);
-
-    /* changelog must be open before it can be deleted */
-    if (cl5GetState() != CL5_STATE_OPEN) {
-        *returncode = 1;
-        if (returntext) {
-            PL_strncpyz(returntext, "changelog is not configured", SLAPI_DSE_RETURNTEXT_SIZE);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_delete - Changelog is not configured\n");
-        goto done;
-    }
-
-    currentDir = cl5GetDir();
-
-    if (currentDir == NULL) {
-        /* something is wrong: we should never be here */
-        *returncode = 1;
-        if (returntext) {
-            PL_strncpyz(returntext, "internal failure", SLAPI_DSE_RETURNTEXT_SIZE);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_delete - NULL directory\n");
-        goto done;
-    }
-
-    /* this call will block until all threads using changelog
-       release changelog by calling cl5RemoveThread () */
-    rc = cl5Close();
-    if (rc != CL5_SUCCESS) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to close changelog; error - %d", rc);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_delete - Failed to close changelog\n");
-        goto done;
-    }
-
-    rc = cl5Delete(currentDir);
-    if (rc != CL5_SUCCESS) {
-        *returncode = 1;
-        if (returntext) {
-            PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "failed to remove changelog; error - %d", rc);
-        }
-
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_config_delete - Failed to remove changelog\n");
-        goto done;
-    }
-
-done:;
-    slapi_rwlock_unlock(s_configLock);
-
-    /* slapi_ch_free accepts NULL pointer */
-    slapi_ch_free((void **)&currentDir);
-
-    if (*returncode == LDAP_SUCCESS) {
-        if (returntext) {
-            returntext[0] = '\0';
-        }
-
-        return SLAPI_DSE_CALLBACK_OK;
-    }
-
-    return SLAPI_DSE_CALLBACK_ERROR;
+    return SLAPI_DSE_CALLBACK_OK;
 }
 
+
 static int
 dont_allow_that(Slapi_PBlock *pb __attribute__((unused)),
                 Slapi_Entry *entryBefore __attribute__((unused)),
@@ -660,31 +391,10 @@ dont_allow_that(Slapi_PBlock *pb __attribute__((unused)),
     return SLAPI_DSE_CALLBACK_ERROR;
 }
 
-static changelog5Config *
-changelog5_dup_config(changelog5Config *config)
-{
-    changelog5Config *dup = (changelog5Config *)slapi_ch_calloc(1, sizeof(changelog5Config));
-
-    if (config->dir)
-        dup->dir = slapi_ch_strdup(config->dir);
-    if (config->maxAge)
-        dup->maxAge = slapi_ch_strdup(config->maxAge);
-
-    dup->maxEntries = config->maxEntries;
-    dup->compactInterval = config->compactInterval;
-    dup->trimInterval = config->trimInterval;
-
-    dup->dbconfig.pageSize = config->dbconfig.pageSize;
-    dup->dbconfig.fileMode = config->dbconfig.fileMode;
-
-    return dup;
-}
-
-
 /*
  * Given the changelog configuration entry, extract the configuration directives.
  */
-static void
+void
 changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config)
 {
     const char *arg;
@@ -698,18 +408,6 @@ changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config)
     if (arg) {
         config->maxEntries = atoi(arg);
     }
-    arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE);
-    if (arg) {
-        if (slapi_is_duration_valid(arg)) {
-            config->compactInterval = (long)slapi_parse_duration(arg);
-        } else {
-            slapi_log_err(SLAPI_LOG_NOTICE, repl_plugin_name_cl,
-                          "changelog5_extract_config - %s: invalid value \"%s\", ignoring the change.\n",
-                          CONFIG_CHANGELOG_COMPACTDB_ATTRIBUTE, arg);
-        }
-    } else {
-        config->compactInterval = CHANGELOGDB_COMPACT_INTERVAL;
-    }
 
     arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_TRIM_ATTRIBUTE);
     if (arg) {
@@ -745,21 +443,56 @@ changelog5_extract_config(Slapi_Entry *entry, changelog5Config *config)
      */
     arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM);
     if (arg) {
-        config->dbconfig.encryptionAlgorithm = slapi_ch_strdup(arg);
+        config->encryptionAlgorithm = slapi_ch_strdup(arg);
     } else {
-        config->dbconfig.encryptionAlgorithm = NULL; /* no encryption */
+        config->encryptionAlgorithm = NULL; /* no encryption */
     }
     /*
      * symmetric key
      */
     arg = slapi_entry_attr_get_ref(entry, CONFIG_CHANGELOG_SYMMETRIC_KEY);
     if (arg) {
-        config->dbconfig.symmetricKey = slapi_ch_strdup(arg);
+        config->symmetricKey = slapi_ch_strdup(arg);
     } else {
-        config->dbconfig.symmetricKey = NULL; /* no symmetric key */
+        config->symmetricKey = NULL; /* no symmetric key */
     }
 }
 
+/* register functions handling attempted operations on the changelog config entries */
+int
+changelog5_register_config_callbacks(const char *dn, Replica *replica)
+{
+    int rc = 0;
+    /* callbacks to handle changes to the new changelog configuration in the main database */
+    rc =slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_add, replica);
+    rc |= slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_modify, replica);
+    rc |= slapi_config_register_callback(SLAPI_OPERATION_MODRDN, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, dont_allow_that, replica);
+    rc |= slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_delete, replica);
+
+    return rc;
+}
+
+int
+changelog5_remove_config_callbacks(const char *dn)
+{
+    int rc = 0;
+
+    rc =slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_add);
+    rc |= slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_modify);
+    rc |= slapi_config_remove_callback(SLAPI_OPERATION_MODRDN, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, dont_allow_that);
+    rc |= slapi_config_remove_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, LDAP_SCOPE_SUBTREE,
+                                   CL_CONFIG_FILTER, cldb_config_delete);
+
+    return rc;
+}
+
 static void
 replace_bslash(char *dir)
 {
@@ -774,18 +507,3 @@ replace_bslash(char *dir)
         bslash = strchr(bslash, '\\');
     }
 }
-
-static int
-notify_replica(Replica *r, void *arg __attribute__((unused)))
-{
-    return replica_log_ruv_elements(r);
-}
-
-static int
-_is_absolutepath(char *dir)
-{
-    if (dir[0] == '/')
-        return 1;
-
-    return 0;
-}

ldap/servers/plugins/replication/cl5_init.c

@@ -20,12 +20,47 @@
 #include "cl5.h"
 #include "repl5.h"
 
+static int _cl5_upgrade_replica(Replica *replica, void *arg);
+static int _cl5_upgrade_replica_config(Replica *replica, changelog5Config *config);
+static int _cl5_upgrade_removedir(char *path);
+static int _cl5_upgrade_removeconfig(void);
+
+/* upgrade changelog*/
+/* the changelog5 configuration maintained all changlog files in a separate directory
+ * now the changlog is part of the instance database.
+ * If this is the first startup with the new version we willi
+ * - try to move the changelog files for each replica to the instance database.
+ * - create a config entry for trimming and encryption in each backend.
+ */
+int
+changelog5_upgrade(void)
+{
+    int rc = 0;
+    changelog5Config config = {};
+
+    changelog5_read_config(&config);
+
+    if (config.dir == NULL) {
+        /* we do not have a valid legacy config, nothing to upgrade */
+        return rc;
+    }
+
+    replica_enumerate_replicas(_cl5_upgrade_replica, (void *)&config);
+
+    rc = _cl5_upgrade_removedir(config.dir);
+
+    rc = _cl5_upgrade_removeconfig();
+
+    changelog5_config_done(&config);
+
+    return rc;
+}
+
 /* initializes changelog*/
 int
-changelog5_init()
+changelog5_init(void)
 {
     int rc;
-    changelog5Config config;
 
     rc = cl5Init();
     if (rc != CL5_SUCCESS) {
@@ -34,41 +69,22 @@ changelog5_init()
         return 1;
     }
 
-    /* read changelog configuration */
+    /* setup callbacks for operations on changelog */
     changelog5_config_init();
-    changelog5_read_config(&config);
-
-    if (config.dir == NULL) {
-        /* changelog is not configured - bail out */
-        /* Note: but still changelog needs to be initialized to allow it
-         * to configure after this point. (don't call cl5Cleanup) */
-        rc = 0; /* OK */
-        goto done;
-    }
 
     /* start changelog */
-    rc = cl5Open(config.dir, &config.dbconfig);
+    rc = cl5Open();
     if (rc != CL5_SUCCESS) {
         slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_init: failed to start changelog at %s\n",
-                      config.dir);
+                      "changelog5_init: failed to start changelog\n");
         rc = 1;
         goto done;
     }
 
-    /* set trimming parameters */
-    rc = cl5ConfigTrimming(config.maxEntries, config.maxAge, config.compactInterval, config.trimInterval);
-    if (rc != CL5_SUCCESS) {
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "changelog5_init: failed to configure changelog trimming\n");
-        rc = 1;
-        goto done;
-    }
 
     rc = 0;
 
 done:
-    changelog5_config_done(&config);
     return rc;
 }
 
@@ -83,3 +99,115 @@ changelog5_cleanup()
     /* cleanup config */
     changelog5_config_cleanup();
 }
+static int
+_cl5_upgrade_replica_config(Replica *replica, changelog5Config *config)
+{
+    int rc = 0;
+    Slapi_Backend *be = slapi_be_select(replica_get_root(replica));
+
+    Slapi_Entry *config_entry = slapi_entry_alloc();
+    slapi_entry_init(config_entry, slapi_ch_strdup("cn=changelog"), NULL);
+    slapi_entry_add_string(config_entry, "objectclass", "top");
+    slapi_entry_add_string(config_entry, "objectclass", "extensibleObject");
+
+    /* keep the changelog trimming config */
+    if (config->maxEntries) {
+        char *maxEnt = slapi_ch_smprintf("%d", config->maxEntries);
+        slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_MAXENTRIES_ATTRIBUTE, maxEnt);
+    }
+    if (strcmp(config->maxAge, CL5_STR_IGNORE)) {
+        slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE, config->maxAge);
+    }
+    if (config->trimInterval != CHANGELOGDB_TRIM_INTERVAL) {
+        /* char *interval = slapi_ch_smprintf("%ld", config->trimInterval); */
+        slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_TRIM_ATTRIBUTE, gen_duration(config->trimInterval));
+    }
+
+    /* if changelog encryption is enabled then in the upgrade mode all backends will have 
+     * an encrypted changelog, store the encryption attrs */
+
+    if (config->encryptionAlgorithm) {
+        slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_ENCRYPTION_ALGORITHM, config->encryptionAlgorithm);
+        slapi_entry_add_string(config_entry, CONFIG_CHANGELOG_SYMMETRIC_KEY, config->symmetricKey);
+    }
+    rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_SET_CONFIG, (void *)config_entry);
+
+    return rc;
+}
+static int
+_cl5_upgrade_replica(Replica *replica, void *arg)
+{
+    int rc = 0;
+    changelog5Config *config = (changelog5Config *)arg;
+
+    /* Move existing database file to backend */
+    char *replGen = replica_get_generation (replica);
+    const char *replName = replica_get_name (replica);
+    char *oldFile = slapi_ch_smprintf("%s/%s_%s.db",
+                                       config->dir, replName, replGen);
+    slapi_ch_free_string(&replGen);
+    if (PR_Access(oldFile, PR_ACCESS_EXISTS) == PR_SUCCESS) {
+        Slapi_Backend *be = slapi_be_select(replica_get_root(replica));
+        char *instancedir;
+        slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir);
+        char *newFile = slapi_ch_smprintf("%s/changelog.db", instancedir);
+
+        rc = slapi_back_ctrl_info(be, BACK_INFO_DBENV_CLDB_UPGRADE, oldFile);
+        slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name_cl,
+                      "_cl5_upgrade_replica: moving file (%s) to (%s) %s\n",
+                      oldFile, newFile, rc?"failed":"succeeded");
+        slapi_ch_free_string(&instancedir);
+    }
+
+    /* Move changelog config to backend config */
+    rc = _cl5_upgrade_replica_config(replica, config);
+
+    return rc;
+}
+static int
+_cl5_upgrade_removedir(char *path)
+{
+    /* this is duplicated from ldbm_delete_dirs, we unfortunately
+     * cannot access the backend functions
+     */
+    PRDir *dirhandle = NULL;
+    PRDirEntry *direntry = NULL;
+    char fullpath[MAXPATHLEN];
+    int rval = 0;
+    PRFileInfo64 info;
+
+    dirhandle = PR_OpenDir(path);
+    if (!dirhandle) {
+        PR_Delete(path);
+        return 0;
+    }
+
+    while (NULL != (direntry =
+                        PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) {
+        if (!direntry->name)
+            break;
+
+        PR_snprintf(fullpath, MAXPATHLEN, "%s/%s", path, direntry->name);
+        rval = PR_GetFileInfo64(fullpath, &info);
+        if (PR_SUCCESS == rval) {
+            PR_Delete(fullpath);
+        }
+    }
+    PR_CloseDir(dirhandle);
+    /* remove the directory itself too */
+    rval += PR_RmDir(path);
+    return rval;
+}
+static int
+_cl5_upgrade_removeconfig(void)
+{
+    int rc = LDAP_SUCCESS;
+
+    Slapi_PBlock *pb = slapi_pblock_new();
+    slapi_delete_internal_set_pb(pb, "cn=changelog5,cn=config", NULL, NULL,
+                                 repl_get_plugin_identity(PLUGIN_MULTIMASTER_REPLICATION), 0);
+    slapi_delete_internal_pb(pb);
+    slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc);
+    slapi_pblock_destroy(pb);
+    return rc;
+}

ldap/servers/plugins/replication/cl5_test.c

@@ -29,7 +29,6 @@
 static void testBasic();
 static void testBackupRestore();
 static void testIteration();
-static void testTrimming();
 static void testPerformance();
 static void testPerformanceMT();
 static void testLDIF();
@@ -57,9 +56,6 @@ testChangelog(TestType type)
     case TEST_ITERATION:
         testIteration();
         break;
-    case TEST_TRIMMING:
-        testTrimming();
-        break;
     case TEST_PERFORMANCE:
         testPerformance();
         break;
@@ -115,7 +111,7 @@ testBackupRestore()
 
     slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Starting backup and recovery test ...\n");
 
-    dir = cl5GetDir();
+    dir = cl5GetLdifDir(NULL);
 
     if (dir) {
         baseDir = getBaseDir(dir);
@@ -127,7 +123,7 @@ testBackupRestore()
             cl5Close();
             rc = cl5Restore(dir, bkDir, NULL);
             if (rc == CL5_SUCCESS)
-                rc = cl5Open(dir, NULL);
+                rc = cl5Open();
 
             /* PR_RmDir (bkDir);*/
         }
@@ -263,45 +259,6 @@ testIteration()
     slapi_ch_free((void **)&replGen);
 }
 
-static void
-testTrimming()
-{
-    PRIntervalTime interval;
-    int count;
-    int rc;
-
-    slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Starting trimming test ...\n");
-
-    rc = populateChangelog(200, NULL);
-
-    if (rc == 0) {
-        interval = PR_SecondsToInterval(2);
-        DS_Sleep(interval);
-
-        rc = populateChangelog(300, NULL);
-
-        if (rc == 0)
-            rc = cl5ConfigTrimming(300, "1d", CHANGELOGDB_COMPACT_INTERVAL, CHANGELOGDB_TRIM_INTERVAL);
-
-        interval = PR_SecondsToInterval(300); /* 5 min is default trimming interval */
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "Trimming test: sleeping for 5 minutes until trimming kicks in\n");
-        DS_Sleep(interval);
-
-        count = cl5GetOperationCount(NULL);
-    }
-
-    if (rc == 0 && count == 300) {
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "Trimming test completed successfully: changelog contains 300 entries\n");
-    } else if (rc == 0) {
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
-                      "Trimming test failed: changelog contains %d entries; expected - 300\n",
-                      count);
-    } else /* general failure */
-        slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, "Trimming test failed\n");
-}
-
 static void
 testPerformance()
 {
@@ -393,7 +350,7 @@ testPerformanceMT()
 static void
 testLDIF()
 {
-    char *clDir = cl5GetDir();
+    char *clDir = cl5GetLdifDir(NULL);
     int rc;
     char *baseDir;
     char ldifFile[MAXPATHLEN];
@@ -411,7 +368,7 @@ testLDIF()
             cl5Close();
             rc = cl5ImportLDIF(clDir, ldifFile, NULL);
             if (rc == CL5_SUCCESS)
-                cl5Open(clDir, NULL);
+                cl5Open();
         }
     }
 

ldap/servers/plugins/replication/cl_crypt.c

@@ -29,44 +29,33 @@
 /*
  * BACK_INFO_CRYPT_INIT
  */
-int
-clcrypt_init(const CL5DBConfig *config, void **clcrypt_handle)
+void *
+clcrypt_init(char *encryptionAlgorithm, Slapi_Backend *be)
 {
     int rc = 0;
-    char *cookie = NULL;
-    Slapi_Backend *be = NULL;
     back_info_crypt_init crypt_init = {0};
+    void *crypt_handle = NULL;
 
     slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name, "-> clcrypt_init\n");
-    /* Encryption is not specified */
-    if (!config->encryptionAlgorithm || !clcrypt_handle) {
+
+    if (!encryptionAlgorithm) {
+        /* Encryption is not specified */
         goto bail;
     }
-    crypt_init.dn = "cn=changelog5,cn=config";
-    crypt_init.encryptionAlgorithm = config->encryptionAlgorithm;
+    crypt_init.dn = "cn=changelog";
+    crypt_init.encryptionAlgorithm = encryptionAlgorithm;
+    crypt_init.be = be;
 
-    be = slapi_get_first_backend(&cookie);
-    while (be) {
-        crypt_init.be = be;
-        rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_INIT,
+    rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_INIT,
                                   (void *)&crypt_init);
-        if (LDAP_SUCCESS == rc) {
-            break; /* Successfully fetched */
-        }
-        be = slapi_get_next_backend(cookie);
-    }
-    slapi_ch_free((void **)&cookie);
 
     if (LDAP_SUCCESS == rc && crypt_init.state_priv) {
-        *clcrypt_handle = crypt_init.state_priv;
-        rc = 0;
-    } else {
-        rc = 1;
+        crypt_handle = crypt_init.state_priv;
     }
 bail:
     slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name,
                   "<- clcrypt_init : %d\n", rc);
-    return rc;
+    return crypt_handle;
 }
 
 /*
@@ -77,38 +66,26 @@ bail:
  *                  :     NULL - failure
  */
 int
-clcrypt_destroy(void *clcrypt_handle)
+clcrypt_destroy(void *clcrypt_handle, Slapi_Backend *be)
 {
     int rc = -1;
-    char *cookie = NULL;
-    Slapi_Backend *be = NULL;
     back_info_crypt_destroy crypt_destroy = {0};
 
     slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name,
                   "-> clcrypt_destroy\n");
     if (NULL == clcrypt_handle) {
         /* Nothing to free */
-        rc = 0;
-        goto bail;
+        return 0;
     }
     crypt_destroy.state_priv = clcrypt_handle;
 
-    be = slapi_get_first_backend(&cookie);
-    while (be) {
-        rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_DESTROY,
-                                  (void *)&crypt_destroy);
-        if (LDAP_SUCCESS == rc) {
-            break; /* Successfully freed */
-        }
-        be = slapi_get_next_backend(cookie);
-    }
-    slapi_ch_free((void **)&cookie);
+    rc = slapi_back_ctrl_info(be, BACK_INFO_CRYPT_DESTROY,
+                              (void *)&crypt_destroy);
     if (LDAP_SUCCESS == rc) {
         rc = 0;
     } else {
         rc = -1;
     }
-bail:
     slapi_log_err(SLAPI_LOG_TRACE, repl_plugin_name,
                   "<- clcrypt_destroy (returning %d)\n", rc);
     return rc;

ldap/servers/plugins/replication/cl_crypt.h

@@ -18,8 +18,8 @@
 #include "nss.h"
 #include "cert.h"
 
-int clcrypt_init(const CL5DBConfig *config, void **clcrypt_handle);
-int clcrypt_destroy(void *clcrypt_handle);
+void *clcrypt_init(char *encryptionAlgorithm, Slapi_Backend *be);
+int clcrypt_destroy(void *clcrypt_handle, Slapi_Backend *be);
 int clcrypt_encrypt_value(void *clcrypt_handle, struct berval *in, struct berval **out);
 int clcrypt_decrypt_value(void *state_priv, struct berval *in, struct berval **out);
 #endif /* _CLCRYPT_H_ */

ldap/servers/plugins/replication/repl5.h

@@ -752,6 +752,8 @@ PRBool ignore_error_and_keep_going(int error);
 void replica_check_release_timeout(Replica *r, Slapi_PBlock *pb);
 void replica_lock_replica(Replica *r);
 void replica_unlock_replica(Replica *r);
+void *replica_get_file_info(Replica *r);
+int replica_set_file_info(Replica *r, void *cl);
 
 /* The functions below handles the state flag */
 /* Current internal state flags */
@@ -807,8 +809,7 @@ typedef struct _cleanruv_purge_data
 {
     int cleaned_rid;
     const Slapi_DN *suffix_sdn;
-    char *replName;
-    char *replGen;
+    Replica *replica;
 } cleanruv_purge_data;
 
 typedef struct _csngen_test_data

ldap/servers/plugins/replication/repl5_init.c

@@ -338,8 +338,8 @@ multimaster_bepreop_init(Slapi_PBlock *pb)
 
     if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepreopdesc) != 0 ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Cleanup) != 0 ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_BACKUP_FN, (void *)cl5WriteRUV) != 0) {
+        /* slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Cleanup) != 0) { */
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_PRE_CLOSE_FN, (void *)cl5Close) != 0) {
         slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_bepreop_init - Failed\n");
         rc = -1;
     }
@@ -385,10 +385,9 @@ multimaster_bepostop_init(Slapi_PBlock *pb)
 
     if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) != 0 ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) != 0 ||
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)cl5Open) != 0 ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN, (void *)multimaster_bepostop_modrdn) != 0 ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN, (void *)multimaster_bepostop_delete) != 0 ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)changelog5_init) != 0 ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV) != 0) {
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_DELETE_FN, (void *)multimaster_bepostop_delete) != 0) {
         slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_bepostop_init - Failed\n");
         rc = -1;
     }
@@ -408,8 +407,9 @@ multimaster_betxn_bepostop_init(Slapi_PBlock *pb)
 
     if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) ||
         slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)changelog5_init) ||
-        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV)) {
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_OPEN_FN, (void *)cl5Open) != 0 ||
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN, (void *)cl5Export) ||
+        slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_IMPORT_FN, (void *)cl5Import)) {
         slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_betxn_bepostop_init - Failed\n");
         rc = -1;
     }
@@ -797,17 +797,16 @@ multimaster_start(Slapi_PBlock *pb)
         /* create replicas */
         multimaster_mtnode_construct_replicas();
 
-        /* Initialise the 5.0 Changelog */
+        /* Upgrade the 5.0 Changelog if it still exists */
+        rc = changelog5_upgrade();
+        if (rc != 0)
+            goto out;
+
+        /* perform initial changelog setup */
         rc = changelog5_init();
         if (rc != 0)
             goto out;
 
-        /* Initialize the replication agreements, unless we're dumping LDIF */
-        if (!is_ldif_dump) {
-            rc = agmtlist_config_init();
-            if (rc != 0)
-                goto out;
-        }
         rc = create_repl_schema_policy();
         if (rc != 0)
             goto out;
@@ -815,9 +814,14 @@ multimaster_start(Slapi_PBlock *pb)
         /* check if the replica's data was reloaded offline and we need
            to reinitialize replica's changelog. This should be done
            after the changelog is initialized */
-
         replica_enumerate_replicas(replica_check_for_data_reload, NULL);
 
+        /* Initialize the replication agreements, unless we're dumping LDIF */
+        if (!is_ldif_dump) {
+            rc = agmtlist_config_init();
+            if (rc != 0)
+                goto out;
+        }
         /* register to be notified when backend state changes */
         slapi_register_backend_state_change((void *)multimaster_be_state_change,
                                             multimaster_be_state_change);

ldap/servers/plugins/replication/repl5_plugins.c

@@ -987,19 +987,15 @@ write_changelog_and_ruv(Slapi_PBlock *pb)
 
     replica_check_release_timeout(r, pb);
 
-    if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) &&
-        (cl5GetState() == CL5_STATE_OPEN)) {
+    if (replica_is_flag_set(r, REPLICA_LOG_CHANGES)) {
         supplier_operation_extension *opext = NULL;
-        const char *repl_name;
-        char *repl_gen;
+        cldb_Handle *cldb  = NULL;
 
         opext = (supplier_operation_extension *)repl_sup_get_ext(REPL_SUP_EXT_OP, op);
         PR_ASSERT(opext);
 
-        /* get replica generation and replica name to pass to the write function */
-        repl_name = replica_get_name(r);
-        repl_gen = opext->repl_gen;
-        PR_ASSERT(repl_name && repl_gen);
+        /* changelog database information to pass to the write function */
+        cldb = replica_get_file_info(r);
 
         /* for replicated operations, we log the original, non-urp data which is
            saved in the operation extension */
@@ -1042,6 +1038,9 @@ write_changelog_and_ruv(Slapi_PBlock *pb)
 
         if (op_params->csn && is_cleaned_rid(csn_get_replicaid(op_params->csn))) {
             /* this RID has been cleaned */
+            if (!operation_is_flag_set(op, OP_FLAG_REPLICATED)) {
+                slapi_ch_free((void **)&op_params->target_address.uniqueid);
+            }
             goto common_return;
         }
 
@@ -1052,6 +1051,9 @@ write_changelog_and_ruv(Slapi_PBlock *pb)
         {
             slapi_log_err(SLAPI_LOG_REPL, "write_changelog_and_ruv",
                           "Skipping internal operation on read-only replica\n");
+            if (!operation_is_flag_set(op, OP_FLAG_REPLICATED)) {
+                slapi_ch_free((void **)&op_params->target_address.uniqueid);
+            }
             goto common_return;
         }
 
@@ -1061,11 +1063,6 @@ write_changelog_and_ruv(Slapi_PBlock *pb)
             op_params->p.p_modify.modify_mods != NULL) {
             void *txn = NULL;
             char csn_str[CSN_STRSIZE];
-            if (cl5_is_diskfull() && !cl5_diskspace_is_available()) {
-                slapi_log_err(SLAPI_LOG_CRIT, repl_plugin_name,
-                              "write_changelog_and_ruv - Skipped due to DISKFULL\n");
-                goto common_return;
-            }
             slapi_pblock_get(pb, SLAPI_TXN, &txn);
             slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name,
                           "write_changelog_and_ruv - Writing change for "
@@ -1074,8 +1071,7 @@ write_changelog_and_ruv(Slapi_PBlock *pb)
                           op_params->target_address.uniqueid,
                           op_params->operation_type,
                           csn_as_string(op_params->csn, PR_FALSE, csn_str));
-            rc = cl5WriteOperationTxn(repl_name, repl_gen, op_params,
-                                      !operation_is_flag_set(op, OP_FLAG_REPLICATED), txn);
+            rc = cl5WriteOperationTxn(cldb, op_params, txn);
             if (rc != CL5_SUCCESS) {
                 /* ONREPL - log error */
                 slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name,

ldap/servers/plugins/replication/repl5_replica.c

@@ -66,6 +66,7 @@ struct replica
     uint64_t agmt_count;               /* Number of agmts */
     Slapi_Counter *release_timeout;    /* The amount of time to wait before releasing active replica */
     uint64_t abort_session;            /* Abort the current replica session */
+    cldb_Handle *cldb;                 /* database info for the changelog */
 };
 
 
@@ -84,6 +85,7 @@ static Slapi_Entry *_replica_get_config_entry(const Slapi_DN *root, const char *
 static int _replica_check_validity(const Replica *r);
 static int _replica_init_from_config(Replica *r, Slapi_Entry *e, char *errortext);
 static int _replica_update_entry(Replica *r, Slapi_Entry *e, char *errortext);
+static int _replica_config_changelog(Replica *r);
 static int _replica_configure_ruv(Replica *r, PRBool isLocked);
 static char *_replica_get_config_dn(const Slapi_DN *root);
 static char *_replica_type_as_string(const Replica *r);
@@ -205,6 +207,7 @@ replica_new_from_entry(Slapi_Entry *e, char *errortext, PRBool is_add_operation,
         rc = LDAP_SUCCESS;
     }
 
+
     /* If smallest csn exists in RUV for our local replica, it's ok to begin iteration */
     PR_ASSERT(object_get_data(r->repl_ruv));
 
@@ -215,6 +218,16 @@ replica_new_from_entry(Slapi_Entry *e, char *errortext, PRBool is_add_operation,
          * during replica initialization
          */
         rc = _replica_update_entry(r, e, errortext);
+        /* add changelog config entry to config 
+         * this is only needed for replicas logging changes,
+         * but for now let it exist for all replicas. Makes handling
+         * of changing replica flags easier
+         */
+        _replica_config_changelog(r);
+        if (r->repl_flags & REPLICA_LOG_CHANGES) {
+            /* Init changelog db file */
+            cldb_SetReplicaDB(r, NULL);
+        }
     } else {
         /*
          * Entry is already in dse.ldif - update it on the disk
@@ -795,6 +808,10 @@ replica_set_ruv(Replica *r, RUV *ruv)
 
     r->repl_ruv = object_new((void *)ruv, (FNFree)ruv_destroy);
 
+    if (r->repl_flags & REPLICA_LOG_CHANGES) {
+        cl5NotifyRUVChange(r);
+    }
+
     replica_unlock(r->repl_lock);
 }
 
@@ -1553,12 +1570,20 @@ replica_reload_ruv(Replica *r)
                                                                    " Recreating the changelog file. This could affect replication with replica's "
                                                                    " consumers in which case the consumers should be reinitialized.\n",
                               slapi_sdn_get_dn(r->repl_root));
-                rc = cl5DeleteDBSync(r);
+
+                /* need to reset changelog db */
+                rc = cldb_RemoveReplicaDB(r);
+                slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                     "replica_reload_ruv: reset cldb for replica\n");
 
                 /* reinstate new ruv */
                 replica_lock(r->repl_lock);
 
                 r->repl_ruv = new_ruv_obj;
+                slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                    "replica_reload_ruv: set cldb for replica\n");
+
+                cldb_SetReplicaDB(r, NULL);
 
                 if (rc == CL5_SUCCESS) {
                     /* log changes to mark starting point for replication */
@@ -1685,7 +1710,10 @@ replica_check_for_data_reload(Replica *r, void *arg __attribute__((unused)))
                                                                        "consumers should be reinitialized.\n",
                                   slapi_sdn_get_dn(r->repl_root));
 
-                    rc = cl5DeleteDBSync(r);
+
+                    /* need to reset changelog db */
+                    rc = cldb_RemoveReplicaDB(r);
+                    cldb_SetReplicaDB(r, NULL);
 
                     if (rc == CL5_SUCCESS) {
                         /* log changes to mark starting point for replication */
@@ -2409,6 +2437,26 @@ _replica_get_config_dn(const Slapi_DN *root)
                            REPLICA_RDN, slapi_sdn_get_dn(root), mp_base);
     return dn;
 }
+/* when a replica is added the changelog config entry is created
+ * it will only the container entry, specifications for trimming 
+ * or encyrption need to be added separately
+ */
+static int
+_replica_config_changelog(Replica *replica)
+{
+    int rc = 0;
+
+    Slapi_Backend *be = slapi_be_select(replica_get_root(replica));
+
+    Slapi_Entry *config_entry = slapi_entry_alloc();
+    slapi_entry_init(config_entry, slapi_ch_strdup("cn=changelog"), NULL);
+    slapi_entry_add_string(config_entry, "objectclass", "top");
+    slapi_entry_add_string(config_entry, "objectclass", "extensibleObject");
+
+    rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_SET_CONFIG, (void *)config_entry);
+
+    return rc;
+}
 
 /* This function retrieves RUV from the root of the replicated tree.
  * The attribute can be missing if
@@ -3499,18 +3547,13 @@ replica_get_referrals_nolock(const Replica *r, char ***referrals)
     }
 }
 
-typedef struct replinfo
-{
-    char *repl_gen;
-    char *repl_name;
-} replinfo;
-
 static int
 replica_log_start_iteration(const ruv_enum_data *rid_data, void *data)
 {
     int rc = 0;
-    replinfo *r_info = (replinfo *)data;
     slapi_operation_parameters op_params;
+    Replica *replica = (Replica *)data;
+    cldb_Handle *cldb = NULL;
 
     if (rid_data->csn == NULL)
         return 0;
@@ -3520,7 +3563,8 @@ replica_log_start_iteration(const ruv_enum_data *rid_data, void *data)
     op_params.target_address.sdn = slapi_sdn_new_ndn_byval(START_ITERATION_ENTRY_DN);
     op_params.target_address.uniqueid = START_ITERATION_ENTRY_UNIQUEID;
     op_params.csn = csn_dup(rid_data->csn);
-    rc = cl5WriteOperation(r_info->repl_name, r_info->repl_gen, &op_params, PR_FALSE);
+    cldb = replica_get_file_info(replica);
+    rc = cl5WriteOperation(cldb, &op_params);
     if (rc == CL5_SUCCESS)
         rc = 0;
     else
@@ -3537,8 +3581,6 @@ replica_log_ruv_elements_nolock(const Replica *r)
 {
     int rc = 0;
     RUV *ruv;
-    char *repl_gen;
-    replinfo r_info;
 
     ruv = (RUV *)object_get_data(r->repl_ruv);
     PR_ASSERT(ruv);
@@ -3546,15 +3588,7 @@ replica_log_ruv_elements_nolock(const Replica *r)
     /* we log it as a delete operation to have the least number of fields
            to set. the entry can be identified by a special target uniqueid and
            special target dn */
-    repl_gen = ruv_get_replica_generation(ruv);
-
-    r_info.repl_name = r->repl_name;
-    r_info.repl_gen = repl_gen;
-
-    rc = ruv_enumerate_elements(ruv, replica_log_start_iteration, &r_info);
-
-    slapi_ch_free((void **)&repl_gen);
-
+    rc = ruv_enumerate_elements(ruv, replica_log_start_iteration, (void *)r);
     return rc;
 }
 
@@ -3777,6 +3811,9 @@ replica_enable_replication(Replica *r)
 
     /* prevent creation of new agreements until the replica is enabled */
     PR_Lock(r->agmt_lock);
+    if (r->repl_flags & REPLICA_LOG_CHANGES) {
+        cldb_SetReplicaDB(r, NULL);
+    }
 
     /* retrieve new ruv */
     rc = replica_reload_ruv(r);
@@ -3863,6 +3900,12 @@ replica_disable_replication(Replica *r)
     slapi_ch_free_string(&locking_purl);
     replica_set_state_flag(r, REPLICA_AGREEMENTS_DISABLED, PR_FALSE);
     PR_Unlock(r->agmt_lock);
+    /* no thread will access the changelog for this replica
+     * remove reference from replica object
+     */
+    if (r->repl_flags & REPLICA_LOG_CHANGES) {
+        cldb_UnSetReplicaDB(r, NULL);
+    }
 
     slapi_log_err(SLAPI_LOG_REPL, repl_plugin_name, "replica_disable_replication - "
                                                     "replica %s is acquired\n",
@@ -4147,3 +4190,12 @@ replica_unlock_replica(Replica *r)
 {
     replica_unlock(r->repl_lock);
 }
+void* replica_get_file_info(Replica *r)
+{
+       return r->cldb;
+}
+int replica_set_file_info(Replica *r, void *cl)
+{
+       r->cldb = (cldb_Handle *)cl;
+       return 0;
+}

ldap/servers/plugins/replication/repl5_replica_config.c

@@ -375,6 +375,10 @@ replica_config_modify(Slapi_PBlock *pb,
         */
         char *new_repl_id = NULL;
         char *new_repl_type = NULL;
+        /* we also need to handle the change of repl_flags and enable or disable 
+         * the changelog
+         */
+        char *new_repl_flag = NULL;
 
         if (*returncode != LDAP_SUCCESS)
             break;
@@ -619,6 +623,10 @@ replica_config_modify(Slapi_PBlock *pb,
             slapi_ch_free_string(&new_repl_type);
             agmtlist_notify_all(pb);
         }
+        if (new_repl_flag) {
+            *returncode = replica_config_change_flags(r, new_repl_flag, errortext, apply_mods);
+            slapi_ch_free_string(&new_repl_flag);
+        }
     }
 
 done:
@@ -760,6 +768,8 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)),
 {
     multimaster_mtnode_extension *mtnode_ext;
     Replica *r;
+    int rc;
+    Slapi_Backend *be;
 
     PR_Lock(s_configLock);
 
@@ -768,7 +778,22 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)),
 
     if (mtnode_ext->replica) {
         /* remove object from the hash */
+        Object *r_obj = mtnode_ext->replica;
+        back_info_config_entry config_entry = {0};
+
         r = (Replica *)object_get_data(mtnode_ext->replica);
+
+        /* retrieves changelog DN and keep it in config_entry.ce */
+        be = slapi_be_select(replica_get_root(r));
+        config_entry.dn = "cn=changelog";
+        rc = slapi_back_ctrl_info(be, BACK_INFO_CLDB_GET_CONFIG, (void *)&config_entry);
+        if (rc !=0 || config_entry.ce == NULL) {
+            slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl,
+                          "replica_config_delete - failed to read config for changelog\n");
+            PR_Unlock(s_configLock);
+            *returncode = LDAP_OPERATIONS_ERROR;
+            return SLAPI_DSE_CALLBACK_ERROR;
+        }
         mtnode_ext->replica = NULL;  /* moving it before deleting the CL because
                                       * deletion can take some time giving the opportunity
                                       * to an operation to start while CL is deleted
@@ -779,8 +804,18 @@ replica_config_delete(Slapi_PBlock *pb __attribute__((unused)),
                                                            "The changelog for replica %s is no longer valid since "
                                                            "the replica config is being deleted.  Removing the changelog.\n",
                       slapi_sdn_get_dn(replica_get_root(r)));
-        cl5DeleteDBSync(r);
+
+        /* As we are removing a replica, all references to it need to be cleared
+         * There are many of them:
+         * - all replicas are stored in a hash table (s_hash): replicat_delete_by_name
+         * - callbacks have been registered with the replica as argument: changelog5_register_config_callbacks
+         * - replica is stored in mapping tree extension mtnode_ext: object_release
+         */
+        cldb_RemoveReplicaDB(r);
         replica_delete_by_name(replica_get_name(r));
+        changelog5_remove_config_callbacks(slapi_entry_get_dn_const(config_entry.ce));
+        slapi_entry_free(config_entry.ce);
+        object_release(r_obj);
     }
 
     PR_Unlock(s_configLock);
@@ -973,7 +1008,7 @@ replica_config_change_type_and_id(Replica *r, const char *new_type, const char *
                     replica_reset_csn_pl(r);
                 }
                 ruv_delete_replica(ruv, oldrid);
-                cl5CleanRUV(oldrid);
+                cl5CleanRUV(oldrid, r);
                 replica_set_csn_assigned(r);
             }
             object_release(ruv_obj);
@@ -1034,6 +1069,13 @@ replica_config_change_flags(Replica *r, const char *new_flags, char *returntext
 
         flags = atol(new_flags);
 
+        if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) && !(flags&REPLICA_LOG_CHANGES)) {
+            /* the replica no longer maintains a changelog, reset */
+           cldb_UnSetReplicaDB(r, NULL);
+        } else if (!replica_is_flag_set(r, REPLICA_LOG_CHANGES) && (flags&REPLICA_LOG_CHANGES)) {
+            /* the replica starts to maintains a changelog, set */
+            cldb_SetReplicaDB(r, NULL);
+        }
         replica_replace_flags(r, flags);
     }
 
@@ -1153,7 +1195,6 @@ static int
 replica_execute_cl2ldif_task(Replica *replica, char *returntext)
 {
     int rc;
-    Replica *rlist[2];
     char fName[MAXPATHLEN];
     char *clDir = NULL;
 
@@ -1167,7 +1208,8 @@ replica_execute_cl2ldif_task(Replica *replica, char *returntext)
 
     /* file is stored in the changelog directory and is named
        <replica name>.ldif */
-    clDir = cl5GetDir();
+    Slapi_Backend *be = slapi_be_select(replica_get_root(replica));
+    clDir = cl5GetLdifDir(be);
     if (NULL == clDir) {
         rc = LDAP_OPERATIONS_ERROR;
         goto bail;
@@ -1177,15 +1219,12 @@ replica_execute_cl2ldif_task(Replica *replica, char *returntext)
         rc = LDAP_OPERATIONS_ERROR;
         goto bail;
     }
-    rlist[0] = replica;
-    rlist[1] = NULL;
-
 
-    PR_snprintf(fName, MAXPATHLEN, "%s/%s.ldif", clDir, replica_get_name(replica));
+    PR_snprintf(fName, MAXPATHLEN, "%s/%s_cl.ldif", clDir, replica_get_name(replica));
     slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name,
                   "replica_execute_cl2ldif_task - Beginning changelog export of replica \"%s\"\n",
                   replica_get_name(replica));
-    rc = cl5ExportLDIF(fName, rlist);
+    rc = cl5ExportLDIF(fName, replica);
     if (rc == CL5_SUCCESS) {
         slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name,
                       "replica_execute_cl2ldif_task - Finished changelog export of replica \"%s\"\n",
@@ -1210,10 +1249,8 @@ static int
 replica_execute_ldif2cl_task(Replica *replica, char *returntext)
 {
     int rc, imprc = 0;
-    Replica *rlist[2];
     char fName[MAXPATHLEN];
     char *clDir = NULL;
-    changelog5Config config;
 
     if (cl5GetState() != CL5_STATE_OPEN) {
         PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "changelog is not open");
@@ -1225,7 +1262,8 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext)
 
     /* file is stored in the changelog directory and is named
        <replica name>.ldif */
-    clDir = cl5GetDir();
+    Slapi_Backend *be = slapi_be_select(replica_get_root(replica));
+    clDir = cl5GetLdifDir(be);
     if (NULL == clDir) {
         rc = LDAP_OPERATIONS_ERROR;
         goto bail;
@@ -1235,8 +1273,6 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext)
         rc = LDAP_OPERATIONS_ERROR;
         goto bail;
     }
-    rlist[0] = replica;
-    rlist[1] = NULL;
 
     PR_snprintf(fName, MAXPATHLEN, "%s/%s.ldif", clDir, replica_get_name(replica));
 
@@ -1254,7 +1290,7 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext)
     slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name,
                   "replica_execute_ldif2cl_task -  Beginning changelog import of replica \"%s\"\n",
                   replica_get_name(replica));
-    imprc = cl5ImportLDIF(clDir, fName, rlist);
+    imprc = cl5ImportLDIF(clDir, fName, replica);
     if (CL5_SUCCESS == imprc) {
         slapi_log_err(SLAPI_LOG_INFO, repl_plugin_name,
                       "replica_execute_ldif2cl_task - Finished changelog import of replica \"%s\"\n",
@@ -1268,21 +1304,18 @@ replica_execute_ldif2cl_task(Replica *replica, char *returntext)
                       "replica_execute_ldif2cl_task - %s\n", returntext);
         imprc = LDAP_OPERATIONS_ERROR;
     }
-    changelog5_read_config(&config);
     /* restart changelog */
-    rc = cl5Open(config.dir, &config.dbconfig);
+    rc = cl5Open();
     if (CL5_SUCCESS == rc) {
         rc = LDAP_SUCCESS;
     } else {
         slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name,
-                      "replica_execute_ldif2cl_task - Failed to start changelog at %s\n",
-                      config.dir ? config.dir : "null config dir");
+                      "replica_execute_ldif2cl_task - Failed to start changelog after import\n");
         rc = LDAP_OPERATIONS_ERROR;
     }
 
 bail:
     slapi_ch_free_string(&clDir);
-    changelog5_config_done(&config);
 
     /* if cl5ImportLDIF returned an error, report it first. */
     return imprc ? imprc : rc;
@@ -1363,7 +1396,7 @@ replica_execute_cleanruv_task(Replica *replica, ReplicaId rid, char *returntext
     /*
      *  Clean the changelog RUV's
      */
-    cl5CleanRUV(rid);
+    cl5CleanRUV(rid, replica);
 
     /*
      * Now purge the changelog.  The purging thread will free the purge_data
@@ -1371,8 +1404,7 @@ replica_execute_cleanruv_task(Replica *replica, ReplicaId rid, char *returntext
     purge_data = (cleanruv_purge_data *)slapi_ch_calloc(1, sizeof(cleanruv_purge_data));
     purge_data->cleaned_rid = rid;
     purge_data->suffix_sdn = replica_get_root(replica);
-    purge_data->replName = (char *)replica_get_name(replica);
-    purge_data->replGen = replica_get_generation(replica);
+    purge_data->replica = replica;
     trigger_cl_purging(purge_data);
 
     if (rc != RUV_SUCCESS) {

ldap/servers/plugins/replication/repl_extop.c

@@ -1159,11 +1159,13 @@ multimaster_extop_EndNSDS50ReplicationRequest(Slapi_PBlock *pb)
                    The best solution I think, would be to "fake" on the supplier
                    an entry that corresponds to the ruv sent to the consumer and then
                    send it as part of the data */
-
+/*
                 if (cl5GetState() == CL5_STATE_OPEN) {
                     cl5DeleteDBSync(connext->replica_acquired);
                 }
-
+                no longer needed, the cl was recreated when replication was reenabled
+                at the end of bulk import
+*/
                 replica_set_ruv(r, connext->supplier_ruv);
                 connext->supplier_ruv = NULL;
 
@@ -1171,7 +1173,8 @@ multimaster_extop_EndNSDS50ReplicationRequest(Slapi_PBlock *pb)
                    smallest csn in the new ruv, so that this replica ca supply
                    other servers.
                 */
-                if (cl5GetState() == CL5_STATE_OPEN) {
+                if (replica_is_flag_set(r, REPLICA_LOG_CHANGES) &&
+                    cl5GetState() == CL5_STATE_OPEN) {
                     replica_log_ruv_elements(r);
                 }
 

ldap/servers/slapd/back-ldbm/archive.c

@@ -412,19 +412,6 @@ ldbm_back_ldbm2archive(Slapi_PBlock *pb)
         }
     }
 
-    return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_BACKUP_FN);
-    if (return_value) {
-        slapi_log_err(SLAPI_LOG_BACKLDBM,
-                      "ldbm_back_ldbm2archive", "pre-backup-plugin failed (%d).\n", return_value);
-        if (is_slapd_running() && run_from_cmdline) {
-            slapi_log_err(SLAPI_LOG_ERR,
-                          "ldbm_back_ldbm2archive", "Standalone db2bak is not supported when a "
-                                                    "multimaster replication enabled server is "
-                                                    "coexisting.\nPlease use db2bak.pl, instead.\n");
-            goto err;
-        }
-    }
-
     /* tell it to archive */
     return_value = dblayer_backup(li, directory, task);
     if (return_value) {
@@ -433,12 +420,6 @@ ldbm_back_ldbm2archive(Slapi_PBlock *pb)
         goto err;
     }
 
-    return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN);
-    if (return_value) {
-        slapi_log_err(SLAPI_LOG_BACKLDBM,
-                      "ldbm_back_ldbm2archive", "post-backup-plugin failed (%d).\n", return_value);
-    }
-
     if (!run_from_cmdline) {
         ldbm_instance *inst;
         Object *inst_obj;

ldap/servers/slapd/back-ldbm/back-ldbm.h

@@ -734,6 +734,7 @@ typedef struct ldbm_instance
     PRLock *inst_handle_list_mutex;
 
     DB *inst_id2entry; /* id2entry for this instance. */
+    DB *inst_changelog; /* changelog for this instance. */
 
     perfctrs_private inst_perf_private; /* Private data for the performance counters specific to this instance */
     attrcrypt_state_private *inst_attrcrypt_state_private;

ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c

@@ -74,7 +74,8 @@ static int commit_good_database(bdb_config *priv, int mode);
 static int read_metadata(struct ldbminfo *li);
 static int count_dbfiles_in_dir(char *directory, int *count, int recurse);
 static int dblayer_override_libdb_functions(void);
-static int dblayer_force_checkpoint(struct ldbminfo *li);
+static int bdb_force_checkpoint(struct ldbminfo *li);
+static int bdb_force_logrenewal(struct ldbminfo *li);
 static int log_flush_threadmain(void *param);
 static int dblayer_delete_transaction_logs(const char *log_dir);
 static int dblayer_is_logfilename(const char *path);
@@ -95,6 +96,7 @@ static PRLock *sync_txn_log_flush = NULL;
 static PRCondVar *sync_txn_log_flush_done = NULL;
 static PRCondVar *sync_txn_log_do_flush = NULL;
 static int bdb_db_remove_ex(bdb_db_env *env, char const path[], char const dbName[], PRBool use_lock);
+static int bdb_db_compact_one_db(DB *db, ldbm_instance *inst);
 static int bdb_restore_file_check(struct ldbminfo *li);
 
 #define MEGABYTE (1024 * 1024)
@@ -2345,9 +2347,12 @@ bdb_get_db(backend *be, char *indexname, int open_flag, struct attrinfo *ai, DB
         goto out;
 
     dbp = *ppDB;
-    return_value = _dblayer_set_db_callbacks(conf, dbp, ai);
-    if (return_value)
-        goto out;
+    if (ai) {
+        return_value = _dblayer_set_db_callbacks(conf, dbp, ai);
+        if (return_value) {
+            goto out;
+        }
+    }
 
     /* The subname argument allows applications to have
      * subdatabases, i.e., multiple databases inside of a single
@@ -2379,9 +2384,12 @@ bdb_get_db(backend *be, char *indexname, int open_flag, struct attrinfo *ai, DB
             goto out;
         }
         dbp = *ppDB;
-        return_value = _dblayer_set_db_callbacks(conf, dbp, ai);
-        if (return_value)
-            goto out;
+        if (ai) {
+            return_value = _dblayer_set_db_callbacks(conf, dbp, ai);
+            if (return_value) {
+                goto out;
+            }
+        }
 
         slapi_ch_free_string(&abs_file_name);
     }
@@ -2457,6 +2465,62 @@ bdb_db_remove(bdb_db_env *env, char const path[], char const dbName[])
     return (bdb_db_remove_ex(env, path, dbName, PR_TRUE));
 }
 
+static int
+bdb_db_compact_one_db(DB *db, ldbm_instance *inst)
+{
+    DBTYPE type;
+    int rc = 0;
+    back_txn txn;
+    DB_COMPACT c_data = {0};
+
+    rc = db->get_type(db, &type);
+    if (rc) {
+        slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db",
+                      "compactdb: failed to determine db type for %s: db error - %d %s\n",
+                      inst->inst_name, rc, db_strerror(rc));
+        return rc;
+    }
+
+    rc = dblayer_txn_begin(inst->inst_be, NULL, &txn);
+    if (rc) {
+        slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: transaction begin failed: %d\n", rc);
+        return rc;
+    }
+    /*
+     * https://docs.oracle.com/cd/E17275_01/html/api_reference/C/BDB-C_APIReference.pdf
+     * "DB_FREELIST_ONLY
+     * Do no page compaction, only returning pages to the filesystem that are already free and at the end
+     * of the file. This flag must be set if the database is a Hash access method database."
+     *
+     */
+
+    uint32_t compact_flags = DB_FREE_SPACE;
+    if (type == DB_HASH) {
+        compact_flags |= DB_FREELIST_ONLY;
+    }
+    rc = db->compact(db, txn.back_txn_txn, NULL /*start*/, NULL /*stop*/,
+                     &c_data, compact_flags, NULL /*end*/);
+    if (rc) {
+        slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db",
+                      "compactdb: failed to compact %s; db error - %d %s\n",
+                      inst->inst_name, rc, db_strerror(rc));
+        if ((rc = dblayer_txn_abort(inst->inst_be, &txn))) {
+            slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: failed to abort txn (%s) db error - %d %s\n",
+                          inst->inst_name, rc, db_strerror(rc));
+        }
+    } else {
+        slapi_log_err(SLAPI_LOG_NOTICE, "bdb_db_compact_one_db",
+                      "compactdb: compact %s - %d pages freed\n",
+                      inst->inst_name, c_data.compact_pages_free);
+        if ((rc = dblayer_txn_commit(inst->inst_be, &txn))) {
+            slapi_log_err(SLAPI_LOG_ERR, "bdb_db_compact_one_db", "compactdb: failed to commit txn (%s) db error - %d %s\n",
+                          inst->inst_name, rc, db_strerror(rc));
+        }
+    }
+
+    return rc;
+}
+
 #define DBLAYER_CACHE_DELAY PR_MillisecondsToInterval(250)
 int
 bdb_rm_db_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_checkpoint)
@@ -2497,7 +2561,7 @@ bdb_rm_db_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_ch
      Force a checkpoint here to break deadlock.
   */
     if (0 == no_force_checkpoint) {
-        dblayer_force_checkpoint(li);
+        bdb_force_checkpoint(li);
     }
 
     if (0 == dblayer_get_index_file(be, a, &db, 0 /* Don't create an index file
@@ -3554,7 +3618,7 @@ bdb_start_checkpoint_thread(struct ldbminfo *li)
 }
 
 /*
- * checkpoint thread -- borrow the timing for compacting id2entry, as well.
+ * checkpoint thread -- borrow the timing for compacting id2entry, and eventually changelog, as well.
  */
 static int
 checkpoint_threadmain(void *param)
@@ -3573,7 +3637,6 @@ checkpoint_threadmain(void *param)
     time_t checkpoint_interval_update = 0;
     time_t compactdb_interval = 0;
     time_t checkpoint_interval = 0;
-    back_txn txn;
 
     PR_ASSERT(NULL != param);
     li = (struct ldbminfo *)param;
@@ -3593,7 +3656,7 @@ checkpoint_threadmain(void *param)
     }
 
     /* work around a problem with newly created environments */
-    dblayer_force_checkpoint(li);
+    bdb_force_checkpoint(li);
 
     PR_Lock(li->li_config_mutex);
     checkpoint_interval = (time_t)BDB_CONFIG(li)->bdb_checkpoint_interval;
@@ -3602,7 +3665,7 @@ checkpoint_threadmain(void *param)
     debug_checkpointing = BDB_CONFIG(li)->bdb_debug_checkpointing;
     PR_Unlock(li->li_config_mutex);
 
-    /* assumes dblayer_force_checkpoint worked */
+    /* assumes bdb_force_checkpoint worked */
     /*
      * Importantly, the use of this api is not affected by backwards time steps
      * and the like. Because this use relative system time, rather than utc,
@@ -3706,7 +3769,6 @@ checkpoint_threadmain(void *param)
             Object *inst_obj;
             ldbm_instance *inst;
             DB *db = NULL;
-            DB_COMPACT c_data = {0};
 
             for (inst_obj = objset_first_obj(li->li_instance_set);
                  inst_obj;
@@ -3719,58 +3781,26 @@ checkpoint_threadmain(void *param)
                 slapi_log_err(SLAPI_LOG_NOTICE, "checkpoint_threadmain", "Compacting DB start: %s\n",
                               inst->inst_name);
 
-                /*
-                 * It's possible for this to heap us after free because when we access db
-                 * *just* as the server shut's down, we don't know it. So we should probably
-                 * do something like wrapping access to the db var in a rwlock, and have "read"
-                 * to access, and take writes to change the state. This would prevent the issue.
-                 */
-                DBTYPE type;
-                rc = db->get_type(db, &type);
+                rc = bdb_db_compact_one_db(db, inst);
                 if (rc) {
                     slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain",
-                                  "compactdb: failed to determine db type for %s: db error - %d %s\n",
-                                  inst->inst_name, rc, db_strerror(rc));
-                    continue;
-                }
-
-                rc = dblayer_txn_begin(inst->inst_be, NULL, &txn);
-                if (rc) {
-                    slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: transaction begin failed: %d\n", rc);
+                                  "compactdb: failed to compact id2entry for %s; db error - %d %s\n",
+                                   inst->inst_name, rc, db_strerror(rc));
                     break;
                 }
-                /*
-                 * https://docs.oracle.com/cd/E17275_01/html/api_reference/C/BDB-C_APIReference.pdf
-                 * "DB_FREELIST_ONLY
-                 * Do no page compaction, only returning pages to the filesystem that are already free and at the end
-                 * of the file. This flag must be set if the database is a Hash access method database."
-                 *
+
+                /* compact changelog db */
+                /* NOTE (LK) this is now done along regular compaction, 
+                 * if it should be configurable add a switch to changelog config
                  */
+                dblayer_get_changelog(inst->inst_be, &db, 0);
 
-                uint32_t compact_flags = DB_FREE_SPACE;
-                if (type == DB_HASH) {
-                    compact_flags |= DB_FREELIST_ONLY;
-                }
-                rc = db->compact(db, txn.back_txn_txn, NULL /*start*/, NULL /*stop*/,
-                                 &c_data, compact_flags, NULL /*end*/);
+                rc = bdb_db_compact_one_db(db, inst);
                 if (rc) {
                     slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain",
-                                  "compactdb: failed to compact %s; db error - %d %s\n",
-                                  inst->inst_name, rc, db_strerror(rc));
-                    if ((rc = dblayer_txn_abort(inst->inst_be, &txn))) {
-                        slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: failed to abort txn (%s) db error - %d %s\n",
-                                      inst->inst_name, rc, db_strerror(rc));
-                        break;
-                    }
-                } else {
-                    slapi_log_err(SLAPI_LOG_NOTICE, "checkpoint_threadmain",
-                                  "compactdb: compact %s - %d pages freed\n",
-                                  inst->inst_name, c_data.compact_pages_free);
-                    if ((rc = dblayer_txn_commit(inst->inst_be, &txn))) {
-                        slapi_log_err(SLAPI_LOG_ERR, "checkpoint_threadmain", "compactdb: failed to commit txn (%s) db error - %d %s\n",
-                                      inst->inst_name, rc, db_strerror(rc));
-                        break;
-                    }
+                                  "compactdb: failed to compact changelog for %s; db error - %d %s\n",
+                                   inst->inst_name, rc, db_strerror(rc));
+                    break;
                 }
             }
             compactdb_interval = compactdb_interval_update;
@@ -3778,7 +3808,7 @@ checkpoint_threadmain(void *param)
         }
     }
     slapi_log_err(SLAPI_LOG_TRACE, "checkpoint_threadmain", "Check point before leaving\n");
-    rval = dblayer_force_checkpoint(li);
+    rval = bdb_force_checkpoint(li);
 error_return:
 
     DECR_THREAD_COUNT(pEnv);
@@ -4036,7 +4066,7 @@ read_metadata(struct ldbminfo *li)
 
 /* handy routine for checkpointing the db */
 static int
-dblayer_force_checkpoint(struct ldbminfo *li)
+bdb_force_checkpoint(struct ldbminfo *li)
 {
     int ret = 0, i;
     dblayer_private *priv = li->li_dblayer_private;
@@ -4051,7 +4081,7 @@ dblayer_force_checkpoint(struct ldbminfo *li)
 
     if (BDB_CONFIG(li)->bdb_enable_transactions) {
 
-        slapi_log_err(SLAPI_LOG_TRACE, "dblayer_force_checkpoint", "Checkpointing database ...\n");
+        slapi_log_err(SLAPI_LOG_TRACE, "bdb_force_checkpoint", "Checkpointing database ...\n");
 
         /*
      * DB workaround. Newly created environments do not know what the
@@ -4063,7 +4093,7 @@ dblayer_force_checkpoint(struct ldbminfo *li)
         for (i = 0; i < 2; i++) {
             ret = dblayer_txn_checkpoint(li, pEnv, PR_FALSE, PR_TRUE);
             if (ret != 0) {
-                slapi_log_err(SLAPI_LOG_ERR, "dblayer_force_checkpoint", "Checkpoint FAILED, error %s (%d)\n",
+                slapi_log_err(SLAPI_LOG_ERR, "bdb_force_checkpoint", "Checkpoint FAILED, error %s (%d)\n",
                               dblayer_strerror(ret), ret);
                 break;
             }
@@ -4073,6 +4103,34 @@ dblayer_force_checkpoint(struct ldbminfo *li)
     return ret;
 }
 
+/* routine to force all existing transaction logs to be cleared
+ * This is necessary if the transaction logs can contain references
+ * to no longer existing files, but would be processed in a fatal
+ * recovery (like in backup/restore).
+ * There is no straight forward way to do this, but the following
+ * scenario should work:
+ *
+ * 1. check for no longer needed transaction logs by
+ *      calling log_archive()
+ * 2. delete these logs (1and2 similar to checkpointing
+ * 3. force a checkpoint
+ * 4. use log_printf() to write a "comment" to the current txn log
+ *      force a checkpoint
+ *      this could be done by writing once about 10MB or
+ *      by writing smaller chunks in a loop
+ * 5. force a checkpoint and check again
+ *  if a txn log to remove exists remove it and we are done
+ *  else repeat step 4
+ *
+ * NOTE: double check if force_checkpoint also does remove txn files
+ * then the check would have to be modified
+ */
+static int
+bdb_force_logrenewal(struct ldbminfo *li)
+{
+    return 0;
+}
+
 static int
 _dblayer_delete_aux_dir(struct ldbminfo *li, char *path)
 {
@@ -4206,6 +4264,12 @@ _dblayer_delete_instance_dir(ldbm_instance *inst, int startdb)
         if (pEnv &&
             /* PL_strcmp takes NULL arg */
             (PL_strcmp(LDBM_FILENAME_SUFFIX, strrchr(direntry->name, '.')) == 0)) {
+            if (strcmp(direntry->name, "changelog.db") == 0) {
+                /* do not delete the changelog, if it no longer
+                 * matches the database it will be recreated later
+                 */
+                continue;
+            }
             rval = bdb_db_remove_ex(pEnv, filename, 0, PR_TRUE);
         } else {
             rval = ldbm_delete_dirs(filename);
@@ -4221,8 +4285,10 @@ _dblayer_delete_instance_dir(ldbm_instance *inst, int startdb)
     }
 done:
     /* remove the directory itself too */
+    /* no
     if (0 == rval)
         PR_RmDir(inst_dirp);
+    */
     if (inst_dirp != inst_dir)
         slapi_ch_free_string(&inst_dirp);
     return rval;
@@ -4236,7 +4302,7 @@ int
 dblayer_delete_instance_dir(backend *be)
 {
     struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
-    int ret = dblayer_force_checkpoint(li);
+    int ret = bdb_force_checkpoint(li);
 
     if (ret != 0) {
         return ret;
@@ -4783,84 +4849,6 @@ out:
     return return_value;
 }
 
-/*
- * Get changelogdir from cn=changelog5,cn=config
- * The value does not have trailing spaces nor slashes.
- * The changelogdir value must be a fullpath.
- */
-static int
-_dblayer_get_changelogdir(struct ldbminfo *li, char **changelogdir)
-{
-    Slapi_PBlock *pb = NULL;
-    Slapi_Entry **entries = NULL;
-    Slapi_Attr *attr = NULL;
-    Slapi_Value *v = NULL;
-    const char *s = NULL;
-    char *attrs[2];
-    int rc = -1;
-
-    if (NULL == li || NULL == changelogdir) {
-        slapi_log_err(SLAPI_LOG_ERR,
-                      "_dblayer_get_changelogdir", "Invalid arg: "
-                                                   "li: 0x%p, changelogdir: 0x%p\n",
-                      li, changelogdir);
-        return rc;
-    }
-    *changelogdir = NULL;
-
-    pb = slapi_pblock_new();
-    attrs[0] = CHANGELOGDIRATTR;
-    attrs[1] = NULL;
-    slapi_search_internal_set_pb(pb, CHANGELOGENTRY,
-                                 LDAP_SCOPE_BASE, "cn=*", attrs, 0, NULL, NULL,
-                                 li->li_identity, 0);
-    slapi_search_internal_pb(pb);
-    slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc);
-
-    if (LDAP_NO_SUCH_OBJECT == rc) {
-        /* No changelog; Most likely standalone or not a master. */
-        rc = LDAP_SUCCESS;
-        goto bail;
-    }
-    if (LDAP_SUCCESS != rc) {
-        slapi_log_err(SLAPI_LOG_ERR,
-                      "_dblayer_get_changelogdir", "Failed to search \"%s\"\n", CHANGELOGENTRY);
-        goto bail;
-    }
-    /* rc == LDAP_SUCCESS */
-    slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries);
-    if (NULL == entries) {
-        /* No changelog */
-        goto bail;
-    }
-    /* There should be only one entry. */
-    rc = slapi_entry_attr_find(entries[0], CHANGELOGDIRATTR, &attr);
-    if (rc || NULL == attr) {
-        /* No changelog dir */
-        rc = LDAP_SUCCESS;
-        goto bail;
-    }
-    rc = slapi_attr_first_value(attr, &v);
-    if (rc || NULL == v) {
-        /* No changelog dir */
-        rc = LDAP_SUCCESS;
-        goto bail;
-    }
-    rc = LDAP_SUCCESS;
-    s = slapi_value_get_string(v);
-    if (NULL == s) {
-        /* No changelog dir */
-        goto bail;
-    }
-    *changelogdir = slapi_ch_strdup(s);
-    /* Remove trailing spaces and '/' if any */
-    normalize_dir(*changelogdir);
-bail:
-    slapi_free_search_results_internal(pb);
-    slapi_pblock_destroy(pb);
-    return rc;
-}
-
 /* Destination Directory is an absolute pathname */
 int
 bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
@@ -4869,6 +4857,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
     bdb_config *conf = NULL;
     char **listA = NULL, **listB = NULL, **listi, **listj, *prefix;
     char *home_dir = NULL;
+    char *db_dir = NULL;
     int return_value = -1;
     char *pathname1;
     char *pathname2;
@@ -4883,6 +4872,9 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
     conf = (bdb_config *)li->li_dblayer_config;
     priv = li->li_dblayer_private;
     PR_ASSERT(NULL != priv);
+
+    db_dir = bdb_get_db_dir(li);
+
     home_dir = bdb_get_home_dir(li, NULL);
     if (NULL == home_dir || '\0' == *home_dir) {
         slapi_log_err(SLAPI_LOG_ERR,
@@ -4919,7 +4911,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
      */
 
     /* do a quick checkpoint */
-    dblayer_force_checkpoint(li);
+    bdb_force_checkpoint(li);
     dblayer_txn_init(li, &txn);
     return_value = dblayer_txn_begin_all(li, NULL, &txn);
     if (return_value) {
@@ -4998,46 +4990,6 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
             if (inst_dirp != inst_dir)
                 slapi_ch_free_string(&inst_dirp);
         }
-        /* Get changelogdir, if any */
-        _dblayer_get_changelogdir(li, &changelogdir);
-        if (changelogdir) {
-            /* dest dir for changelog: dest_dir/repl_changelog_backup  */
-            char *changelog_destdir = slapi_ch_smprintf("%s/%s",
-                                                        dest_dir, CHANGELOG_BACKUPDIR);
-            return_value = bdb_copy_directory(li, task, changelogdir,
-                                                  changelog_destdir,
-                                                  0 /* backup */,
-                                                  &cnt, 0, 1);
-            if (return_value) {
-                slapi_log_err(SLAPI_LOG_ERR,
-                              "dblayer_backup", "Error in copying directory "
-                                                "(%s -> %s): err=%d\n",
-                              changelogdir, changelog_destdir, return_value);
-                if (task) {
-                    slapi_task_log_notice(task,
-                                          "Backup: error in copying directory "
-                                          "(%s -> %s): err=%d\n",
-                                          changelogdir, changelog_destdir, return_value);
-                }
-                slapi_ch_free_string(&changelog_destdir);
-                goto bail;
-            }
-            /* Copy DBVERSION */
-            pathname1 = slapi_ch_smprintf("%s/%s",
-                                          changelogdir, DBVERSION_FILENAME);
-            pathname2 = slapi_ch_smprintf("%s/%s",
-                                          changelog_destdir, DBVERSION_FILENAME);
-            return_value = dblayer_copyfile(pathname1, pathname2,
-                                            0, priv->dblayer_file_mode);
-            slapi_ch_free_string(&pathname2);
-            slapi_ch_free_string(&changelog_destdir);
-            if (0 > return_value) {
-                slapi_log_err(SLAPI_LOG_ERR, "dblayer_backup", "Failed to copy file %s\n", pathname1);
-                slapi_ch_free_string(&pathname1);
-                goto bail;
-            }
-            slapi_ch_free_string(&pathname1);
-        }
         if (conf->bdb_enable_transactions) {
             /* now, get the list of logfiles that still exist */
             return_value = LOG_ARCHIVE(((bdb_db_env *)priv->dblayer_env)->bdb_DB_ENV,
@@ -5088,7 +5040,7 @@ bdb_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task)
                     (0 != strlen(conf->bdb_log_directory))) {
                     prefix = conf->bdb_log_directory;
                 } else {
-                    prefix = home_dir;
+                    prefix = db_dir;
                 }
                 /* log files have the same filename len(100 is a safety net:) */
                 p1len = strlen(prefix) + strlen(*listB) + 100;
@@ -5368,13 +5320,6 @@ bdb_restore(struct ldbminfo *li, char *src_dir, Slapi_Task *task)
             {
                 tmp_rval = PR_GetFileInfo64(filename1, &info);
                 if (tmp_rval == PR_SUCCESS && PR_FILE_DIRECTORY == info.type) {
-                    /* Is it CHANGELOG_BACKUPDIR? */
-                    if (0 == strcmp(CHANGELOG_BACKUPDIR, direntry->name)) {
-                        /* Yes, this is a changelog backup. */
-                        /* Get the changelog path */
-                        _dblayer_get_changelogdir(li, &changelogdir);
-                        continue;
-                    }
                     inst = ldbm_instance_find_by_name(li, (char *)direntry->name);
                     if (inst == NULL) {
                         slapi_log_err(SLAPI_LOG_ERR,
@@ -6019,6 +5964,14 @@ bdb_get_info(Slapi_Backend *be, int cmd, void **info)
         }
         break;
     }
+    case BACK_INFO_INSTANCE_DIR: {
+        if (li) {
+            ldbm_instance *inst = (ldbm_instance *)be->be_instance_info;
+            *(char **)info = dblayer_get_full_inst_dir(li, inst, NULL, 0);
+            rc = 0;
+        }
+        break;
+    }
     case BACK_INFO_LOG_DIRECTORY: {
         if (li) {
             *(char **)info = bdb_config_db_logdirectory_get_ext((void *)li);
@@ -6034,6 +5987,21 @@ bdb_get_info(Slapi_Backend *be, int cmd, void **info)
         rc = get_suffix_key(be, (struct _back_info_index_key *)info);
         break;
     }
+    case BACK_INFO_DBENV_CLDB: {
+        ldbm_instance *inst = (ldbm_instance *) be->be_instance_info;
+        if (inst->inst_changelog) {
+            rc = 0;
+        } else {
+            DB *db;
+            rc = dblayer_get_changelog(be, &db,DB_CREATE);
+        }
+        if (rc == 0) {
+           *(DB **)info = inst->inst_changelog;
+        } else {
+            *(DB **)info = NULL;
+        }
+        break;
+    }
     default:
         break;
     }
@@ -6069,7 +6037,13 @@ bdb_back_ctrl(Slapi_Backend *be, int cmd, void *info)
     switch (cmd) {
     case BACK_INFO_CRYPT_INIT: {
         back_info_crypt_init *crypt_init = (back_info_crypt_init *)info;
-        rc = back_crypt_init(crypt_init->be, crypt_init->dn,
+        Slapi_DN configdn;
+        slapi_sdn_init(&configdn);
+        be_getbasedn(be, &configdn);
+        char *crypt_dn = slapi_ch_smprintf("%s,%s",
+                        crypt_init->dn,
+                        slapi_sdn_get_dn(&configdn));
+        rc = back_crypt_init(crypt_init->be, crypt_dn,
                              crypt_init->encryptionAlgorithm,
                              &(crypt_init->state_priv));
         break;
@@ -6091,6 +6065,106 @@ bdb_back_ctrl(Slapi_Backend *be, int cmd, void *info)
                                       &(crypt_value->out));
         break;
     }
+    case BACK_INFO_DBENV_CLDB_REMOVE: {
+        DB *db = (DB *)info;
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        ldbm_instance *inst = (ldbm_instance *) be->be_instance_info;
+        if (li) {
+            dblayer_private *priv = (dblayer_private *)li->li_dblayer_private;
+            if (priv && priv->dblayer_env) {
+                char *instancedir;
+                slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir);
+                char *path = slapi_ch_smprintf("%s/changelog.db", instancedir);
+                db->close(db, 0);
+                rc = bdb_db_remove_ex((bdb_db_env *)priv->dblayer_env, path, NULL, PR_TRUE);
+                inst->inst_changelog = NULL;
+                slapi_ch_free_string(&instancedir);
+            }
+        }
+        break;
+    }
+    case BACK_INFO_DBENV_CLDB_UPGRADE: {
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        char *oldFile = (char *)info;
+        if (li) {
+            dblayer_private *priv = (dblayer_private *)li->li_dblayer_private;
+            if (priv && priv->dblayer_env) {
+                DB_ENV *pEnv = ((bdb_db_env *)priv->dblayer_env)->bdb_DB_ENV;
+                if (pEnv) {
+                    char *instancedir;
+                    slapi_back_get_info(be, BACK_INFO_INSTANCE_DIR, (void **)&instancedir);
+                    char *newFile = slapi_ch_smprintf("%s/changelog.db", instancedir);
+                    rc = pEnv->dbrename(pEnv, 0, oldFile, 0, newFile, 0);
+                    slapi_ch_free_string(&instancedir);
+                    bdb_force_logrenewal(li);
+                }
+            }
+        }
+        break;
+    }
+    case BACK_INFO_CLDB_GET_CONFIG: {
+        /* get a config entry relative to the
+         * backend config entry
+         * Caller must free the returned entry (config->ce)
+         * If it fails config->ce is left unchanged
+         */
+        back_info_config_entry *config = (back_info_config_entry *)info;
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        Slapi_DN configdn;
+        slapi_sdn_init(&configdn);
+        be_getbasedn(be, &configdn);
+        char *config_dn = slapi_ch_smprintf("%s,%s",
+                        config->dn,
+                        slapi_sdn_get_dn(&configdn));
+        Slapi_PBlock *search_pb = slapi_pblock_new();
+        slapi_search_internal_set_pb(search_pb, config_dn, LDAP_SCOPE_BASE, "objectclass=*",
+                                     NULL, 0, NULL, NULL, li->li_identity, 0);
+        slapi_search_internal_pb(search_pb);
+        slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &rc);
+        if (LDAP_SUCCESS == rc ) {
+            Slapi_Entry **entries;
+            slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries);
+            if (entries && entries[0]) {
+                config->ce = slapi_entry_dup(entries[0]);
+            } else {
+                rc = -1;
+            }
+        }
+        slapi_free_search_results_internal(search_pb);
+        slapi_pblock_destroy(search_pb);
+        slapi_ch_free_string(&config_dn);
+        break;
+    }
+    case BACK_INFO_CLDB_SET_CONFIG: {
+        /* This control option allows a plugin to set a backend configuration
+         * entry without knowing the location of the backend config.
+         * It passes an entry with a relative dn and this dn is expanded by the
+         * backend config dn.
+         */
+        Slapi_DN fulldn;
+        Slapi_DN configdn;
+        struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+        Slapi_Entry *config_entry = (Slapi_Entry *)info;
+
+        slapi_sdn_init(&configdn);
+        be_getbasedn(be, &configdn);
+        char *newdn = slapi_ch_smprintf("%s,%s",
+                        slapi_entry_get_dn_const(config_entry),
+                        slapi_sdn_get_dn(&configdn));
+        slapi_sdn_init(&fulldn);
+        slapi_sdn_init_dn_byref(&fulldn, newdn);
+        slapi_entry_set_sdn(config_entry, &fulldn);
+        slapi_ch_free_string(&newdn);
+
+        Slapi_PBlock *pb = slapi_pblock_new();
+        slapi_pblock_init(pb);
+        slapi_add_entry_internal_set_pb(pb, config_entry, NULL,
+                                        li->li_identity, 0);
+        slapi_add_internal_pb(pb);
+        slapi_pblock_get(pb, SLAPI_PLUGIN_INTOP_RESULT, &rc);
+        slapi_pblock_destroy(pb);
+        break;
+    }
     default:
         break;
     }

ldap/servers/slapd/back-ldbm/db-bdb/bdb_ldif2db.c

@@ -706,6 +706,7 @@ bdb_db2ldif(Slapi_PBlock *pb)
     int decrypt = 0;
     int32_t dump_replica = 0;
     int dump_uniqueid = 1;
+    int dump_changelog = 0;
     int fd = STDOUT_FILENO;
     IDList *idl = NULL; /* optimization for -s include lists */
     int cnt = 0, lastcnt = 0;
@@ -816,6 +817,7 @@ bdb_db2ldif(Slapi_PBlock *pb)
     slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &fname);
     slapi_pblock_get(pb, SLAPI_DB2LDIF_PRINTKEY, &printkey);
     slapi_pblock_get(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &dump_uniqueid);
+    slapi_pblock_get(pb, SLAPI_LDIF_CHANGELOG, &dump_changelog);
 
     /* tsk, overloading printkey.  shame on me. */
     ok_index = !(printkey & EXPORT_ID2ENTRY_ONLY);
@@ -1286,6 +1288,12 @@ bdb_db2ldif(Slapi_PBlock *pb)
                       "export %s: Processed %d entries (100%%).\n",
                       inst->inst_name, cnt);
     }
+    if (run_from_cmdline && dump_changelog) {
+        return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN);
+        slapi_log_err(SLAPI_LOG_INFO, "ldbm_back_ldbm2ldif",
+                      "export changelog for %s.\n", inst->inst_name);
+    }
+
 bye:
     if (idl) {
         idl_free(&idl);

ldap/servers/slapd/back-ldbm/dblayer.c

@@ -92,6 +92,7 @@
 
 #define NEWDIR_MODE 0755
 #define DB_REGION_PREFIX "__db."
+#define BE_CHANGELOG_FILE "changelog"
 
 
 static int dblayer_post_restore = 0;
@@ -395,6 +396,35 @@ dblayer_release_id2entry(backend *be __attribute__((unused)), DB *pDB __attribut
     return 0;
 }
 
+int
+dblayer_close_changelog(backend *be)
+{
+    ldbm_instance *inst;
+    DB *pDB = NULL;
+    int return_value = 0;
+
+    PR_ASSERT(NULL != be);
+    inst = (ldbm_instance *) be->be_instance_info;
+    PR_ASSERT(NULL != inst);
+
+    pDB = inst->inst_changelog;
+    if (pDB) {
+        return_value = pDB->close(pDB,0);
+        inst->inst_changelog = NULL;
+    }
+    return return_value;
+}
+
+int
+dblayer_erase_changelog_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_chkpt)
+{
+    if ((NULL == be) || (NULL == be->be_database)) {
+        return 0;
+    }
+    /* TBD (LK) */
+    return 0;
+}
+
 int
 dblayer_close_indexes(backend *be)
 {
@@ -464,6 +494,7 @@ dblayer_instance_close(backend *be)
     }
 
     return_value = dblayer_close_indexes(be);
+    return_value |= dblayer_close_changelog(be);
 
     /* Now close id2entry if it's open */
     pDB = inst->inst_id2entry;
@@ -604,6 +635,52 @@ dblayer_get_index_file(backend *be, struct attrinfo *a, DB **ppDB, int open_flag
     return return_value;
 }
 
+int dblayer_get_changelog(backend *be, DB** ppDB, int open_flags)
+{
+    ldbm_instance *inst = (ldbm_instance *) be->be_instance_info;
+    int return_value = -1;
+    DB *pDB = NULL;
+
+    *ppDB = NULL;
+
+    if (inst->inst_changelog) {
+        /* This means that the pointer is valid, so we should return it. */
+        *ppDB = inst->inst_changelog;
+        return 0;
+    }
+
+    /* only one thread should open the chgangelog, we can use the mutex
+     * for opening the index files.
+     */
+    PR_Lock(inst->inst_handle_list_mutex);
+    if (inst->inst_changelog) {
+        /* another thread set the handle while we were waiting on the lock */
+        *ppDB = inst->inst_changelog;
+        PR_Unlock(inst->inst_handle_list_mutex);
+        return 0;
+    }
+
+    /* attrinfo handle is still blank, and we have the mutex: open the
+     * index file and stuff it in the attrinfo.
+     */
+    return_value = dblayer_open_file(be, BE_CHANGELOG_FILE, open_flags,
+                                   NULL, &pDB);
+    if (0 == return_value) {
+        /* Opened it OK */
+        inst->inst_changelog = pDB;
+        /* And, most importantly, return something to the caller!*/
+        *ppDB = pDB;
+    } else {
+        /* Did not open it OK ! */
+        /* Do nothing, because return value and fact that we didn't
+         * store a DB* in the attrinfo is enough
+         */
+    }
+    PR_Unlock(inst->inst_handle_list_mutex);
+
+    return return_value;
+}
+
 /*
  * Unlock the db lib mutex here if we need to.
  */

ldap/servers/slapd/back-ldbm/proto-back-ldbm.h

@@ -82,6 +82,7 @@ int dblayer_get_index_file(backend *be, struct attrinfo *a, DB **ppDB, int creat
 int dblayer_release_index_file(backend *be, struct attrinfo *a, DB *pDB);
 int dblayer_erase_index_file(backend *be, struct attrinfo *a, PRBool use_lock, int no_force_chkpt);
 int dblayer_get_id2entry(backend *be, DB **ppDB);
+int dblayer_get_changelog(backend *be, DB** ppDB, int create);
 int dblayer_release_id2entry(backend *be, DB *pDB);
 int dblayer_txn_init(struct ldbminfo *li, back_txn *txn);
 int dblayer_txn_begin(backend *be, back_txnid parent_txn, back_txn *txn);

ldap/servers/slapd/backend.c

@@ -270,6 +270,17 @@ slapi_be_gettype(Slapi_Backend *be)
     return r;
 }
 
+Slapi_DN *
+be_getbasedn(Slapi_Backend *be, Slapi_DN *dn)
+{
+    if (be->be_state == BE_STATE_DELETED) {
+        slapi_sdn_set_ndn_byval(dn, NULL);
+    } else {
+        slapi_sdn_set_ndn_byref(dn, be->be_basedn);
+    }
+    return dn;
+}
+
 Slapi_DN *
 be_getconfigdn(Slapi_Backend *be, Slapi_DN *dn)
 {

ldap/servers/slapd/main.c

@@ -89,6 +89,7 @@ struct main_config
     char *archive_name;
     int db2ldif_dump_replica;
     int db2ldif_dump_uniqueid;
+    int ldif_include_changelog;
     int ldif2db_generate_uniqueid;
     char *ldif2db_namespaceid;
     int importexport_encrypt;
@@ -517,6 +518,7 @@ main(int argc, char **argv)
     /* Set a number of defaults */
     mcfg.slapd_exemode = SLAPD_EXEMODE_UNKNOWN;
     mcfg.ldif_printkey = EXPORT_PRINTKEY | EXPORT_APPENDMODE;
+    mcfg.ldif_include_changelog = 0;
     mcfg.db2ldif_dump_uniqueid = 1;
     mcfg.ldif2db_generate_uniqueid = SLAPI_UNIQUEID_GENERATE_TIME_BASED;
     mcfg.ldif2db_removedupvals = 1;
@@ -1181,7 +1183,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
      *
      */
 
-    char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qV";
+    char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qRV";
     struct opt_ext long_options_db2ldif[] = {
         {"version", ArgNone, 'v'},
         {"debug", ArgRequired, 'd'},
@@ -1195,6 +1197,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
         {"noUniqueIds", ArgNone, 'u'},
         {"configDir", ArgRequired, 'D'},
         {"encrypt", ArgOptional, 'E'},
+        {"includechangelog", ArgNone, 'R'},
         {"nowrap", ArgNone, 'U'},
         {"minimalEncode", ArgNone, 'm'},
         {"oneOutputFile", ArgNone, 'o'},
@@ -1600,6 +1603,20 @@ process_command_line(int argc, char **argv, struct main_config *mcfg)
 
             break;
 
+        case 'R': /* db2ldif  and ldif2db only */
+            if (mcfg->slapd_exemode != SLAPD_EXEMODE_DB2LDIF &&
+                mcfg->slapd_exemode != SLAPD_EXEMODE_LDIF2DB) {
+                usage(mcfg->myname, mcfg->extraname, mcfg->slapd_exemode);
+                exit(1);
+            }
+
+            /*
+             * import/export should handle changelog.
+             */
+            mcfg->ldif_include_changelog = 1;
+
+            break;
+
         case 'C':
             if (mcfg->slapd_exemode == SLAPD_EXEMODE_LDIF2DB) {
                 /* used to mean "Cool new import" (which is now
@@ -2177,6 +2194,7 @@ slapd_exemode_db2ldif(int argc, char **argv, struct main_config *mcfg)
         slapi_pblock_set(pb, SLAPI_BACKEND_INSTANCE_NAME, *instp);
         slapi_pblock_set_ldif_dump_replica(pb, mcfg->db2ldif_dump_replica);
         slapi_pblock_set(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &(mcfg->db2ldif_dump_uniqueid));
+        slapi_pblock_set(pb, SLAPI_LDIF_CHANGELOG, &(mcfg->ldif_include_changelog));
         int32_t task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE;
         slapi_pblock_set(pb, SLAPI_TASK_FLAGS, &task_flags);
         int32_t is_running = 0;

ldap/servers/slapd/pblock.c

@@ -1221,12 +1221,6 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value)
         }
         (*(IFP *)value) = pblock->pb_plugin->plg_bepreclose;
         break;
-    case SLAPI_PLUGIN_BE_PRE_BACKUP_FN:
-        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPREOPERATION) {
-            return (-1);
-        }
-        (*(IFP *)value) = pblock->pb_plugin->plg_beprebackup;
-        break;
 
     /* backend postoperation plugin */
     case SLAPI_PLUGIN_BE_POST_MODIFY_FN:
@@ -1259,11 +1253,18 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value)
         }
         (*(IFP *)value) = pblock->pb_plugin->plg_bepostopen;
         break;
-    case SLAPI_PLUGIN_BE_POST_BACKUP_FN:
+
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
         if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
             return (-1);
         }
-        (*(IFP *)value) = pblock->pb_plugin->plg_bepostbackup;
+        (*(IFP *)value) = pblock->pb_plugin->plg_bepostexport;
+        break;
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        (*(IFP *)value) = pblock->pb_plugin->plg_bepostimport;
         break;
 
     /* internal preoperation plugin */
@@ -2079,6 +2080,14 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value)
             (*(int *)value) = 0;
         }
         break;
+    case SLAPI_LDIF_CHANGELOG:
+        if (pblock->pb_task != NULL) {
+            (*(int *)value) = pblock->pb_task->ldif_include_changelog;
+        } else {
+            (*(int *)value) = 0;
+        }
+        break;
+
     /* dbverify */
     case SLAPI_DBVERIFY_DBDIR:
         if (pblock->pb_task != NULL) {
@@ -3110,12 +3119,6 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value)
         }
         pblock->pb_plugin->plg_bepreclose = (IFP)value;
         break;
-    case SLAPI_PLUGIN_BE_PRE_BACKUP_FN:
-        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPREOPERATION) {
-            return (-1);
-        }
-        pblock->pb_plugin->plg_beprebackup = (IFP)value;
-        break;
 
     /* backend postoperation plugin */
     case SLAPI_PLUGIN_BE_POST_MODIFY_FN:
@@ -3148,11 +3151,17 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value)
         }
         pblock->pb_plugin->plg_bepostopen = (IFP)value;
         break;
-    case SLAPI_PLUGIN_BE_POST_BACKUP_FN:
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
         if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
             return (-1);
         }
-        pblock->pb_plugin->plg_bepostbackup = (IFP)value;
+        pblock->pb_plugin->plg_bepostexport = (IFP)value;
+        break;
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
+        if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) {
+            return (-1);
+        }
+        pblock->pb_plugin->plg_bepostimport = (IFP)value;
         break;
 
     /* internal preoperation plugin */
@@ -3867,6 +3876,11 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value)
         pblock->pb_task->import_state = *((int *)value);
         break;
 
+    case SLAPI_LDIF_CHANGELOG:
+        _pblock_assert_pb_task(pblock);
+        pblock->pb_task->ldif_include_changelog = *((int *)value);
+        break;
+
     case SLAPI_LDIF2DB_ENCRYPT:
     case SLAPI_DB2LDIF_DECRYPT:
         _pblock_assert_pb_task(pblock);

ldap/servers/slapd/pblock_v3.h

@@ -60,6 +60,7 @@ typedef struct _slapi_pblock_task
     char *ldif_namespaceid; /* used for name based uniqueid generation */
     int ldif_dump_replica;
     int ldif_dump_uniqueid;     /* dump uniqueid during db2ldif */
+    int ldif_include_changelog;     /* include changelog for import/export */
     int ldif_generate_uniqueid; /* generate uniqueid during db2ldif */
     int ldif_encrypt;           /* used to enable encrypt/decrypt on import and export */
     int seq_type;

ldap/servers/slapd/plugin.c

@@ -350,7 +350,6 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction)
     case SLAPI_PLUGIN_BE_PRE_ADD_FN:
     case SLAPI_PLUGIN_BE_PRE_DELETE_FN:
     case SLAPI_PLUGIN_BE_PRE_CLOSE_FN:
-    case SLAPI_PLUGIN_BE_PRE_BACKUP_FN:
         plugin_list_number = PLUGIN_LIST_BEPREOPERATION;
         do_op = 1; /* always allow backend callbacks (even during startup) */
         break;
@@ -359,7 +358,8 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction)
     case SLAPI_PLUGIN_BE_POST_ADD_FN:
     case SLAPI_PLUGIN_BE_POST_DELETE_FN:
     case SLAPI_PLUGIN_BE_POST_OPEN_FN:
-    case SLAPI_PLUGIN_BE_POST_BACKUP_FN:
+    case SLAPI_PLUGIN_BE_POST_EXPORT_FN:
+    case SLAPI_PLUGIN_BE_POST_IMPORT_FN:
         plugin_list_number = PLUGIN_LIST_BEPOSTOPERATION;
         do_op = 1; /* always allow backend callbacks (even during startup) */
         break;
@@ -3565,8 +3565,8 @@ plugin_invoke_plugin_pb(struct slapdplugin *plugin, int operation, Slapi_PBlock
         operation == SLAPI_PLUGIN_CLEANUP_FN ||
         operation == SLAPI_PLUGIN_BE_PRE_CLOSE_FN ||
         operation == SLAPI_PLUGIN_BE_POST_OPEN_FN ||
-        operation == SLAPI_PLUGIN_BE_PRE_BACKUP_FN ||
-        operation == SLAPI_PLUGIN_BE_POST_BACKUP_FN)
+        operation == SLAPI_PLUGIN_BE_POST_EXPORT_FN ||
+        operation == SLAPI_PLUGIN_BE_POST_IMPORT_FN)
         return PR_TRUE;
 
     slapi_pblock_get(pb, SLAPI_OPERATION, &pb_op);

ldap/servers/slapd/proto-slap.h

@@ -180,6 +180,7 @@ void be_done(Slapi_Backend *be);
 void be_addsuffix(Slapi_Backend *be, const Slapi_DN *suffix);
 Slapi_DN *be_getconfigdn(Slapi_Backend *be, Slapi_DN *dn);
 Slapi_DN *be_getmonitordn(Slapi_Backend *be, Slapi_DN *dn);
+Slapi_DN *be_getbasedn(Slapi_Backend *be, Slapi_DN *dn);
 int be_writeconfig(Slapi_Backend *be);
 void global_backend_lock_init(void);
 int global_backend_lock_requested(void);

ldap/servers/slapd/slap.h

@@ -1179,14 +1179,12 @@ struct slapdplugin
             IFP plg_un_bepre_delete;           /* delete */
             IFP plg_un_bepre_delete_tombstone; /* tombstone creation */
             IFP plg_un_bepre_close;            /* close */
-            IFP plg_un_bepre_backup;           /* backup */
         } plg_un_bepre;
 #define plg_bepremodify plg_un.plg_un_bepre.plg_un_bepre_modify
 #define plg_bepremodrdn plg_un.plg_un_bepre.plg_un_bepre_modrdn
 #define plg_bepreadd plg_un.plg_un_bepre.plg_un_bepre_add
 #define plg_bepredelete plg_un.plg_un_bepre.plg_un_bepre_delete
 #define plg_bepreclose plg_un.plg_un_bepre.plg_un_bepre_close
-#define plg_beprebackup plg_un.plg_un_bepre.plg_un_bepre_backup
 
         /* backend post-operation plugin structure */
         struct plg_un_bepost_operation
@@ -1196,14 +1194,16 @@ struct slapdplugin
             IFP plg_un_bepost_add;    /* add */
             IFP plg_un_bepost_delete; /* delete */
             IFP plg_un_bepost_open;   /* open */
-            IFP plg_un_bepost_backup; /* backup */
+            IFP plg_un_bepost_import; /* import */
+            IFP plg_un_bepost_export; /* export */
         } plg_un_bepost;
 #define plg_bepostmodify plg_un.plg_un_bepost.plg_un_bepost_modify
 #define plg_bepostmodrdn plg_un.plg_un_bepost.plg_un_bepost_modrdn
 #define plg_bepostadd plg_un.plg_un_bepost.plg_un_bepost_add
 #define plg_bepostdelete plg_un.plg_un_bepost.plg_un_bepost_delete
 #define plg_bepostopen plg_un.plg_un_bepost.plg_un_bepost_open
-#define plg_bepostbackup plg_un.plg_un_bepost.plg_un_bepost_backup
+#define plg_bepostimport plg_un.plg_un_bepost.plg_un_bepost_import
+#define plg_bepostexport plg_un.plg_un_bepost.plg_un_bepost_export
 
         /* internal  pre-operation plugin structure */
         struct plg_un_internal_pre_operation

ldap/servers/slapd/slapi-plugin.h

@@ -7075,7 +7075,6 @@ typedef struct slapi_plugindesc
 #define SLAPI_PLUGIN_BE_PRE_MODRDN_FN 452
 #define SLAPI_PLUGIN_BE_PRE_DELETE_FN 453
 #define SLAPI_PLUGIN_BE_PRE_CLOSE_FN  454
-#define SLAPI_PLUGIN_BE_PRE_BACKUP_FN 455
 
 /* preoperation plugin to the backend - just after transaction creation */
 #define SLAPI_PLUGIN_BE_TXN_PRE_ADD_FN              460
@@ -7112,7 +7111,8 @@ typedef struct slapi_plugindesc
 #define SLAPI_PLUGIN_BE_POST_MODRDN_FN 552
 #define SLAPI_PLUGIN_BE_POST_DELETE_FN 553
 #define SLAPI_PLUGIN_BE_POST_OPEN_FN   554
-#define SLAPI_PLUGIN_BE_POST_BACKUP_FN 555
+#define SLAPI_PLUGIN_BE_POST_EXPORT_FN 556
+#define SLAPI_PLUGIN_BE_POST_IMPORT_FN 557
 
 /* postoperation plugin to the backend - just before transaction commit */
 #define SLAPI_PLUGIN_BE_TXN_POST_ADD_FN    560
@@ -7413,6 +7413,7 @@ typedef enum _slapi_op_note_t {
 #define SLAPI_DB2LDIF_FILE 184
 /* dump uniqueid */
 #define SLAPI_DB2LDIF_DUMP_UNIQUEID  176
+#define SLAPI_LDIF_CHANGELOG  1761
 #define SLAPI_DB2LDIF_SERVER_RUNNING 197
 
 /* db2ldif/ldif2db/bak2db/db2bak arguments */
@@ -7726,6 +7727,7 @@ int slapi_check_account_lock(Slapi_PBlock *pb, Slapi_Entry *bind_target_entry, i
  *
  * \note Implemented cmd:
  * BACK_INFO_DBENV - Get the dbenv
+ * BACK_INFO_DBENV_CLBD - Get the changelog db for the backend
  * BACK_INFO_DBENV_OPENFLAGS - Get the dbenv openflags
  * BACK_INFO_INDEXPAGESIZE - Get the index page size
  */
@@ -7764,6 +7766,12 @@ int slapi_back_ctrl_info(Slapi_Backend *be, int cmd, void *info);
 enum
 {
     BACK_INFO_DBENV,               /* Get the dbenv */
+    BACK_INFO_DBENV_CLDB,          /* Get the changelog */
+    BACK_INFO_DBENV_CLDB_REMOVE,   /* Remove the changelog */
+    BACK_INFO_DBENV_CLDB_RESET,    /* Recreate the changelog */
+    BACK_INFO_DBENV_CLDB_UPGRADE,  /* Move an old cl file to the instance database */
+    BACK_INFO_CLDB_SET_CONFIG,     /* Set the CL configuration for a backend database */
+    BACK_INFO_CLDB_GET_CONFIG,     /* Get the CL configuration for a backend database */
     BACK_INFO_DB_PAGESIZE,         /* Get the db page size */
     BACK_INFO_INDEXPAGESIZE,       /* Get the index page size */
     BACK_INFO_DBENV_OPENFLAGS,     /* Get the dbenv openflags */
@@ -7771,7 +7779,8 @@ enum
     BACK_INFO_CRYPT_DESTROY,       /* Ctrl: clcrypt_destroy */
     BACK_INFO_CRYPT_ENCRYPT_VALUE, /* Ctrl: clcrypt_encrypt_value */
     BACK_INFO_CRYPT_DECRYPT_VALUE, /* Ctrl: clcrypt_decrypt_value */
-    BACK_INFO_DIRECTORY,           /* Get the directory path */
+    BACK_INFO_DIRECTORY,           /* Get the db directory path */
+    BACK_INFO_INSTANCE_DIR,        /* Get the path to an instance */
     BACK_INFO_LOG_DIRECTORY,       /* Get the txn log directory */
     BACK_INFO_INDEX_KEY,           /* Get the status of a key in an index */
     BACK_INFO_DB_DIRECTORY,        /* Get the db directory */
@@ -7809,6 +7818,13 @@ struct _back_info_crypt_value
 };
 typedef struct _back_info_crypt_value back_info_crypt_value;
 
+struct _back_info_config_entry
+{
+    char *dn;           /* input  -- part of dn below backend config entry */
+    Slapi_Entry *ce;    /* output -- requested config entry */
+};
+typedef struct _back_info_config_entry back_info_config_entry;
+
 #define BACK_CRYPT_OUTBUFF_EXTLEN 16
 
 /**

ldap/servers/slapd/task.c

@@ -2514,6 +2514,9 @@ task_fixup_tombstones_add(Slapi_PBlock *pb,
     task_data->base = base;
     task_data->task = task;
 
+    /* Stash a pointer to our data in the task */
+    slapi_task_set_data(task, task_data);
+
     if ((stripcsn = slapi_entry_attr_get_ref(e, TASK_TOMBSTONE_FIXUP_STRIPCSN))) {
         if (strcasecmp(stripcsn, "yes") == 0 || strcasecmp(stripcsn, "on") == 0) {
             task_data->stripcsn = 1;

ldap/servers/slapd/time.c

@@ -688,8 +688,8 @@ gen_duration(long duration)
     char *duration_str = NULL;
     long remainder = 0;
     long devided = duration;
-    int devider[] = {60, 60, 24, 0};
-    char *unit[] = {"", "M", "H", "D", NULL};
+    int devider[] = {60, 60, 24, 7, 0};
+    char *unit[] = {"", "m", "h", "d", "w", NULL};
     int i = 0;
 
     if (0 > duration) {

ldap/servers/slapd/tools/dbscan.c

@@ -1047,6 +1047,9 @@ is_changelog(char *filename)
     } else {
         ptr++;
     }
+
+    if (0 == strcmp(ptr,"changelog.db")) return 1;
+
     for (; ptr && *ptr; ptr++) {
         if ('.' == *ptr) {
             if (0 == strncmp(ptr, ".db", 3)) {

src/lib389/lib389/__init__.py

@@ -182,6 +182,8 @@ def wrapper(f, name):
 
 
 def pid_exists(pid):
+    if not pid:
+        return False
     if pid <= 0:
         return False
     try:

src/lib389/lib389/agreement.py

@@ -446,6 +446,13 @@ class Agreement(DSLdapObject):
         """
         return self.get_attr_val_utf8('nsDS5ReplicaWaitForAsyncResults')
 
+    def set_flowcontrolwindow(self, value):
+        """Set nsds5ReplicaFlowControlWindow to value.
+
+        :param value: During total update Number of entries to send without waiting ack
+        :type value: str
+        """
+        self.replace('nsds5ReplicaFlowControlWindow', value)
 
 class WinsyncAgreement(Agreement):
     """A replication agreement from this server instance to

src/lib389/lib389/config.py

@@ -473,8 +473,12 @@ class CertmapLegacy(object):
                     output += "%s:%s %s\n" % (name, v, certmap[v])
         # Now write it out
         certmap = os.path.join(self._instance.get_config_dir(), 'certmap.conf')
+        if not os.access(certmap, os.W_OK):
+            os.chmod(certmap, 0o660)
         with open(certmap, 'w') as f:
             f.write(output)
+        if os.access(certmap, os.W_OK):
+            os.chmod(certmap, 0o440)
 
 
 class LDBMConfig(DSLdapObject):

src/lib389/lib389/replica.py

@@ -1685,8 +1685,8 @@ class Replicas(DSLdapObjects):
 
         repl_roots = []
         try:
-            cl = Changelog5(self._instance)
-            cl_dir = cl.get_attr_val_utf8_l("nsslapd-changelogdir")
+            # Changelog is now dumped in the ldif directory
+            cl_dir = self._instance.get_ldif_dir()
         except ldap.NO_SUCH_OBJECT:
             raise ValueError("Changelog entry was not found. Probably, the replication is not enabled on this instance")
 
@@ -1878,7 +1878,8 @@ class ReplicationManager(object):
         # So this can wrap it and make it easy.
         self._log.debug("Creating first master on %s" % instance.ldapuri)
 
-        self._ensure_changelog(instance)
+        # With changelog now integrated with the main database
+        # The config cn=changelog5,cn=config entry is no longer needed
 
         rgroup_dn = self._create_service_account(instance, instance)
 
@@ -1991,6 +1992,7 @@ class ReplicationManager(object):
             'nsDS5ReplicaHost': to_instance.host,
             'nsDS5ReplicaPort': str(to_instance.port),
             'nsDS5ReplicaCredentials': repl_manager_password,
+            'nsds5ReplicaFlowControlWindow': '100',
         })
         # Do a replica refresh.
         temp_agmt.begin_reinit()
@@ -2030,7 +2032,7 @@ class ReplicationManager(object):
         from_r = from_replicas.get(self._suffix)
 
         # Ensure we have a cl
-        self._ensure_changelog(to_instance)
+        # self._ensure_changelog(to_instance)
 
         # Create our credentials
         repl_dn = self._create_service_account(from_instance, to_instance)
@@ -2101,7 +2103,7 @@ class ReplicationManager(object):
         from_r = from_replicas.get(self._suffix)
 
         # Ensure we have a changelog
-        self._ensure_changelog(to_instance)
+        # self._ensure_changelog(to_instance)
 
         # Create replica on to_instance, with bootstrap details.
         to_r = to_replicas.create(properties={
@@ -2417,7 +2419,9 @@ class ReplicationManager(object):
             if change == desc:
                 self._log.info("SUCCESS: Replication from %s to %s is working" % (from_instance.ldapuri, to_instance.ldapuri))
                 return True
+            self._log.info("Retry: Replication from %s to %s is NOT working (expect %s / got description=%s)" % (from_instance.ldapuri, to_instance.ldapuri, change, desc))
             time.sleep(1)
+        self._log.info("FAIL: Replication from %s to %s is NOT working (expect %s / got description=%s)" % (from_instance.ldapuri, to_instance.ldapuri, change, desc))
         raise Exception("Replication did not sync in time!")
 
 

src/lib389/lib389/topologies.py

@@ -108,6 +108,10 @@ def _create_instances(topo_dict, suffix):
             if role == ReplicaRole.HUB:
                 hs[instance.serverid] = instance
                 instances.update(hs)
+            if DEBUGGING:
+                instance.config.set('nsslapd-accesslog-logbuffering','off')
+                instance.config.set('nsslapd-errorlog-level','8192')
+                instance.config.set('nsslapd-auditlog-logging-enabled','on')
             log.info("Instance with parameters {} was created.".format(args_instance))
 
     if "standalone1" in instances and len(instances) == 1:

src/lib389/lib389/utils.py

@@ -1108,6 +1108,11 @@ def ds_is_newer(*ver, instance=None):
     """
     return ds_is_related('newer', *ver, instance=instance)
 
+def ds_supports_new_changelog():
+    """
+    Return True if the current version of ns-slapd supports changelogs under cn=changelog,cn=<backend>,cn=ldbm..
+    """
+    return ds_is_newer('1.4.4.3')
 
 def gentime_to_datetime(gentime):
     """Convert Generalized time to datetime object

test/libslapd/pblock/pblock_accessors.txt

@@ -109,13 +109,11 @@ SLAPI_PLUGIN_ACL_SYNTAX_CHECK
 SLAPI_PLUGIN_ARGC
 SLAPI_PLUGIN_ARGV
 SLAPI_PLUGIN_BE_POST_ADD_FN
-SLAPI_PLUGIN_BE_POST_BACKUP_FN
 SLAPI_PLUGIN_BE_POST_DELETE_FN
 SLAPI_PLUGIN_BE_POST_MODIFY_FN
 SLAPI_PLUGIN_BE_POST_MODRDN_FN
 SLAPI_PLUGIN_BE_POST_OPEN_FN
 SLAPI_PLUGIN_BE_PRE_ADD_FN
-SLAPI_PLUGIN_BE_PRE_BACKUP_FN
 SLAPI_PLUGIN_BE_PRE_CLOSE_FN
 SLAPI_PLUGIN_BE_PRE_DELETE_FN
 SLAPI_PLUGIN_BE_PRE_MODIFY_FN

test/libslapd/pblock/pblock_accessors_freq.txt

@@ -220,8 +220,6 @@ SLAPI_PLUGIN_ARGV
 21
 SLAPI_PLUGIN_BE_POST_ADD_FN
 12
-SLAPI_PLUGIN_BE_POST_BACKUP_FN
-12
 SLAPI_PLUGIN_BE_POST_DELETE_FN
 13
 SLAPI_PLUGIN_BE_POST_MODIFY_FN
@@ -232,8 +230,6 @@ SLAPI_PLUGIN_BE_POST_OPEN_FN
 12
 SLAPI_PLUGIN_BE_PRE_ADD_FN
 14
-SLAPI_PLUGIN_BE_PRE_BACKUP_FN
-11
 SLAPI_PLUGIN_BE_PRE_CLOSE_FN
 11
 SLAPI_PLUGIN_BE_PRE_DELETE_FN