Ticket 3 - lib389 - config test

Bug Description:  python 3 suppor for config test

Fix Description:  Fix python 3 support for config test, correct
a small issue in removal, and improve some logging of large values

https://pagure.io/lib389/issue/3

Author: wibrown

Review by: spichugi (Thanks!)

dirsrvtests/tests/suites/config/config_test.py

@@ -14,6 +14,10 @@ from lib389.topologies import topology_m2
 
 from lib389._constants import DN_CONFIG, DEFAULT_SUFFIX
 
+from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES
+
+from lib389.config import LDBMConfig
+
 USER_DN = 'uid=test_user,%s' % DEFAULT_SUFFIX
 
 logging.getLogger(__name__).setLevel(logging.INFO)
@@ -32,37 +36,7 @@ def big_file():
     return TEMP_BIG_FILE
 
 
[email protected]
-def test_user(topology_m2):
-    """Add and remove test user"""
-
-    try:
-        topology_m2.ms["master1"].add_s(Entry((USER_DN, {
-            'uid': 'test_user',
-            'givenName': 'test_user',
-            'objectclass': ['top', 'person',
-                            'organizationalPerson',
-                            'inetorgperson'],
-            'cn': 'test_user',
-            'sn': 'test_user'})))
-        time.sleep(1)
-    except ldap.LDAPError as e:
-        log.fatal('Failed to add user (%s): error (%s)' % (USER_DN,
-                                                           e.message['desc']))
-        raise
-
-    def fin():
-        try:
-            topology_m2.ms["master1"].delete_s(USER_DN)
-            time.sleep(1)
-        except ldap.LDAPError as e:
-            log.fatal('Failed to delete user (%s): error (%s)' % (
-                USER_DN,
-                e.message['desc']))
-            raise
-
-
-def test_maxbersize_repl(topology_m2, test_user, big_file):
+def test_maxbersize_repl(topology_m2, big_file):
     """maxbersize is ignored in the replicated operations.
 
     :id: ad57de60-7d56-4323-bbca-5556e5cdb126
@@ -80,64 +54,38 @@ def test_maxbersize_repl(topology_m2, test_user, big_file):
         4. The big value is successfully replicated to master2
     """
 
+    users_m1 = UserAccounts(topology_m2.ms["master1"], DEFAULT_SUFFIX)
+    users_m2 = UserAccounts(topology_m2.ms["master2"], DEFAULT_SUFFIX)
+
+    user_m1 = users_m1.create(properties=TEST_USER_PROPERTIES)
+    time.sleep(2)
+    user_m2 = users_m2.get(dn=user_m1.dn)
+
     log.info("Set nsslapd-maxbersize: 20K to master2")
-    try:
-        topology_m2.ms["master2"].modify_s("cn=config", [(ldap.MOD_REPLACE,
-                                                          'nsslapd-maxbersize', '20480')])
-    except ldap.LDAPError as e:
-        log.error('Failed to set nsslapd-maxbersize == 20480: error ' +
-                  e.message['desc'])
-        raise
+    topology_m2.ms["master2"].config.set('nsslapd-maxbersize', '20480')
 
-    topology_m2.ms["master2"].restart(20)
+    topology_m2.ms["master2"].restart()
 
     log.info('Try to add attribute with a big value to master2 - expect to FAIL')
     with pytest.raises(ldap.SERVER_DOWN):
-        topology_m2.ms["master2"].modify_s(USER_DN, [(ldap.MOD_REPLACE,
-                                                      'jpegphoto', big_file)])
+        user_m2.add('jpegphoto', big_file)
 
-    topology_m2.ms["master2"].restart(20)
-    topology_m2.ms["master1"].restart(20)
+    topology_m2.ms["master2"].restart()
+    topology_m2.ms["master1"].restart()
 
     log.info('Try to add attribute with a big value to master1 - expect to PASS')
-    try:
-        topology_m2.ms["master1"].modify_s(USER_DN, [(ldap.MOD_REPLACE,
-                                                      'jpegphoto', big_file)])
-    except ldap.SERVER_DOWN as e:
-        log.fatal('Failed to add a big attribute, error: ' + e.message['desc'])
-        raise
+    user_m1.add('jpegphoto', big_file)
 
-    time.sleep(1)
+    time.sleep(2)
 
     log.info('Check if a big value was successfully added to master1')
-    try:
-        entries = topology_m2.ms["master1"].search_s(USER_DN, ldap.SCOPE_BASE,
-                                                     '(cn=*)',
-                                                     ['jpegphoto'])
-        assert entries[0].data['jpegphoto']
-    except ldap.LDAPError as e:
-        log.fatal('Search failed, error: ' + e.message['desc'])
-        raise
+
+    photo_m1 = user_m1.get_attr_vals('jpegphoto')
 
     log.info('Check if a big value was successfully replicated to master2')
-    try:
-        entries = topology_m2.ms["master2"].search_s(USER_DN, ldap.SCOPE_BASE,
-                                                     '(cn=*)',
-                                                     ['jpegphoto'])
-        assert entries[0].data['jpegphoto']
-    except ldap.LDAPError as e:
-        log.fatal('Search failed, error: ' + e.message['desc'])
-        raise
-
-    log.info("Set nsslapd-maxbersize: 2097152 (default) to master2")
-    try:
-        topology_m2.ms["master2"].modify_s("cn=config", [(ldap.MOD_REPLACE,
-                                                          'nsslapd-maxbersize', '2097152')])
-    except ldap.LDAPError as e:
-        log.error('Failed to set nsslapd-maxbersize == 2097152 error ' +
-                  e.message['desc'])
-        raise
+    photo_m2 = user_m2.get_attr_vals('jpegphoto')
 
+    assert photo_m2 == photo_m1
 
 def test_config_listen_backport_size(topology_m2):
     """Check that nsslapd-listen-backlog-size acted as expected
@@ -158,46 +106,16 @@ def test_config_listen_backport_size(topology_m2):
         5. nsslapd-listen-backlog-size should be successfully set
     """
 
-    try:
-        entry = topology_m2.ms["master1"].search_s(DN_CONFIG, ldap.SCOPE_BASE, 'objectclass=top',
-                                                   ['nsslapd-listen-backlog-size'])
-        default_val = entry[0].data['nsslapd-listen-backlog-size'][0]
-        assert default_val, 'Failed to get nsslapd-listen-backlog-size from config'
-    except ldap.LDAPError as e:
-        log.fatal('Failed to search config, error: ' + e.message('desc'))
-        raise
-
-    try:
-        topology_m2.ms["master1"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
-                                                        'nsslapd-listen-backlog-size',
-                                                        '256')])
-    except ldap.LDAPError as e:
-        log.fatal('Failed to modify config, error: ' + e.message('desc'))
-        raise
-
-    try:
-        topology_m2.ms["master1"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
-                                                        'nsslapd-listen-backlog-size',
-                                                        '-1')])
-    except ldap.LDAPError as e:
-        log.fatal('Failed to modify config(negative value), error: ' +
-                  e.message('desc'))
-        raise
+    default_val = topology_m2.ms["master1"].config.get_attr_val_bytes('nsslapd-listen-backlog-size')
+
+    topology_m2.ms["master1"].config.replace('nsslapd-listen-backlog-size', '256')
+
+    topology_m2.ms["master1"].config.replace('nsslapd-listen-backlog-size', '-1')
 
     with pytest.raises(ldap.LDAPError):
-        topology_m2.ms["master1"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
-                                                        'nsslapd-listen-backlog-size',
-                                                        'ZZ')])
-        log.fatal('Invalid value was successfully added')
+        topology_m2.ms["master1"].config.replace('nsslapd-listen-backlog-size', 'ZZ')
 
-    # Cleanup - undo what we've done
-    try:
-        topology_m2.ms["master1"].modify_s(DN_CONFIG, [(ldap.MOD_REPLACE,
-                                                        'nsslapd-listen-backlog-size',
-                                                        default_val)])
-    except ldap.LDAPError as e:
-        log.fatal('Failed to reset config, error: ' + e.message('desc'))
-        raise
+    topology_m2.ms["master1"].config.replace('nsslapd-listen-backlog-size', default_val)
 
 
 def test_config_deadlock_policy(topology_m2):
@@ -220,47 +138,25 @@ def test_config_deadlock_policy(topology_m2):
         5. nsslapd-db-deadlock-policy should be successfully set
     """
 
-    LDBM_DN = 'cn=config,cn=ldbm database,cn=plugins,cn=config'
-    default_val = '9'
+    default_val = b'9'
+
+    ldbmconfig = LDBMConfig(topology_m2.ms["master1"])
+
+    deadlock_policy = ldbmconfig.get_attr_val_bytes('nsslapd-db-deadlock-policy')
+    assert deadlock_policy == default_val
 
-    try:
-        entry = topology_m2.ms["master1"].search_s(LDBM_DN, ldap.SCOPE_BASE, 'objectclass=top',
-                                                   ['nsslapd-db-deadlock-policy'])
-        val = entry[0].data['nsslapd-db-deadlock-policy'][0]
-        assert val, 'Failed to get nsslapd-db-deadlock-policy from config'
-        assert val == default_val, 'The wrong default value was present'
-    except ldap.LDAPError as e:
-        log.fatal('Failed to search config, error: ' + e.message('desc'))
-        raise
 
     # Try a range of valid values
     for val in ('0', '5', '9'):
-        try:
-            topology_m2.ms["master1"].modify_s(LDBM_DN, [(ldap.MOD_REPLACE,
-                                                          'nsslapd-db-deadlock-policy',
-                                                          val)])
-        except ldap.LDAPError as e:
-            log.fatal('Failed to modify config: nsslapd-db-deadlock-policy to (%s), error: %s' %
-                      (val, e.message('desc')))
-            raise
+        ldbmconfig.replace('nsslapd-db-deadlock-policy', val)
 
     # Try a range of invalid values
     for val in ('-1', '10'):
         with pytest.raises(ldap.LDAPError):
-            topology_m2.ms["master1"].modify_s(LDBM_DN, [(ldap.MOD_REPLACE,
-                                                          'nsslapd-db-deadlock-policy',
-                                                          val)])
-            log.fatal('Able to add invalid value to nsslapd-db-deadlock-policy(%s)' % (val))
+            ldbmconfig.replace('nsslapd-db-deadlock-policy', val)
 
     # Cleanup - undo what we've done
-    try:
-        topology_m2.ms["master1"].modify_s(LDBM_DN, [(ldap.MOD_REPLACE,
-                                                      'nsslapd-db-deadlock-policy',
-                                                      default_val)])
-    except ldap.LDAPError as e:
-        log.fatal('Failed to reset nsslapd-db-deadlock-policy to the default value(%s), error: %s' %
-                  (default_val, e.message('desc')))
-        raise
+    ldbmconfig.replace('nsslapd-db-deadlock-policy', deadlock_policy)
 
 
 if __name__ == '__main__':

src/lib389/lib389/__init__.py

@@ -1274,16 +1274,20 @@ class DirSrv(SimpleLDAPObject, object):
             # TODO: use the status call instead!!!!
             pid = pid_from_file(self.ds_paths.pid_file)
             if pid is None:
+                self.log.debug("No pidfile found for %s" % self.serverid)
                 # No pidfile yet ...
                 self.state = DIRSRV_STATE_OFFLINE
                 return False
             if pid == 0:
+                self.log.debug("Pid of 0 not valid for %s" % self.serverid)
                 self.state = DIRSRV_STATE_OFFLINE
                 raise ValueError
             # Wait
             if not pid_exists(pid):
+                self.log.debug("Pid of %s is not running for %s" % (pid, self.serverid))
                 self.state = DIRSRV_STATE_OFFLINE
                 return False
+            self.log.debug("Pid of %s for %s and running" % (pid, self.serverid))
             return True
 
     def restart(self, timeout=120, post_open=True):

src/lib389/lib389/_mapped_object.py

@@ -295,7 +295,10 @@ class DSLdapObject(DSLogging):
         :raises: ValueError - if instance is not online
         """
 
-        self._log.debug("%s set(%r, %r)" % (self._dn, key, value))
+        if value is None or len(value) < 512:
+            self._log.debug("%s set(%r, %r)" % (self._dn, key, value))
+        else:
+            self._log.debug("%s set(%r, value too large)" % (self._dn, key))
         if self._instance.state != DIRSRV_STATE_ONLINE:
             raise ValueError("Invalid state. Cannot set properties on instance that is not ONLINE.")
 
@@ -804,6 +807,7 @@ class DSLdapObjects(DSLogging):
         # This will yield and & filter for objectClass with as many terms as needed.
         filterstr = self._get_objectclass_filter()
         self._log.debug('_gen_dn filter = %s' % filterstr)
+        self._log.debug('_gen_dn dn = %s' % dn)
         return self._instance.search_ext_s(
             base=dn,
             scope=ldap.SCOPE_BASE,

src/lib389/lib389/config.py

@@ -350,4 +350,25 @@ class CertmapLegacy(object):
             f.write(output)
 
 
+class LDBMConfig(DSLdapObject):
+    """
+        Manage "cn=config,cn=ldbm database,cn=plugins,cn=config" including:
+        - Performance related tunings
+        - DB backend settings
+
+        :param instance: An instance
+        :type instance: lib389.DirSrv
+        :param batch: Not implemented
+        :type batch: bool
+    """
+
+    def __init__(self, conn, batch=False):
+        super(LDBMConfig, self).__init__(instance=conn, batch=batch)
+        self._dn = DN_CONFIG_LDBM
+        config_compare_exclude = []
+        self._rdn_attribute = 'cn'
+        self._lint_functions = []
+        self._protected = True
+
+
 

src/lib389/lib389/instance/remove.py

@@ -26,11 +26,12 @@ def remove_ds_instance(dirsrv):
     remove_paths['db_dir'] = dirsrv.ds_paths.db_dir
     ### WARNING: The changelogdb isn't removed. we assume it's in:
     # db_dir ../changelogdb. So remove that too!
-    remove_paths['changelogdb_dir'] = os.path.join(dirsrv.ds_paths.db_dir, '../changelogdb')
+    # abspath will resolve the ".." down.
+    remove_paths['changelogdb_dir'] = os.path.abspath(os.path.join(dirsrv.ds_paths.db_dir, '../changelogdb'))
     remove_paths['ldif_dir'] = dirsrv.ds_paths.ldif_dir
     remove_paths['lock_dir'] = dirsrv.ds_paths.lock_dir
     remove_paths['log_dir'] = dirsrv.ds_paths.log_dir
-    remove_paths['run_dir'] = dirsrv.ds_paths.run_dir
+    # remove_paths['run_dir'] = dirsrv.ds_paths.run_dir
 
     marker_path = "%s/sysconfig/dirsrv-%s" % (dirsrv.ds_paths.sysconf_dir, dirsrv.serverid)
 
@@ -44,9 +45,8 @@ def remove_ds_instance(dirsrv):
     # for path in ('backup_dir', 'cert_dir', 'config_dir', 'db_dir',
     #             'ldif_dir', 'lock_dir', 'log_dir', 'run_dir'):
     for path_k in remove_paths:
-        if os.path.exists(remove_paths[path_k]):
-            _log.debug("Removing %s" % remove_paths[path_k])
-            shutil.rmtree(remove_paths[path_k])
+        _log.debug("Removing %s" % remove_paths[path_k])
+        shutil.rmtree(remove_paths[path_k], ignore_errors=True)
 
     # Finally remove the sysconfig marker.
     os.remove(marker_path)