# --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2021 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). # See LICENSE for details. # --- END COPYRIGHT BLOCK --- # import logging import pytest import datetime import subprocess from multiprocessing import Process, Queue from lib389 import pid_from_file from lib389.utils import ldap, os from lib389._constants import DEFAULT_SUFFIX, ReplicaRole from lib389.cli_base import LogCapture from lib389.idm.user import UserAccounts from lib389.idm.organizationalunit import OrganizationalUnits from lib389.tasks import AccessLog from lib389.backend import Backends from lib389.ldclt import Ldclt from lib389.dbgen import dbgen_users from lib389.tasks import ImportTask from lib389.index import Indexes from lib389.plugins import AttributeUniquenessPlugin from lib389.config import BDB_LDBMConfig from lib389.monitor import MonitorLDBM from lib389.topologies import create_topology, _remove_ssca_db pytestmark = pytest.mark.tier2 db_locks_monitoring_ack = pytest.mark.skipif(not os.environ.get('DB_LOCKS_MONITORING_ACK', False), reason="DB locks monitoring tests may take hours if the feature is not present or another failure exists. " "Also, the feature requires a big amount of space as we set nsslapd-db-locks to 1300000.") DEBUGGING = os.getenv('DEBUGGING', default=False) if DEBUGGING: logging.getLogger(__name__).setLevel(logging.DEBUG) else: logging.getLogger(__name__).setLevel(logging.INFO) log = logging.getLogger(__name__) def _kill_ns_slapd(inst): pid = str(pid_from_file(inst.ds_paths.pid_file)) cmd = ['kill', '-9', pid] subprocess.Popen(cmd, stdout=subprocess.PIPE) @pytest.fixture(scope="function") def topology_st_fn(request): """Create DS standalone instance for each test case""" topology = create_topology({ReplicaRole.STANDALONE: 1}) def fin(): # Kill the hanging process at the end of test to prevent failures in the following tests if DEBUGGING: [_kill_ns_slapd(inst) for inst in topology] else: [_kill_ns_slapd(inst) for inst in topology] assert _remove_ssca_db(topology) [inst.stop() for inst in topology if inst.exists()] [inst.delete() for inst in topology if inst.exists()] request.addfinalizer(fin) topology.logcap = LogCapture() return topology @pytest.fixture(scope="function") def setup_attruniq_index_be_import(topology_st_fn): """Enable Attribute Uniqueness, disable indexes and import 120000 entries to the default backend """ inst = topology_st_fn.standalone inst.config.loglevel([AccessLog.DEFAULT, AccessLog.INTERNAL], service='access') inst.config.set('nsslapd-plugin-logging', 'on') inst.restart() attruniq = AttributeUniquenessPlugin(inst, dn="cn=attruniq,cn=plugins,cn=config") attruniq.create(properties={'cn': 'attruniq'}) for cn in ['uid', 'cn', 'sn', 'uidNumber', 'gidNumber', 'homeDirectory', 'givenName', 'description']: attruniq.add_unique_attribute(cn) attruniq.add_unique_subtree(DEFAULT_SUFFIX) attruniq.enable_all_subtrees() attruniq.enable() indexes = Indexes(inst) for cn in ['uid', 'cn', 'sn', 'uidNumber', 'gidNumber', 'homeDirectory', 'givenName', 'description']: indexes.ensure_state(properties={ 'cn': cn, 'nsSystemIndex': 'false', 'nsIndexType': 'none'}) bdb_config = BDB_LDBMConfig(inst) bdb_config.replace("nsslapd-db-locks", "130000") inst.restart() ldif_dir = inst.get_ldif_dir() import_ldif = ldif_dir + '/perf_import.ldif' # Valid online import import_task = ImportTask(inst) dbgen_users(inst, 120000, import_ldif, DEFAULT_SUFFIX, entry_name="userNew") import_task.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX) import_task.wait() assert import_task.is_complete() def create_user_wrapper(q, users): try: users.create_test_user() except Exception as ex: q.put(ex) def spawn_worker_thread(function, users, log, timeout, info): log.info(f"Starting the thread - {info}") q = Queue() p = Process(target=function, args=(q,users,)) p.start() log.info(f"Waiting for {timeout} seconds for the thread to finish") p.join(timeout) if p.is_alive(): log.info("Killing the thread as it's still running") p.terminate() p.join() raise RuntimeError(f"Function call was aborted: {info}") result = q.get() if isinstance(result, Exception): raise result else: return result @db_locks_monitoring_ack @pytest.mark.parametrize("lock_threshold", [("70"), ("80"), ("95")]) def test_exhaust_db_locks_basic(topology_st_fn, setup_attruniq_index_be_import, lock_threshold): """Test that when all of the locks are exhausted the instance still working and database is not corrupted :id: 299108cc-04d8-4ddc-b58e-99157fccd643 :setup: Standalone instance with Attr Uniq plugin and user indexes disabled :steps: 1. Set nsslapd-db-locks to 11000 2. Check that we stop acquiring new locks when the threshold is reached 3. Check that we can regulate a pause interval for DB locks monitoring thread 4. Make sure the feature works for different backends on the same suffix :expectedresults: 1. Success 2. Success 3. Success 4. Success """ inst = topology_st_fn.standalone ADDITIONAL_SUFFIX = 'ou=newpeople,dc=example,dc=com' backends = Backends(inst) backends.create(properties={'nsslapd-suffix': ADDITIONAL_SUFFIX, 'name': ADDITIONAL_SUFFIX[-3:]}) ous = OrganizationalUnits(inst, DEFAULT_SUFFIX) ous.create(properties={'ou': 'newpeople'}) bdb_config = BDB_LDBMConfig(inst) bdb_config.replace("nsslapd-db-locks", "11000") # Restart server inst.restart() for lock_enabled in ["on", "off"]: for lock_pause in ["100", "500", "1000"]: bdb_config.replace("nsslapd-db-locks-monitoring-enabled", lock_enabled) bdb_config.replace("nsslapd-db-locks-monitoring-threshold", lock_threshold) bdb_config.replace("nsslapd-db-locks-monitoring-pause", lock_pause) inst.restart() if lock_enabled == "off": raised_exception = (RuntimeError, ldap.SERVER_DOWN) else: raised_exception = ldap.OPERATIONS_ERROR users = UserAccounts(inst, DEFAULT_SUFFIX) with pytest.raises(raised_exception): spawn_worker_thread(create_user_wrapper, users, log, 30, f"Adding user with monitoring enabled='{lock_enabled}'; " f"threshold='{lock_threshold}'; pause='{lock_pause}'.") # Restart because we already run out of locks and the next unindexed searches will fail eventually if lock_enabled == "off": _kill_ns_slapd(inst) inst.restart() users = UserAccounts(inst, ADDITIONAL_SUFFIX, rdn=None) with pytest.raises(raised_exception): spawn_worker_thread(create_user_wrapper, users, log, 30, f"Adding user with monitoring enabled='{lock_enabled}'; " f"threshold='{lock_threshold}'; pause='{lock_pause}'.") # In case feature is disabled - restart for the clean up if lock_enabled == "off": _kill_ns_slapd(inst) inst.restart() @db_locks_monitoring_ack def test_exhaust_db_locks_big_pause(topology_st_fn, setup_attruniq_index_be_import): """Test that DB lock pause setting increases the wait interval value for the monitoring thread :id: 7d5bf838-5d4e-4ad5-8c03-5716afb84ea6 :setup: Standalone instance with Attr Uniq plugin and user indexes disabled :steps: 1. Set nsslapd-db-locks to 20000 while using the default threshold value (95%) 2. Set nsslapd-db-locks-monitoring-pause to 10000 (10 seconds) 3. Make sure that the pause is successfully increased a few times in a row :expectedresults: 1. Success 2. Success 3. Success """ inst = topology_st_fn.standalone bdb_config = BDB_LDBMConfig(inst) bdb_config.replace("nsslapd-db-locks", "20000") lock_pause = bdb_config.get_attr_val_int("nsslapd-db-locks-monitoring-pause") assert lock_pause == 500 lock_pause = "10000" bdb_config.replace("nsslapd-db-locks-monitoring-pause", lock_pause) # Restart server inst.restart() lock_enabled = bdb_config.get_attr_val_utf8_l("nsslapd-db-locks-monitoring-enabled") lock_threshold = bdb_config.get_attr_val_int("nsslapd-db-locks-monitoring-threshold") assert lock_enabled == "on" assert lock_threshold == 90 users = UserAccounts(inst, DEFAULT_SUFFIX) start = datetime.datetime.now() with pytest.raises(ldap.OPERATIONS_ERROR): spawn_worker_thread(create_user_wrapper, users, log, 30, f"Adding user with monitoring enabled='{lock_enabled}'; " f"threshold='{lock_threshold}'; pause='{lock_pause}'. Expect it to 'Work'") end = datetime.datetime.now() time_delta = end - start if time_delta.seconds < 9: raise RuntimeError("nsslapd-db-locks-monitoring-pause attribute doesn't function correctly. " f"Finished the execution in {time_delta.seconds} seconds") # In case something has failed - restart for the clean up inst.restart()