Issue 50873 - Fix issues with healthcheck tool

Description:
I finished remaining tests for healthcheck tool.
I moved some of the tests to separate files because one large file was becoming messy.
Also the test in health_sync_test.py is separate because it is time sensitive to reproduce.
Running it with other tests in one file can cause delay and not catching the error code.

Created requirements.txt to install libfaketime.
Updated topologies.py for LogCapture.

Relates: https://pagure.io/389-ds-base/issue/50873

Reviewed by: vashirov (Thanks!)

dirsrvtests/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-libfaketime

dirsrvtests/tests/suites/healthcheck/health_config_test.py

@@ -0,0 +1,326 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+import pytest
+import os
+import subprocess
+
+from lib389.backend import Backends
+from lib389.cos import CosTemplates, CosPointerDefinitions
+from lib389.index import Index
+from lib389.plugins import ReferentialIntegrityPlugin
+from lib389.utils import *
+from lib389._constants import *
+from lib389.cli_base import FakeArgs
+from lib389.topologies import topology_st
+from lib389.cli_ctl.health import health_check_run
+from lib389.paths import Paths
+
+
+CMD_OUTPUT = 'No issues found.'
+JSON_OUTPUT = '[]'
+
+ds_paths = Paths()
+pytestmark = pytest.mark.skipif(ds_paths.perl_enabled and (os.getenv('PYINSTALL') is None),
+                                reason="These tests need to use python installer")
+
+if DEBUGGING:
+    logging.getLogger(__name__).setLevel(logging.DEBUG)
+else:
+    logging.getLogger(__name__).setLevel(logging.INFO)
+log = logging.getLogger(__name__)
+
+
+def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searched_code2=None):
+    args = FakeArgs()
+    args.instance = instance.serverid
+    args.verbose = instance.verbose
+    args.list_errors = False
+    args.list_checks = False
+    args.check = None
+    args.dry_run = False
+
+    if json:
+        log.info('Use healthcheck with --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+    else:
+        log.info('Use healthcheck without --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+
+    log.info('Clear the log')
+    topology.logcap.flush()
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_logging_format_should_be_revised(topology_st):
+    """Check if HealthCheck returns DSCLE0001 code
+
+    :id: 277d7980-123b-481b-acba-d90921b9f5ac
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Set nsslapd-logging-hr-timestamps-enabled to ‘off’
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. Set nsslapd-logging-hr-timestamps-enabled to ‘on’
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSCLE0001 code and related details
+        4. Healthcheck reports DSCLE0001 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSCLE0001'
+
+    standalone = topology_st.standalone
+
+    log.info('Set nsslapd-logging-hr-timestamps-enabled to off')
+    standalone.config.set('nsslapd-logging-hr-timestamps-enabled', 'off')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE)
+
+    log.info('Set nsslapd-logging-hr-timestamps-enabled to off')
+    standalone.config.set('nsslapd-logging-hr-timestamps-enabled', 'on')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_RI_plugin_is_misconfigured(topology_st):
+    """Check if HealthCheck returns DSRILE0001 code
+
+    :id: de2e90a2-89fe-472c-acdb-e13cbca5178d
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Configure the instance with Integrity Plugin
+        3. Set the referint-update-delay attribute of the RI plugin, to a value upper than 0
+        4. Use HealthCheck without --json option
+        5. Use HealthCheck with --json option
+        6. Set the referint-update-delay attribute to 0
+        7. Use HealthCheck without --json option
+        8. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Healthcheck reports DSRILE0001 code and related details
+        5. Healthcheck reports DSRILE0001 code and related details
+        6. Success
+        7. Healthcheck reports no issue found
+        8. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSRILE0001'
+
+    standalone = topology_st.standalone
+
+    plugin = ReferentialIntegrityPlugin(standalone)
+    plugin.disable()
+    plugin.enable()
+
+    log.info('Set the referint-update-delay attribute to a value upper than 0')
+    plugin.replace('referint-update-delay', '5')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE)
+
+    log.info('Set the referint-update-delay attribute back to 0')
+    plugin.replace('referint-update-delay', '0')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_RI_plugin_missing_indexes(topology_st):
+    """Check if HealthCheck returns DSRILE0002 code
+
+    :id: 05c55e37-bb3e-48d1-bbe8-29c980f94f10
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Configure the instance with Integrity Plugin
+        3. Change the index type of the member attribute index to ‘approx’
+        4. Use HealthCheck without --json option
+        5. Use HealthCheck with --json option
+        6. Set the index type of the member attribute index to ‘eq’
+        7. Use HealthCheck without --json option
+        8. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Healthcheck reports DSRILE0002 code and related details
+        5. Healthcheck reports DSRILE0002 code and related details
+        6. Success
+        7. Healthcheck reports no issue found
+        8. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSRILE0002'
+    MEMBER_DN = 'cn=member,cn=index,cn=userroot,cn=ldbm database,cn=plugins,cn=config'
+
+    standalone = topology_st.standalone
+
+    log.info('Enable RI plugin')
+    plugin = ReferentialIntegrityPlugin(standalone)
+    plugin.disable()
+    plugin.enable()
+
+    log.info('Change the index type of the member attribute index to approx')
+    index = Index(topology_st.standalone, MEMBER_DN)
+    index.replace('nsIndexType', 'approx')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE)
+
+    log.info('Set the index type of the member attribute index back to eq')
+    index.replace('nsIndexType', 'eq')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_virtual_attr_incorrectly_indexed(topology_st):
+    """Check if HealthCheck returns DSVIRTLE0001 code
+
+    :id: 1055173b-21aa-4aaa-9e91-4dc6c5e0c01f
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Create a CoS definition entry
+        3. Create the matching CoS template entry, with postalcode as virtual attribute
+        4. Create an index for postalcode
+        5. Use HealthCheck without --json option
+        6. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Success
+        5. Healthcheck reports DSVIRTLE0001 code and related details
+        6. Healthcheck reports DSVIRTLE0001 code and related details
+    """
+
+    RET_CODE = 'DSVIRTLE0001'
+
+    standalone = topology_st.standalone
+    postal_index_properties = {
+        'cn': 'postalcode',
+        'nsSystemIndex': 'False',
+        'nsIndexType': ['eq', 'sub', 'pres'],
+    }
+
+    log.info('Add cosPointer, cosTemplate and test entry to default suffix, where virtual attribute is postal code')
+    cos_pointer_properties = {
+        'cn': 'cosPointer',
+        'description': 'cosPointer example',
+        'cosTemplateDn': 'cn=cosTemplateExample,ou=People,dc=example,dc=com',
+        'cosAttribute': 'postalcode',
+    }
+    cos_pointer_definitions = CosPointerDefinitions(standalone, DEFAULT_SUFFIX, 'ou=People')
+    cos_pointer_definitions.create(properties=cos_pointer_properties)
+
+    log.info('Create CoS template')
+    cos_template_properties = {
+        'cn': 'cosTemplateExample',
+        'postalcode': '117'
+    }
+    cos_templates = CosTemplates(standalone, DEFAULT_SUFFIX, 'ou=People')
+    cos_templates.create(properties=cos_template_properties)
+
+    log.info('Create an index for postalcode')
+    backends = Backends(topology_st.standalone)
+    ur_indexes = backends.get('userRoot').get_indexes()
+    ur_indexes.create(properties=postal_index_properties)
+
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
[email protected](ds_is_older("1.4.2.4"), reason="May fail because of bug 1796050")
+def test_healthcheck_low_disk_space(topology_st):
+    """Check if HealthCheck returns DSDSLE0001 code
+
+    :id: 144b335d-077e-430c-9c0e-cd6b0f2f73c1
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Get the free disk space for /
+        3. Use fallocate to create a file large enough for the use % be up 90%
+        4. Use HealthCheck without --json option
+        5. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        3. Healthcheck reports DSDSLE0001 code and related details
+        4. Healthcheck reports DSDSLE0001 code and related details
+    """
+
+    RET_CODE = 'DSDSLE0001'
+
+    standalone = topology_st.standalone
+    file = '{}/foo'.format(standalone.ds_paths.log_dir)
+
+    log.info('Count the disk space to allocate')
+    total_size = int(re.findall(r'\d+', str(os.statvfs(standalone.ds_paths.log_dir)))[2]) * 4096
+    avail_size = round(int(re.findall(r'\d+', str(os.statvfs(standalone.ds_paths.log_dir)))[3]) * 4096)
+    used_size = total_size - avail_size
+    count_total_percent = total_size * 0.92
+    final_value = count_total_percent - used_size
+
+    log.info('Create a file large enough for the use % be up 90%')
+    subprocess.call(['fallocate', '-l', str(round(final_value)), file])
+
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+    log.info('Remove created file')
+    os.remove(file)
+
+
+if __name__ == '__main__':
+    # Run isolated
+    # -s for DEBUG mode
+    CURRENT_FILE = os.path.realpath(__file__)

dirsrvtests/tests/suites/healthcheck/health_repl_test.py

@@ -0,0 +1,270 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+import pytest
+import os
+import subprocess
+import distro
+
+from lib389.idm.user import UserAccounts
+from lib389.replica import Changelog5, ReplicationManager, Replicas
+from lib389.utils import *
+from lib389._constants import *
+from lib389.cli_base import FakeArgs
+from lib389.topologies import topology_m2, topology_m3
+from lib389.cli_ctl.health import health_check_run
+from lib389.paths import Paths
+
+CMD_OUTPUT = 'No issues found.'
+JSON_OUTPUT = '[]'
+
+ds_paths = Paths()
+pytestmark = pytest.mark.skipif(ds_paths.perl_enabled and (os.getenv('PYINSTALL') is None),
+                                reason="These tests need to use python installer")
+
+if DEBUGGING:
+    logging.getLogger(__name__).setLevel(logging.DEBUG)
+else:
+    logging.getLogger(__name__).setLevel(logging.INFO)
+log = logging.getLogger(__name__)
+
+
+def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searched_code2=None):
+    args = FakeArgs()
+    args.instance = instance.serverid
+    args.verbose = instance.verbose
+    args.list_errors = False
+    args.list_checks = False
+    args.check = None
+    args.dry_run = False
+
+    if json:
+        log.info('Use healthcheck with --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+    else:
+        log.info('Use healthcheck without --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+
+    log.info('Clear the log')
+    topology.logcap.flush()
+
+
+def set_changelog_trimming(instance):
+    log.info('Get the changelog enteries')
+    inst_changelog = Changelog5(instance)
+
+    log.info('Set nsslapd-changelogmaxage to 30d')
+    inst_changelog.add('nsslapd-changelogmaxage', '30')
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication_replica_not_reachable(topology_m2):
+    """Check if HealthCheck returns DSREPLLE0005 code
+
+    :id: d452a564-7b82-4c1a-b331-a71abbd82a10
+    :setup: Replicated topology
+    :steps:
+        1. Create a replicated topology
+        2. On M1, set nsds5replicaport for the replication agreement to an unreachable port on the replica
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. On M1, set nsds5replicaport for the replication agreement to a reachable port number
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSREPLLE0005 code and related details
+        4. Healthcheck reports DSREPLLE0005 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSREPLLE0005'
+
+    M1 = topology_m2.ms['master1']
+    M2 = topology_m2.ms['master2']
+
+    set_changelog_trimming(M1)
+
+    log.info('Set nsds5replicaport for the replication agreement to an unreachable port')
+    repl = ReplicationManager(DEFAULT_SUFFIX)
+    repl.wait_for_replication(M1, M2)
+
+    replica_m1 = Replicas(M1).get(DEFAULT_SUFFIX)
+    agmt_m1 = replica_m1.get_agreements().list()[0]
+    agmt_m1.replace('nsds5replicaport', '4389')
+
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=True)
+
+    log.info('Set nsds5replicaport for the replication agreement to a reachable port')
+    agmt_m1.replace('nsDS5ReplicaPort', '{}'.format(M2.port))
+    repl.wait_for_replication(M1, M2)
+
+    run_healthcheck_and_flush_log(topology_m2, M1, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, JSON_OUTPUT, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_changelog_trimming_not_configured(topology_m2):
+    """Check if HealthCheck returns DSCLLE0001 code
+
+    :id: c2165032-88ba-4978-a4ca-2fecfd8c35d8
+    :setup: Replicated topology
+    :steps:
+        1. Create a replicated topology
+        2. On M1, check that value of nsslapd-changelogmaxage from cn=changelog5,cn=config is None
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. On M1, set nsslapd-changelogmaxage to 30d
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSCLLE0001 code and related details
+        4. Healthcheck reports DSCLLE0001 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    M1 = topology_m2.ms['master1']
+    M2 = topology_m2.ms['master2']
+
+    RET_CODE = 'DSCLLE0001'
+
+    log.info('Get the changelog entries for M1')
+    changelog_m1 = Changelog5(M1)
+
+    log.info('Check nsslapd-changelogmaxage value')
+    if changelog_m1.get_attr_val('nsslapd-changelogmaxage') is not None:
+        changelog_m1.remove_all('nsslapd-changelogmaxage')
+
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=True)
+
+    set_changelog_trimming(M1)
+
+    run_healthcheck_and_flush_log(topology_m2, M1, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, JSON_OUTPUT, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication_presence_of_conflict_entries(topology_m2):
+    """Check if HealthCheck returns DSREPLLE0002 code
+
+    :id: 43abc6c6-2075-42eb-8fa3-aa092ff64cba
+    :setup: Replicated topology
+    :steps:
+        1. Create a replicated topology
+        2. Create conflict entries : different entries renamed to the same dn
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSREPLLE0002 code and related details
+        4. Healthcheck reports DSREPLLE0002 code and related details
+    """
+
+    RET_CODE = 'DSREPLLE0002'
+
+    M1 = topology_m2.ms['master1']
+    M2 = topology_m2.ms['master2']
+
+    repl = ReplicationManager(DEFAULT_SUFFIX)
+    repl.wait_for_replication(M1, M2)
+
+    topology_m2.pause_all_replicas()
+
+    log.info("Create conflict entries")
+    test_users_m1 = UserAccounts(M1, DEFAULT_SUFFIX)
+    test_users_m2 = UserAccounts(M2, DEFAULT_SUFFIX)
+    user_num = 1000
+    test_users_m1.create_test_user(user_num, 2000)
+    test_users_m2.create_test_user(user_num, 2000)
+
+    topology_m2.resume_all_replicas()
+
+    repl.test_replication_topology(topology_m2)
+
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, RET_CODE, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication_out_of_sync_broken(topology_m3):
+    """Check if HealthCheck returns DSREPLLE0001 code
+
+    :id: b5ae7cae-de0f-4206-95a4-f81538764bea
+    :setup: 3 MMR topology
+    :steps:
+        1. Create a 3 masters full-mesh topology, on M2 and M3 don’t set nsds5BeginReplicaRefresh:start
+        2. Perform modifications on M1
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSREPLLE0001 code and related details
+        4. Healthcheck reports DSREPLLE0001 code and related details
+    """
+
+    RET_CODE = 'DSREPLLE0001'
+
+    M1 = topology_m3.ms['master1']
+    M2 = topology_m3.ms['master2']
+    M3 = topology_m3.ms['master3']
+
+    log.info('Break master2 and master3')
+    replicas = Replicas(M2)
+    replica = replicas.list()[0]
+    replica.replace('nsds5ReplicaBindDNGroup', 'cn=repl')
+
+    replicas = Replicas(M3)
+    replica = replicas.list()[0]
+    replica.replace('nsds5ReplicaBindDNGroup', 'cn=repl')
+
+    log.info('Perform update on master1')
+    test_users_m1 = UserAccounts(M1, DEFAULT_SUFFIX)
+    test_users_m1.create_test_user(1005, 2000)
+
+    run_healthcheck_and_flush_log(topology_m3, M1, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_m3, M1, RET_CODE, json=True)
+
+
+if __name__ == '__main__':
+    # Run isolated
+    # -s for DEBUG mode
+    CURRENT_FILE = os.path.realpath(__file__)

dirsrvtests/tests/suites/healthcheck/health_security_test.py

@@ -0,0 +1,360 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+import pytest
+import os
+import subprocess
+import distro
+
+
+from datetime import *
+from lib389.config import Encryption
+from lib389.utils import *
+from lib389._constants import *
+from lib389.cli_base import FakeArgs
+from lib389.topologies import topology_st
+from lib389.cli_ctl.health import health_check_run
+from lib389.paths import Paths
+
+CMD_OUTPUT = 'No issues found.'
+JSON_OUTPUT = '[]'
+
+ds_paths = Paths()
+pytestmark = pytest.mark.skipif(ds_paths.asan_enabled or ds_paths.perl_enabled and (os.getenv('PYINSTALL') is None),
+                                reason="These tests can only be run with python installer and disabled ASAN")
+
+libfaketime = pytest.importorskip('libfaketime')
+libfaketime.reexec_if_needed()
+
+if DEBUGGING:
+    logging.getLogger(__name__).setLevel(logging.DEBUG)
+else:
+    logging.getLogger(__name__).setLevel(logging.INFO)
+log = logging.getLogger(__name__)
+
+
+def is_fips():
+    if os.path.exists('/proc/sys/crypto/fips_enabled'):
+        with open('/proc/sys/crypto/fips_enabled', 'r') as f:
+            state = f.readline().strip()
+            if state == '1':
+                return True
+            else:
+                return False
+
+
+def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searched_code2=None):
+    args = FakeArgs()
+    args.instance = instance.serverid
+    args.verbose = instance.verbose
+    args.list_errors = False
+    args.list_checks = False
+    args.check = None
+    args.dry_run = False
+
+    if json:
+        log.info('Use healthcheck with --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+    else:
+        log.info('Use healthcheck without --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+
+    log.info('Clear the log')
+    topology.logcap.flush()
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_insecure_pwd_hash_configured(topology_st):
+    """Check if HealthCheck returns DSCLE0002 code
+
+    :id: 6baf949c-a5eb-4f4e-83b4-8302e677758a
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Configure an insecure passwordStorageScheme (as SHA) for the instance
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. Set passwordStorageScheme and nsslapd-rootpwstoragescheme to PBKDF2_SHA256
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSCLE0002 code and related details
+        4. Healthcheck reports DSCLE0002 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSCLE0002'
+
+    standalone = topology_st.standalone
+
+    log.info('Configure an insecure passwordStorageScheme (SHA)')
+    standalone.config.set('passwordStorageScheme', 'SHA')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE)
+
+    if is_fips():
+        log.info('Set passwordStorageScheme and nsslapd-rootpwstoragescheme to SSHA512 in FIPS mode')
+        standalone.config.set('passwordStorageScheme', 'SSHA512')
+        standalone.config.set('nsslapd-rootpwstoragescheme', 'SSHA512')
+    else:
+        log.info('Set passwordStorageScheme and nsslapd-rootpwstoragescheme to PBKDF2_SHA256')
+        standalone.config.set('passwordStorageScheme', 'PBKDF2_SHA256')
+        standalone.config.set('nsslapd-rootpwstoragescheme', 'PBKDF2_SHA256')
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_min_allowed_tls_version_too_low(topology_st):
+    """Check if HealthCheck returns DSELE0001 code
+
+    :id: a4be3390-9508-4827-8f82-e4e21081caab
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Set the TLS minimum version to TLS1.0
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. Set the TLS minimum version to TLS1.2
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSELE0001 code and related details
+        4. Healthcheck reports DSELE0001 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSELE0001'
+    HIGHER_VS = 'TLS1.2'
+    SMALL_VS = 'TLS1.0'
+    RHEL = 'Red Hat Enterprise Linux'
+
+    standalone = topology_st.standalone
+
+    standalone.enable_tls()
+
+    # We have to update-crypto-policies to LEGACY, otherwise we can't set TLS1.0
+    log.info('Updating crypto policies')
+    assert subprocess.check_call(['update-crypto-policies', '--set', 'LEGACY']) == 0
+
+    log.info('Set the TLS minimum version to TLS1.0')
+    enc = Encryption(standalone)
+    enc.replace('sslVersionMin', SMALL_VS)
+    standalone.restart()
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=RET_CODE)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=RET_CODE)
+
+    log.info('Set the TLS minimum version to TLS1.2')
+    enc.replace('sslVersionMin', HIGHER_VS)
+    standalone.restart()
+
+    run_healthcheck_and_flush_log(topology_st, standalone, json=False, searched_code=CMD_OUTPUT)
+    run_healthcheck_and_flush_log(topology_st, standalone, json=True, searched_code=JSON_OUTPUT)
+
+    if RHEL in distro.linux_distribution():
+        log.info('Set crypto-policies back to DEFAULT')
+        assert subprocess.check_call(['update-crypto-policies', '--set', 'DEFAULT']) == 0
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_resolvconf_bad_file_perm(topology_st):
+    """Check if HealthCheck returns DSPERMLE0001 code
+
+    :id: 8572b9e9-70e7-49e9-b745-864f6f2468a8
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Change the /etc/resolv.conf file permissions to 444
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. set /etc/resolv.conf permissions to 644
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSPERMLE0001 code and related details
+        4. Healthcheck reports DSPERMLE0001 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSPERMLE0001'
+
+    standalone = topology_st.standalone
+
+    log.info('Change the /etc/resolv.conf file permissions to 444')
+    os.chmod('/etc/resolv.conf', 0o444)
+
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+    log.info('Change the /etc/resolv.conf file permissions to 644')
+    os.chmod('/etc/resolv.conf', 0o644)
+
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_pwdfile_bad_file_perm(topology_st):
+    """Check if HealthCheck returns DSPERMLE0002 code
+
+    :id: ec137d66-bad6-4eed-90bd-fc1d572bbe1f
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Change the /etc/dirsrv/slapd-xxx/pwdfile.txt permissions to 000
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+        5. Change the /etc/dirsrv/slapd-xxx/pwdfile.txt permissions to 400
+        6. Use HealthCheck without --json option
+        7. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSPERMLE0002 code and related details
+        4. Healthcheck reports DSPERMLE0002 code and related details
+        5. Success
+        6. Healthcheck reports no issue found
+        7. Healthcheck reports no issue found
+    """
+
+    RET_CODE = 'DSPERMLE0002'
+
+    standalone = topology_st.standalone
+    cert_dir = standalone.ds_paths.cert_dir
+
+    log.info('Change the /etc/dirsrv/slapd-{}/pwdfile.txt permissions to 000'.format(standalone.serverid))
+    os.chmod('{}/pwdfile.txt'.format(cert_dir), 0o000)
+
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+    log.info('Change the /etc/dirsrv/slapd-{}/pwdfile.txt permissions to 400'.format(standalone.serverid))
+    os.chmod('{}/pwdfile.txt'.format(cert_dir), 0o400)
+
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_certif_expiring_within_30d(topology_st):
+    """Check if HealthCheck returns DSCERTLE0001 code
+
+    :id: c2165032-88ba-4978-a4ca-2fecfd8c35d8
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Use libfaketime to tell the process the date is within 30 days before certificate expiration
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSCERTLE0001 code and related details
+        4. Healthcheck reports DSCERTLE0001 code and related details
+    """
+
+    RET_CODE = 'DSCERTLE0001'
+
+    standalone = topology_st.standalone
+
+    standalone.enable_tls()
+
+    # Cert is valid two years from today, so we count the date that is within 30 days before certificate expiration
+    date_future = datetime.now() + timedelta(days=701)
+
+    with libfaketime.fake_time(date_future):
+        run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+        run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+    # Try again with real time just to make sure no issues were found
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
+
+
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_certif_expired(topology_st):
+    """Check if HealthCheck returns DSCERTLE0002 code
+
+    :id: ceff2c22-62c0-4fd9-b737-930a88458d68
+    :setup: Standalone instance
+    :steps:
+        1. Create DS instance
+        2. Use libfaketime to tell the process the date is after certificate expiration
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Healthcheck reports DSCERTLE0002 code and related details
+        4. Healthcheck reports DSCERTLE0002 code and related details
+    """
+
+    RET_CODE = 'DSCERTLE0002'
+
+    standalone = topology_st.standalone
+
+    standalone.enable_tls()
+
+    # Cert is valid two years from today, so we count the date that is after expiration
+    date_future = datetime.now() + timedelta(days=731)
+
+    with libfaketime.fake_time(date_future):
+        run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+        run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
+
+    # Try again with real time just to make sure no issues were found
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
+
+
+if __name__ == '__main__':
+    # Run isolated
+    # -s for DEBUG mode
+    CURRENT_FILE = os.path.realpath(__file__)

dirsrvtests/tests/suites/healthcheck/health_sync_test.py

@@ -0,0 +1,137 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+import pytest
+import os
+
+from datetime import *
+from lib389.agreement import Agreements
+from lib389.idm.user import UserAccounts
+from lib389.utils import *
+from lib389._constants import *
+from lib389.cli_base import FakeArgs
+from lib389.topologies import topology_m3
+from lib389.cli_ctl.health import health_check_run
+from lib389.paths import Paths
+
+
+ds_paths = Paths()
+pytestmark = pytest.mark.skipif(ds_paths.perl_enabled and (os.getenv('PYINSTALL') is None),
+                                reason="These tests need to use python installer")
+
+if DEBUGGING:
+    logging.getLogger(__name__).setLevel(logging.DEBUG)
+else:
+    logging.getLogger(__name__).setLevel(logging.INFO)
+log = logging.getLogger(__name__)
+
+
+def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searched_code2=None):
+    args = FakeArgs()
+    args.instance = instance.serverid
+    args.verbose = instance.verbose
+    args.list_errors = False
+    args.list_checks = False
+    args.check = None
+    args.dry_run = False
+
+    if json:
+        log.info('Use healthcheck with --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+    else:
+        log.info('Use healthcheck without --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+
+    log.info('Clear the log')
+    topology.logcap.flush()
+
+
+# This test is in separate file because it is timeout specific
[email protected]
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication_out_of_sync_not_broken(topology_m3):
+    """Check if HealthCheck returns DSREPLLE0003 code
+
+    :id: 8305000d-ba4d-4c00-8331-be0e8bd92150
+    :setup: 3 MMR topology
+    :steps:
+        1. Create a 3 masters full-mesh topology, all replicas being synchronized
+        2. Stop M1
+        3. Perform an update on M2 and M3.
+        4. Check M2 and M3 are synchronized.
+        5. From M2, reinitialize the M3 agreement
+        6. Stop M2 and M3
+        7. Restart M1
+        8. Start M3
+        9. Use HealthCheck without --json option
+        10. Use HealthCheck with --json option
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Success
+        5. Success
+        6. Success
+        7. Success
+        8. Success
+        9. Healthcheck reports DSREPLLE0003 code and related details
+        10. Healthcheck reports DSREPLLE0003 code and related details
+    """
+
+    RET_CODE = 'DSREPLLE0003'
+
+    M1 = topology_m3.ms['master1']
+    M2 = topology_m3.ms['master2']
+    M3 = topology_m3.ms['master3']
+
+    log.info('Stop master1')
+    M1.stop()
+
+    log.info('Perform update on master2 and master3')
+    test_users_m2 = UserAccounts(M2, DEFAULT_SUFFIX)
+    test_users_m3 = UserAccounts(M3, DEFAULT_SUFFIX)
+    test_users_m2.create_test_user(1000, 2000)
+    test_users_m3.create_test_user(1001, 2000)
+
+    log.info('Init M2->M3 agreement')
+    agmt = Agreements(M2).list()[1]
+    agmt.begin_reinit()
+    agmt.wait_reinit()
+
+    log.info('Stop M2 and M3')
+    M2.stop()
+    M3.stop()
+
+    log.info('Start M1 first, then M3')
+    M1.start()
+    M3.start()
+
+    run_healthcheck_and_flush_log(topology_m3, M3, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_m3, M3, RET_CODE, json=True)
+
+
+if __name__ == '__main__':
+    # Run isolated
+    # -s for DEBUG mode
+    CURRENT_FILE = os.path.realpath(__file__)

dirsrvtests/tests/suites/healthcheck/healthcheck_test.py

@@ -6,16 +6,22 @@
 # See LICENSE for details.
 # --- END COPYRIGHT BLOCK ---
 #
+
 import pytest
 import os
-import subprocess
+
+from lib389.backend import Backends
+from lib389.mappingTree import MappingTrees
+from lib389.replica import Changelog5
 from lib389.utils import *
 from lib389._constants import *
 from lib389.cli_base import FakeArgs
-from lib389.topologies import topology_st, topology_no_sample
+from lib389.topologies import topology_st, topology_no_sample, topology_m2
 from lib389.cli_ctl.health import health_check_run
 from lib389.paths import Paths
 
+CMD_OUTPUT = 'No issues found.'
+JSON_OUTPUT = '[]'
 
 ds_paths = Paths()
 pytestmark = pytest.mark.skipif(ds_paths.perl_enabled and (os.getenv('PYINSTALL') is None),
@@ -28,6 +34,48 @@ else:
 log = logging.getLogger(__name__)
 
 
+def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searched_code2=None):
+    args = FakeArgs()
+    args.instance = instance.serverid
+    args.verbose = instance.verbose
+    args.list_errors = False
+    args.list_checks = False
+    args.check = None
+    args.dry_run = False
+
+    if json:
+        log.info('Use healthcheck with --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+    else:
+        log.info('Use healthcheck without --json option')
+        args.json = json
+        health_check_run(instance, topology.logcap.log, args)
+        assert topology.logcap.contains(searched_code)
+        log.info('Healthcheck returned searched code: %s' % searched_code)
+
+        if searched_code2 is not None:
+            assert topology.logcap.contains(searched_code2)
+            log.info('Healthcheck returned searched code: %s' % searched_code2)
+
+    log.info('Clear the log')
+    topology.logcap.flush()
+
+
+def set_changelog_trimming(instance):
+    log.info('Get the changelog enteries')
+    inst_changelog = Changelog5(instance)
+
+    log.info('Set nsslapd-changelogmaxage to 30d')
+    inst_changelog.add('nsslapd-changelogmaxage', '30')
+
+
 @pytest.mark.ds50873
 @pytest.mark.bz1685160
 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
@@ -47,516 +95,228 @@ def test_healthcheck_standalone(topology_st):
     """
 
     standalone = topology_st.standalone
-    cmd_output = 'No issues found.'
-    json_ouput = '[]'
-
-    args = FakeArgs()
-    args.instance = standalone.serverid
-    args.verbose = standalone.verbose
-
-    log.info("Use healthcheck without --json option")
-    args.json = False
-    health_check_run(standalone, topology_st.logcap.log, args)
-    assert topology_st.logcap.contains(cmd_output)
 
-    log.info('Use healthcheck with --json option')
-    args.json = True
-    health_check_run(standalone, topology_st.logcap.log, args)
-    assert topology_st.logcap.contains(json_ouput)
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT,json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
 
 
 @pytest.mark.ds50873
[email protected]796343
[email protected]
 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
-def test_health_check_database_not_initialized(topology_no_sample):
-    """Check if HealthCheck returns DSBLE0003 code
+def test_healthcheck_standalone_tls(topology_st):
+    """Check functionality of HealthCheck Tool on TLS enabled standalone instance with no errors
 
-    :id: 716b1ff1-94bd-4780-98b8-96ff8ef21e30
+    :id: 4844b446-3939-4fbd-b14b-293b20bb8be0
     :setup: Standalone instance
     :steps:
-        1. Create DS instance without example entries
-        2. Use HealthCheck without --json option
-        3. Use HealthCheck with --json option
+        1. Create DS instance
+        2. Enable TLS
+        3. Use HealthCheck without --json option
+        4. Use HealthCheck with --json option
     :expectedresults:
         1. Success
-        2. HealthCheck should return code DSBLE0003
-        3. HealthCheck should return code DSBLE0003
+        2. Success
+        3. Success
+        4. Success
     """
 
-    ret_code = 'DSBLE0003'
-    standalone = topology_no_sample.standalone
-
-    args = FakeArgs()
-    args.instance = standalone.serverid
-    args.verbose = standalone.verbose
-
-    log.info("Use healthcheck without --json option")
-    args.json = False
-    health_check_run(standalone, topology_no_sample.logcap.log, args)
-    assert topology_no_sample.logcap.contains(ret_code)
-    log.info("HealthCheck returned DSBLE0003")
+    standalone = topology_st.standalone
+    standalone.enable_tls()
 
-    log.info('Use healthcheck with --json option')
-    args.json = True
-    health_check_run(standalone, topology_no_sample.logcap.log, args)
-    assert topology_no_sample.logcap.contains(ret_code)
-    log.info("HealthCheck with --json argument returned DSBLE0003")
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT,json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
 
 
 @pytest.mark.ds50873
 @pytest.mark.bz1685160
[email protected].skip(reason="Not implemented")
-def test_healthcheck_replication(request):
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication(topology_m2):
     """Check functionality of HealthCheck Tool on replication instance with no errors
 
     :id: 9ee6d491-d6d7-4c2c-ac78-70d08f054166
     :setup: 2 MM topology
     :steps:
         1. Create a two masters replication topology
-        2. Use HealthCheck without --json option
+        2. Set nsslapd-changelogmaxage to 30d
+        3. Use HealthCheck without --json option
         3. Use HealthCheck with --json option
     :expectedresults:
         1. Success
         2. Success
         3. Success
+        4. Success
     """
 
+    M1 = topology_m2.ms['master1']
+    M2 = topology_m2.ms['master2']
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_backend_missing_mapping_tree(request):
-    """Check if HealthCheck returns DSBLE0001 and DSBLE0002 code
+    # If we don't set changelog trimming, we will get error DSCLLE0001
+    set_changelog_trimming(M1)
+    set_changelog_trimming(M2)
 
-    :id: 4c83ffcf-01a4-4ec8-a3d2-01022b566225
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Disable the dc=example,dc=com backend suffix entry in the mapping tree
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. Enable the dc=example,dc=com backend suffix entry in the mapping tree
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSBLE0001 and DSBLE0002 codes and related details
-        4. Healthcheck reports DSBLE0001 and DSBLE0002 codes and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
-    """
+    log.info('Run healthcheck for master1')
+    run_healthcheck_and_flush_log(topology_m2, M1, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, JSON_OUTPUT, json=True)
+
+    log.info('Run healthcheck for master2')
+    run_healthcheck_and_flush_log(topology_m2, M2, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M2, JSON_OUTPUT, json=True)
 
 
 @pytest.mark.ds50873
 @pytest.mark.bz1685160
[email protected].skip(reason="Not implemented")
-def test_healthcheck_virtual_attr_incorrectly_indexed(request):
-    """Check if HealthCheck returns DSVIRTLE0001 code
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_replication_tls(topology_m2):
+    """Check functionality of HealthCheck Tool on replication instance with no errors
 
-    :id: 1055173b-21aa-4aaa-9e91-4dc6c5e0c01f
-    :setup: Standalone instance
+    :id: 9ee6d491-d6d7-4c2c-ac78-70d08f054166
+    :setup: 2 MM topology
     :steps:
-        1. Create DS instance from template file
-        2. Create a CoS definition entry
-        3. Create the matching CoS template entry, with postalcode as virtual attribute
-        4. Create an index for postalcode
-        5. Use HealthCheck without --json option
-        6. Use HealthCheck with --json option
+        1. Create a two masters replication topology
+        2. Enable TLS
+        3. Set nsslapd-changelogmaxage to 30d
+        4. Use HealthCheck without --json option
+        5. Use HealthCheck with --json option
     :expectedresults:
         1. Success
         2. Success
         3. Success
         4. Success
-        5. Healthcheck reports DSVIRTLE0001 code and related details
-        6. Healthcheck reports DSVIRTLE0001 code and related details
-    """
-
-
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_logging_format_should_be_revised(request):
-    """Check if HealthCheck returns DSCLE0001 code
-
-    :id: 277d7980-123b-481b-acba-d90921b9f5ac
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Set nsslapd-logging-hr-timestamps-enabled to ‘off’
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. Set nsslapd-logging-hr-timestamps-enabled to ‘on’
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSCLE0001 code and related details
-        4. Healthcheck reports DSCLE0001 code and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
-    """
-
-
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_insecure_pwd_hash_configured(request):
-    """Check if HealthCheck returns DSCLE0002 code
-
-    :id: 6baf949c-a5eb-4f4e-83b4-8302e677758a
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Configure an insecure passwordStorageScheme (as SHA) for the instance
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. Set passwordStorageScheme and nsslapd-rootpwstoragescheme to PBKDF2_SHA256
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSCLE0002 code and related details
-        4. Healthcheck reports DSCLE0002 code and related details
         5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
     """
 
+    M1 = topology_m2.ms['master1']
+    M2 = topology_m2.ms['master2']
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_min_allowed_tls_version_too_low(request):
-    """Check if HealthCheck returns DSELE0001 code
-
-    :id: a4be3390-9508-4827-8f82-e4e21081caab
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Set the TLS minimum version to TLS1.0
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. Set the TLS minimum version to TLS1.2
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSELE0001 code and related details
-        4. Healthcheck reports DSELE0001 code and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
-    """
-
+    M1.enable_tls()
+    M2.enable_tls()
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_RI_plugin_is_misconfigured(request):
-    """Check if HealthCheck returns DSRILE0001 code
+    log.info('Run healthcheck for master1')
+    run_healthcheck_and_flush_log(topology_m2, M1, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M1, JSON_OUTPUT, json=True)
 
-    :id: de2e90a2-89fe-472c-acdb-e13cbca5178d
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Configure the instance with Integrity Plugin
-        3. Set the referint-update-delay attribute of the RI plugin, to a value upper than 0
-        4. Use HealthCheck without --json option
-        5. Use HealthCheck with --json option
-        6. Set the referint-update-delay attribute to 0
-        7. Use HealthCheck without --json option
-        8. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Success
-        4. Healthcheck reports DSRILE0001 code and related details
-        5. Healthcheck reports DSRILE0001 code and related details
-        6. Success
-        7. Healthcheck reports no issue found
-        8. Healthcheck reports no issue found
-    """
+    log.info('Run healthcheck for master2')
+    run_healthcheck_and_flush_log(topology_m2, M2, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_m2, M2, JSON_OUTPUT, json=True)
 
 
 @pytest.mark.ds50873
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_RI_plugin_missing_indexes(request):
-    """Check if HealthCheck returns DSRILE0002 code
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
[email protected](reason="Will fail because of bz1837315. Set proper version after bug is fixed")
+def test_healthcheck_unable_to_query_backend(topology_st):
+    """Check if HealthCheck returns DSBLE0002 code
 
-    :id: 05c55e37-bb3e-48d1-bbe8-29c980f94f10
+    :id: 716b1ff1-94bd-4780-98b8-96ff8ef21e30
     :setup: Standalone instance
     :steps:
-        1. Create DS instance from template file
-        2. Configure the instance with Integrity Plugin
-        3. Change the index type of the member attribute index to ‘approx’
+        1. Create DS instance
+        2. Create a new root suffix and database
+        3. Disable new suffix
         4. Use HealthCheck without --json option
         5. Use HealthCheck with --json option
-        6. Set the index type of the member attribute index to ‘eq’
-        7. Use HealthCheck without --json option
-        8. Use HealthCheck with --json option
     :expectedresults:
         1. Success
         2. Success
         3. Success
-        4. Healthcheck reports DSRILE0002 code and related details
-        5. Healthcheck reports DSRILE0002 code and related details
-        6. Success
-        7. Healthcheck reports no issue found
-        8. Healthcheck reports no issue found
-    """
-
-
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_replication_out_of_sync_broken(request):
-    """Check if HealthCheck returns DSREPLLE0001 code
-
-    :id: b5ae7cae-de0f-4206-95a4-f81538764bea
-    :setup: 3 MMR topology
-    :steps:
-        1. Create a 3 masters full-mesh topology, on M2 and M3 don’t set nsds5BeginReplicaRefresh:start
-        2. Perform modifications on M1
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSREPLLE0001 code and related details
-        4. Healthcheck reports DSREPLLE0001 code and related details
-    """
-
-
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_replication_presence_of_conflict_entries(request):
-    """Check if HealthCheck returns DSREPLLE0002 code
-
-    :id: 43abc6c6-2075-42eb-8fa3-aa092ff64cba
-    :setup: Replicated topology
-    :steps:
-        1. Create a replicated topology
-        2. Create conflict entries : different entries renamed to the same dn
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSREPLLE0002 code and related details
-        4. Healthcheck reports DSREPLLE0002 code and related details
+        4. HealthCheck should return code DSBLE0002
+        5. HealthCheck should return code DSBLE0002
     """
 
+    RET_CODE = 'DSBLE0002'
+    NEW_SUFFIX = 'dc=test,dc=com'
+    NEW_BACKEND = 'userData'
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_replication_out_of_sync_not_broken(request):
-    """Check if HealthCheck returns DSREPLLE0003 code
-
-    :id: 8305000d-ba4d-4c00-8331-be0e8bd92150
-    :setup: 3 MMR topology
-    :steps:
-        1. Create a 3 masters full-mesh topology, all replicas being synchronized
-        2. stop M1
-        3. Perform an update on M2 and M3.
-        4. Check M2 and M3 are synchronized.
-        5. From M2, reinitialize the M3 agreement
-        6. Stop M2
-        7. Restart M1
-        8. Use HealthCheck without --json option
-        9. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Success
-        4. Success
-        5. Success
-        6. Success
-        7. Success
-        8. Healthcheck reports DSREPLLE0003 code and related details
-        9. Healthcheck reports DSREPLLE0003 code and related details
-    """
+    standalone = topology_st.standalone
 
+    backends = Backends(standalone)
+    backends.create(properties={
+        'cn': NEW_BACKEND,
+        'nsslapd-suffix': NEW_SUFFIX,
+    })
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_replication_presence_of_conflict_entries(request):
-    """Check if HealthCheck returns DSREPLLE0005 code
+    mts = MappingTrees(standalone)
+    mt_new = mts.get(NEW_SUFFIX)
+    mt_new.replace('nsslapd-state', 'disabled')
 
-    :id: d452a564-7b82-4c1a-b331-a71abbd82a10
-    :setup: Replicated topology
-    :steps:
-        1. Create a replicated topology
-        2. On M1, set nsds5replicaport for the replication agreement to an unreachable port on the replica
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. On M1, set nsds5replicaport for the replication agreement to a reachable port number
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSREPLLE0005 code and related details
-        4. Healthcheck reports DSREPLLE0005 code and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
-    """
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE, json=True)
 
 
 @pytest.mark.ds50873
 @pytest.mark.bz1685160
[email protected](reason="Not implemented")
-def test_healthcheck_changelog_trimming_not_configured(request):
-    """Check if HealthCheck returns DSCLLE0001 code
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
[email protected](reason="Will fail because of bz1835619 and bz1837315. Set proper version after bugs are fixed")
+def test_healthcheck_backend_missing_mapping_tree(topology_st):
+    """Check if HealthCheck returns DSBLE0001 and DSBLE0003 code
 
-    :id: c2165032-88ba-4978-a4ca-2fecfd8c35d8
-    :setup: Replicated topology
+    :id: 4c83ffcf-01a4-4ec8-a3d2-01022b566225
+    :setup: Standalone instance
     :steps:
-        1. Create a replicated topology
-        2. On M1, remove nsslapd-changelogmaxage from cn=changelog5,cn=config
+        1. Create DS instance
+        2. Disable the dc=example,dc=com backend suffix entry in the mapping tree
         3. Use HealthCheck without --json option
         4. Use HealthCheck with --json option
-        5. On M1, set nsslapd-changelogmaxage to 30d
+        5. Enable the dc=example,dc=com backend suffix entry in the mapping tree
         6. Use HealthCheck without --json option
         7. Use HealthCheck with --json option
     :expectedresults:
         1. Success
         2. Success
-        3. Healthcheck reports DSCLLE0001 code and related details
-        4. Healthcheck reports DSCLLE0001 code and related details
+        3. Healthcheck reports DSBLE0001 and DSBLE0003 codes and related details
+        4. Healthcheck reports DSBLE0001 and DSBLE0003 codes and related details
         5. Success
         6. Healthcheck reports no issue found
         7. Healthcheck reports no issue found
     """
 
+    RET_CODE1 = 'DSBLE0001'
+    RET_CODE2 = 'DSBLE0003'
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_certif_expiring_within_30d(request):
-    """Check if HealthCheck returns DSCERTLE0001 code
-
-    :id: c2165032-88ba-4978-a4ca-2fecfd8c35d8
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Use libfaketime to tell the process the date is within 30 days before certificate expiration
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSCERTLE0001 code and related details
-        4. Healthcheck reports DSCERTLE0001 code and related details
-    """
-
-
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_certif_expired(request):
-    """Check if HealthCheck returns DSCERTLE0002 code
+    standalone = topology_st.standalone
 
-    :id: ceff2c22-62c0-4fd9-b737-930a88458d68
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Use libfaketime to tell the process the date is after certificate expiration
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSCERTLE0002 code and related details
-        4. Healthcheck reports DSCERTLE0002 code and related details
-    """
+    log.info('Delete the dc=example,dc=com backend suffix entry in the mapping tree')
+    mts = MappingTrees(standalone)
+    mt = mts.get(DEFAULT_SUFFIX)
+    mt.delete()
 
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE1, json=False, searched_code2=RET_CODE2)
+    run_healthcheck_and_flush_log(topology_st, standalone, RET_CODE1, json=True, searched_code2=RET_CODE2)
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_low_disk_space(request):
-    """Check if HealthCheck returns DSDSLE0001 code
+    log.info('Create the dc=example,dc=com backend suffix entry')
+    mts.create(properties={
+        'cn': DEFAULT_SUFFIX,
+        'nsslapd-state': 'backend',
+        'nsslapd-backend': 'userRoot',
+    })
 
-    :id: 144b335d-077e-430c-9c0e-cd6b0f2f73c1
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Get the free disk space for /
-        3. Use fallocate -l to create a file large enough for the use % be up 90%
-        4. Use HealthCheck without --json option
-        5. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. free disk space value
-        3. Success
-        3. Healthcheck reports DSDSLE0001 code and related details
-        4. Healthcheck reports DSDSLE0001 code and related details
-    """
+    run_healthcheck_and_flush_log(topology_st, standalone, CMD_OUTPUT, json=False)
+    run_healthcheck_and_flush_log(topology_st, standalone, JSON_OUTPUT, json=True)
 
 
 @pytest.mark.ds50873
[email protected]685160
[email protected].skip(reason="Not implemented")
-def test_healthcheck_resolvconf_bad_file_perm(request):
-    """Check if HealthCheck returns DSPERMLE0001 code
[email protected]
[email protected](ds_is_older("1.4.1"), reason="Not implemented")
+def test_healthcheck_database_not_initialized(topology_no_sample):
+    """Check if HealthCheck returns DSBLE0003 code
 
-    :id: 8572b9e9-70e7-49e9-b745-864f6f2468a8
+    :id: 716b1ff1-94bd-4780-98b8-96ff8ef21e30
     :setup: Standalone instance
     :steps:
-        1. Create DS instance from template file
-        2. Change the /etc/resolv.conf file permissions to 444
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. set /etc/resolv.conf permissions to 644
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
+        1. Create DS instance without example entries
+        2. Use HealthCheck without --json option
+        3. Use HealthCheck with --json option
     :expectedresults:
         1. Success
-        2. Success
-        3. Healthcheck reports DSPERMLE0001 code and related details
-        4. Healthcheck reports DSPERMLE0001 code and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
+        2. HealthCheck should return code DSBLE0003
+        3. HealthCheck should return code DSBLE0003
     """
 
+    RET_CODE = 'DSBLE0003'
+    standalone = topology_no_sample.standalone
 
[email protected]
[email protected]
[email protected](reason="Not implemented")
-def test_healthcheck_security_bad_file_perm(request):
-    """Check if HealthCheck returns DSPERMLE0002 code
-
-    :id: ec137d66-bad6-4eed-90bd-fc1d572bbe1f
-    :setup: Standalone instance
-    :steps:
-        1. Create DS instance from template file
-        2. Change the /etc/dirsrv/slapd-xxx/pwdfile.txt permissions to 000
-        3. Use HealthCheck without --json option
-        4. Use HealthCheck with --json option
-        5. Change the /etc/dirsrv/slapd-xxx/pwdfile.txt permissions to 400
-        6. Use HealthCheck without --json option
-        7. Use HealthCheck with --json option
-    :expectedresults:
-        1. Success
-        2. Success
-        3. Healthcheck reports DSPERMLE0002 code and related details
-        4. Healthcheck reports DSPERMLE0002 code and related details
-        5. Success
-        6. Healthcheck reports no issue found
-        7. Healthcheck reports no issue found
-    """
+    run_healthcheck_and_flush_log(topology_no_sample, standalone, RET_CODE, json=False)
+    run_healthcheck_and_flush_log(topology_no_sample, standalone, RET_CODE, json=True)
 
 
 if __name__ == '__main__':

src/lib389/lib389/topologies.py

@@ -325,6 +325,7 @@ def topology_st_gssapi(request):
 
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -342,7 +343,9 @@ def topology_no_sample(request):
         if DEBUGGING:
             topology.standalone.stop()
         else:
-            topology.standalone.delete(pyinstall=PYINSTALL)
+            assert _remove_ssca_db(topology)
+            if topology.standalone.exists():
+                topology.standalone.delete(pyinstall=PYINSTALL)
 
     request.addfinalizer(fin)
 
@@ -364,6 +367,7 @@ def topology_i2(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -381,6 +385,7 @@ def topology_i3(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 @pytest.fixture(scope="module")
@@ -397,6 +402,7 @@ def topology_m1(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 @pytest.fixture(scope="module")
@@ -414,6 +420,7 @@ def topology_m1c1(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -431,6 +438,7 @@ def topology_m2(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -448,6 +456,7 @@ def topology_m3(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -465,6 +474,7 @@ def topology_m4(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -483,6 +493,7 @@ def topology_m2c2(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology
 
 
@@ -519,4 +530,5 @@ def topology_m1h1c1(request):
             [inst.delete(pyinstall=PYINSTALL) for inst in topology if inst.exists()]
     request.addfinalizer(fin)
 
+    topology.logcap = LogCapture()
     return topology