Issue 4747 - Remove unstable/unstatus tests from PRCI (#4748)

Bug description:
	Some tests (17) in the tests suite (dirsrvtest/tests/suites)
	are failing although there is no regression.
	It needs (long) investigations to status if failures
	are due to a bug in the tests or in DS core.
	Until those investigations are completes, test suites
	loose a large part of its value to detect regression.
	Indeed those failing tests may hide a real regression.

Fix description:
	Flag failing tests with pytest.mark.flaky(max_runs=2, min_passes=1)
	Additional action will be to create upstream 17 ticket to
	status on each failing tests

relates: https://github.com/389ds/389-ds-base/issues/4747

Reviewed by: Simon Pichugin, Viktor Ashirov (many thanks for your
reviews and help)

Platforms tested: F33

.github/workflows/pytest.yml

@@ -65,7 +65,7 @@ jobs:
         set -x
         CID=$(sudo docker run -d -h server.example.com --privileged --rm -v /sys/fs/cgroup:/sys/fs/cgroup:rw,rslave -v ${PWD}:/workspace quay.io/389ds/ci-images:test)
         sudo docker exec $CID sh -c "dnf install -y -v dist/rpms/*rpm"
-        sudo docker exec $CID py.test --junit-xml=pytest.xml -v dirsrvtests/tests/suites/${{ matrix.suite }}
+        sudo docker exec $CID py.test  --suppress-no-test-exit-code  -m "not flaky" --junit-xml=pytest.xml -v dirsrvtests/tests/suites/${{ matrix.suite }}
 
     - name: Make the results file readable by all
       if: always()

dirsrvtests/tests/suites/acl/keywords_test.py

@@ -223,7 +223,8 @@ def test_user_binds_without_any_password_and_cannot_access_the_data(topo, add_us
     with pytest.raises(ldap.INSUFFICIENT_ACCESS):
         org.replace("seeAlso", "cn=1")
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_user_can_access_the_data_when_connecting_from_any_machine(
         topo, add_user, aci_of_user
 ):
@@ -253,6 +254,8 @@ def test_user_can_access_the_data_when_connecting_from_any_machine(
     OrganizationalUnit(conn, DNS_OU_KEY).replace("seeAlso", "cn=1")
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_user_can_access_the_data_when_connecting_from_internal_ds_network_only(
         topo, add_user, aci_of_user
 ):
@@ -283,7 +286,8 @@ def test_user_can_access_the_data_when_connecting_from_internal_ds_network_only(
     # Perform Operation
     OrganizationalUnit(conn, DNS_OU_KEY).replace("seeAlso", "cn=1")
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_user_can_access_the_data_when_connecting_from_some_network_only(
         topo, add_user, aci_of_user
 ):
@@ -314,7 +318,8 @@ def test_user_can_access_the_data_when_connecting_from_some_network_only(
     # Perform Operation
     OrganizationalUnit(conn, DNS_OU_KEY).replace("seeAlso", "cn=1")
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_from_an_unauthorized_network(topo, add_user, aci_of_user):
     """User cannot access the data when connecting from an unauthorized network as per the ACI.
 
@@ -341,7 +346,8 @@ def test_from_an_unauthorized_network(topo, add_user, aci_of_user):
     # Perform Operation
     OrganizationalUnit(conn, DNS_OU_KEY).replace("seeAlso", "cn=1")
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_user_cannot_access_the_data_when_connecting_from_an_unauthorized_network_2(
         topo, add_user, aci_of_user):
     """User cannot access the data when connecting from an unauthorized network as per the ACI.
@@ -430,6 +436,8 @@ def test_dnsalias_keyword_test_nodns_cannot(topo, add_user, aci_of_user):
     with pytest.raises(ldap.INSUFFICIENT_ACCESS):
         org.replace("seeAlso", "cn=1")
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.ds50378
 @pytest.mark.bz1710848
 @pytest.mark.parametrize("ip_addr", ['127.0.0.1', "[::1]"])

dirsrvtests/tests/suites/clu/dsctl_acceptance_test.py

@@ -14,7 +14,8 @@ from lib389.topologies import topology_st as topo
 
 log = logging.getLogger(__name__)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_custom_path(topo):
     """Test that a custom path, backup directory, is correctly used by lib389
     when the server is stopped.

dirsrvtests/tests/suites/clu/repl_monitor_test.py

@@ -90,6 +90,8 @@ def get_hostnames_from_log(port1, port2):
         host_m2 = match.group(2)
     return (host_m1, host_m2)
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.ds50545
 @pytest.mark.bz1739718
 @pytest.mark.skipif(ds_is_older("1.4.0"), reason="Not implemented")

dirsrvtests/tests/suites/dynamic_plugins/dynamic_plugins_test.py

@@ -68,7 +68,8 @@ def check_replicas(topology_m2):
 
     log.info('Data is consistent across the replicas.\n')
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_acceptance(topology_m2):
     """Exercise each plugin and its main features, while
     changing the configuration without restarting the server.
@@ -140,7 +141,8 @@ def test_acceptance(topology_m2):
     ############################################################################
     check_replicas(topology_m2)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_memory_corruption(topology_m2):
     """Check the plugins for memory corruption issues while
     dynamic plugins option is enabled
@@ -242,6 +244,8 @@ def test_memory_corruption(topology_m2):
     ############################################################################
     check_replicas(topology_m2)
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.tier2
 def test_stress(topology_m2):
     """Test plugins while under a big load. Perform the test 5 times

dirsrvtests/tests/suites/fourwaymmr/fourwaymmr_test.py

@@ -145,7 +145,8 @@ def test_delete_a_few_entries_in_m4(topo_m4, _cleanupentris):
         topo_m4.ms["supplier4"], topo_m4.ms["supplier3"], 30
     )
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_replicated_multivalued_entries(topo_m4):
     """Replicated multivalued entries are ordered the same way on all consumers
 

dirsrvtests/tests/suites/healthcheck/health_config_test.py

@@ -329,6 +329,7 @@ def test_healthcheck_low_disk_space(topology_st):
     os.remove(file)
 
 
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.ds50791
 @pytest.mark.bz1843567
 @pytest.mark.xfail(ds_is_older("1.4.3.8"), reason="Not implemented")

dirsrvtests/tests/suites/healthcheck/health_sync_test.py

@@ -61,6 +61,8 @@ def run_healthcheck_and_flush_log(topology, instance, searched_code, json, searc
 @pytest.mark.ds50873
 @pytest.mark.bz1685160
 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_healthcheck_replication_out_of_sync_not_broken(topology_m3):
     """Check if HealthCheck returns DSREPLLE0003 code
 

dirsrvtests/tests/suites/import/import_test.py

@@ -14,6 +14,7 @@ import os
 import pytest
 import time
 import glob
+import logging
 from lib389.topologies import topology_st as topo
 from lib389._constants import DEFAULT_SUFFIX, TaskWarning
 from lib389.dbgen import dbgen_users
@@ -28,6 +29,12 @@ from lib389.idm.account import Accounts
 
 pytestmark = pytest.mark.tier1
 
+DEBUGGING = os.getenv("DEBUGGING", default=False)
+if DEBUGGING:
+    logging.getLogger(__name__).setLevel(logging.DEBUG)
+else:
+    logging.getLogger(__name__).setLevel(logging.INFO)
+log = logging.getLogger(__name__)
 
 def _generate_ldif(topo, no_no):
     """
@@ -350,7 +357,8 @@ def _toggle_private_import_mem(request, topo):
             ('nsslapd-db-private-import-mem', 'off'))
     request.addfinalizer(finofaci)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean):
     """With nsslapd-db-private-import-mem: on is faster import.
 
@@ -382,16 +390,19 @@ def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean):
     # Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0
     config = LDBMConfig(topo.standalone)
     # Measure offline import time duration total_time1
-    total_time1 = _import_offline(topo, 20)
+    total_time1 = _import_offline(topo, 1000)
     # Now nsslapd-db-private-import-mem:off
     config.replace('nsslapd-db-private-import-mem', 'off')
     accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
     for i in accounts.filter('(uid=*)'):
         UserAccount(topo.standalone, i.dn).delete()
     # Measure offline import time duration total_time2
-    total_time2 = _import_offline(topo, 20)
+    total_time2 = _import_offline(topo, 1000)
     # total_time1 < total_time2
+    log.info("total_time1 = %f" % total_time1)
+    log.info("total_time2 = %f" % total_time2)
     assert total_time1 < total_time2
+
     # Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1
     config.replace_many(
         ('nsslapd-db-private-import-mem', 'on'),
@@ -399,14 +410,16 @@ def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean):
     for i in accounts.filter('(uid=*)'):
         UserAccount(topo.standalone, i.dn).delete()
     # Measure offline import time duration total_time1
-    total_time1 = _import_offline(topo, 20)
+    total_time1 = _import_offline(topo, 1000)
     # Now nsslapd-db-private-import-mem:off
     config.replace('nsslapd-db-private-import-mem', 'off')
     for i in accounts.filter('(uid=*)'):
         UserAccount(topo.standalone, i.dn).delete()
     # Measure offline import time duration total_time2
-    total_time2 = _import_offline(topo, 20)
+    total_time2 = _import_offline(topo, 1000)
     # total_time1 < total_time2
+    log.info("toral_time1 = %f" % total_time1)
+    log.info("total_time2 = %f" % total_time2)
     assert total_time1 < total_time2
 
 

dirsrvtests/tests/suites/indexes/regression_test.py

@@ -81,6 +81,8 @@ def set_small_idlistscanlimit(request, topo):
 
     request.addfinalizer(fin)
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.skipif(ds_is_older("1.4.4.4"), reason="Not implemented")
 def test_reindex_task_creates_abandoned_index_file(topo):
     """

dirsrvtests/tests/suites/paged_results/paged_results_test.py

@@ -510,7 +510,8 @@ def test_search_with_timelimit(topology_st, create_user):
     finally:
         del_users(users_list)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.parametrize('aci_subject',
                          ('dns = "{}"'.format(HOSTNAME),
                           'ip = "{}"'.format(IP_ADDRESS)),

dirsrvtests/tests/suites/password/regression_test.py

@@ -217,6 +217,8 @@ def test_global_vs_local(topo, passw_policy, create_user, user_pasw):
     # reset password
     create_user.set('userPassword', PASSWORD)
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 @pytest.mark.ds49789
 def test_unhashed_pw_switch(topo_supplier):
     """Check that nsslapd-unhashed-pw-switch works corrently

dirsrvtests/tests/suites/plugins/accpol_test.py

@@ -520,7 +520,8 @@ def test_glinact_limit(topology_st, accpol_global):
     modify_attr(topology_st, ACCP_CONF, 'accountInactivityLimit', '12')
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glnologin_attr(topology_st, accpol_global):
     """Verify if user account is inactivated based on createTimeStamp attribute, no lastLoginTime attribute present
 
@@ -610,7 +611,8 @@ def test_glnologin_attr(topology_st, accpol_global):
     account_status(topology_st, suffix, subtree, userid, nousrs, 0, "Enabled")
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glnoalt_stattr(topology_st, accpol_global):
     """Verify if user account can be inactivated based on lastLoginTime attribute, altstateattrname set to 1.1
 
@@ -656,6 +658,8 @@ def test_glnoalt_stattr(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glattr_modtime(topology_st, accpol_global):
     """Verify if user account can be inactivated based on modifyTimeStamp attribute
 
@@ -705,6 +709,8 @@ def test_glattr_modtime(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glnoalt_nologin(topology_st, accpol_global):
     """Verify if account policy plugin works if we set altstateattrname set to 1.1 and alwaysrecordlogin to NO
 
@@ -763,6 +769,8 @@ def test_glnoalt_nologin(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glinact_nsact(topology_st, accpol_global):
     """Verify if user account can be activated using dsidm.
 
@@ -813,6 +821,8 @@ def test_glinact_nsact(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glinact_acclock(topology_st, accpol_global):
     """Verify if user account is activated when account is unlocked by passwordlockoutduration.
 
@@ -869,6 +879,8 @@ def test_glinact_acclock(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_glnact_pwexp(topology_st, accpol_global):
     """Verify if user account is activated when password is reset after password is expired
 
@@ -952,6 +964,8 @@ def test_glnact_pwexp(topology_st, accpol_global):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_locact_inact(topology_st, accpol_local):
     """Verify if user account is inactivated when accountInactivityLimit is exceeded.
 
@@ -996,6 +1010,8 @@ def test_locact_inact(topology_st, accpol_local):
     del_users(topology_st, suffix, subtree, userid, nousrs)
 
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_locinact_modrdn(topology_st, accpol_local):
     """Verify if user account is inactivated when moved from ou=groups to ou=people subtree.
 

dirsrvtests/tests/suites/plugins/managed_entry_test.py

@@ -112,7 +112,8 @@ class WithObjectClass(Account):
         self._rdn_attribute = 'uid'
         self._create_objectclasses = ['top', 'person', 'inetorgperson']
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_mentry01(topo, _create_inital):
     """Test Managed Entries basic functionality
 

dirsrvtests/tests/suites/plugins/memberof_test.py

@@ -2658,7 +2658,8 @@ def test_complex_group_scenario_9(topology_st):
     verify_post_025(topology_st, memofegrp020_1, memofegrp020_2, memofegrp020_3, memofegrp020_4, memofegrp020_5,
                     memofuser1, memofuser2, memofuser3, memofuser4)
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_memberof_auto_add_oc(topology_st):
     """Test the auto add objectclass (OC) feature. The plugin should add a predefined
     objectclass that will allow memberOf to be added to an entry.

dirsrvtests/tests/suites/replication/cleanallruv_test.py

@@ -224,7 +224,7 @@ def test_clean(topology_m4, m4rid):
 
     log.info('test_clean PASSED, restoring supplier 4...')
 
-
+@pytest.mark.flaky(max_runs=2, min_passes=1)
 def test_clean_restart(topology_m4, m4rid):
     """Check that cleanallruv task works properly after a restart
 
@@ -296,6 +296,7 @@ def test_clean_restart(topology_m4, m4rid):
     log.info('test_clean_restart PASSED, restoring supplier 4...')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_clean_force(topology_m4, m4rid):
     """Check that multiple tasks with a 'force' option work properly
 
@@ -354,6 +355,7 @@ def test_clean_force(topology_m4, m4rid):
     log.info('test_clean_force PASSED, restoring supplier 4...')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_abort(topology_m4, m4rid):
     """Test the abort task basic functionality
 
@@ -409,6 +411,7 @@ def test_abort(topology_m4, m4rid):
     log.info('test_abort PASSED, restoring supplier 4...')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_abort_restart(topology_m4, m4rid):
     """Test the abort task can handle a restart, and then resume
 
@@ -487,6 +490,7 @@ def test_abort_restart(topology_m4, m4rid):
     log.info('test_abort_restart PASSED, restoring supplier 4...')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_abort_certify(topology_m4, m4rid):
     """Test the abort task with a replica-certify-all option
 
@@ -556,6 +560,7 @@ def test_abort_certify(topology_m4, m4rid):
     log.info('test_abort_certify PASSED, restoring supplier 4...')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_stress_clean(topology_m4, m4rid):
     """Put each server(m1 - m4) under a stress, and perform the entire clean process
 
@@ -643,6 +648,7 @@ def test_stress_clean(topology_m4, m4rid):
     ldbm_config.set('nsslapd-readonly', 'off')
 
 
[email protected](max_runs=2, min_passes=1)
 def test_multiple_tasks_with_force(topology_m4, m4rid):
     """Check that multiple tasks with a 'force' option work properly
 

dirsrvtests/tests/suites/replication/encryption_cl5_test.py

@@ -82,7 +82,8 @@ def _check_unhashed_userpw_encrypted(inst, change_type, user_dn, user_pw, is_enc
                 assert user_pw_attr in entry, 'Changelog entry does not contain clear text password'
     assert count, 'Operation type and DN of the entry not matched in changelog'
 
-
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_algorithm_unhashed(topology_with_tls):
     """Check encryption algorithm AES
     And check unhashed#user#password attribute for encryption.

dirsrvtests/tests/suites/retrocl/basic_test.py

@@ -30,6 +30,8 @@ ATTR_CARLICENSE = 'carLicense'
 
 log = logging.getLogger(__name__)
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_retrocl_exclude_attr_add(topology_st):
     """ Test exclude attribute feature of the retrocl plugin for add operation
 
@@ -160,6 +162,8 @@ def test_retrocl_exclude_attr_add(topology_st):
         log.fatal("Changelog search failed, error: " +str(e))
         assert False
 
+#unstable or unstatus tests, skipped for now
[email protected](max_runs=2, min_passes=1)
 def test_retrocl_exclude_attr_mod(topology_st):
     """ Test exclude attribute feature of the retrocl plugin for mod operation