Issue 4661 - RFE - allow importing openldap schemas (#4662)

Bug Description: Many applications only publish schemas in
openldap formats. We should be able to import them.

Fix Description: Add a dsconf tool that allows online
importing of these schemas. This uses the migration framework
underneath so that we avoid code duplication.

fixes: https://github.com/389ds/389-ds-base/issues/4661

Author: William Brown <[email protected]>

Review by: @mreynolds389 (Thanks!)

dirsrvtests/tests/suites/openldap_2_389/migrate_hdb_test.py

@@ -39,7 +39,7 @@ def test_migrate_openldap_hdb(topology_st):
     config = olConfig(config_path)
     ldifs = {}
 
-    migration = Migration(config, inst, ldifs)
+    migration = Migration(inst, config.schema, config.databases, ldifs)
 
     print("==== migration plan ====")
     print(migration.__unicode__())

dirsrvtests/tests/suites/openldap_2_389/migrate_test.py

@@ -67,7 +67,7 @@ def test_migrate_openldap_slapdd(topology_st):
         "dc=example,dc=net": os.path.join(DATADIR1, 'example_net.slapcat.ldif'),
     }
 
-    migration = Migration(config, inst, ldifs)
+    migration = Migration(inst, config.schema, config.databases, ldifs)
 
     print("==== migration plan ====")
     print(migration.__unicode__())
@@ -105,7 +105,7 @@ def test_migrate_openldap_slapdd_skip_elements(topology_st):
 
     # 1.3.6.1.4.1.5322.13.1.1 is namedObject, so check that isn't there
 
-    migration = Migration(config, inst, ldifs,
+    migration = Migration(inst, config.schema, config.databases, ldifs,
         skip_schema_oids=['1.3.6.1.4.1.5322.13.1.1'],
         skip_overlays=[olOverlayType.UNIQUE],
     )

src/lib389/cli/dscontainer

@@ -272,7 +272,8 @@ def begin_magic():
         # Create the marker to say we exist. This is also a good writable permissions
         # test for the volume.
         basedn = '# basedn = dc=example,dc=com'
-        if suffix := os.getenv("SUFFIX_NAME"):
+        suffix = os.getenv("SUFFIX_NAME")
+        if suffix is not None:
             basedn = f'basedn = {suffix}'
         config_file = """
 [localhost]

src/lib389/cli/openldap_to_ds

@@ -181,7 +181,11 @@ def do_migration(inst, log, args, skip_overlays):
     ldifmeta = LdifMetadata(args.slapd_ldif, log)
 
     # Create the migration plan.
-    migration = Migration(config, inst, ldifmeta.get_suffixes(),
+    migration = Migration(
+        inst,
+        config.schema,
+        config.databases,
+        ldifmeta.get_suffixes(),
         skip_schema_oids,
         skip_overlays,
         skip_entry_attributes,

src/lib389/lib389/cli_conf/schema.py

@@ -9,6 +9,8 @@
 from json import dumps as dump_json
 from lib389.cli_base import _get_arg
 from lib389.schema import Schema, AttributeUsage, ObjectclassKind
+from lib389.migrate.openldap.config import olSchema
+from lib389.migrate.plan import Migration
 
 
 def _validate_dual_args(enable_arg, disable_arg):
@@ -221,6 +223,19 @@ def get_syntaxes(inst, basedn, log, args):
             print("%s (%s)", name, id)
 
 
+def import_openldap_schema_file(inst, basedn, log, args):
+    log = log.getChild('import_openldap_schema_file')
+    log.debug(f"Parsing {args.schema_file} ...")
+    olschema = olSchema([args.schema_file], log)
+    migration = Migration(inst, olschema)
+    if args.confirm:
+        migration.execute_plan(log)
+        log.info("🎉 Schema migration complete!")
+    else:
+        migration.display_plan_review(log)
+        log.info("No actions taken. To apply migration plan, use '--confirm'")
+
+
 def _get_parameters(args, type):
     if type not in ('attributetypes', 'objectclasses'):
         raise ValueError("Wrong parser type: %s" % type)
@@ -383,3 +398,11 @@ def create_parser(subparsers):
     validate_parser.add_argument('DN', help="Base DN that contains entries to validate")
     validate_parser.add_argument('-f', '--filter', help='Filter for entries to validate.\n'
                                                         'If omitted, all entries with filter "(objectclass=*)" are validated')
+
+    import_oldap_schema_parser = schema_subcommands.add_parser('import-openldap-file',
+                                                    help='Import an openldap formatted dynamic schema ldifs. These will contain values like olcAttributeTypes and olcObjectClasses.')
+    import_oldap_schema_parser.set_defaults(func=import_openldap_schema_file)
+    import_oldap_schema_parser.add_argument('schema_file', help="Path to the openldap dynamic schema ldif to import")
+    import_oldap_schema_parser.add_argument('--confirm',
+                                            default=False, action='store_true',
+                                            help="Confirm that you want to apply these schema migration actions to the 389-ds instance. By default no actions are taken.")

src/lib389/lib389/migrate/openldap/config.py

@@ -26,8 +26,12 @@ class SimpleParser(LDIFParser):
         self.entries.append((dn, entry))
 
 
-def ldif_parse(path, rpath):
-    with open(os.path.join(path, rpath), 'r') as f:
+def ldif_parse(path, rpath=None):
+    if rpath is None:
+        jpath = path
+    else:
+        jpath = os.path.join(path, rpath)
+    with open(jpath, 'r') as f:
         sp = SimpleParser(f)
         sp.parse()
         return sp.entries
@@ -243,16 +247,14 @@ obsolete {self.obsolete} -> {ds_obj.obsolete}""")
         return False
 
 class olSchema(object):
-    def __init__(self, path, log):
+    def __init__(self, schemas, log):
         self.log = log
-        self.log.debug(f"olSchema path -> {path}")
-        schemas = sorted(os.listdir(path))
         self.log.debug(f"olSchemas -> {schemas}")
 
         self.raw_schema = []
 
         for schema in schemas:
-            entries = ldif_parse(path, schema)
+            entries = ldif_parse(schema)
             assert len(entries) == 1
             self.raw_schema.append(entries.pop())
         # self.log.debug(f"raw_schema -> {self.raw_schema}")
@@ -282,8 +284,14 @@ class olConfig(object):
         self.config_entry = config_entries.pop()
         self.log.debug(self.config_entry)
 
+        schema_path = os.path.join(path, 'cn=config/cn=schema/')
+        self.log.debug(f"olConfig schema path -> {schema_path}")
+        schemas = [
+            os.path.join(schema_path, x)
+            for x in sorted(os.listdir(schema_path))
+        ]
         # Parse all the child values.
-        self.schema = olSchema(os.path.join(path, 'cn=config/cn=schema/'), self.log)
+        self.schema = olSchema(schemas, self.log)
 
         dbs = sorted([
             os.path.split(x)[1].replace('.ldif', '')

src/lib389/lib389/migrate/plan.py

@@ -411,7 +411,7 @@ class PluginUnknownManual(MigrationAction):
 
 
 class Migration(object):
-    def __init__(self, olconfig, inst, ldifs=None, skip_schema_oids=[], skip_overlays=[], skip_entry_attributes=[]):
+    def __init__(self, inst, olschema=None, oldatabases=None, ldifs=None, skip_schema_oids=[], skip_overlays=[], skip_entry_attributes=[]):
         """Generate a migration plan from an openldap config, the instance to migrate too
         and an optional dictionary of { suffix: ldif_path }.
 
@@ -420,7 +420,8 @@ class Migration(object):
         accepted. Plan modification is "out of scope", but possible as the array could
         be manipulated in place.
         """
-        self.olconfig = olconfig
+        self.olschema = olschema
+        self.oldatabases = oldatabases
         self.inst = inst
         self.plan = []
         self.ldifs = ldifs
@@ -497,6 +498,8 @@ class Migration(object):
         return buff
 
     def _gen_schema_plan(self):
+        if self.olschema is None:
+            return
         # Get the server schema so that we can query it repeatedly.
         schema = Schema(self.inst)
         schema_attrs = schema.get_attributetypes()
@@ -505,7 +508,7 @@ class Migration(object):
         resolver = Resolver(schema_attrs)
 
         # Examine schema attrs
-        for attr in self.olconfig.schema.attrs:
+        for attr in self.olschema.attrs:
             # If we have been instructed to ignore this oid, skip.
             if attr.oid in self._schema_oid_do_not_migrate:
                 continue
@@ -529,7 +532,7 @@ class Migration(object):
                 self.plan.append(SchemaAttributeAmbiguous(attr, overlaps))
 
         # Examine schema classes
-        for obj in self.olconfig.schema.classes:
+        for obj in self.olschema.classes:
             # If we have been instructed to ignore this oid, skip.
             if obj.oid in self._schema_oid_do_not_migrate:
                 continue
@@ -603,9 +606,12 @@ class Migration(object):
     def _gen_db_plan(self):
         # Create/Manage dbs
         # Get the set of current dbs.
+        if self.oldatabases is None:
+            return
+
         backends = Backends(self.inst)
 
-        for db in self.olconfig.databases:
+        for db in self.oldatabases:
             # Get the suffix
             suffix = db.suffix
             try:
@@ -637,6 +643,10 @@ class Migration(object):
         if log is None:
             log = logger
 
+        # Do we have anything to do?
+        if len(self.plan) == 0:
+            raise Exception("Migration has no actions to perform")
+
         count = 1
 
         # First apply everything
@@ -657,6 +667,8 @@ class Migration(object):
 
     def display_plan_review(self, log):
         """Given an output log sink, display the migration plan"""
+        if len(self.plan) == 0:
+            raise Exception("Migration has no actions to perform")
         for item in self.plan:
             item.display_plan(log)