From 61e3cb0ef14aaae497b34ec73d5b5d5aba07a2d7 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Jul 15 2019 13:27:05 +0000
Subject: Port to libmodulemd v2


Fixes: https://pagure.io/pungi/issue/1225
JIRA: COMPOSE-3662
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

---

diff --git a/pungi/__init__.py b/pungi/__init__.py
index 12ada28..22aaf88 100644
--- a/pungi/__init__.py
+++ b/pungi/__init__.py
@@ -6,9 +6,9 @@ import re
 
 try:
     import gi
-    gi.require_version('Modulemd', '1.0') # noqa
+    gi.require_version('Modulemd', '2.0') # noqa
     from gi.repository import Modulemd
-except:
+except (ImportError, ValueError):
     Modulemd = None
 
 
diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py
index 6489eab..197d7e8 100644
--- a/pungi/phases/createrepo.py
+++ b/pungi/phases/createrepo.py
@@ -36,7 +36,7 @@ from .base import PhaseBase
 from ..util import (
     find_old_compose,
     get_arch_variant_data,
-    iter_module_defaults,
+    collect_module_defaults,
     temp_dir,
 )
 from pungi import Modulemd
@@ -196,22 +196,21 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
 
     # call modifyrepo to inject modulemd if needed
     if pkg_type == "rpm" and arch in variant.arch_mmds and Modulemd is not None:
-        modules = []
+        mod_index = Modulemd.ModuleIndex()
         metadata = []
 
         for module_id, mmd in variant.arch_mmds[arch].items():
             if modules_metadata:
-                module_rpms = mmd.peek_rpm_artifacts().dup()
+                module_rpms = mmd.get_rpm_artifacts()
                 metadata.append((module_id, module_rpms))
-            modules.append(mmd)
+            mod_index.add_module_stream(mmd)
 
-        module_names = set([x.get_name() for x in modules])
-        for mmddef in iter_module_defaults(compose.paths.work.module_defaults_dir()):
-            if mmddef.peek_module_name() in module_names:
-                modules.append(mmddef)
+        module_names = set(mod_index.get_module_names())
+        defaults_dir = compose.paths.work.module_defaults_dir()
+        collect_module_defaults(defaults_dir, module_names, mod_index)
 
         log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
-        add_modular_metadata(repo, repo_dir, modules, log_file)
+        add_modular_metadata(repo, repo_dir, mod_index, log_file)
 
         for module_id, module_rpms in metadata:
             modulemd_path = os.path.join(
@@ -230,11 +229,12 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None)
     compose.log_info("[DONE ] %s" % msg)
 
 
-def add_modular_metadata(repo, repo_path, mmd, log_file):
+def add_modular_metadata(repo, repo_path, mod_index, log_file):
     """Add modular metadata into a repository."""
     with temp_dir() as tmp_dir:
         modules_path = os.path.join(tmp_dir, "modules.yaml")
-        Modulemd.dump(mmd, modules_path)
+        with open(modules_path, "w") as f:
+            f.write(mod_index.dump_to_string())
 
         cmd = repo.get_modifyrepo_cmd(
             os.path.join(repo_path, "repodata"),
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index 67c8ad0..d7438a3 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -29,10 +29,10 @@ import pungi.wrappers.kojiwrapper
 
 from pungi import Modulemd
 from pungi.compose import get_ordered_variant_uids
-from pungi.arch import get_compatible_arches, split_name_arch, tree_arch_to_yum_arch
+from pungi.arch import get_compatible_arches, split_name_arch
 from pungi.phases.base import PhaseBase
 from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
-                        makedirs, iter_module_defaults)
+                        makedirs, collect_module_defaults)
 from pungi.phases.createrepo import add_modular_metadata
 
 
@@ -377,23 +377,18 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map):
 
     # Add modular metadata into the repo
     if variant.arch_mmds:
-        mmds = []
+        mod_index = Modulemd.ModuleIndex()
         for mmd in variant.arch_mmds[arch].values():
-            # Set the arch field, but no other changes are needed.
-            repo_mmd = mmd.copy()
-            repo_mmd.set_arch(tree_arch_to_yum_arch(arch))
-            mmds.append(repo_mmd)
+            mod_index.add_module_stream(mmd)
 
-        module_names = set([x.get_name() for x in mmds])
+        module_names = set(mod_index.get_module_names())
         defaults_dir = compose.paths.work.module_defaults_dir()
-        for mmddef in iter_module_defaults(defaults_dir):
-            if mmddef.peek_module_name() in module_names:
-                mmds.append(mmddef)
+        collect_module_defaults(defaults_dir, module_names, mod_index)
 
         log_file = compose.paths.log.log_file(
             arch, "lookaside_repo_modules_%s" % (variant.uid)
         )
-        add_modular_metadata(cr, repo, mmds, log_file)
+        add_modular_metadata(cr, repo, mod_index, log_file)
 
     compose.log_info('[DONE ] %s', msg)
 
diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py
index af33296..8f69964 100644
--- a/pungi/phases/gather/methods/method_hybrid.py
+++ b/pungi/phases/gather/methods/method_hybrid.py
@@ -142,7 +142,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
     def prepare_modular_packages(self):
         for var in self.compose.all_variants.values():
             for mmd in var.arch_mmds.get(self.arch, {}).values():
-                self.modular_packages.update(mmd.get_rpm_artifacts().dup())
+                self.modular_packages.update(mmd.get_rpm_artifacts())
 
     def prepare_langpacks(self, arch, variant):
         if not self.compose.has_comps:
@@ -223,7 +223,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
 
         modules = []
         for mmd in variant.arch_mmds.get(arch, {}).values():
-            modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream()))
+            modules.append("%s:%s" % (mmd.get_module_name(), mmd.get_stream_name()))
 
         input_packages = []
         for pkg_name, pkg_arch in packages:
@@ -399,10 +399,10 @@ def get_platform(compose, variant, arch):
 
     for var in compose.all_variants.values():
         for mmd in var.arch_mmds.get(arch, {}).values():
-            for dep in mmd.peek_dependencies():
-                streams = dep.peek_requires().get("platform")
+            for dep in mmd.get_dependencies():
+                streams = dep.get_runtime_streams("platform")
                 if streams:
-                    platforms.update(streams.dup())
+                    platforms.update(streams)
 
     if len(platforms) > 1:
         raise RuntimeError("There are conflicting requests for platform.")
diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py
index 2dc818e..8e5de32 100644
--- a/pungi/phases/gather/sources/source_module.py
+++ b/pungi/phases/gather/sources/source_module.py
@@ -38,7 +38,7 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
 
         compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True)
 
-        for nsvc, mmd in variant.arch_mmds[arch].items():
+        for nsvc, module_stream in variant.arch_mmds[arch].items():
             available_rpms = sum(
                 (
                     variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, [])
@@ -46,7 +46,7 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
                 ),
                 [],
             )
-            to_include = set(mmd.get_rpm_artifacts().get())
+            to_include = set(module_stream.get_rpm_artifacts())
             for rpm_obj in available_rpms:
                 if rpm_obj.nevra in to_include:
                     packages.add((rpm_obj, None))
diff --git a/pungi/phases/init.py b/pungi/phases/init.py
index 7ba51a2..d06a927 100644
--- a/pungi/phases/init.py
+++ b/pungi/phases/init.py
@@ -229,8 +229,8 @@ def validate_module_defaults(path):
     """
     seen_defaults = collections.defaultdict(set)
 
-    for mmddef in iter_module_defaults(path):
-        seen_defaults[mmddef.peek_module_name()].add(mmddef.peek_default_stream())
+    for module_name, defaults in iter_module_defaults(path):
+        seen_defaults[module_name].add(defaults.get_default_stream())
 
     errors = []
     for module_name, defaults in seen_defaults.items():
diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
index 60e7a76..553b605 100644
--- a/pungi/phases/pkgset/common.py
+++ b/pungi/phases/pkgset/common.py
@@ -20,9 +20,10 @@ from kobo.shortcuts import run, relative_path
 from kobo.threads import run_in_threads
 
 import pungi.phases.pkgset.pkgsets
+from pungi import Modulemd
 from pungi.arch import get_valid_arches
 from pungi.wrappers.createrepo import CreaterepoWrapper
-from pungi.util import is_arch_multilib, find_old_compose, iter_module_defaults
+from pungi.util import is_arch_multilib, find_old_compose, collect_module_defaults
 from pungi.phases.createrepo import add_modular_metadata
 
 
@@ -118,12 +119,14 @@ def _create_arch_repo(worker_thread, args, task_num):
                                   update_md_path=repo_dir_global, checksum=createrepo_checksum)
     run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True)
     # Add modulemd to the repo for all modules in all variants on this architecture.
-    mmds = list(iter_module_defaults(compose.paths.work.module_defaults_dir()))
-    for variant in compose.get_variants(arch=arch):
-        mmds.extend(variant.arch_mmds.get(arch, {}).values())
-    if mmds:
+    if Modulemd:
+        mod_index = collect_module_defaults(compose.paths.work.module_defaults_dir())
+
+        for variant in compose.get_variants(arch=arch):
+            for module_stream in variant.arch_mmds.get(arch, {}).values():
+                mod_index.add_module_stream(module_stream)
         add_modular_metadata(
-            repo, repo_dir, mmds, compose.paths.log.log_file(arch, "arch_repo_modulemd")
+            repo, repo_dir, mod_index, compose.paths.log.log_file(arch, "arch_repo_modulemd")
         )
 
     compose.log_info("[DONE ] %s" % msg)
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index f3e7db8..1a7bf3a 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -247,7 +247,9 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=
 
     for arch in variant.arches:
         try:
-            mmd = Modulemd.Module.new_from_file(mmds["modulemd.%s.txt" % arch])
+            mmd = Modulemd.ModuleStream.read_file(
+                mmds["modulemd.%s.txt" % arch], strict=True
+            )
             variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
         except KeyError:
             # There is no modulemd for this arch. This could mean an arch was
@@ -662,11 +664,9 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                         # Not current tag, skip it
                         continue
                     for arch_modules in variant.arch_mmds.values():
-                        arch_mmd = arch_modules[nsvc]
-                        if arch_mmd:
-                            for rpm_nevra in arch_mmd.get_rpm_artifacts().get():
-                                nevra = parse_nvra(rpm_nevra)
-                                modular_packages.add((nevra["name"], nevra["arch"]))
+                        for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts():
+                            nevra = parse_nvra(rpm_nevra)
+                            modular_packages.add((nevra["name"], nevra["arch"]))
 
             pkgset.populate(
                 compose_tag,
diff --git a/pungi/util.py b/pungi/util.py
index d1a3ab6..c2c9a94 100644
--- a/pungi/util.py
+++ b/pungi/util.py
@@ -927,12 +927,38 @@ def parse_koji_event(event):
 
 
 def iter_module_defaults(path):
-    """Given a path to a directory with yaml files, yield each module default in there.
+    """Given a path to a directory with yaml files, yield each module default
+    in there as a pair (module_name, ModuleDefaults instance).
     """
+    # It is really tempting to merge all the module indexes into a single one
+    # and work with it. However that does not allow for detecting conflicting
+    # defaults. That should not happen in practice, but better safe than sorry.
+    # Once libmodulemd can report the error, this code can be simplifed by a
+    # lot. It's implemented in
+    # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
+    # and released in 2.6.0
     for file in glob.glob(os.path.join(path, "*.yaml")):
-        for mmddef in Modulemd.objects_from_file(file):
-            if isinstance(mmddef, Modulemd.Defaults):
-                yield mmddef
+        index = Modulemd.ModuleIndex()
+        index.update_from_file(file, strict=False)
+        for module_name in index.get_module_names():
+            yield module_name, index.get_module(module_name).get_defaults()
+
+
+def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None):
+    """Load module defaults into index.
+
+    If `modules_to_load` is passed in, it should be a set of module names. Only
+    defaults for these modules will be loaded.
+
+    If `mod_index` is passed in, it will be updated and returned. If it was
+    not, a new ModuleIndex will be created and returned
+    """
+    mod_index = mod_index or Modulemd.ModuleIndex()
+    for module_name, defaults in iter_module_defaults(defaults_dir):
+        if not modules_to_load or module_name in modules_to_load:
+            mod_index.add_defaults(defaults)
+
+    return mod_index
 
 
 def load_config(file_path, defaults={}):
