From 61e3cb0ef14aaae497b34ec73d5b5d5aba07a2d7 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Jul 15 2019 13:27:05 +0000 Subject: Port to libmodulemd v2 Fixes: https://pagure.io/pungi/issue/1225 JIRA: COMPOSE-3662 Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/__init__.py b/pungi/__init__.py index 12ada28..22aaf88 100644 --- a/pungi/__init__.py +++ b/pungi/__init__.py @@ -6,9 +6,9 @@ import re try: import gi - gi.require_version('Modulemd', '1.0') # noqa + gi.require_version('Modulemd', '2.0') # noqa from gi.repository import Modulemd -except: +except (ImportError, ValueError): Modulemd = None diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 6489eab..197d7e8 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -36,7 +36,7 @@ from .base import PhaseBase from ..util import ( find_old_compose, get_arch_variant_data, - iter_module_defaults, + collect_module_defaults, temp_dir, ) from pungi import Modulemd @@ -196,22 +196,21 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) # call modifyrepo to inject modulemd if needed if pkg_type == "rpm" and arch in variant.arch_mmds and Modulemd is not None: - modules = [] + mod_index = Modulemd.ModuleIndex() metadata = [] for module_id, mmd in variant.arch_mmds[arch].items(): if modules_metadata: - module_rpms = mmd.peek_rpm_artifacts().dup() + module_rpms = mmd.get_rpm_artifacts() metadata.append((module_id, module_rpms)) - modules.append(mmd) + mod_index.add_module_stream(mmd) - module_names = set([x.get_name() for x in modules]) - for mmddef in iter_module_defaults(compose.paths.work.module_defaults_dir()): - if mmddef.peek_module_name() in module_names: - modules.append(mmddef) + module_names = set(mod_index.get_module_names()) + defaults_dir = compose.paths.work.module_defaults_dir() + collect_module_defaults(defaults_dir, module_names, mod_index) log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant) - add_modular_metadata(repo, repo_dir, modules, log_file) + add_modular_metadata(repo, repo_dir, mod_index, log_file) for module_id, module_rpms in metadata: modulemd_path = os.path.join( @@ -230,11 +229,12 @@ def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None) compose.log_info("[DONE ] %s" % msg) -def add_modular_metadata(repo, repo_path, mmd, log_file): +def add_modular_metadata(repo, repo_path, mod_index, log_file): """Add modular metadata into a repository.""" with temp_dir() as tmp_dir: modules_path = os.path.join(tmp_dir, "modules.yaml") - Modulemd.dump(mmd, modules_path) + with open(modules_path, "w") as f: + f.write(mod_index.dump_to_string()) cmd = repo.get_modifyrepo_cmd( os.path.join(repo_path, "repodata"), diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 67c8ad0..d7438a3 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -29,10 +29,10 @@ import pungi.wrappers.kojiwrapper from pungi import Modulemd from pungi.compose import get_ordered_variant_uids -from pungi.arch import get_compatible_arches, split_name_arch, tree_arch_to_yum_arch +from pungi.arch import get_compatible_arches, split_name_arch from pungi.phases.base import PhaseBase from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data, - makedirs, iter_module_defaults) + makedirs, collect_module_defaults) from pungi.phases.createrepo import add_modular_metadata @@ -377,23 +377,18 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map): # Add modular metadata into the repo if variant.arch_mmds: - mmds = [] + mod_index = Modulemd.ModuleIndex() for mmd in variant.arch_mmds[arch].values(): - # Set the arch field, but no other changes are needed. - repo_mmd = mmd.copy() - repo_mmd.set_arch(tree_arch_to_yum_arch(arch)) - mmds.append(repo_mmd) + mod_index.add_module_stream(mmd) - module_names = set([x.get_name() for x in mmds]) + module_names = set(mod_index.get_module_names()) defaults_dir = compose.paths.work.module_defaults_dir() - for mmddef in iter_module_defaults(defaults_dir): - if mmddef.peek_module_name() in module_names: - mmds.append(mmddef) + collect_module_defaults(defaults_dir, module_names, mod_index) log_file = compose.paths.log.log_file( arch, "lookaside_repo_modules_%s" % (variant.uid) ) - add_modular_metadata(cr, repo, mmds, log_file) + add_modular_metadata(cr, repo, mod_index, log_file) compose.log_info('[DONE ] %s', msg) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index af33296..8f69964 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -142,7 +142,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): def prepare_modular_packages(self): for var in self.compose.all_variants.values(): for mmd in var.arch_mmds.get(self.arch, {}).values(): - self.modular_packages.update(mmd.get_rpm_artifacts().dup()) + self.modular_packages.update(mmd.get_rpm_artifacts()) def prepare_langpacks(self, arch, variant): if not self.compose.has_comps: @@ -223,7 +223,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): modules = [] for mmd in variant.arch_mmds.get(arch, {}).values(): - modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream())) + modules.append("%s:%s" % (mmd.get_module_name(), mmd.get_stream_name())) input_packages = [] for pkg_name, pkg_arch in packages: @@ -399,10 +399,10 @@ def get_platform(compose, variant, arch): for var in compose.all_variants.values(): for mmd in var.arch_mmds.get(arch, {}).values(): - for dep in mmd.peek_dependencies(): - streams = dep.peek_requires().get("platform") + for dep in mmd.get_dependencies(): + streams = dep.get_runtime_streams("platform") if streams: - platforms.update(streams.dup()) + platforms.update(streams) if len(platforms) > 1: raise RuntimeError("There are conflicting requests for platform.") diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py index 2dc818e..8e5de32 100644 --- a/pungi/phases/gather/sources/source_module.py +++ b/pungi/phases/gather/sources/source_module.py @@ -38,7 +38,7 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True) - for nsvc, mmd in variant.arch_mmds[arch].items(): + for nsvc, module_stream in variant.arch_mmds[arch].items(): available_rpms = sum( ( variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, []) @@ -46,7 +46,7 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): ), [], ) - to_include = set(mmd.get_rpm_artifacts().get()) + to_include = set(module_stream.get_rpm_artifacts()) for rpm_obj in available_rpms: if rpm_obj.nevra in to_include: packages.add((rpm_obj, None)) diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 7ba51a2..d06a927 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -229,8 +229,8 @@ def validate_module_defaults(path): """ seen_defaults = collections.defaultdict(set) - for mmddef in iter_module_defaults(path): - seen_defaults[mmddef.peek_module_name()].add(mmddef.peek_default_stream()) + for module_name, defaults in iter_module_defaults(path): + seen_defaults[module_name].add(defaults.get_default_stream()) errors = [] for module_name, defaults in seen_defaults.items(): diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py index 60e7a76..553b605 100644 --- a/pungi/phases/pkgset/common.py +++ b/pungi/phases/pkgset/common.py @@ -20,9 +20,10 @@ from kobo.shortcuts import run, relative_path from kobo.threads import run_in_threads import pungi.phases.pkgset.pkgsets +from pungi import Modulemd from pungi.arch import get_valid_arches from pungi.wrappers.createrepo import CreaterepoWrapper -from pungi.util import is_arch_multilib, find_old_compose, iter_module_defaults +from pungi.util import is_arch_multilib, find_old_compose, collect_module_defaults from pungi.phases.createrepo import add_modular_metadata @@ -118,12 +119,14 @@ def _create_arch_repo(worker_thread, args, task_num): update_md_path=repo_dir_global, checksum=createrepo_checksum) run(cmd, logfile=compose.paths.log.log_file(arch, "arch_repo"), show_cmd=True) # Add modulemd to the repo for all modules in all variants on this architecture. - mmds = list(iter_module_defaults(compose.paths.work.module_defaults_dir())) - for variant in compose.get_variants(arch=arch): - mmds.extend(variant.arch_mmds.get(arch, {}).values()) - if mmds: + if Modulemd: + mod_index = collect_module_defaults(compose.paths.work.module_defaults_dir()) + + for variant in compose.get_variants(arch=arch): + for module_stream in variant.arch_mmds.get(arch, {}).values(): + mod_index.add_module_stream(module_stream) add_modular_metadata( - repo, repo_dir, mmds, compose.paths.log.log_file(arch, "arch_repo_modulemd") + repo, repo_dir, mod_index, compose.paths.log.log_file(arch, "arch_repo_modulemd") ) compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index f3e7db8..1a7bf3a 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -247,7 +247,9 @@ def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules= for arch in variant.arches: try: - mmd = Modulemd.Module.new_from_file(mmds["modulemd.%s.txt" % arch]) + mmd = Modulemd.ModuleStream.read_file( + mmds["modulemd.%s.txt" % arch], strict=True + ) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: # There is no modulemd for this arch. This could mean an arch was @@ -662,11 +664,9 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): # Not current tag, skip it continue for arch_modules in variant.arch_mmds.values(): - arch_mmd = arch_modules[nsvc] - if arch_mmd: - for rpm_nevra in arch_mmd.get_rpm_artifacts().get(): - nevra = parse_nvra(rpm_nevra) - modular_packages.add((nevra["name"], nevra["arch"])) + for rpm_nevra in arch_modules[nsvc].get_rpm_artifacts(): + nevra = parse_nvra(rpm_nevra) + modular_packages.add((nevra["name"], nevra["arch"])) pkgset.populate( compose_tag, diff --git a/pungi/util.py b/pungi/util.py index d1a3ab6..c2c9a94 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -927,12 +927,38 @@ def parse_koji_event(event): def iter_module_defaults(path): - """Given a path to a directory with yaml files, yield each module default in there. + """Given a path to a directory with yaml files, yield each module default + in there as a pair (module_name, ModuleDefaults instance). """ + # It is really tempting to merge all the module indexes into a single one + # and work with it. However that does not allow for detecting conflicting + # defaults. That should not happen in practice, but better safe than sorry. + # Once libmodulemd can report the error, this code can be simplifed by a + # lot. It's implemented in + # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d + # and released in 2.6.0 for file in glob.glob(os.path.join(path, "*.yaml")): - for mmddef in Modulemd.objects_from_file(file): - if isinstance(mmddef, Modulemd.Defaults): - yield mmddef + index = Modulemd.ModuleIndex() + index.update_from_file(file, strict=False) + for module_name in index.get_module_names(): + yield module_name, index.get_module(module_name).get_defaults() + + +def collect_module_defaults(defaults_dir, modules_to_load=None, mod_index=None): + """Load module defaults into index. + + If `modules_to_load` is passed in, it should be a set of module names. Only + defaults for these modules will be loaded. + + If `mod_index` is passed in, it will be updated and returned. If it was + not, a new ModuleIndex will be created and returned + """ + mod_index = mod_index or Modulemd.ModuleIndex() + for module_name, defaults in iter_module_defaults(defaults_dir): + if not modules_to_load or module_name in modules_to_load: + mod_index.add_defaults(defaults) + + return mod_index def load_config(file_path, defaults={}):