From 3422e6d5c103ec951d3a3cdc3abca2c0fd226d86 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 9 Apr 2022 10:19:13 -0400 Subject: [PATCH] Insert generated image.json into the ostree commit [retry] This is part of https://github.com/coreos/fedora-coreos-tracker/issues/1151 Our generated disk images are largely just a "shell" around the egg of an ostree commit. There is almost nothing that lives in the disk image that isn't in the commit. (This is especially true now that a preparatory commit previous to this moved the *content* of our static `grub.cfg` into `image.json`) In the original coreos-assembler design I'd tried to cleanly separate builds of the ostree from disk image builds, but also support linking them together (with matching version numbers, etc.) The separate `image.yaml` was part of this. This...mostly worked. This change furthers that separation by having image builds input from *just the ostree commit*. Crucially we would no longer need the config git repository to perform an image build. And this in turn unlocks truly better separating ostree builds from disk image builds in the pipeline *and* supporting downstream tooling generating disk images from custom containers. One neat thing here is we will finally fix a longstanding issue where coreos-assembler fails when just the `image.yaml` changes: Closes: #972 --- src/cmd-buildextend-live | 22 ++++++++++------------ src/cmd-buildextend-metal | 5 +++++ src/cmd-generate-hashlist | 2 +- src/cmd-sign | 2 +- src/cmdlib.sh | 20 +++++++++++++++++--- src/cosalib/cmdlib.py | 13 ++++++++++++- src/cosalib/ova.py | 6 ++++-- src/cosalib/qemuvariants.py | 7 ++++++- 8 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 486fd26545..c057b9765d 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -18,7 +18,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds -from cosalib.cmdlib import run_verbose, sha256sum_file, generate_image_json +from cosalib.cmdlib import run_verbose, sha256sum_file from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob from cosalib.meta import GenericBuildMeta @@ -46,9 +46,6 @@ if not args.build: args.build = builds.get_latest() print(f"Targeting build: {args.build}") -image_json = generate_image_json('src/config/image.yaml') -squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression'] - srcdir_prefix = "src/config/live/" if not os.path.isdir(srcdir_prefix): @@ -58,6 +55,15 @@ workdir = os.path.abspath(os.getcwd()) builddir = builds.get_build_dir(args.build) buildmeta_path = os.path.join(builddir, 'meta.json') buildmeta = GenericBuildMeta(workdir=workdir, build=args.build) +repo = os.path.join(workdir, 'tmp/repo') + +# Grab the commit hash for this build +buildmeta_commit = buildmeta['ostree-commit'] + +import_ostree_commit(workdir, builddir, buildmeta) +with open(os.path.join(workdir, 'tmp/image.json')) as f: + image_json = json.load(f) +squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression'] base_name = buildmeta['name'] if base_name == "rhcos" and args.fast: @@ -71,12 +77,6 @@ if os.path.exists(build_semaphore): raise Exception( f"{build_semaphore} exists: another process is building live") - -# Grab the commit hash for this build -buildmeta_commit = buildmeta['ostree-commit'] - -repo = os.path.join(workdir, 'tmp/repo') - # Don't run if it's already been done, unless forced if 'live-iso' in buildmeta['images'] and not args.force: print(f"'live' has already been built for {args.build}. Skipping.") @@ -696,8 +696,6 @@ boot print(f"Updated: {buildmeta_path}") -import_ostree_commit(repo, builddir, buildmeta) - # lock and build with open(build_semaphore, 'w') as f: f.write(f"{time.time_ns()}") diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 87f6dac3cd..251b05ba2d 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -117,6 +117,11 @@ commit=$(meta_key ostree-commit) ostree_repo=${tmprepo} # Ensure that we have the cached unpacked commit import_ostree_commit_for_build "${build}" +# Note this overwrote the bits generated in prepare_build +# for image_json. In the future we expect to split prepare_build +# into prepare_ostree_build and prepare_diskimage_build; the +# latter path would only run this. +image_json=${workdir}/tmp/image.json image_format=raw if [[ $image_type == qemu ]]; then diff --git a/src/cmd-generate-hashlist b/src/cmd-generate-hashlist index 2b3a339f55..3164bf97d9 100755 --- a/src/cmd-generate-hashlist +++ b/src/cmd-generate-hashlist @@ -80,7 +80,7 @@ class HashListV1(dict): checkout = 'tmp/repo/tmp/keylime-checkout' import_ostree_commit( - 'tmp/repo', + os.getcwd(), self._metadata.build_dir, self._metadata, force=True) diff --git a/src/cmd-sign b/src/cmd-sign index 975f921db0..35f9b3bb78 100755 --- a/src/cmd-sign +++ b/src/cmd-sign @@ -134,7 +134,7 @@ def robosign_ostree(args, s3, build, gpgkey): # Ensure we have an unpacked repo with the ostree content if not os.path.exists('tmp/repo'): subprocess.check_call(['ostree', '--repo=tmp/repo', 'init', '--mode=archive']) - import_ostree_commit('tmp/repo', builddir, build, force=True) + import_ostree_commit(os.getcwd(), builddir, build, force=True) repo = OSTree.Repo.new(Gio.File.new_for_path('tmp/repo')) repo.open(None) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index b8f7631589..59c78ecbe1 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -398,6 +398,16 @@ EOF done fi + # Store the fully rendered disk image config (image.json) inside + # the ostree commit, so it can later be extracted by disk image + # builds. + local imagejsondir="${tmp_overridesdir}/imagejson" + export ostree_image_json="/usr/share/coreos-assembler/image.json" + mkdir -p "${imagejsondir}/usr/share/coreos-assembler/" + cp "${image_json}" "${imagejsondir}${ostree_image_json}" + commit_overlay cosa-image-json "${imagejsondir}" + layers="${layers} cosa-image-json" + local_overrides_lockfile="${tmp_overridesdir}/local-overrides.json" if [ -n "${with_cosa_overrides}" ] && [[ -n $(ls "${overridesdir}/rpm/"*.rpm 2> /dev/null) ]]; then (cd "${overridesdir}"/rpm && rm -rf .repodata && createrepo_c .) @@ -884,6 +894,7 @@ builds.bump_timestamp() print('Build ${buildid} was inserted ${arch:+for $arch}')") } +# Prepare the image.json as part of an ostree image build write_image_json() { local srcfile=$1; shift local outfile=$1; shift @@ -894,7 +905,9 @@ from cosalib import cmdlib cmdlib.write_image_json('${srcfile}', '${outfile}')") } -# Shell wrapper for the Python import_ostree_commit +# API to prepare image builds. +# Ensures that the tmp/repo ostree repo is initialized, +# and also writes tmp/image.json. import_ostree_commit_for_build() { local buildid=$1; shift (python3 -c " @@ -902,9 +915,10 @@ import sys sys.path.insert(0, '${DIR}') from cosalib import cmdlib from cosalib.builds import Builds -builds = Builds('${workdir:-$(pwd)}') +workdir = '${workdir:-$(pwd)}' +builds = Builds(workdir) builddir = builds.get_build_dir('${buildid}') buildmeta = builds.get_build_meta('${buildid}') -cmdlib.import_ostree_commit('${workdir:-$(pwd)}/tmp/repo', builddir, buildmeta) +cmdlib.import_ostree_commit(workdir, builddir, buildmeta) ") } diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 33e605d2cd..c51df75c27 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -233,6 +233,12 @@ def rm_allow_noent(path): pass +def extract_image_json(workdir, commit): + repo = os.path.join(workdir, 'tmp/repo') + with open(os.path.join(workdir, 'tmp/image.json'), 'w') as f: + subprocess.check_call(['ostree', f'--repo={repo}', 'cat', commit, '/usr/share/coreos-assembler/image.json'], stdout=f) + + # In coreos-assembler, we are strongly oriented towards the concept of a single # versioned "build" object that has artifacts. But rpm-ostree (among other things) # really natively wants to operate on unpacked ostree repositories. So, we maintain @@ -241,7 +247,8 @@ def rm_allow_noent(path): # a metal image, we may not have preserved that cache. # # Call this function to ensure that the ostree commit for a given build is in tmp/repo. -def import_ostree_commit(repo, buildpath, buildmeta, force=False): +def import_ostree_commit(workdir, buildpath, buildmeta, force=False): + repo = os.path.join(workdir, 'tmp/repo') commit = buildmeta['ostree-commit'] tarfile = os.path.join(buildpath, buildmeta['images']['ostree']['path']) # create repo in case e.g. tmp/ was cleared out; idempotent @@ -255,6 +262,7 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): stderr=subprocess.DEVNULL) == 0 and not os.path.isfile(commitpartial) and not force): + extract_image_json(workdir, commit) return print(f"Extracting {commit}") @@ -279,6 +287,9 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): 'ostree-unverified-image:oci-archive:' + tarfile]) subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']]) + # Also extract image.json since it's commonly needed by image builds + extract_image_json(workdir, commit) + def get_basearch(): try: diff --git a/src/cosalib/ova.py b/src/cosalib/ova.py index 6949534da2..b11a202db8 100644 --- a/src/cosalib/ova.py +++ b/src/cosalib/ova.py @@ -5,12 +5,13 @@ import logging as log import os.path import sys +import json cosa_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, f"{cosa_dir}/cosalib") sys.path.insert(0, cosa_dir) -from cosalib.cmdlib import generate_image_json, image_info +from cosalib.cmdlib import image_info from cosalib.qemuvariants import QemuVariantImage @@ -86,7 +87,8 @@ def generate_ovf_parameters(self, vmdk, cpu=2, memory=4096): Returns a dictionary with the parameters needed to create an OVF file based on the qemu, vmdk, image.yaml, and info from the build metadata """ - image_json = generate_image_json('src/config/image.yaml') + with open(os.path.join(self._workdir, 'tmp/image.json')) as f: + image_json = json.load(f) system_type = 'vmx-{}'.format(image_json['vmware-hw-version']) os_type = image_json['vmware-os-type'] diff --git a/src/cosalib/qemuvariants.py b/src/cosalib/qemuvariants.py index c683bf66c3..38f56517a9 100644 --- a/src/cosalib/qemuvariants.py +++ b/src/cosalib/qemuvariants.py @@ -20,7 +20,8 @@ get_basearch, image_info, run_verbose, - sha256sum_file + sha256sum_file, + import_ostree_commit ) # BASEARCH is the current machine architecture @@ -236,6 +237,10 @@ def mutate_image(self): :param callback: callback function for extra processing image :type callback: function """ + + # Disk image builds may require an unpacked ostree repo and tmp/image.json in general. + import_ostree_commit(self._workdir, self.build_dir, self.meta) + work_img = os.path.join(self._tmpdir, f"{self.image_name_base}.{self.image_format}") final_img = os.path.join(os.path.abspath(self.build_dir),