diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 486fd26545..c057b9765d 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -18,7 +18,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds -from cosalib.cmdlib import run_verbose, sha256sum_file, generate_image_json +from cosalib.cmdlib import run_verbose, sha256sum_file from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob from cosalib.meta import GenericBuildMeta @@ -46,9 +46,6 @@ if not args.build: args.build = builds.get_latest() print(f"Targeting build: {args.build}") -image_json = generate_image_json('src/config/image.yaml') -squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression'] - srcdir_prefix = "src/config/live/" if not os.path.isdir(srcdir_prefix): @@ -58,6 +55,15 @@ workdir = os.path.abspath(os.getcwd()) builddir = builds.get_build_dir(args.build) buildmeta_path = os.path.join(builddir, 'meta.json') buildmeta = GenericBuildMeta(workdir=workdir, build=args.build) +repo = os.path.join(workdir, 'tmp/repo') + +# Grab the commit hash for this build +buildmeta_commit = buildmeta['ostree-commit'] + +import_ostree_commit(workdir, builddir, buildmeta) +with open(os.path.join(workdir, 'tmp/image.json')) as f: + image_json = json.load(f) +squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression'] base_name = buildmeta['name'] if base_name == "rhcos" and args.fast: @@ -71,12 +77,6 @@ if os.path.exists(build_semaphore): raise Exception( f"{build_semaphore} exists: another process is building live") - -# Grab the commit hash for this build -buildmeta_commit = buildmeta['ostree-commit'] - -repo = os.path.join(workdir, 'tmp/repo') - # Don't run if it's already been done, unless forced if 'live-iso' in buildmeta['images'] and not args.force: print(f"'live' has already been built for {args.build}. Skipping.") @@ -696,8 +696,6 @@ boot print(f"Updated: {buildmeta_path}") -import_ostree_commit(repo, builddir, buildmeta) - # lock and build with open(build_semaphore, 'w') as f: f.write(f"{time.time_ns()}") diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 87f6dac3cd..251b05ba2d 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -117,6 +117,11 @@ commit=$(meta_key ostree-commit) ostree_repo=${tmprepo} # Ensure that we have the cached unpacked commit import_ostree_commit_for_build "${build}" +# Note this overwrote the bits generated in prepare_build +# for image_json. In the future we expect to split prepare_build +# into prepare_ostree_build and prepare_diskimage_build; the +# latter path would only run this. +image_json=${workdir}/tmp/image.json image_format=raw if [[ $image_type == qemu ]]; then diff --git a/src/cmd-generate-hashlist b/src/cmd-generate-hashlist index 2b3a339f55..3164bf97d9 100755 --- a/src/cmd-generate-hashlist +++ b/src/cmd-generate-hashlist @@ -80,7 +80,7 @@ class HashListV1(dict): checkout = 'tmp/repo/tmp/keylime-checkout' import_ostree_commit( - 'tmp/repo', + os.getcwd(), self._metadata.build_dir, self._metadata, force=True) diff --git a/src/cmd-sign b/src/cmd-sign index 975f921db0..35f9b3bb78 100755 --- a/src/cmd-sign +++ b/src/cmd-sign @@ -134,7 +134,7 @@ def robosign_ostree(args, s3, build, gpgkey): # Ensure we have an unpacked repo with the ostree content if not os.path.exists('tmp/repo'): subprocess.check_call(['ostree', '--repo=tmp/repo', 'init', '--mode=archive']) - import_ostree_commit('tmp/repo', builddir, build, force=True) + import_ostree_commit(os.getcwd(), builddir, build, force=True) repo = OSTree.Repo.new(Gio.File.new_for_path('tmp/repo')) repo.open(None) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index b8f7631589..59c78ecbe1 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -398,6 +398,16 @@ EOF done fi + # Store the fully rendered disk image config (image.json) inside + # the ostree commit, so it can later be extracted by disk image + # builds. + local imagejsondir="${tmp_overridesdir}/imagejson" + export ostree_image_json="/usr/share/coreos-assembler/image.json" + mkdir -p "${imagejsondir}/usr/share/coreos-assembler/" + cp "${image_json}" "${imagejsondir}${ostree_image_json}" + commit_overlay cosa-image-json "${imagejsondir}" + layers="${layers} cosa-image-json" + local_overrides_lockfile="${tmp_overridesdir}/local-overrides.json" if [ -n "${with_cosa_overrides}" ] && [[ -n $(ls "${overridesdir}/rpm/"*.rpm 2> /dev/null) ]]; then (cd "${overridesdir}"/rpm && rm -rf .repodata && createrepo_c .) @@ -884,6 +894,7 @@ builds.bump_timestamp() print('Build ${buildid} was inserted ${arch:+for $arch}')") } +# Prepare the image.json as part of an ostree image build write_image_json() { local srcfile=$1; shift local outfile=$1; shift @@ -894,7 +905,9 @@ from cosalib import cmdlib cmdlib.write_image_json('${srcfile}', '${outfile}')") } -# Shell wrapper for the Python import_ostree_commit +# API to prepare image builds. +# Ensures that the tmp/repo ostree repo is initialized, +# and also writes tmp/image.json. import_ostree_commit_for_build() { local buildid=$1; shift (python3 -c " @@ -902,9 +915,10 @@ import sys sys.path.insert(0, '${DIR}') from cosalib import cmdlib from cosalib.builds import Builds -builds = Builds('${workdir:-$(pwd)}') +workdir = '${workdir:-$(pwd)}' +builds = Builds(workdir) builddir = builds.get_build_dir('${buildid}') buildmeta = builds.get_build_meta('${buildid}') -cmdlib.import_ostree_commit('${workdir:-$(pwd)}/tmp/repo', builddir, buildmeta) +cmdlib.import_ostree_commit(workdir, builddir, buildmeta) ") } diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 33e605d2cd..c51df75c27 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -233,6 +233,12 @@ def rm_allow_noent(path): pass +def extract_image_json(workdir, commit): + repo = os.path.join(workdir, 'tmp/repo') + with open(os.path.join(workdir, 'tmp/image.json'), 'w') as f: + subprocess.check_call(['ostree', f'--repo={repo}', 'cat', commit, '/usr/share/coreos-assembler/image.json'], stdout=f) + + # In coreos-assembler, we are strongly oriented towards the concept of a single # versioned "build" object that has artifacts. But rpm-ostree (among other things) # really natively wants to operate on unpacked ostree repositories. So, we maintain @@ -241,7 +247,8 @@ def rm_allow_noent(path): # a metal image, we may not have preserved that cache. # # Call this function to ensure that the ostree commit for a given build is in tmp/repo. -def import_ostree_commit(repo, buildpath, buildmeta, force=False): +def import_ostree_commit(workdir, buildpath, buildmeta, force=False): + repo = os.path.join(workdir, 'tmp/repo') commit = buildmeta['ostree-commit'] tarfile = os.path.join(buildpath, buildmeta['images']['ostree']['path']) # create repo in case e.g. tmp/ was cleared out; idempotent @@ -255,6 +262,7 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): stderr=subprocess.DEVNULL) == 0 and not os.path.isfile(commitpartial) and not force): + extract_image_json(workdir, commit) return print(f"Extracting {commit}") @@ -279,6 +287,9 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): 'ostree-unverified-image:oci-archive:' + tarfile]) subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']]) + # Also extract image.json since it's commonly needed by image builds + extract_image_json(workdir, commit) + def get_basearch(): try: diff --git a/src/cosalib/ova.py b/src/cosalib/ova.py index 6949534da2..b11a202db8 100644 --- a/src/cosalib/ova.py +++ b/src/cosalib/ova.py @@ -5,12 +5,13 @@ import logging as log import os.path import sys +import json cosa_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, f"{cosa_dir}/cosalib") sys.path.insert(0, cosa_dir) -from cosalib.cmdlib import generate_image_json, image_info +from cosalib.cmdlib import image_info from cosalib.qemuvariants import QemuVariantImage @@ -86,7 +87,8 @@ def generate_ovf_parameters(self, vmdk, cpu=2, memory=4096): Returns a dictionary with the parameters needed to create an OVF file based on the qemu, vmdk, image.yaml, and info from the build metadata """ - image_json = generate_image_json('src/config/image.yaml') + with open(os.path.join(self._workdir, 'tmp/image.json')) as f: + image_json = json.load(f) system_type = 'vmx-{}'.format(image_json['vmware-hw-version']) os_type = image_json['vmware-os-type'] diff --git a/src/cosalib/qemuvariants.py b/src/cosalib/qemuvariants.py index c683bf66c3..38f56517a9 100644 --- a/src/cosalib/qemuvariants.py +++ b/src/cosalib/qemuvariants.py @@ -20,7 +20,8 @@ get_basearch, image_info, run_verbose, - sha256sum_file + sha256sum_file, + import_ostree_commit ) # BASEARCH is the current machine architecture @@ -236,6 +237,10 @@ def mutate_image(self): :param callback: callback function for extra processing image :type callback: function """ + + # Disk image builds may require an unpacked ostree repo and tmp/image.json in general. + import_ostree_commit(self._workdir, self.build_dir, self.meta) + work_img = os.path.join(self._tmpdir, f"{self.image_name_base}.{self.image_format}") final_img = os.path.join(os.path.abspath(self.build_dir),