Insert generated image.json into the ostree commit [retry]

src/cmd-buildextend-live

@@ -18,7 +18,7 @@ import time
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from cosalib.builds import Builds
-from cosalib.cmdlib import run_verbose, sha256sum_file, generate_image_json
+from cosalib.cmdlib import run_verbose, sha256sum_file
 from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob
 from cosalib.meta import GenericBuildMeta
 
@@ -46,9 +46,6 @@ if not args.build:
     args.build = builds.get_latest()
 print(f"Targeting build: {args.build}")
 
-image_json = generate_image_json('src/config/image.yaml')
-squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression']
-
 srcdir_prefix = "src/config/live/"
 
 if not os.path.isdir(srcdir_prefix):
@@ -58,6 +55,15 @@ workdir = os.path.abspath(os.getcwd())
 builddir = builds.get_build_dir(args.build)
 buildmeta_path = os.path.join(builddir, 'meta.json')
 buildmeta = GenericBuildMeta(workdir=workdir, build=args.build)
+repo = os.path.join(workdir, 'tmp/repo')
+
+# Grab the commit hash for this build
+buildmeta_commit = buildmeta['ostree-commit']
+
+import_ostree_commit(workdir, builddir, buildmeta)
+with open(os.path.join(workdir, 'tmp/image.json')) as f:
+    image_json = json.load(f)
+squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression']
 
 base_name = buildmeta['name']
 if base_name == "rhcos" and args.fast:
@@ -71,12 +77,6 @@ if os.path.exists(build_semaphore):
     raise Exception(
         f"{build_semaphore} exists: another process is building live")
 
-
-# Grab the commit hash for this build
-buildmeta_commit = buildmeta['ostree-commit']
-
-repo = os.path.join(workdir, 'tmp/repo')
-
 # Don't run if it's already been done, unless forced
 if 'live-iso' in buildmeta['images'] and not args.force:
     print(f"'live' has already been built for {args.build}. Skipping.")
@@ -696,8 +696,6 @@ boot
     print(f"Updated: {buildmeta_path}")
 
 
-import_ostree_commit(repo, builddir, buildmeta)
-
 # lock and build
 with open(build_semaphore, 'w') as f:
     f.write(f"{time.time_ns()}")

src/cmd-buildextend-metal

@@ -117,6 +117,11 @@ commit=$(meta_key ostree-commit)
 ostree_repo=${tmprepo}
 # Ensure that we have the cached unpacked commit
 import_ostree_commit_for_build "${build}"
+# Note this overwrote the bits generated in prepare_build
+# for image_json.  In the future we expect to split prepare_build
+# into prepare_ostree_build and prepare_diskimage_build; the
+# latter path would only run this.
+image_json=${workdir}/tmp/image.json
 
 image_format=raw
 if [[ $image_type == qemu ]]; then

src/cmd-generate-hashlist

@@ -80,7 +80,7 @@ class HashListV1(dict):
         checkout = 'tmp/repo/tmp/keylime-checkout'
 
         import_ostree_commit(
-            'tmp/repo',
+            os.getcwd(),
             self._metadata.build_dir,
             self._metadata,
             force=True)

src/cmd-sign

@@ -134,7 +134,7 @@ def robosign_ostree(args, s3, build, gpgkey):
     # Ensure we have an unpacked repo with the ostree content
     if not os.path.exists('tmp/repo'):
         subprocess.check_call(['ostree', '--repo=tmp/repo', 'init', '--mode=archive'])
-    import_ostree_commit('tmp/repo', builddir, build, force=True)
+    import_ostree_commit(os.getcwd(), builddir, build, force=True)
     repo = OSTree.Repo.new(Gio.File.new_for_path('tmp/repo'))
     repo.open(None)
 

src/cmdlib.sh

@@ -387,6 +387,16 @@ EOF
         done
     fi
 
+    # Store the fully rendered disk image config (image.json) inside
+    # the ostree commit, so it can later be extracted by disk image
+    # builds.
+    local imagejsondir="${tmp_overridesdir}/imagejson"
+    export ostree_image_json="/usr/share/coreos-assembler/image.json"
+    mkdir -p "${imagejsondir}/usr/share/coreos-assembler/"
+    cp "${image_json}" "${imagejsondir}${ostree_image_json}"
+    commit_overlay cosa-image-json "${imagejsondir}"
+    layers="${layers} cosa-image-json"
+
     local_overrides_lockfile="${tmp_overridesdir}/local-overrides.json"
     if [ -n "${with_cosa_overrides}" ] && [[ -n $(ls "${overridesdir}/rpm/"*.rpm 2> /dev/null) ]]; then
         (cd "${overridesdir}"/rpm && rm -rf .repodata && createrepo_c .)
@@ -869,6 +879,7 @@ builds.bump_timestamp()
 print('Build ${buildid} was inserted ${arch:+for $arch}')")
 }
 
+# Prepare the image.json as part of an ostree image build
 write_image_json() {
     local srcfile=$1; shift
     local outfile=$1; shift
@@ -879,17 +890,20 @@ from cosalib import cmdlib
 cmdlib.write_image_json('${srcfile}', '${outfile}')")
 }
 
-# Shell wrapper for the Python import_ostree_commit
+# API to prepare image builds.
+# Ensures that the tmp/repo ostree repo is initialized,
+# and also writes tmp/image.json.
 import_ostree_commit_for_build() {
     local buildid=$1; shift
     (python3 -c "
 import sys
 sys.path.insert(0, '${DIR}')
 from cosalib import cmdlib
 from cosalib.builds import Builds
-builds = Builds('${workdir:-$(pwd)}')
+workdir = '${workdir:-$(pwd)}'
+builds = Builds(workdir)
 builddir = builds.get_build_dir('${buildid}')
 buildmeta = builds.get_build_meta('${buildid}')
-cmdlib.import_ostree_commit('${workdir:-$(pwd)}/tmp/repo', builddir, buildmeta)
+cmdlib.import_ostree_commit(workdir, builddir, buildmeta)
 ")
 }

src/cosalib/cmdlib.py

@@ -233,6 +233,12 @@ def rm_allow_noent(path):
         pass
 
 
+def extract_image_json(workdir, commit):
+    repo = os.path.join(workdir, 'tmp/repo')
+    with open(os.path.join(workdir, 'tmp/image.json'), 'w') as f:
+        subprocess.check_call(['ostree', f'--repo={repo}', 'cat', commit, '/usr/share/coreos-assembler/image.json'], stdout=f)
+
+
 # In coreos-assembler, we are strongly oriented towards the concept of a single
 # versioned "build" object that has artifacts.  But rpm-ostree (among other things)
 # really natively wants to operate on unpacked ostree repositories.  So, we maintain
@@ -241,7 +247,8 @@ def rm_allow_noent(path):
 # a metal image, we may not have preserved that cache.
 #
 # Call this function to ensure that the ostree commit for a given build is in tmp/repo.
-def import_ostree_commit(repo, buildpath, buildmeta, force=False):
+def import_ostree_commit(workdir, buildpath, buildmeta, force=False):
+    repo = os.path.join(workdir, 'tmp/repo')
     commit = buildmeta['ostree-commit']
     tarfile = os.path.join(buildpath, buildmeta['images']['ostree']['path'])
     # create repo in case e.g. tmp/ was cleared out; idempotent
@@ -255,6 +262,7 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False):
                         stderr=subprocess.DEVNULL) == 0
             and not os.path.isfile(commitpartial)
             and not force):
+        extract_image_json(workdir, commit)
         return
 
     print(f"Extracting {commit}")
@@ -279,6 +287,9 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False):
                                    'ostree-unverified-image:oci-archive:' + tarfile])
             subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']])
 
+    # Also extract image.json since it's commonly needed by image builds
+    extract_image_json(workdir, commit)
+
 
 def get_basearch():
     try:

src/cosalib/ova.py

@@ -5,12 +5,13 @@
 import logging as log
 import os.path
 import sys
+import json
 
 cosa_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, f"{cosa_dir}/cosalib")
 sys.path.insert(0, cosa_dir)
 
-from cosalib.cmdlib import generate_image_json, image_info
+from cosalib.cmdlib import image_info
 from cosalib.qemuvariants import QemuVariantImage
 
 
@@ -86,7 +87,8 @@ def generate_ovf_parameters(self, vmdk, cpu=2, memory=4096):
         Returns a dictionary with the parameters needed to create an OVF file
         based on the qemu, vmdk, image.yaml, and info from the build metadata
         """
-        image_json = generate_image_json('src/config/image.yaml')
+        with open(os.path.join(self._workdir, 'tmp/image.json')) as f:
+            image_json = json.load(f)
 
         system_type = 'vmx-{}'.format(image_json['vmware-hw-version'])
         os_type = image_json['vmware-os-type']

src/cosalib/qemuvariants.py

@@ -20,7 +20,8 @@
     get_basearch,
     image_info,
     run_verbose,
-    sha256sum_file
+    sha256sum_file,
+    import_ostree_commit
 )
 
 # BASEARCH is the current machine architecture
@@ -236,6 +237,10 @@ def mutate_image(self):
         :param callback: callback function for extra processing image
         :type callback: function
         """
+
+        # Disk image builds may require an unpacked ostree repo and tmp/image.json in general.
+        import_ostree_commit(self._workdir, self.build_dir, self.meta)
+
         work_img = os.path.join(self._tmpdir,
                                 f"{self.image_name_base}.{self.image_format}")
         final_img = os.path.join(os.path.abspath(self.build_dir),