Don't pull-local over 9p during rpm-ostree compose

Instead of having rpm-ostree effectively doing a `pull-local` under the
hood over 9p, change things so we compose in a local repo, export the
commit to an archive and only copy *that* over 9p.

This should greatly help with pipelines hitting ENOMEM due to
transferring many small files over 9p:

openshift/os#594

An initial approach exported to OCI archive instead, but encapsulating
and unencapsulating are more expensive operations. Unfortunately, we
still need to unpack into `tmp/repo` given that many follow-up commands
expect the commit to be available in `tmp/repo`. If we can drop that
assumption (and get rid of `tmp/repo` entirely), we could refactor
things further so that `compose.sh` creates the final chunked OCI
artifact upfront.

In local testing, this adds about 30s to `cosa build`. We still compress
just once, and we get hardlinks pulling out of the tarball.

src/cmd-build

@@ -175,6 +175,9 @@ if [ -n "${previous_commit}" ]; then
         ostree refs --repo="${tmprepo}" --create "${ref}" "${previous_commit}"
     fi
 
+    # also make sure the previous build ref exists
+    ostree refs --repo="${tmprepo}" --create "${previous_build}" "${previous_commit}" --force
+
     # Corner-case here: if the previous build was for a different ref, then we
     # want to make sure rpm-ostree doesn't select the same version. Do this by
     # pretending the ref is currently pointing at the last commit on the

src/cmdlib.sh

@@ -493,21 +493,38 @@ runcompose_tree() {
 
     rm -f "${changed_stamp}"
     # shellcheck disable=SC2086
-    set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${repo}" \
-            --write-composejson-to "${composejson}" \
+    set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree \
             --touch-if-changed "${changed_stamp}" --cachedir="${workdir}"/cache \
             ${COSA_RPMOSTREE_ARGS:-} --unified-core "${manifest}" "$@"
 
     echo "Running: $*"
 
     # this is the heart of the privs vs no privs dual path
     if has_privileges; then
+        set - "$@" --repo "${repo}" --write-composejson-to "${composejson}"
         # we hardcode a umask of 0022 here to make sure that composes are run
         # with a consistent value, regardless of the environment
         (umask 0022 && sudo -E "$@")
         sudo chown -R -h "${USER}":"${USER}" "${tmprepo}"
     else
-        runvm_with_cache "$@"
+        local tarball="${workdir}/tmp/repo/commit.tar"
+        rm -f "${tarball}"
+        runvm_with_cache /usr/lib/coreos-assembler/compose.sh \
+            "${tarball}" "${composejson}" "$@"
+        if [ ! -f "${tarball}" ]; then
+            return
+        fi
+        local commit
+        commit=$(jq -r '.["ostree-commit"]' < "${composejson}")
+        local import_repo="${workdir}/tmp/repo-import"
+        rm -rf "${import_repo}" && mkdir "${import_repo}"
+        tar -C "${import_repo}" -xf "${tarball}" && rm -f "${tarball}"
+        # this is archive to archive so will hardlink
+        ostree pull-local --repo "${repo}" "${import_repo}" "${commit}"
+        if [ -n "${ref}" ]; then
+            ostree refs --repo "${repo}" "${commit}" --create "${ref}" --force
+        fi
+        rm -rf "${import_repo}"
     fi
 }
 

src/compose.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -euo pipefail
+
+output_tarball=$1; shift
+output_composejson=$1; shift
+
+tarball=cache/output.tar
+composejson=cache/compose.json
+
+repo=cache/repo
+rm -rf "${repo}" "${composejson}"
+ostree init --repo="${repo}" --mode=archive-z2
+
+# we do need to pull at least the overlay bits over 9p, but it shouldn't be that
+# many files
+ostree refs --repo tmp/repo overlay --list | \
+    xargs -r ostree pull-local --repo "${repo}" tmp/repo
+# And import commit metadata for all refs; this will bring in the previous
+# build if there is one. Ideally, we'd import the SELinux policy too to take
+# advantage of https://github.com/coreos/rpm-ostree/pull/1704 but that's yet
+# more files over 9p and our pipelines don't have cache anyway (and devs likely
+# use the privileged path).
+ostree refs --repo tmp/repo | \
+    xargs -r ostree pull-local --commit-metadata-only --repo "${repo}" tmp/repo
+
+# run rpm-ostree
+"$@" --repo "${repo}" --write-composejson-to "${composejson}"
+
+if [ ! -f "${composejson}" ]; then
+    # no commit was produced; we're done
+    exit 0
+fi
+
+tar -f "${tarball}" -C "${repo}" -c .
+
+# this is key bit where we move the OSTree content over 9p
+mv "${tarball}" "${output_tarball}"
+mv "${composejson}" "${output_composejson}"

src/vmdeps.txt

@@ -27,3 +27,5 @@ gdisk xfsprogs e2fsprogs dosfstools btrfs-progs
 
 # needed for basic CA support
 ca-certificates
+
+tar