diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53289a9fa29..76c6e921035 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,7 +54,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git zstd sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo \ diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 35d624b3660..4d4df968ea7 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -34,7 +34,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq python3 zstd sudo systemctl stop dnsmasq sudo systemctl mask dnsmasq diff --git a/.github/workflows/setup-flatcar-sdk.sh b/.github/workflows/setup-flatcar-sdk.sh index 419b8017183..d362ba23594 100755 --- a/.github/workflows/setup-flatcar-sdk.sh +++ b/.github/workflows/setup-flatcar-sdk.sh @@ -9,7 +9,7 @@ fi sudo ln -sfn /bin/bash /bin/sh sudo apt-get update sudo apt-get install -y ca-certificates curl git gnupg lbzip2 lsb-release \ - qemu-user-static + qemu-user-static zstd sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg diff --git a/.github/workflows/update-sdk.yaml b/.github/workflows/update-sdk.yaml index 8c039463a6d..4d8e2b7bd9b 100644 --- a/.github/workflows/update-sdk.yaml +++ b/.github/workflows/update-sdk.yaml @@ -58,7 +58,7 @@ jobs: run: | sudo rm /bin/sh sudo ln -s /bin/bash /bin/sh - sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git jq openssh-client rsync + sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git jq openssh-client rsync zstd sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo \ diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 5773a4c1c6c..39fdb46939c 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -19,9 +19,9 @@ GC_BUCKET="flatcar-linux" DEFAULT_HTTP_IMAGE_URL_TEMPLATE="@PROTO@://${BUILDCACHE_SERVER}/images/@ARCH@/@VERNUM@" -if ! command -v pigz > /dev/null; then - # No PIGZ on Flatcar - PIGZ="docker run --rm -i ghcr.io/flatcar/pigz --fast" +if ! command -v zstd > /dev/null; then + # we require zstd and it is included by default on flatcar + echo >&2 "zstd could not be found. unpacking container image may fail." fi CI_GIT_AUTHOR="flatcar-ci" diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index e6bfa05edbc..3129b3ad4a3 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -7,7 +7,6 @@ # CI automation common functions. source ci-automation/ci-config.env -: ${PIGZ:=pigz} : ${docker:=docker} : ${TEST_WORK_DIR:='__TESTS__'} @@ -155,10 +154,10 @@ function docker_image_to_buildcache() { local version="$2" # strip potential container registry prefix - local tarball="$(basename "$image")-${version}.tar.gz" + local tarball="$(basename "$image")-${version}.tar.zst" local id_file="$(basename "$image")-${version}.id" - $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" + $docker save "${image}":"${version}" | zstd -T0 -o "${tarball}" # Cut the "sha256:" prefix that is present in Docker but not in Podman $docker image inspect "${image}":"${version}" | jq -r '.[].Id' | sed 's/^sha256://' > "${id_file}" create_digests "${SIGNER:-}" "${tarball}" "${id_file}" @@ -180,7 +179,8 @@ function docker_commit_to_buildcache() { function docker_image_from_buildcache() { local name="$1" local version="$2" - local tgz="${name}-${version}.tar.gz" + local compr="${3:-zst}" + local tgz="${name}-${version}.tar.${compr}" local id_file="${name}-${version}.id" local id_file_url="https://${BUILDCACHE_SERVER}/containers/${version}/${id_file}" local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}" @@ -214,7 +214,8 @@ function docker_image_from_buildcache() { --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ --remote-name "${url_release}" - cat "${tgz}" | $PIGZ -d -c | $docker load + # zstd can handle zlib as well :) + zstd -d -c ${tgz} | $docker load rm "${tgz}" } @@ -229,7 +230,8 @@ function docker_image_from_registry_or_buildcache() { fi echo "Falling back to tar ball download..." >&2 - docker_image_from_buildcache "${image}" "${version}" + docker_image_from_buildcache "${image}" "${version}" zst || \ + docker_image_from_buildcache "${image}" "${version}" gz } # --