From 54273d3630bf3e99e54698e3a21915eb81f4e336 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Jan 2024 18:04:42 +0100 Subject: [PATCH 1/7] distro/fedora: add bootable container package set The package set mostly replicates the package selection from centos-bootc [1]. Comments are also retained from the original source files to assist in future decisions for modifications. The two differences from the centos-bootc package set are: - Kernel not specified: it is selected automatically along with user package selection. - Most of the packages specified in the user-experience [2] file aren't added. We want to keep the image definition minimal and add or iterate on the package set going forward as required. [1] https://github.com/CentOS/centos-bootc/tree/4d5e1d86fbe1028f47e08d5533b4825e3b42dc68 [2] https://github.com/CentOS/centos-bootc/blob/4d5e1d86fbe1028f47e08d5533b4825e3b42dc68/tier-1/user-experience.yaml --- pkg/distro/fedora/package_sets.go | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/pkg/distro/fedora/package_sets.go b/pkg/distro/fedora/package_sets.go index f440e12b1e..30ebe4c9cf 100644 --- a/pkg/distro/fedora/package_sets.go +++ b/pkg/distro/fedora/package_sets.go @@ -188,6 +188,104 @@ func iotCommitPackageSet(t *imageType) rpmmd.PackageSet { return ps } +func bootableContainerPackageSet(t *imageType) rpmmd.PackageSet { + // Replicating package selection from centos-bootc: + // https://github.com/CentOS/centos-bootc/ + ps := rpmmd.PackageSet{ + Include: []string{ + "acl", + "attr", // used by admins interactively + "bootc", + "bootupd", + "chrony", // NTP support + "container-selinux", + "container-selinux", + "crun", + "cryptsetup", + "dnf", + "e2fsprogs", + "fwupd", // if you're using linux-firmware, you probably also want fwupd + "iproute", "iproute-tc", // route manipulation and QoS + "iptables", "nftables", // firewall manipulation + "iptables-services", // additional firewall support + "kbd", // i18n + "keyutils", // Manipulating the kernel keyring; used by bootc + "libsss_sudo", // allow communication between sudo and SSSD for caching sudo rules by SSSD + "linux-firmware", // linux-firmware now a recommends so let's explicitly include it + "logrotate", // There are things that write outside of the journal still (such as the classic wtmp, etc.). auditd also writes outside the journal but it has its own log rotation. Anything package layered will also tend to expect files dropped in /etc/logrotate.d to work. Really, this is a legacy thing, but if we don't have it then people's disks will slowly fill up with logs. + "lsof", + "lvm2", // Storage configuration/management + "nano", // default editor + "ncurses", // provides terminal tools like clear, reset, tput, and tset + "NetworkManager-cloud-setup", // support for cloud quirks and dynamic config in real rootfs: https://github.com/coreos/fedora-coreos-tracker/issues/320 + "NetworkManager", "hostname", // standard tools for configuring network/hostname + "NetworkManager-team", "teamd", // teaming https://github.com/coreos/fedora-coreos-config/pull/289 and http://bugzilla.redhat.com/1758162 + "NetworkManager-tui", // interactive Networking configuration during coreos-install + "nfs-utils-coreos", "iptables-nft", // minimal NFS client + "nss-altfiles", + "openssh-clients", + "openssh-server", + "openssl", + "ostree", + "passwd", "shadow-utils", // User configuration + "podman", + "rpm-ostree", + "selinux-policy-targeted", + "sg3_utils", + "skopeo", + "socat", "net-tools", "bind-utils", // interactive network tools for admins + "sssd-client", "sssd-ad", "sssd-ipa", "sssd-krb5", "sssd-ldap", // SSSD backends + "stalld", // Boost starving threads https://github.com/coreos/fedora-coreos-tracker/issues/753 + "subscription-manager", // To ensure we can enable client certs to access RHEL content + "sudo", + "systemd", + "systemd-resolved", // resolved was broken out to its own package in rawhide/f35 + "tpm2-tools", // needed for tpm2 bound luks + "WALinuxAgent-udev", // udev rules for Azure (rhbz#1748432) + "xfsprogs", + "zram-generator", // zram-generator (but not zram-generator-defaults) for F33 change + }, + Exclude: []string{ + "cowsay", // just in case + "grubby", + "initscripts", // make sure initscripts doesn't get pulled back in https://github.com/coreos/fedora-coreos-tracker/issues/220#issuecomment-611566254 + "NetworkManager-initscripts-ifcfg-rh", // do not use legacy ifcfg config format in NetworkManager See https://github.com/coreos/fedora-coreos-config/pull/1991 + "nodejs", + "perl", + "perl-interpreter", + "plymouth", // for (datacenter/cloud oriented) servers, we want to see the details by default. https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/HSMISZ3ETWQ4ETVLWZQJ55ARZT27AAV3/ + "systemd-networkd", // we use NetworkManager + }, + } + + switch t.Arch().Name() { + case arch.ARCH_AARCH64.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + "ostree-grub2", + }, + }) + case arch.ARCH_PPC64LE.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + "librtas", + "powerpc-utils-core", + "ppc64-diag-rtas", + }, + }) + case arch.ARCH_X86_64.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + }, + }) + } + + return ps +} + // INSTALLER PACKAGE SET func installerPackageSet(t *imageType) rpmmd.PackageSet { From 33238281ff7a901120162eac084d182a10a8bd2b Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 16 Nov 2023 14:35:01 +0100 Subject: [PATCH 2/7] distro/fedora: new image type: iot-bootable-container Add new image type and its image function called "iot-bootable-container". The image function is very similar to the iot commit but enables the BootContainer flag so it exports a bootable base container instead of a tarball of the ostree repo. The iot-bootable-container image type is supported on F39+ only. --- pkg/distro/fedora/distro.go | 72 +++++++++++++++++++++++++++++++++++++ pkg/distro/fedora/images.go | 26 ++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/pkg/distro/fedora/distro.go b/pkg/distro/fedora/distro.go index 0388603a8c..4a4d66ba4d 100644 --- a/pkg/distro/fedora/distro.go +++ b/pkg/distro/fedora/distro.go @@ -131,6 +131,20 @@ var ( exports: []string{"commit-archive"}, } + iotBootableContainer = imageType{ + name: "iot-bootable-container", + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + packageSets: map[string]packageSetFunc{ + osPkgsKey: bootableContainerPackageSet, + }, + rpmOstree: true, + image: bootableContainerImage, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "ostree-commit", "ostree-encapsulate"}, + exports: []string{"ostree-encapsulate"}, + } + iotOCIImgType = imageType{ name: "iot-container", nameAliases: []string{"fedora-iot-container"}, @@ -843,6 +857,7 @@ func newDistro(version int) distro.Distro { }, iotSimplifiedInstallerImgType, ) + aarch64.addImageTypes( &platform.Aarch64{ BasePlatform: platform.BasePlatform{ @@ -868,6 +883,63 @@ func newDistro(version int) distro.Distro { ) } + if !common.VersionLessThan(rd.Releasever(), "39") { + // bootc was introduced in F39 + x86_64.addImageTypes( + &platform.X86{ + BasePlatform: platform.BasePlatform{ + FirmwarePackages: []string{ + "biosdevname", + "iwlwifi-dvm-firmware", + "iwlwifi-mvm-firmware", + "microcode_ctl", + }, + }, + BIOS: true, + UEFIVendor: "fedora", + }, + iotBootableContainer, + ) + aarch64.addImageTypes( + &platform.Aarch64{ + BasePlatform: platform.BasePlatform{ + FirmwarePackages: []string{ + "arm-image-installer", + "bcm283x-firmware", + "brcmfmac-firmware", + "iwlwifi-mvm-firmware", + "realtek-firmware", + "uboot-images-armv8", + }, + }, + UEFIVendor: "fedora", + }, + iotBootableContainer, + ) + + ppc64le.addImageTypes( + &platform.PPC64LE{ + BIOS: true, + BasePlatform: platform.BasePlatform{ + ImageFormat: platform.FORMAT_QCOW2, + QCOW2Compat: "1.1", + }, + }, + iotBootableContainer, + ) + + s390x.addImageTypes( + &platform.S390X{ + Zipl: true, + BasePlatform: platform.BasePlatform{ + ImageFormat: platform.FORMAT_QCOW2, + QCOW2Compat: "1.1", + }, + }, + iotBootableContainer, + ) + } + ppc64le.addImageTypes( &platform.PPC64LE{ BIOS: true, diff --git a/pkg/distro/fedora/images.go b/pkg/distro/fedora/images.go index 05e7c3a102..b1f29c6338 100644 --- a/pkg/distro/fedora/images.go +++ b/pkg/distro/fedora/images.go @@ -423,6 +423,32 @@ func iotCommitImage(workload workload.Workload, return img, nil } +func bootableContainerImage(workload workload.Workload, + t *imageType, + bp *blueprint.Blueprint, + options distro.ImageOptions, + packageSets map[string]rpmmd.PackageSet, + containers []container.SourceSpec, + rng *rand.Rand) (image.ImageKind, error) { + + parentCommit, commitRef := makeOSTreeParentCommit(options.OSTree, t.OSTreeRef()) + img := image.NewOSTreeArchive(commitRef) + + d := t.arch.distro + + img.Platform = t.platform + img.OSCustomizations = osCustomizations(t, packageSets[osPkgsKey], containers, bp.Customizations) + img.Environment = t.environment + img.Workload = workload + img.OSTreeParent = parentCommit + img.OSVersion = d.osVersion + img.Filename = t.Filename() + img.InstallWeakDeps = false + img.BootContainer = true + + return img, nil +} + func iotContainerImage(workload workload.Workload, t *imageType, bp *blueprint.Blueprint, From fa2c83e8e67d4a256b9f8c0fffc0f967d7a66d63 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Sat, 13 Jan 2024 17:47:43 +0100 Subject: [PATCH 3/7] fedora: update image type lists in tests Add iot-bootable-container to all the test lists for Fedora 39 and 40. Also use the full distro name in each call to t.Run() instead of just the name "fedora" to make each iteration easier to distinguish. --- pkg/distro/fedora/distro_test.go | 82 +++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/pkg/distro/fedora/distro_test.go b/pkg/distro/fedora/distro_test.go index 8f23dff345..9928be96e2 100644 --- a/pkg/distro/fedora/distro_test.go +++ b/pkg/distro/fedora/distro_test.go @@ -215,6 +215,14 @@ func TestFilenameFromType(t *testing.T) { }, }, "39": { + { + name: "iot-bootable-container", + args: args{"iot-bootable-container"}, + want: wantResult{ + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + }, + }, { name: "iot-simplified-installer", args: args{"iot-simplified-installer"}, @@ -225,6 +233,14 @@ func TestFilenameFromType(t *testing.T) { }, }, "40": { + { + name: "iot-bootable-container", + args: args{"iot-bootable-container"}, + want: wantResult{ + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + }, + }, { name: "iot-simplified-installer", args: args{"iot-simplified-installer"}, @@ -236,7 +252,7 @@ func TestFilenameFromType(t *testing.T) { }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { allTests := append(tests, verTypes[dist.distro.Releasever()]...) for _, tt := range allTests { t.Run(tt.name, func(t *testing.T) { @@ -292,7 +308,7 @@ func TestImageType_BuildPackages(t *testing.T) { "aarch64": aarch64BuildPackages, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { d := dist.distro for _, archLabel := range d.ListArches() { archStruct, err := d.GetArch(archLabel) @@ -344,8 +360,14 @@ func TestImageType_Name(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -365,14 +387,20 @@ func TestImageType_Name(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, mapping := range imgMap { arch, err := dist.distro.GetArch(mapping.arch) if assert.NoError(t, err) { @@ -534,8 +562,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -557,8 +591,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -567,6 +607,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { "container", "qcow2", }, + verTypes: map[string][]string{ + "39": { + "iot-bootable-container", + }, + "40": { + "iot-bootable-container", + }, + }, }, { arch: "s390x", @@ -574,11 +622,19 @@ func TestArchitecture_ListImageTypes(t *testing.T) { "container", "qcow2", }, + verTypes: map[string][]string{ + "39": { + "iot-bootable-container", + }, + "40": { + "iot-bootable-container", + }, + }, }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, mapping := range imgMap { arch, err := dist.distro.GetArch(mapping.arch) require.NoError(t, err) @@ -624,7 +680,7 @@ func TestFedora37_GetArch(t *testing.T) { } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, a := range arches { actualArch, err := dist.distro.GetArch(a.name) if a.errorExpected { From 76636780c392fb2e13284783e400b35322aeff4b Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 12 Jan 2024 15:49:32 +0100 Subject: [PATCH 4/7] test: add new image type to config map --- test/config-map.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/config-map.json b/test/config-map.json index 1dc0bbad87..eaca1d725e 100644 --- a/test/config-map.json +++ b/test/config-map.json @@ -66,7 +66,7 @@ "edge-vsphere" ] }, - "./configs/edge-ostree-pull-fips.json": { + "./configs/edge-ostree-pull-fips.json": { "distros": [ "rhel-93", "rhel-94" @@ -91,7 +91,6 @@ "edge-ami" ] }, - "./configs/embed-containers-2.json": { "image-types": [ "edge-container" @@ -117,6 +116,7 @@ "gce", "gce-rhui", "image-installer", + "iot-bootable-container", "iot-container", "live-installer", "minimal-raw", From 6a327b345ae261a3018daa9db8d6d37f001f994e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 15 Jan 2024 22:55:56 +0100 Subject: [PATCH 5/7] test: add AMI build and boot test for iot-bootable-container To boot test a iot-bootable-container, build an AMI using bootc-image-builder then use our boot-aws command to upload and boot the image in AWS. Since we can't build containers from the local registry yet, we push it to the gitlab registry and pull it back down. --- test/scripts/boot-image | 92 +++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 13 deletions(-) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index 5368bfe287..27b79d47d1 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -61,25 +61,85 @@ def ensure_uncompressed(filepath): yield filepath -def boot_ami(distro, arch, image_type, image_path): +def cmd_boot_aws(arch, image_name, privkey, pubkey, image_path): aws_config = get_aws_config() + cmd = ["go", "run", "./cmd/boot-aws", "run", + "--access-key-id", aws_config["key_id"], + "--secret-access-key", aws_config["secret_key"], + "--region", aws_config["region"], + "--bucket", aws_config["bucket"], + "--arch", arch, + "--ami-name", image_name, + "--s3-key", f"images/boot/{image_name}", + "--username", "osbuild", + "--ssh-privkey", privkey, + "--ssh-pubkey", pubkey, + image_path, "test/scripts/base-host-check.sh"] + testlib.runcmd_nc(cmd) + + +def boot_ami(distro, arch, image_type, image_path): with ensure_uncompressed(image_path) as raw_image_path: with create_ssh_key() as (privkey, pubkey): image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) - cmd = ["go", "run", "./cmd/boot-aws", "run", - "--access-key-id", aws_config["key_id"], - "--secret-access-key", aws_config["secret_key"], - "--region", aws_config["region"], - "--bucket", aws_config["bucket"], - "--arch", arch, - "--ami-name", image_name, - "--s3-key", f"images/boot/{image_name}", - "--username", "osbuild", - "--ssh-privkey", privkey, - "--ssh-pubkey", pubkey, - raw_image_path, "test/scripts/base-host-check.sh"] + cmd_boot_aws(arch, image_name, privkey, pubkey, raw_image_path) + + +def boot_container(distro, arch, image_type, image_path, manifest_id): + """ + Use bootc-image-builder to build an AMI and boot it. + """ + # push container to registry so we can build it with BIB + # remove when BIB can pull from containers-storage: https://github.com/osbuild/bootc-image-builder/pull/120 + container_name = f"iot-bootable-container:{distro}-{arch}-{manifest_id}" + cmd = ["./tools/ci/push-container.sh", image_path, container_name] + testlib.runcmd_nc(cmd) + container_ref = f"{testlib.REGISTRY}/{container_name}" + + with TemporaryDirectory() as tmpdir: + with create_ssh_key() as (privkey_file, pubkey_file): + with open(pubkey_file, encoding="utf-8") as pubkey_fp: + pubkey = pubkey_fp.read() + + # write a config to create a user + config_file = os.path.join(tmpdir, "config.json") + with open(config_file, "w", encoding="utf-8") as cfg_fp: + config = { + "blueprint": { + "customizations": { + "user": [ + { + "name": "osbuild", + "key": pubkey, + "groups": [ + "wheel" + ] + } + ] + } + } + } + json.dump(config, cfg_fp) + + # build an AMI + cmd = ["sudo", "podman", "run", + "--rm", "-it", + "--privileged", + "--pull=newer", + "--security-opt", "label=type:unconfined_t", + "-v", f"{tmpdir}:/output", + "-v", f"{config_file}:/config.json", + "quay.io/centos-bootc/bootc-image-builder:latest", + "--type=ami", + "--config=/config.json", + container_ref] testlib.runcmd_nc(cmd) + # boot it + image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) + raw_image_path = f"{tmpdir}/image/disk.raw" + cmd_boot_aws(arch, image_name, privkey_file, pubkey_file, raw_image_path) + def find_image_file(build_path: str) -> str: """ @@ -124,6 +184,12 @@ def main(): match image_type: case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami": boot_ami(distro, arch, image_type, image_path) + case "iot-bootable-container": + info_file_path = os.path.join(search_path, "info.json") + with open(info_file_path, encoding="utf-8") as info_fp: + build_info = json.load(info_fp) + manifest_id = build_info["manifest-checksum"] + boot_container(distro, arch, image_type, image_path, manifest_id) case _: # skip print(f"{image_type} boot tests are not supported yet") From e08a2af8678e14907075586f953aef05f6fc9b8e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 15 Jan 2024 22:58:51 +0100 Subject: [PATCH 6/7] test: add iot-bootable-container to CAN_BOOT_TEST list --- test/scripts/imgtestlib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 8cd959aa33..d625347391 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -30,6 +30,7 @@ "ec2-ha", "ec2-sap", "edge-ami", + "iot-bootable-container", ] From c37674b3c3ae9b5fcb2474aa229f222d42be03ca Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 17 Jan 2024 17:24:22 +0100 Subject: [PATCH 7/7] test: chmod a+rwX the bootc-image-builder artifact Make the build directory and all its contents world rwXable. This isn't needed for the test itself, the artifact is already readable, but the original permissions make the cleanup of the testing environment fail. --- test/scripts/boot-image | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index 27b79d47d1..ebaca52f7c 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -137,6 +137,9 @@ def boot_container(distro, arch, image_type, image_path, manifest_id): # boot it image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) + + # Build artifacts are owned by root. Make them world accessible. + testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", tmpdir]) raw_image_path = f"{tmpdir}/image/disk.raw" cmd_boot_aws(arch, image_name, privkey_file, pubkey_file, raw_image_path)