diff --git a/pkg/distro/fedora/distro.go b/pkg/distro/fedora/distro.go index 0388603a8c..4a4d66ba4d 100644 --- a/pkg/distro/fedora/distro.go +++ b/pkg/distro/fedora/distro.go @@ -131,6 +131,20 @@ var ( exports: []string{"commit-archive"}, } + iotBootableContainer = imageType{ + name: "iot-bootable-container", + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + packageSets: map[string]packageSetFunc{ + osPkgsKey: bootableContainerPackageSet, + }, + rpmOstree: true, + image: bootableContainerImage, + buildPipelines: []string{"build"}, + payloadPipelines: []string{"os", "ostree-commit", "ostree-encapsulate"}, + exports: []string{"ostree-encapsulate"}, + } + iotOCIImgType = imageType{ name: "iot-container", nameAliases: []string{"fedora-iot-container"}, @@ -843,6 +857,7 @@ func newDistro(version int) distro.Distro { }, iotSimplifiedInstallerImgType, ) + aarch64.addImageTypes( &platform.Aarch64{ BasePlatform: platform.BasePlatform{ @@ -868,6 +883,63 @@ func newDistro(version int) distro.Distro { ) } + if !common.VersionLessThan(rd.Releasever(), "39") { + // bootc was introduced in F39 + x86_64.addImageTypes( + &platform.X86{ + BasePlatform: platform.BasePlatform{ + FirmwarePackages: []string{ + "biosdevname", + "iwlwifi-dvm-firmware", + "iwlwifi-mvm-firmware", + "microcode_ctl", + }, + }, + BIOS: true, + UEFIVendor: "fedora", + }, + iotBootableContainer, + ) + aarch64.addImageTypes( + &platform.Aarch64{ + BasePlatform: platform.BasePlatform{ + FirmwarePackages: []string{ + "arm-image-installer", + "bcm283x-firmware", + "brcmfmac-firmware", + "iwlwifi-mvm-firmware", + "realtek-firmware", + "uboot-images-armv8", + }, + }, + UEFIVendor: "fedora", + }, + iotBootableContainer, + ) + + ppc64le.addImageTypes( + &platform.PPC64LE{ + BIOS: true, + BasePlatform: platform.BasePlatform{ + ImageFormat: platform.FORMAT_QCOW2, + QCOW2Compat: "1.1", + }, + }, + iotBootableContainer, + ) + + s390x.addImageTypes( + &platform.S390X{ + Zipl: true, + BasePlatform: platform.BasePlatform{ + ImageFormat: platform.FORMAT_QCOW2, + QCOW2Compat: "1.1", + }, + }, + iotBootableContainer, + ) + } + ppc64le.addImageTypes( &platform.PPC64LE{ BIOS: true, diff --git a/pkg/distro/fedora/distro_test.go b/pkg/distro/fedora/distro_test.go index 8f23dff345..9928be96e2 100644 --- a/pkg/distro/fedora/distro_test.go +++ b/pkg/distro/fedora/distro_test.go @@ -215,6 +215,14 @@ func TestFilenameFromType(t *testing.T) { }, }, "39": { + { + name: "iot-bootable-container", + args: args{"iot-bootable-container"}, + want: wantResult{ + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + }, + }, { name: "iot-simplified-installer", args: args{"iot-simplified-installer"}, @@ -225,6 +233,14 @@ func TestFilenameFromType(t *testing.T) { }, }, "40": { + { + name: "iot-bootable-container", + args: args{"iot-bootable-container"}, + want: wantResult{ + filename: "iot-bootable-container.tar", + mimeType: "application/x-tar", + }, + }, { name: "iot-simplified-installer", args: args{"iot-simplified-installer"}, @@ -236,7 +252,7 @@ func TestFilenameFromType(t *testing.T) { }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { allTests := append(tests, verTypes[dist.distro.Releasever()]...) for _, tt := range allTests { t.Run(tt.name, func(t *testing.T) { @@ -292,7 +308,7 @@ func TestImageType_BuildPackages(t *testing.T) { "aarch64": aarch64BuildPackages, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { d := dist.distro for _, archLabel := range d.ListArches() { archStruct, err := d.GetArch(archLabel) @@ -344,8 +360,14 @@ func TestImageType_Name(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -365,14 +387,20 @@ func TestImageType_Name(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, mapping := range imgMap { arch, err := dist.distro.GetArch(mapping.arch) if assert.NoError(t, err) { @@ -534,8 +562,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -557,8 +591,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { }, verTypes: map[string][]string{ "38": {"iot-simplified-installer"}, - "39": {"iot-simplified-installer"}, - "40": {"iot-simplified-installer"}, + "39": { + "iot-bootable-container", + "iot-simplified-installer", + }, + "40": { + "iot-bootable-container", + "iot-simplified-installer", + }, }, }, { @@ -567,6 +607,14 @@ func TestArchitecture_ListImageTypes(t *testing.T) { "container", "qcow2", }, + verTypes: map[string][]string{ + "39": { + "iot-bootable-container", + }, + "40": { + "iot-bootable-container", + }, + }, }, { arch: "s390x", @@ -574,11 +622,19 @@ func TestArchitecture_ListImageTypes(t *testing.T) { "container", "qcow2", }, + verTypes: map[string][]string{ + "39": { + "iot-bootable-container", + }, + "40": { + "iot-bootable-container", + }, + }, }, } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, mapping := range imgMap { arch, err := dist.distro.GetArch(mapping.arch) require.NoError(t, err) @@ -624,7 +680,7 @@ func TestFedora37_GetArch(t *testing.T) { } for _, dist := range fedoraFamilyDistros { - t.Run(dist.name, func(t *testing.T) { + t.Run(dist.distro.Name(), func(t *testing.T) { for _, a := range arches { actualArch, err := dist.distro.GetArch(a.name) if a.errorExpected { diff --git a/pkg/distro/fedora/images.go b/pkg/distro/fedora/images.go index 05e7c3a102..b1f29c6338 100644 --- a/pkg/distro/fedora/images.go +++ b/pkg/distro/fedora/images.go @@ -423,6 +423,32 @@ func iotCommitImage(workload workload.Workload, return img, nil } +func bootableContainerImage(workload workload.Workload, + t *imageType, + bp *blueprint.Blueprint, + options distro.ImageOptions, + packageSets map[string]rpmmd.PackageSet, + containers []container.SourceSpec, + rng *rand.Rand) (image.ImageKind, error) { + + parentCommit, commitRef := makeOSTreeParentCommit(options.OSTree, t.OSTreeRef()) + img := image.NewOSTreeArchive(commitRef) + + d := t.arch.distro + + img.Platform = t.platform + img.OSCustomizations = osCustomizations(t, packageSets[osPkgsKey], containers, bp.Customizations) + img.Environment = t.environment + img.Workload = workload + img.OSTreeParent = parentCommit + img.OSVersion = d.osVersion + img.Filename = t.Filename() + img.InstallWeakDeps = false + img.BootContainer = true + + return img, nil +} + func iotContainerImage(workload workload.Workload, t *imageType, bp *blueprint.Blueprint, diff --git a/pkg/distro/fedora/package_sets.go b/pkg/distro/fedora/package_sets.go index f440e12b1e..30ebe4c9cf 100644 --- a/pkg/distro/fedora/package_sets.go +++ b/pkg/distro/fedora/package_sets.go @@ -188,6 +188,104 @@ func iotCommitPackageSet(t *imageType) rpmmd.PackageSet { return ps } +func bootableContainerPackageSet(t *imageType) rpmmd.PackageSet { + // Replicating package selection from centos-bootc: + // https://github.com/CentOS/centos-bootc/ + ps := rpmmd.PackageSet{ + Include: []string{ + "acl", + "attr", // used by admins interactively + "bootc", + "bootupd", + "chrony", // NTP support + "container-selinux", + "container-selinux", + "crun", + "cryptsetup", + "dnf", + "e2fsprogs", + "fwupd", // if you're using linux-firmware, you probably also want fwupd + "iproute", "iproute-tc", // route manipulation and QoS + "iptables", "nftables", // firewall manipulation + "iptables-services", // additional firewall support + "kbd", // i18n + "keyutils", // Manipulating the kernel keyring; used by bootc + "libsss_sudo", // allow communication between sudo and SSSD for caching sudo rules by SSSD + "linux-firmware", // linux-firmware now a recommends so let's explicitly include it + "logrotate", // There are things that write outside of the journal still (such as the classic wtmp, etc.). auditd also writes outside the journal but it has its own log rotation. Anything package layered will also tend to expect files dropped in /etc/logrotate.d to work. Really, this is a legacy thing, but if we don't have it then people's disks will slowly fill up with logs. + "lsof", + "lvm2", // Storage configuration/management + "nano", // default editor + "ncurses", // provides terminal tools like clear, reset, tput, and tset + "NetworkManager-cloud-setup", // support for cloud quirks and dynamic config in real rootfs: https://github.com/coreos/fedora-coreos-tracker/issues/320 + "NetworkManager", "hostname", // standard tools for configuring network/hostname + "NetworkManager-team", "teamd", // teaming https://github.com/coreos/fedora-coreos-config/pull/289 and http://bugzilla.redhat.com/1758162 + "NetworkManager-tui", // interactive Networking configuration during coreos-install + "nfs-utils-coreos", "iptables-nft", // minimal NFS client + "nss-altfiles", + "openssh-clients", + "openssh-server", + "openssl", + "ostree", + "passwd", "shadow-utils", // User configuration + "podman", + "rpm-ostree", + "selinux-policy-targeted", + "sg3_utils", + "skopeo", + "socat", "net-tools", "bind-utils", // interactive network tools for admins + "sssd-client", "sssd-ad", "sssd-ipa", "sssd-krb5", "sssd-ldap", // SSSD backends + "stalld", // Boost starving threads https://github.com/coreos/fedora-coreos-tracker/issues/753 + "subscription-manager", // To ensure we can enable client certs to access RHEL content + "sudo", + "systemd", + "systemd-resolved", // resolved was broken out to its own package in rawhide/f35 + "tpm2-tools", // needed for tpm2 bound luks + "WALinuxAgent-udev", // udev rules for Azure (rhbz#1748432) + "xfsprogs", + "zram-generator", // zram-generator (but not zram-generator-defaults) for F33 change + }, + Exclude: []string{ + "cowsay", // just in case + "grubby", + "initscripts", // make sure initscripts doesn't get pulled back in https://github.com/coreos/fedora-coreos-tracker/issues/220#issuecomment-611566254 + "NetworkManager-initscripts-ifcfg-rh", // do not use legacy ifcfg config format in NetworkManager See https://github.com/coreos/fedora-coreos-config/pull/1991 + "nodejs", + "perl", + "perl-interpreter", + "plymouth", // for (datacenter/cloud oriented) servers, we want to see the details by default. https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/HSMISZ3ETWQ4ETVLWZQJ55ARZT27AAV3/ + "systemd-networkd", // we use NetworkManager + }, + } + + switch t.Arch().Name() { + case arch.ARCH_AARCH64.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + "ostree-grub2", + }, + }) + case arch.ARCH_PPC64LE.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + "librtas", + "powerpc-utils-core", + "ppc64-diag-rtas", + }, + }) + case arch.ARCH_X86_64.String(): + ps.Append(rpmmd.PackageSet{ + Include: []string{ + "irqbalance", + }, + }) + } + + return ps +} + // INSTALLER PACKAGE SET func installerPackageSet(t *imageType) rpmmd.PackageSet { diff --git a/test/config-map.json b/test/config-map.json index 1dc0bbad87..eaca1d725e 100644 --- a/test/config-map.json +++ b/test/config-map.json @@ -66,7 +66,7 @@ "edge-vsphere" ] }, - "./configs/edge-ostree-pull-fips.json": { + "./configs/edge-ostree-pull-fips.json": { "distros": [ "rhel-93", "rhel-94" @@ -91,7 +91,6 @@ "edge-ami" ] }, - "./configs/embed-containers-2.json": { "image-types": [ "edge-container" @@ -117,6 +116,7 @@ "gce", "gce-rhui", "image-installer", + "iot-bootable-container", "iot-container", "live-installer", "minimal-raw", diff --git a/test/scripts/boot-image b/test/scripts/boot-image index 5368bfe287..ebaca52f7c 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -61,25 +61,88 @@ def ensure_uncompressed(filepath): yield filepath -def boot_ami(distro, arch, image_type, image_path): +def cmd_boot_aws(arch, image_name, privkey, pubkey, image_path): aws_config = get_aws_config() + cmd = ["go", "run", "./cmd/boot-aws", "run", + "--access-key-id", aws_config["key_id"], + "--secret-access-key", aws_config["secret_key"], + "--region", aws_config["region"], + "--bucket", aws_config["bucket"], + "--arch", arch, + "--ami-name", image_name, + "--s3-key", f"images/boot/{image_name}", + "--username", "osbuild", + "--ssh-privkey", privkey, + "--ssh-pubkey", pubkey, + image_path, "test/scripts/base-host-check.sh"] + testlib.runcmd_nc(cmd) + + +def boot_ami(distro, arch, image_type, image_path): with ensure_uncompressed(image_path) as raw_image_path: with create_ssh_key() as (privkey, pubkey): image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) - cmd = ["go", "run", "./cmd/boot-aws", "run", - "--access-key-id", aws_config["key_id"], - "--secret-access-key", aws_config["secret_key"], - "--region", aws_config["region"], - "--bucket", aws_config["bucket"], - "--arch", arch, - "--ami-name", image_name, - "--s3-key", f"images/boot/{image_name}", - "--username", "osbuild", - "--ssh-privkey", privkey, - "--ssh-pubkey", pubkey, - raw_image_path, "test/scripts/base-host-check.sh"] + cmd_boot_aws(arch, image_name, privkey, pubkey, raw_image_path) + + +def boot_container(distro, arch, image_type, image_path, manifest_id): + """ + Use bootc-image-builder to build an AMI and boot it. + """ + # push container to registry so we can build it with BIB + # remove when BIB can pull from containers-storage: https://github.com/osbuild/bootc-image-builder/pull/120 + container_name = f"iot-bootable-container:{distro}-{arch}-{manifest_id}" + cmd = ["./tools/ci/push-container.sh", image_path, container_name] + testlib.runcmd_nc(cmd) + container_ref = f"{testlib.REGISTRY}/{container_name}" + + with TemporaryDirectory() as tmpdir: + with create_ssh_key() as (privkey_file, pubkey_file): + with open(pubkey_file, encoding="utf-8") as pubkey_fp: + pubkey = pubkey_fp.read() + + # write a config to create a user + config_file = os.path.join(tmpdir, "config.json") + with open(config_file, "w", encoding="utf-8") as cfg_fp: + config = { + "blueprint": { + "customizations": { + "user": [ + { + "name": "osbuild", + "key": pubkey, + "groups": [ + "wheel" + ] + } + ] + } + } + } + json.dump(config, cfg_fp) + + # build an AMI + cmd = ["sudo", "podman", "run", + "--rm", "-it", + "--privileged", + "--pull=newer", + "--security-opt", "label=type:unconfined_t", + "-v", f"{tmpdir}:/output", + "-v", f"{config_file}:/config.json", + "quay.io/centos-bootc/bootc-image-builder:latest", + "--type=ami", + "--config=/config.json", + container_ref] testlib.runcmd_nc(cmd) + # boot it + image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) + + # Build artifacts are owned by root. Make them world accessible. + testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", tmpdir]) + raw_image_path = f"{tmpdir}/image/disk.raw" + cmd_boot_aws(arch, image_name, privkey_file, pubkey_file, raw_image_path) + def find_image_file(build_path: str) -> str: """ @@ -124,6 +187,12 @@ def main(): match image_type: case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami": boot_ami(distro, arch, image_type, image_path) + case "iot-bootable-container": + info_file_path = os.path.join(search_path, "info.json") + with open(info_file_path, encoding="utf-8") as info_fp: + build_info = json.load(info_fp) + manifest_id = build_info["manifest-checksum"] + boot_container(distro, arch, image_type, image_path, manifest_id) case _: # skip print(f"{image_type} boot tests are not supported yet") diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 8cd959aa33..d625347391 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -30,6 +30,7 @@ "ec2-ha", "ec2-sap", "edge-ami", + "iot-bootable-container", ]