From b4d9c99d69eeb786c8c1359ce78b799d72dcf858 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 1 Feb 2022 20:29:26 -0500 Subject: [PATCH 1/3] mantle/platform: rename IgnitionNetworkKargs to AppendFirstbootKernelArgs We want to expose this to external tests and AppendFirstbootKernelArgs is a bit more explicit about what it's doing, which is applying kernel arguments that will apply to only the first boot of the machine. --- mantle/cmd/kola/qemuexec.go | 2 +- mantle/platform/qemu.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index e5d0e5573d..04545c4739 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -265,7 +265,7 @@ func runQemuExec(cmd *cobra.Command, args []string) error { } builder.ForceConfigInjection = forceConfigInjection if len(knetargs) > 0 { - builder.IgnitionNetworkKargs = knetargs + builder.AppendFirstbootKernelArgs = knetargs } builder.AppendKernelArgs = strings.Join(kargs, " ") builder.Firmware = kola.QEMUOptions.Firmware diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 1b5a360d07..2f9af59abc 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -406,8 +406,8 @@ type QemuBuilder struct { // AppendKernelArgs are appended to the bootloader config AppendKernelArgs string - // IgnitionNetworkKargs are written to /boot/ignition - IgnitionNetworkKargs string + // AppendFirstbootKernelArgs are written to /boot/ignition + AppendFirstbootKernelArgs string Hostname string @@ -874,8 +874,8 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { return errors.Wrapf(err, "rendering ignition") } requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection - if requiresInjection || builder.IgnitionNetworkKargs != "" || builder.AppendKernelArgs != "" { - if err := setupPreboot(builder.ConfigFile, builder.IgnitionNetworkKargs, builder.AppendKernelArgs, + if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" { + if err := setupPreboot(builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, disk.dstFileName, disk.SectorSize); err != nil { return errors.Wrapf(err, "ignition injection with guestfs failed") } From 87eda63f8cee17626d1a1a048e8ea073cf7f2ea6 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 1 Feb 2022 20:34:32 -0500 Subject: [PATCH 2/3] mantle/platform: qemuexec: rename --knetargs to --firstbootkargs This is a little more inline with what's being done, which is to apply the given arguments to the first boot of the machine only. --- mantle/cmd/kola/qemuexec.go | 16 ++++++++-------- mantle/platform/qemu.go | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 04545c4739..0586d4bcae 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -47,11 +47,11 @@ var ( usernet bool cpuCountHost bool - hostname string - ignition string - butane string - kargs []string - knetargs string + hostname string + ignition string + butane string + kargs []string + firstbootkargs string ignitionFragments []string bindro []string @@ -75,7 +75,7 @@ const maxAdditionalNics = 16 func init() { root.AddCommand(cmdQemuExec) - cmdQemuExec.Flags().StringVarP(&knetargs, "knetargs", "", "", "Arguments for Ignition networking on kernel commandline") + cmdQemuExec.Flags().StringVarP(&firstbootkargs, "firstbootkargs", "", "", "Additional first boot kernel arguments") cmdQemuExec.Flags().StringArrayVar(&kargs, "kargs", nil, "Additional kernel arguments applied") cmdQemuExec.Flags().BoolVarP(&usernet, "usernet", "U", false, "Enable usermode networking") cmdQemuExec.Flags().StringSliceVar(&ignitionFragments, "add-ignition", nil, "Append well-known Ignition fragment: [\"autologin\", \"autoresize\"]") @@ -264,8 +264,8 @@ func runQemuExec(cmd *cobra.Command, args []string) error { config.Mount9p(dest, false) } builder.ForceConfigInjection = forceConfigInjection - if len(knetargs) > 0 { - builder.AppendFirstbootKernelArgs = knetargs + if len(firstbootkargs) > 0 { + builder.AppendFirstbootKernelArgs = firstbootkargs } builder.AppendKernelArgs = strings.Join(kargs, " ") builder.Firmware = kola.QEMUOptions.Firmware diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 2f9af59abc..ea9bec074c 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -713,7 +713,7 @@ func (gf *coreosGuestfish) destroy() { } // setupPreboot performs changes necessary before the disk is booted -func setupPreboot(confPath, knetargs, kargs string, diskImagePath string, diskSectorSize int) error { +func setupPreboot(confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { gf, err := newGuestfish(diskImagePath, diskSectorSize) if err != nil { return err @@ -731,8 +731,8 @@ func setupPreboot(confPath, knetargs, kargs string, diskImagePath string, diskSe } // See /boot/grub2/grub.cfg - if knetargs != "" { - grubStr := fmt.Sprintf("set ignition_network_kcmdline='%s'\n", knetargs) + if firstbootkargs != "" { + grubStr := fmt.Sprintf("set ignition_network_kcmdline='%s'\n", firstbootkargs) if err := exec.Command("guestfish", gf.remote, "write", "/ignition.firstboot", grubStr).Run(); err != nil { return errors.Wrapf(err, "guestfish write") } From 97249401af27d1596a3189e6a1dee03d1369a32f Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 1 Feb 2022 20:54:33 -0500 Subject: [PATCH 3/3] kola/external-tests: add appendFirstbootKernelArgs support in kola.json Add the ability to specify firstboot kernel arguments to transiently append to the default kernel arguments for the first boot of the machine. This will be useful when testing workflows that use kernel arguments for network configuration. For now, this only supports the qemu platform. --- docs/kola/external-tests.md | 6 +- mantle/kola/harness.go | 57 ++++++++++--------- mantle/kola/register/register.go | 3 + mantle/platform/machine/aws/cluster.go | 4 ++ mantle/platform/machine/azure/cluster.go | 3 + mantle/platform/machine/do/cluster.go | 3 + mantle/platform/machine/esx/cluster.go | 3 + mantle/platform/machine/gcloud/cluster.go | 3 + mantle/platform/machine/openstack/cluster.go | 3 + mantle/platform/machine/packet/cluster.go | 3 + mantle/platform/machine/qemuiso/cluster.go | 3 + mantle/platform/machine/unprivqemu/cluster.go | 3 + mantle/platform/platform.go | 15 ++--- 13 files changed, 74 insertions(+), 35 deletions(-) diff --git a/docs/kola/external-tests.md b/docs/kola/external-tests.md index f37ac80f5b..f4d1701a9e 100644 --- a/docs/kola/external-tests.md +++ b/docs/kola/external-tests.md @@ -195,7 +195,8 @@ Here's an example `kola.json`: "minMemory": 4096, "minDisk": 15, "additionalNics": 2, - "appendKernelArgs": "ip=bond0:dhcp bond=bond0:ens5,ens6:mode=active-backup,miimon=100" + "appendKernelArgs": "enforcing=0" + "appendFirstbootKernelArgs": "ip=bond0:dhcp bond=bond0:ens5,ens6:mode=active-backup,miimon=100" "timeoutMin": 8, "exclusive": true, "conflicts": ["ext.config.some-test", "podman.some-other-test"] @@ -244,6 +245,9 @@ to `qemuexec`. It is currently only supported on `qemu-unpriv`. The `appendKernelArgs` key has the same semantics at the `--kargs` argument to `qemuexec`. It is currently only supported on `qemu-unpriv`. +The `appendFirstbootKernelArgs` key has the same semantics at the `--firstbootkargs` +argument to `qemuexec`. It is currently only supported on `qemu-unpriv`. + The `timeoutMin` key takes a positive integer and specifies a timeout for the test in minutes. After the specified amount of time, the test will be interrupted. diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index ad7bf2bd0b..0d5d83f000 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -747,19 +747,20 @@ func RunUpgradeTests(patterns []string, rerun bool, pltfrm, outputDir string, pr // externalTestMeta is parsed from kola.json in external tests type externalTestMeta struct { - Architectures string `json:"architectures,omitempty"` - Platforms string `json:"platforms,omitempty"` - Distros string `json:"distros,omitempty"` - Tags string `json:"tags,omitempty"` - RequiredTag string `json:"requiredTag,omitempty"` - AdditionalDisks []string `json:"additionalDisks,omitempty"` - MinMemory int `json:"minMemory,omitempty"` - MinDiskSize int `json:"minDisk,omitempty"` - AdditionalNics int `json:"additionalNics,omitempty"` - AppendKernelArgs string `json:"appendKernelArgs,omitempty"` - Exclusive bool `json:"exclusive"` - TimeoutMin int `json:"timeoutMin"` - Conflicts []string `json:"conflicts"` + Architectures string `json:"architectures,omitempty"` + Platforms string `json:"platforms,omitempty"` + Distros string `json:"distros,omitempty"` + Tags string `json:"tags,omitempty"` + RequiredTag string `json:"requiredTag,omitempty"` + AdditionalDisks []string `json:"additionalDisks,omitempty"` + MinMemory int `json:"minMemory,omitempty"` + MinDiskSize int `json:"minDisk,omitempty"` + AdditionalNics int `json:"additionalNics,omitempty"` + AppendKernelArgs string `json:"appendKernelArgs,omitempty"` + AppendFirstbootKernelArgs string `json:"appendFirstbootKernelArgs,omitempty"` + Exclusive bool `json:"exclusive"` + TimeoutMin int `json:"timeoutMin"` + Conflicts []string `json:"conflicts"` } // metadataFromTestBinary extracts JSON-in-comment like: @@ -923,13 +924,14 @@ ExecStart=%s DependencyDir: destDirs, Tags: []string{"external"}, - AdditionalDisks: targetMeta.AdditionalDisks, - MinMemory: targetMeta.MinMemory, - MinDiskSize: targetMeta.MinDiskSize, - AdditionalNics: targetMeta.AdditionalNics, - AppendKernelArgs: targetMeta.AppendKernelArgs, - NonExclusive: !targetMeta.Exclusive, - Conflicts: targetMeta.Conflicts, + AdditionalDisks: targetMeta.AdditionalDisks, + MinMemory: targetMeta.MinMemory, + MinDiskSize: targetMeta.MinDiskSize, + AdditionalNics: targetMeta.AdditionalNics, + AppendKernelArgs: targetMeta.AppendKernelArgs, + AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs, + NonExclusive: !targetMeta.Exclusive, + Conflicts: targetMeta.Conflicts, Run: func(c cluster.TestCluster) { mach := c.Machines()[0] @@ -1342,13 +1344,14 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig var userdata *conf.UserData = t.UserData options := platform.MachineOptions{ - MultiPathDisk: t.MultiPathDisk, - AdditionalDisks: t.AdditionalDisks, - MinMemory: t.MinMemory, - MinDiskSize: t.MinDiskSize, - AdditionalNics: t.AdditionalNics, - AppendKernelArgs: t.AppendKernelArgs, - SkipStartMachine: true, + MultiPathDisk: t.MultiPathDisk, + AdditionalDisks: t.AdditionalDisks, + MinMemory: t.MinMemory, + MinDiskSize: t.MinDiskSize, + AdditionalNics: t.AdditionalNics, + AppendKernelArgs: t.AppendKernelArgs, + AppendFirstbootKernelArgs: t.AppendFirstbootKernelArgs, + SkipStartMachine: true, } // Providers sometimes fail to bring up a machine within a diff --git a/mantle/kola/register/register.go b/mantle/kola/register/register.go index 74e87cf740..a14a86f665 100644 --- a/mantle/kola/register/register.go +++ b/mantle/kola/register/register.go @@ -87,6 +87,9 @@ type Test struct { // Additional kernel arguments to append to the defaults. AppendKernelArgs string + // Additional first boot kernel arguments to append to the defaults. + AppendFirstbootKernelArgs string + // ExternalTest is a path to a binary that will be uploaded ExternalTest string // DependencyDir is a path to directory that will be uploaded, normally used by external tests diff --git a/mantle/platform/machine/aws/cluster.go b/mantle/platform/machine/aws/cluster.go index 0e93bafc62..b3dbce1442 100644 --- a/mantle/platform/machine/aws/cluster.go +++ b/mantle/platform/machine/aws/cluster.go @@ -53,6 +53,10 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, errors.New("platform aws does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform aws does not support appending firstboot kernel arguments") + } + conf, err := ac.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_EC2_IPV4_PUBLIC}", "$private_ipv4": "${COREOS_EC2_IPV4_LOCAL}", diff --git a/mantle/platform/machine/azure/cluster.go b/mantle/platform/machine/azure/cluster.go index 97729d2d9c..32df106119 100644 --- a/mantle/platform/machine/azure/cluster.go +++ b/mantle/platform/machine/azure/cluster.go @@ -56,6 +56,9 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendKernelArgs != "" { return nil, errors.New("platform azure does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform azure does not support appending firstboot kernel arguments") + } conf, err := ac.RenderUserData(userdata, map[string]string{ "$private_ipv4": "${COREOS_AZURE_IPV4_DYNAMIC}", diff --git a/mantle/platform/machine/do/cluster.go b/mantle/platform/machine/do/cluster.go index 8edf41ea42..9fd79d5d6f 100644 --- a/mantle/platform/machine/do/cluster.go +++ b/mantle/platform/machine/do/cluster.go @@ -49,6 +49,9 @@ func (dc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendKernelArgs != "" { return nil, errors.New("platform do does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform do does not support appending firstboot kernel arguments") + } conf, err := dc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_DIGITALOCEAN_IPV4_PUBLIC_0}", diff --git a/mantle/platform/machine/esx/cluster.go b/mantle/platform/machine/esx/cluster.go index a6ee8960b9..023586db15 100644 --- a/mantle/platform/machine/esx/cluster.go +++ b/mantle/platform/machine/esx/cluster.go @@ -53,6 +53,9 @@ func (ec *cluster) NewMachineWithOptions(userdata *platformConf.UserData, option if options.AppendKernelArgs != "" { return nil, errors.New("platform esx does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform esx does not support appending firstboot kernel arguments") + } conf, err := ec.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_ESX_IPV4_PUBLIC_0}", diff --git a/mantle/platform/machine/gcloud/cluster.go b/mantle/platform/machine/gcloud/cluster.go index 1b96668079..bdabd886c5 100644 --- a/mantle/platform/machine/gcloud/cluster.go +++ b/mantle/platform/machine/gcloud/cluster.go @@ -49,6 +49,9 @@ func (gc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendKernelArgs != "" { return nil, errors.New("platform gce does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform gce does not support appending firstboot kernel arguments") + } conf, err := gc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_GCE_IP_EXTERNAL_0}", diff --git a/mantle/platform/machine/openstack/cluster.go b/mantle/platform/machine/openstack/cluster.go index e3c206f844..59fcf2ec00 100644 --- a/mantle/platform/machine/openstack/cluster.go +++ b/mantle/platform/machine/openstack/cluster.go @@ -47,6 +47,9 @@ func (oc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendKernelArgs != "" { return nil, errors.New("platform openstack does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform openstack does not support appending firstboot kernel arguments") + } conf, err := oc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_OPENSTACK_IPV4_PUBLIC}", diff --git a/mantle/platform/machine/packet/cluster.go b/mantle/platform/machine/packet/cluster.go index 5d3c0c8b43..533faa57df 100644 --- a/mantle/platform/machine/packet/cluster.go +++ b/mantle/platform/machine/packet/cluster.go @@ -49,6 +49,9 @@ func (pc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendKernelArgs != "" { return nil, errors.New("platform packet does not support appending kernel arguments") } + if options.AppendFirstbootKernelArgs != "" { + return nil, errors.New("platform packet does not support appending firstboot kernel arguments") + } conf, err := pc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_PACKET_IPV4_PUBLIC_0}", diff --git a/mantle/platform/machine/qemuiso/cluster.go b/mantle/platform/machine/qemuiso/cluster.go index 14efca73f5..d3008ccb45 100644 --- a/mantle/platform/machine/qemuiso/cluster.go +++ b/mantle/platform/machine/qemuiso/cluster.go @@ -136,6 +136,9 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.AppendKernelArgs != "" { builder.AppendKernelArgs = options.AppendKernelArgs } + if options.AppendFirstbootKernelArgs != "" { + builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs + } inst, err := builder.Exec() if err != nil { diff --git a/mantle/platform/machine/unprivqemu/cluster.go b/mantle/platform/machine/unprivqemu/cluster.go index e412704c0c..8a8828ff54 100644 --- a/mantle/platform/machine/unprivqemu/cluster.go +++ b/mantle/platform/machine/unprivqemu/cluster.go @@ -163,6 +163,9 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.AppendKernelArgs != "" { builder.AppendKernelArgs = options.AppendKernelArgs } + if options.AppendFirstbootKernelArgs != "" { + builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs + } if !qc.RuntimeConf().InternetAccess { builder.RestrictNetworking = true } diff --git a/mantle/platform/platform.go b/mantle/platform/platform.go index 03657000a0..daea38e487 100644 --- a/mantle/platform/platform.go +++ b/mantle/platform/platform.go @@ -153,13 +153,14 @@ type Flight interface { } type MachineOptions struct { - MultiPathDisk bool - AdditionalDisks []string - MinMemory int - MinDiskSize int - AdditionalNics int - AppendKernelArgs string - SkipStartMachine bool // Skip platform.StartMachine on machine bringup + MultiPathDisk bool + AdditionalDisks []string + MinMemory int + MinDiskSize int + AdditionalNics int + AppendKernelArgs string + AppendFirstbootKernelArgs string + SkipStartMachine bool // Skip platform.StartMachine on machine bringup } // SystemdDropin is a userdata type agnostic struct representing a systemd dropin