From a70833e72412db445c320107841d4718bbd5b745 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 13 Oct 2023 15:56:40 -0400 Subject: [PATCH 1/2] kola/qemuexec: add `--netboot` option For local testing, it's useful to have an easy way to netboot a system. Enable this by exposing QEMU's builtin support for TFTP serving and BOOTP option. For iPXE, one can just directly pass the iPXE script. For PXELINUX/GRUB, you'll likely want to prepare a netboot directory with your artifacts. Probably this should be streamlined more in the future, and also deduped more with the related bits in `metal.go`. But anyway, for now this is immediately useful in helping to test root on iSCSI locally (via iPXE's `sanboot` option). --- docs/cosa/run.md | 88 +++++++++++++++++++++++++++++++++++++ mantle/cmd/kola/qemuexec.go | 13 ++++-- mantle/platform/qemu.go | 33 ++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) diff --git a/docs/cosa/run.md b/docs/cosa/run.md index e6460a8b9e..9f44f1b3d0 100644 --- a/docs/cosa/run.md +++ b/docs/cosa/run.md @@ -192,3 +192,91 @@ TARGET SOURCE FSTYPE OPTIONS installed system automatically just as the live environment itself was set up.) This is equivalent to our `kola testiso` multipath tests. + +## Netbooting + +You can use the `--netboot` option to boot via BOOTP (e.g. iPXE, PXELINUX, GRUB). + +### iPXE + +This is the simplest since it's the default firmware and doesn't require +chaining. You can just point to the iPXE script, e.g.: + +``` +$ cat tmp/ipxe/boot.ipxe +#!ipxe +kernel / initrd=main coreos.live.rootfs_url= ignition.firstboot ignition.platform.id=metal console=ttyS0 ignition.config.url= +initrd --name main / +boot +$ cosa run -c --netboot tmp/ipxe/boot.ipxe +``` + +(That example requires hosting the rootfs separately, but you can also combine with the initrd.) + +Or doing an iSCSI boot: + +``` +#!ipxe +sanboot iscsi:192.168.10.1::::iqn.2023-10.coreos.target.vm:coreos +``` + +See [this section](https://docs.fedoraproject.org/en-US/fedora-coreos/live-booting/#_booting_via_ipxe) of the official docs for more info. + +### PXELINUX + +Point to the `pxelinux.0` binary, likely symlinked, e.g.: + +``` +$ tree tmp/pxelinux/ +tmp/pxelinux/ +├── fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img +├── fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 +├── fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img -> ../../builds/latest/x86_64/fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img +├── ldlinux.c32 -> /usr/share/syslinux/ldlinux.c32 +├── pxelinux.0 -> /usr/share/syslinux/pxelinux.0 +└── pxelinux.cfg + └── default + +2 directories, 6 files +$ cat tmp/pxelinux/pxelinux.cfg/default +DEFAULT pxeboot +TIMEOUT 20 +PROMPT 0 +LABEL pxeboot + KERNEL fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 + APPEND initrd=fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img,fedora-coreos-38.20231010.dev.0-live-rootfs.x86_64.img ignition.firstboot ignition.platform.id=metal ignition.config.url= console=ttyS0 +IPAPPEND 2 + +$ cosa run -c --netboot tmp/pxelinux/pxelinux.0 -m 4096 +``` + +See [this section](https://docs.fedoraproject.org/en-US/fedora-coreos/live-booting/#_booting_via_pxe) of the official docs for more info. + +### GRUB + +Create the netboot dir if not already created: + +``` +$ mkdir tmp/grub-netboot +$ grub2-mknetdir --net-directory tmp/grub-netboot +``` + +Create your GRUB config, e.g.: + +``` +$ cat tmp/grub-netboot/boot/grub2/grub.cfg +default=0 +timeout=1 +menuentry "CoreOS (BIOS/UEFI)" { + echo "Loading kernel" + linux /fedora-coreos-38.20231010.dev.0-live-kernel-x86_64 coreos.live.rootfs_url= ignition.firstboot ignition.platform.id=metal console=ttyS0 ignition.config.url= + echo "Loading initrd" + initrd fedora-coreos-38.20231010.dev.0-live-initramfs.x86_64.img +} +``` + +And point to it and the `core.0` binary: + +``` +$ cosa run -c --netboot-dir tmp/grub-netboot --netboot boot/grub2/i386-pc/core.0 -m 4096 +``` diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index fc07543794..134a1f53b2 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -70,6 +70,9 @@ var ( sshCommand string additionalNics int + + netboot string + netbootDir string ) const maxAdditionalNics = 16 @@ -97,7 +100,8 @@ func init() { cmdQemuExec.Flags().StringVarP(&consoleFile, "console-to-file", "", "", "Filepath in which to save serial console logs") cmdQemuExec.Flags().IntVarP(&additionalNics, "additional-nics", "", 0, "Number of additional NICs to add") cmdQemuExec.Flags().StringVarP(&sshCommand, "ssh-command", "x", "", "Command to execute instead of spawning a shell") - + cmdQemuExec.Flags().StringVarP(&netboot, "netboot", "", "", "Filepath to BOOTP program (e.g. PXELINUX/GRUB binary or iPXE script") + cmdQemuExec.Flags().StringVarP(&netbootDir, "netboot-dir", "", "", "Directory to serve over TFTP (default: BOOTP parent dir). If specified, --netboot is relative to this dir.") } func renderFragments(fragments []string, c *conf.Conf) error { @@ -315,7 +319,7 @@ func runQemuExec(cmd *cobra.Command, args []string) error { if kola.QEMUOptions.Firmware != "" { builder.Firmware = kola.QEMUOptions.Firmware } - if kola.QEMUOptions.DiskImage != "" { + if kola.QEMUOptions.DiskImage != "" && netboot == "" { if err := builder.AddBootDisk(buildDiskFromOptions()); err != nil { return err } @@ -323,7 +327,7 @@ func runQemuExec(cmd *cobra.Command, args []string) error { return err } } - if kola.QEMUIsoOptions.IsoPath != "" { + if kola.QEMUIsoOptions.IsoPath != "" && netboot == "" { err := builder.AddIso(kola.QEMUIsoOptions.IsoPath, "bootindex=3", kola.QEMUIsoOptions.AsDisk) if err != nil { return err @@ -358,6 +362,9 @@ func runQemuExec(cmd *cobra.Command, args []string) error { } builder.EnableUsermodeNetworking(h) } + if netboot != "" { + builder.SetNetbootP(netboot, netbootDir) + } if additionalNics != 0 { if additionalNics < 0 || additionalNics > maxAdditionalNics { return errors.Wrapf(nil, "additional-nics value cannot be negative or greater than %d", maxAdditionalNics) diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 31d0ccaab7..305cd5cc0f 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -481,6 +481,8 @@ type QemuBuilder struct { RestrictNetworking bool requestedHostForwardPorts []HostForwardPort additionalNics int + netbootP string + netbootDir string finalized bool diskID uint @@ -602,6 +604,12 @@ func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) { builder.requestedHostForwardPorts = h } +func (builder *QemuBuilder) SetNetbootP(filename, dir string) { + builder.UsermodeNetworking = true + builder.netbootP = filename + builder.netbootDir = dir +} + func (builder *QemuBuilder) AddAdditionalNics(additionalNics int) { builder.additionalNics = additionalNics } @@ -629,6 +637,31 @@ func (builder *QemuBuilder) setupNetworking() error { if builder.RestrictNetworking { netdev += ",restrict=on" } + if builder.netbootP != "" { + // do an early stat so we fail with a nicer error now instead of in the VM + if _, err := os.Stat(filepath.Join(builder.netbootDir, builder.netbootP)); err != nil { + return err + } + tftpDir := "" + relpath := "" + if builder.netbootDir == "" { + absPath, err := filepath.Abs(builder.netbootP) + if err != nil { + return err + } + tftpDir = filepath.Dir(absPath) + relpath = filepath.Base(absPath) + } else { + absPath, err := filepath.Abs(builder.netbootDir) + if err != nil { + return err + } + tftpDir = absPath + relpath = builder.netbootP + } + netdev += fmt.Sprintf(",tftp=%s,bootfile=/%s", tftpDir, relpath) + builder.Append("-boot", "order=n") + } builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", "netdev=eth0")) return nil From 412fa2db995f9a196bbd5e5a925bc2946816dcbd Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 13 Oct 2023 17:27:00 -0400 Subject: [PATCH 2/2] kola/qemuexec: allow changing guest network This is useful when you're nesting VMs and you want the first VM to be able to access the host. The default host address that QEMU assigns (e.g. 10.0.2.2, modifiable via the `host=...` netdev knob) doesn't always work because it's not actually an IP address owned by the host, but proxied by QEMU itself. So the source appears to come from localhost, but in some contexts (e.g. iSCSI), we need the host and the guest to agree that the same IP refers to the host. With this, one can start the first VM as usual (e.g. `cosa run`) and the second VM within with e.g. `cosa run --usernet-addr 10.0.3.0/24` and be able to talk back to the outer VM via the valid address 10.0.2.15. To be clear, this can all be done with passthrough QEMU args, so this is just about making it more convenient. --- mantle/cmd/kola/qemuexec.go | 7 +++++-- mantle/platform/machine/qemu/cluster.go | 4 ++-- mantle/platform/machine/qemuiso/cluster.go | 4 ++-- mantle/platform/qemu.go | 7 ++++++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 134a1f53b2..93082545ef 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -73,6 +73,8 @@ var ( netboot string netbootDir string + + usernetAddr string ) const maxAdditionalNics = 16 @@ -102,6 +104,7 @@ func init() { cmdQemuExec.Flags().StringVarP(&sshCommand, "ssh-command", "x", "", "Command to execute instead of spawning a shell") cmdQemuExec.Flags().StringVarP(&netboot, "netboot", "", "", "Filepath to BOOTP program (e.g. PXELINUX/GRUB binary or iPXE script") cmdQemuExec.Flags().StringVarP(&netbootDir, "netboot-dir", "", "", "Directory to serve over TFTP (default: BOOTP parent dir). If specified, --netboot is relative to this dir.") + cmdQemuExec.Flags().StringVarP(&usernetAddr, "usernet-addr", "", "", "Guest IP network (QEMU default is '10.0.2.0/24')") } func renderFragments(fragments []string, c *conf.Conf) error { @@ -356,11 +359,11 @@ func runQemuExec(cmd *cobra.Command, args []string) error { if cpuCountHost { builder.Processors = -1 } - if usernet { + if usernet || usernetAddr != "" { h := []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, } - builder.EnableUsermodeNetworking(h) + builder.EnableUsermodeNetworking(h, usernetAddr) } if netboot != "" { builder.SetNetbootP(netboot, netbootDir) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 6d2a82204b..28d159f015 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -175,12 +175,12 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } if len(options.HostForwardPorts) > 0 { - builder.EnableUsermodeNetworking(options.HostForwardPorts) + builder.EnableUsermodeNetworking(options.HostForwardPorts, "") } else { h := []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, } - builder.EnableUsermodeNetworking(h) + builder.EnableUsermodeNetworking(h, "") } if options.AdditionalNics > 0 { builder.AddAdditionalNics(options.AdditionalNics) diff --git a/mantle/platform/machine/qemuiso/cluster.go b/mantle/platform/machine/qemuiso/cluster.go index b430bb35a6..9223192149 100644 --- a/mantle/platform/machine/qemuiso/cluster.go +++ b/mantle/platform/machine/qemuiso/cluster.go @@ -124,12 +124,12 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } if len(options.HostForwardPorts) > 0 { - builder.EnableUsermodeNetworking(options.HostForwardPorts) + builder.EnableUsermodeNetworking(options.HostForwardPorts, "") } else { h := []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, } - builder.EnableUsermodeNetworking(h) + builder.EnableUsermodeNetworking(h, "") } if options.AdditionalNics > 0 { diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 305cd5cc0f..3a15340505 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -478,6 +478,7 @@ type QemuBuilder struct { ignitionRendered bool UsermodeNetworking bool + usermodeNetworkingAddr string RestrictNetworking bool requestedHostForwardPorts []HostForwardPort additionalNics int @@ -599,9 +600,10 @@ func virtio(arch, device, args string) string { // EnableUsermodeNetworking configure forwarding for all requested ports, // via usermode network helpers. -func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort) { +func (builder *QemuBuilder) EnableUsermodeNetworking(h []HostForwardPort, usernetAddr string) { builder.UsermodeNetworking = true builder.requestedHostForwardPorts = h + builder.usermodeNetworkingAddr = usernetAddr } func (builder *QemuBuilder) SetNetbootP(filename, dir string) { @@ -637,6 +639,9 @@ func (builder *QemuBuilder) setupNetworking() error { if builder.RestrictNetworking { netdev += ",restrict=on" } + if builder.usermodeNetworkingAddr != "" { + netdev += ",net=" + builder.usermodeNetworkingAddr + } if builder.netbootP != "" { // do an early stat so we fail with a nicer error now instead of in the VM if _, err := os.Stat(filepath.Join(builder.netbootDir, builder.netbootP)); err != nil {