libvirt: support setting dnsmasq options through the install config

Since libvirt 5.6.0, there is an option to pass in dnsmasq options through
the libvirt network [1]. This is something that is useful because:
  - this solves [2] through the installer itself without a workaround of setting
    the dnsmasq options manually. this will also enable easy multi cluster deploys
    on a system by just specifying the domain mapping in the install-config
  - can also be used to point to a loadbalancer in cases like the multi-arch
    CI/CD systems where there are multiple clusters running on each system and it
    uses a load balancer.

Note that there is also an option to just inject an xslt, but this cannot be done after the network
is created and this method is much cleaner since the terraform provider itself supports this option

[1] https://libvirt.org/formatnetwork.html#elementsNamespaces
[2] openshift#1007

pkg/asset/cluster/tfvars.go

@@ -367,12 +367,15 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			return err
 		}
 		data, err = libvirttfvars.TFVars(
-			masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
-			string(*rhcosImage),
-			&installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
-			installConfig.Config.Platform.Libvirt.Network.IfName,
-			masterCount,
-			installConfig.Config.ControlPlane.Architecture,
+			libvirttfvars.TFVarsSources{
+				MasterConfig:   masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
+				OsImage:        string(*rhcosImage),
+				MachineCIDR:    &installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
+				Bridge:         installConfig.Config.Platform.Libvirt.Network.IfName,
+				MasterCount:    masterCount,
+				Architecture:   installConfig.Config.ControlPlane.Architecture,
+				DnsmasqOptions: installConfig.Config.Platform.Libvirt.Network.DnsmasqOptions,
+			},
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/tfvars/libvirt/libvirt.go

@@ -18,29 +18,42 @@ import (
 )
 
 type config struct {
-	URI             string   `json:"libvirt_uri,omitempty"`
-	Image           string   `json:"os_image,omitempty"`
-	IfName          string   `json:"libvirt_network_if"`
-	MasterIPs       []string `json:"libvirt_master_ips,omitempty"`
-	BootstrapIP     string   `json:"libvirt_bootstrap_ip,omitempty"`
-	MasterMemory    string   `json:"libvirt_master_memory,omitempty"`
-	MasterVcpu      string   `json:"libvirt_master_vcpu,omitempty"`
-	BootstrapMemory int      `json:"libvirt_bootstrap_memory,omitempty"`
-	MasterDiskSize  string   `json:"libvirt_master_size,omitempty"`
+	URI             string            `json:"libvirt_uri,omitempty"`
+	Image           string            `json:"os_image,omitempty"`
+	IfName          string            `json:"libvirt_network_if"`
+	MasterIPs       []string          `json:"libvirt_master_ips,omitempty"`
+	BootstrapIP     string            `json:"libvirt_bootstrap_ip,omitempty"`
+	MasterMemory    string            `json:"libvirt_master_memory,omitempty"`
+	MasterVcpu      string            `json:"libvirt_master_vcpu,omitempty"`
+	BootstrapMemory int               `json:"libvirt_bootstrap_memory,omitempty"`
+	MasterDiskSize  string            `json:"libvirt_master_size,omitempty"`
+	DnsmasqOptions  map[string]string `json:"libvirt_dnsmasq_options,omitempty"`
+}
+
+// TFVarsSources contains the parameters to be converted into Terraform variables
+type TFVarsSources struct {
+	MasterConfig   *v1beta1.LibvirtMachineProviderConfig
+	OsImage        string
+	MachineCIDR    *net.IPNet
+	Bridge         string
+	MasterCount    int
+	Architecture   types.Architecture
+	DnsmasqOptions map[string]string
 }
 
 // TFVars generates libvirt-specific Terraform variables.
-func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, machineCIDR *net.IPNet, bridge string, masterCount int, architecture types.Architecture) ([]byte, error) {
-	bootstrapIP, err := cidr.Host(machineCIDR, 10)
+func TFVars(sources TFVarsSources) ([]byte, error) {
+	bootstrapIP, err := cidr.Host(sources.MachineCIDR, 10)
 	if err != nil {
 		return nil, errors.Errorf("failed to generate bootstrap IP: %v", err)
 	}
 
-	masterIPs, err := generateIPs("master", machineCIDR, masterCount, 11)
+	masterIPs, err := generateIPs("master", sources.MachineCIDR, sources.MasterCount, 11)
 	if err != nil {
 		return nil, err
 	}
 
+	osImage := sources.OsImage
 	url, err := url.Parse(osImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse image url")
@@ -59,17 +72,18 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string,
 	}
 
 	cfg := &config{
-		URI:          masterConfig.URI,
-		Image:        osImage,
-		IfName:       bridge,
-		BootstrapIP:  bootstrapIP.String(),
-		MasterIPs:    masterIPs,
-		MasterMemory: strconv.Itoa(masterConfig.DomainMemory),
-		MasterVcpu:   strconv.Itoa(masterConfig.DomainVcpu),
+		URI:            sources.MasterConfig.URI,
+		Image:          osImage,
+		IfName:         sources.Bridge,
+		BootstrapIP:    bootstrapIP.String(),
+		MasterIPs:      masterIPs,
+		MasterMemory:   strconv.Itoa(sources.MasterConfig.DomainMemory),
+		MasterVcpu:     strconv.Itoa(sources.MasterConfig.DomainVcpu),
+		DnsmasqOptions: sources.DnsmasqOptions,
 	}
 
-	if masterConfig.Volume.VolumeSize != nil {
-		cfg.MasterDiskSize = masterConfig.Volume.VolumeSize.String()
+	if sources.MasterConfig.Volume.VolumeSize != nil {
+		cfg.MasterDiskSize = sources.MasterConfig.Volume.VolumeSize.String()
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/types/libvirt/platform.go

@@ -33,4 +33,15 @@ type Network struct {
 	// +kubebuilder:default="tt0"
 	// +optional
 	IfName string `json:"if,omitempty"`
+
+	// The Dnsmasq options to specify (libvirt 5.6.0 and beyond)
+	// This is useful to force a domain to a specific IP, specifically
+	// in the libvirt case to force a domain to a particular worker node
+	// or an LB if one is setup.
+	// Refer to: https://github.com/openshift/installer/issues/1007
+	//
+	// dnsmasqoptions:
+	//    address: "/.apps.<domain>/<cluster-subnet-prefix>.51"
+	// +optional
+	DnsmasqOptions map[string]string `json:"dnsmasqoptions,omitempty"`
 }