libvirt: support setting network dnsmasq options through the install …

…config

Since libvirt 5.6.0, there is an option to pass in dnsmasq options through the libvirt network [1].
Leveraging this would be useful for the multi-arch automation as we have a hacky workaround today
to inject host records [2] which waits for the libvirt network to be created and then updates it with the
records (which is need in libvirt because of [3]). With this dnsmasq option which can be specified in the
libvirt network at creation time, we can point to the .1 address and have a load balancer forward the
traffic which would be much cleaner.

With this change, the option can be specified through the install config yaml in the network section as pairs
of option name and values. An example:

```
platform:
  libvirt:
    network:
      dnsmasqoptions:
        address: "/.apps.tt.testing/192.168.126.51"
      if: tt0
```
The terraform provider supports rendering these options through a datasource and injecting them into the network xml.
Since this config is optional, not specifying it will continue to work as before without issues.

[1] https://libvirt.org/formatnetwork.html#elementsNamespaces
[2] https://github.com/openshift/release/blob/master/ci-operator/templates/openshift/installer/cluster-launch-installer-remote-libvirt-e2e.yaml#L498
[2] openshift#1007

data/data/libvirt/main.tf

@@ -78,7 +78,7 @@ resource "libvirt_network" "net" {
       content {
         option_name  = options.value.option_name
         option_value = options.value.option_value
-        }
+      }
     }
   }
 
@@ -141,7 +141,8 @@ data "libvirt_network_dns_host_template" "masters_int" {
 }
 
 data "libvirt_network_dnsmasq_options_template" "options" {
-  count = length(var.libvirt_dnsmasq_options)
+  count        = length(var.libvirt_dnsmasq_options)
   option_name  = keys(var.libvirt_dnsmasq_options)[count.index]
   option_value = values(var.libvirt_dnsmasq_options)[count.index]
-}
+}
+

pkg/asset/cluster/tfvars.go

@@ -367,12 +367,15 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			return err
 		}
 		data, err = libvirttfvars.TFVars(
-			masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
-			string(*rhcosImage),
-			&installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
-			installConfig.Config.Platform.Libvirt.Network.IfName,
-			masterCount,
-			installConfig.Config.ControlPlane.Architecture,
+			libvirttfvars.TFVarsSources{
+				MasterConfig:   masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
+				OsImage:        string(*rhcosImage),
+				MachineCIDR:    &installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
+				Bridge:         installConfig.Config.Platform.Libvirt.Network.IfName,
+				MasterCount:    masterCount,
+				Architecture:   installConfig.Config.ControlPlane.Architecture,
+				DnsmasqOptions: installConfig.Config.Platform.Libvirt.Network.DnsmasqOptions,
+			},
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/tfvars/libvirt/libvirt.go

@@ -18,29 +18,42 @@ import (
 )
 
 type config struct {
-	URI             string   `json:"libvirt_uri,omitempty"`
-	Image           string   `json:"os_image,omitempty"`
-	IfName          string   `json:"libvirt_network_if"`
-	MasterIPs       []string `json:"libvirt_master_ips,omitempty"`
-	BootstrapIP     string   `json:"libvirt_bootstrap_ip,omitempty"`
-	MasterMemory    string   `json:"libvirt_master_memory,omitempty"`
-	MasterVcpu      string   `json:"libvirt_master_vcpu,omitempty"`
-	BootstrapMemory int      `json:"libvirt_bootstrap_memory,omitempty"`
-	MasterDiskSize  string   `json:"libvirt_master_size,omitempty"`
+	URI             string            `json:"libvirt_uri,omitempty"`
+	Image           string            `json:"os_image,omitempty"`
+	IfName          string            `json:"libvirt_network_if"`
+	MasterIPs       []string          `json:"libvirt_master_ips,omitempty"`
+	BootstrapIP     string            `json:"libvirt_bootstrap_ip,omitempty"`
+	MasterMemory    string            `json:"libvirt_master_memory,omitempty"`
+	MasterVcpu      string            `json:"libvirt_master_vcpu,omitempty"`
+	BootstrapMemory int               `json:"libvirt_bootstrap_memory,omitempty"`
+	MasterDiskSize  string            `json:"libvirt_master_size,omitempty"`
+	DnsmasqOptions  map[string]string `json:"libvirt_dnsmasq_options,omitempty"`
+}
+
+// TFVarsSources contains the parameters to be converted into Terraform variables
+type TFVarsSources struct {
+	MasterConfig   *v1beta1.LibvirtMachineProviderConfig
+	OsImage        string
+	MachineCIDR    *net.IPNet
+	Bridge         string
+	MasterCount    int
+	Architecture   types.Architecture
+	DnsmasqOptions map[string]string
 }
 
 // TFVars generates libvirt-specific Terraform variables.
-func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, machineCIDR *net.IPNet, bridge string, masterCount int, architecture types.Architecture) ([]byte, error) {
-	bootstrapIP, err := cidr.Host(machineCIDR, 10)
+func TFVars(sources TFVarsSources) ([]byte, error) {
+	bootstrapIP, err := cidr.Host(sources.MachineCIDR, 10)
 	if err != nil {
 		return nil, errors.Errorf("failed to generate bootstrap IP: %v", err)
 	}
 
-	masterIPs, err := generateIPs("master", machineCIDR, masterCount, 11)
+	masterIPs, err := generateIPs("master", sources.MachineCIDR, sources.MasterCount, 11)
 	if err != nil {
 		return nil, err
 	}
 
+	osImage := sources.OsImage
 	url, err := url.Parse(osImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse image url")
@@ -59,17 +72,18 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string,
 	}
 
 	cfg := &config{
-		URI:          masterConfig.URI,
-		Image:        osImage,
-		IfName:       bridge,
-		BootstrapIP:  bootstrapIP.String(),
-		MasterIPs:    masterIPs,
-		MasterMemory: strconv.Itoa(masterConfig.DomainMemory),
-		MasterVcpu:   strconv.Itoa(masterConfig.DomainVcpu),
+		URI:            sources.MasterConfig.URI,
+		Image:          osImage,
+		IfName:         sources.Bridge,
+		BootstrapIP:    bootstrapIP.String(),
+		MasterIPs:      masterIPs,
+		MasterMemory:   strconv.Itoa(sources.MasterConfig.DomainMemory),
+		MasterVcpu:     strconv.Itoa(sources.MasterConfig.DomainVcpu),
+		DnsmasqOptions: sources.DnsmasqOptions,
 	}
 
-	if masterConfig.Volume.VolumeSize != nil {
-		cfg.MasterDiskSize = masterConfig.Volume.VolumeSize.String()
+	if sources.MasterConfig.Volume.VolumeSize != nil {
+		cfg.MasterDiskSize = sources.MasterConfig.Volume.VolumeSize.String()
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/types/libvirt/platform.go

@@ -33,4 +33,13 @@ type Network struct {
 	// +kubebuilder:default="tt0"
 	// +optional
 	IfName string `json:"if,omitempty"`
+
+	// The Dnsmasq options to specify (libvirt 5.6.0 and beyond)
+	// This is useful to force a domain to a specific IP, specifically
+	// in the libvirt case to force a domain to a particular worker node
+	// or an LB if one is setup.
+	// Refer to: https://github.com/openshift/installer/issues/1007
+	//
+	// +optional
+	DnsmasqOptions map[string]string `json:"dnsmasqoptions,omitempty"`
 }