libvirt: support setting network dnsmasq options through the install …

…config

Since libvirt 5.6.0, there is an option to pass in dnsmasq options through the libvirt network [1].
Leveraging this would be useful for the multi-arch automation as we have a hacky workaround today
to inject host records [2] which waits for the libvirt network to be created and then updates it with the
records (which is need in libvirt because of [3]). With this dnsmasq option which can be specified in the
libvirt network at creation time, we can point to the .1 address and have a load balancer forward the
traffic which would be much cleaner.

With this change, the option can be specified through the install config yaml in the network section as pairs
of option name and values. An example:

```
platform:
  libvirt:
    network:
      dnsmasqoptions:
        address: "/.apps.tt.testing/192.168.126.51"
      if: tt0
```
The terraform provider supports rendering these options through a datasource and injecting them into the network xml.
Since this config is optional, not specifying it will continue to work as before without issues.

[1] https://libvirt.org/formatnetwork.html#elementsNamespaces
[2] https://github.com/openshift/release/blob/master/ci-operator/templates/openshift/installer/cluster-launch-installer-remote-libvirt-e2e.yaml#L498
[2] openshift#1007

data/data/install.openshift.io_installconfigs.yaml

@@ -1479,6 +1479,13 @@ spec:
                   network:
                     description: Network
                     properties:
+                      dnsmasqOptions:
+                        additionalProperties:
+                          type: string
+                        description: DnsmasqOptions is the dnsmasq options to be used
+                          when installing on libvirt. This is useful to force domains
+                          to a specific IP or to a loadbalancer.
+                        type: object
                       if:
                         default: tt0
                         description: The interface make used for the network. Default

data/data/libvirt/main.tf

@@ -78,7 +78,7 @@ resource "libvirt_network" "net" {
       content {
         option_name  = options.value.option_name
         option_value = options.value.option_value
-        }
+      }
     }
   }
 
@@ -141,7 +141,8 @@ data "libvirt_network_dns_host_template" "masters_int" {
 }
 
 data "libvirt_network_dnsmasq_options_template" "options" {
-  count = length(var.libvirt_dnsmasq_options)
+  count        = length(var.libvirt_dnsmasq_options)
   option_name  = keys(var.libvirt_dnsmasq_options)[count.index]
   option_value = values(var.libvirt_dnsmasq_options)[count.index]
-}
+}
+

pkg/asset/cluster/tfvars.go

@@ -367,12 +367,15 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			return err
 		}
 		data, err = libvirttfvars.TFVars(
-			masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
-			string(*rhcosImage),
-			&installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
-			installConfig.Config.Platform.Libvirt.Network.IfName,
-			masterCount,
-			installConfig.Config.ControlPlane.Architecture,
+			libvirttfvars.TFVarsSources{
+				MasterConfig:   masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
+				OsImage:        string(*rhcosImage),
+				MachineCIDR:    &installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
+				Bridge:         installConfig.Config.Platform.Libvirt.Network.IfName,
+				MasterCount:    masterCount,
+				Architecture:   installConfig.Config.ControlPlane.Architecture,
+				DnsmasqOptions: installConfig.Config.Platform.Libvirt.Network.DnsmasqOptions,
+			},
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/tfvars/libvirt/libvirt.go

@@ -18,29 +18,42 @@ import (
 )
 
 type config struct {
-	URI             string   `json:"libvirt_uri,omitempty"`
-	Image           string   `json:"os_image,omitempty"`
-	IfName          string   `json:"libvirt_network_if"`
-	MasterIPs       []string `json:"libvirt_master_ips,omitempty"`
-	BootstrapIP     string   `json:"libvirt_bootstrap_ip,omitempty"`
-	MasterMemory    string   `json:"libvirt_master_memory,omitempty"`
-	MasterVcpu      string   `json:"libvirt_master_vcpu,omitempty"`
-	BootstrapMemory int      `json:"libvirt_bootstrap_memory,omitempty"`
-	MasterDiskSize  string   `json:"libvirt_master_size,omitempty"`
+	URI             string            `json:"libvirt_uri,omitempty"`
+	Image           string            `json:"os_image,omitempty"`
+	IfName          string            `json:"libvirt_network_if"`
+	MasterIPs       []string          `json:"libvirt_master_ips,omitempty"`
+	BootstrapIP     string            `json:"libvirt_bootstrap_ip,omitempty"`
+	MasterMemory    string            `json:"libvirt_master_memory,omitempty"`
+	MasterVcpu      string            `json:"libvirt_master_vcpu,omitempty"`
+	BootstrapMemory int               `json:"libvirt_bootstrap_memory,omitempty"`
+	MasterDiskSize  string            `json:"libvirt_master_size,omitempty"`
+	DnsmasqOptions  map[string]string `json:"libvirt_dnsmasq_options,omitempty"`
+}
+
+// TFVarsSources contains the parameters to be converted into Terraform variables
+type TFVarsSources struct {
+	MasterConfig   *v1beta1.LibvirtMachineProviderConfig
+	OsImage        string
+	MachineCIDR    *net.IPNet
+	Bridge         string
+	MasterCount    int
+	Architecture   types.Architecture
+	DnsmasqOptions map[string]string
 }
 
 // TFVars generates libvirt-specific Terraform variables.
-func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, machineCIDR *net.IPNet, bridge string, masterCount int, architecture types.Architecture) ([]byte, error) {
-	bootstrapIP, err := cidr.Host(machineCIDR, 10)
+func TFVars(sources TFVarsSources) ([]byte, error) {
+	bootstrapIP, err := cidr.Host(sources.MachineCIDR, 10)
 	if err != nil {
 		return nil, errors.Errorf("failed to generate bootstrap IP: %v", err)
 	}
 
-	masterIPs, err := generateIPs("master", machineCIDR, masterCount, 11)
+	masterIPs, err := generateIPs("master", sources.MachineCIDR, sources.MasterCount, 11)
 	if err != nil {
 		return nil, err
 	}
 
+	osImage := sources.OsImage
 	url, err := url.Parse(osImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse image url")
@@ -59,17 +72,18 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string,
 	}
 
 	cfg := &config{
-		URI:          masterConfig.URI,
-		Image:        osImage,
-		IfName:       bridge,
-		BootstrapIP:  bootstrapIP.String(),
-		MasterIPs:    masterIPs,
-		MasterMemory: strconv.Itoa(masterConfig.DomainMemory),
-		MasterVcpu:   strconv.Itoa(masterConfig.DomainVcpu),
+		URI:            sources.MasterConfig.URI,
+		Image:          osImage,
+		IfName:         sources.Bridge,
+		BootstrapIP:    bootstrapIP.String(),
+		MasterIPs:      masterIPs,
+		MasterMemory:   strconv.Itoa(sources.MasterConfig.DomainMemory),
+		MasterVcpu:     strconv.Itoa(sources.MasterConfig.DomainVcpu),
+		DnsmasqOptions: sources.DnsmasqOptions,
 	}
 
-	if masterConfig.Volume.VolumeSize != nil {
-		cfg.MasterDiskSize = masterConfig.Volume.VolumeSize.String()
+	if sources.MasterConfig.Volume.VolumeSize != nil {
+		cfg.MasterDiskSize = sources.MasterConfig.Volume.VolumeSize.String()
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/types/libvirt/platform.go

@@ -33,4 +33,11 @@ type Network struct {
 	// +kubebuilder:default="tt0"
 	// +optional
 	IfName string `json:"if,omitempty"`
+
+	// DnsmasqOptions is the dnsmasq options to be used when installing on
+	// libvirt. This is useful to force domains to a specific IP or to a
+	// loadbalancer.
+	//
+	// +optional
+	DnsmasqOptions map[string]string `json:"dnsmasqOptions,omitempty"`
 }