libvirt: support setting network dnsmasq options through the install …

…config

Since libvirt 5.6.0, there is an option to pass in dnsmasq options through the libvirt network [1]. This addresses the following problems:

- eliminate the need for hacking routes in the cluster (the workaround mentioned in [3]) so that libvirt's dnsmasq does not manage the domain (and so the requests from inside the cluster will go up the chain to the host itself).
- eliminate the hacky workaround used in the multi-arch CI automation to inject `*.apps` entries in the libvirt network that point to a single worker node [2]. Instead of waiting for the libvirt networks to come up and update entries, we can set this before the installation itself through the install config.
- another issue this solves - with the above mentioned workaround, having multiple worker nodes becomes problematic when running upgrade tests. Having the route to just one worker node would fail the upgrade when that worker node is down. With this change, we could now point to the .1 address and have a load balancer forward traffic to any worker node.

With this change, the option can be specified through the install config yaml in the network section as pairs of option name and values. An example:
```
platform:
  libvirt:
    network:
      dnsmasqOptions:
        address: "/.apps.tt.testing/192.168.126.51"
      if: tt0
```
The terraform provider supports rendering these options through a datasource and injecting them into the network xml.
Since this config is optional, not specifying it will continue to work as before without issues.

[1] https://libvirt.org/formatnetwork.html#elementsNamespaces
[2] https://github.com/openshift/release/blob/master/ci-operator/templates/openshift/installer/cluster-launch-installer-remote-libvirt-e2e.yaml#L532-L554
[2] openshift#1007

data/data/install.openshift.io_installconfigs.yaml

@@ -1479,6 +1479,13 @@ spec:
                   network:
                     description: Network
                     properties:
+                      dnsmasqOptions:
+                        additionalProperties:
+                          type: string
+                        description: DnsmasqOptions is the dnsmasq options to be used
+                          when installing on libvirt. This is useful to force domains
+                          to a specific IP or to a loadbalancer.
+                        type: object
                       if:
                         default: tt0
                         description: The interface make used for the network. Default

data/data/libvirt/main.tf

@@ -78,7 +78,7 @@ resource "libvirt_network" "net" {
       content {
         option_name  = options.value.option_name
         option_value = options.value.option_value
-        }
+      }
     }
   }
 
@@ -141,7 +141,7 @@ data "libvirt_network_dns_host_template" "masters_int" {
 }
 
 data "libvirt_network_dnsmasq_options_template" "options" {
-  count = length(var.libvirt_dnsmasq_options)
+  count        = length(var.libvirt_dnsmasq_options)
   option_name  = keys(var.libvirt_dnsmasq_options)[count.index]
   option_value = values(var.libvirt_dnsmasq_options)[count.index]
 }

pkg/asset/cluster/tfvars.go

@@ -367,12 +367,15 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			return err
 		}
 		data, err = libvirttfvars.TFVars(
-			masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
-			string(*rhcosImage),
-			&installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
-			installConfig.Config.Platform.Libvirt.Network.IfName,
-			masterCount,
-			installConfig.Config.ControlPlane.Architecture,
+			libvirttfvars.TFVarsSources{
+				MasterConfig:   masters[0].Spec.ProviderSpec.Value.Object.(*libvirtprovider.LibvirtMachineProviderConfig),
+				OsImage:        string(*rhcosImage),
+				MachineCIDR:    &installConfig.Config.Networking.MachineNetwork[0].CIDR.IPNet,
+				Bridge:         installConfig.Config.Platform.Libvirt.Network.IfName,
+				MasterCount:    masterCount,
+				Architecture:   installConfig.Config.ControlPlane.Architecture,
+				DnsmasqOptions: installConfig.Config.Platform.Libvirt.Network.DnsmasqOptions,
+			},
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/tfvars/libvirt/libvirt.go

@@ -18,29 +18,42 @@ import (
 )
 
 type config struct {
-	URI             string   `json:"libvirt_uri,omitempty"`
-	Image           string   `json:"os_image,omitempty"`
-	IfName          string   `json:"libvirt_network_if"`
-	MasterIPs       []string `json:"libvirt_master_ips,omitempty"`
-	BootstrapIP     string   `json:"libvirt_bootstrap_ip,omitempty"`
-	MasterMemory    string   `json:"libvirt_master_memory,omitempty"`
-	MasterVcpu      string   `json:"libvirt_master_vcpu,omitempty"`
-	BootstrapMemory int      `json:"libvirt_bootstrap_memory,omitempty"`
-	MasterDiskSize  string   `json:"libvirt_master_size,omitempty"`
+	URI             string            `json:"libvirt_uri,omitempty"`
+	Image           string            `json:"os_image,omitempty"`
+	IfName          string            `json:"libvirt_network_if"`
+	MasterIPs       []string          `json:"libvirt_master_ips,omitempty"`
+	BootstrapIP     string            `json:"libvirt_bootstrap_ip,omitempty"`
+	MasterMemory    string            `json:"libvirt_master_memory,omitempty"`
+	MasterVcpu      string            `json:"libvirt_master_vcpu,omitempty"`
+	BootstrapMemory int               `json:"libvirt_bootstrap_memory,omitempty"`
+	MasterDiskSize  string            `json:"libvirt_master_size,omitempty"`
+	DnsmasqOptions  map[string]string `json:"libvirt_dnsmasq_options,omitempty"`
+}
+
+// TFVarsSources contains the parameters to be converted into Terraform variables
+type TFVarsSources struct {
+	MasterConfig   *v1beta1.LibvirtMachineProviderConfig
+	OsImage        string
+	MachineCIDR    *net.IPNet
+	Bridge         string
+	MasterCount    int
+	Architecture   types.Architecture
+	DnsmasqOptions map[string]string
 }
 
 // TFVars generates libvirt-specific Terraform variables.
-func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, machineCIDR *net.IPNet, bridge string, masterCount int, architecture types.Architecture) ([]byte, error) {
-	bootstrapIP, err := cidr.Host(machineCIDR, 10)
+func TFVars(sources TFVarsSources) ([]byte, error) {
+	bootstrapIP, err := cidr.Host(sources.MachineCIDR, 10)
 	if err != nil {
 		return nil, errors.Errorf("failed to generate bootstrap IP: %v", err)
 	}
 
-	masterIPs, err := generateIPs("master", machineCIDR, masterCount, 11)
+	masterIPs, err := generateIPs("master", sources.MachineCIDR, sources.MasterCount, 11)
 	if err != nil {
 		return nil, err
 	}
 
+	osImage := sources.OsImage
 	url, err := url.Parse(osImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse image url")
@@ -59,17 +72,18 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string,
 	}
 
 	cfg := &config{
-		URI:          masterConfig.URI,
-		Image:        osImage,
-		IfName:       bridge,
-		BootstrapIP:  bootstrapIP.String(),
-		MasterIPs:    masterIPs,
-		MasterMemory: strconv.Itoa(masterConfig.DomainMemory),
-		MasterVcpu:   strconv.Itoa(masterConfig.DomainVcpu),
+		URI:            sources.MasterConfig.URI,
+		Image:          osImage,
+		IfName:         sources.Bridge,
+		BootstrapIP:    bootstrapIP.String(),
+		MasterIPs:      masterIPs,
+		MasterMemory:   strconv.Itoa(sources.MasterConfig.DomainMemory),
+		MasterVcpu:     strconv.Itoa(sources.MasterConfig.DomainVcpu),
+		DnsmasqOptions: sources.DnsmasqOptions,
 	}
 
-	if masterConfig.Volume.VolumeSize != nil {
-		cfg.MasterDiskSize = masterConfig.Volume.VolumeSize.String()
+	if sources.MasterConfig.Volume.VolumeSize != nil {
+		cfg.MasterDiskSize = sources.MasterConfig.Volume.VolumeSize.String()
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/types/libvirt/platform.go

@@ -33,4 +33,11 @@ type Network struct {
 	// +kubebuilder:default="tt0"
 	// +optional
 	IfName string `json:"if,omitempty"`
+
+	// DnsmasqOptions is the dnsmasq options to be used when installing on
+	// libvirt. This is useful to force domains to a specific IP or to a
+	// loadbalancer.
+	//
+	// +optional
+	DnsmasqOptions map[string]string `json:"dnsmasqOptions,omitempty"`
 }