provider/cloudstack: use nm to get metadata address

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	cloud.google.com/go v0.58.0
 	cloud.google.com/go/storage v1.9.0
+	github.com/Wifx/gonetworkmanager v0.3.0
 	github.com/aws/aws-sdk-go v1.30.28
 	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd/v22 v22.0.0

go.sum

@@ -38,6 +38,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Wifx/gonetworkmanager v0.3.0 h1:Gn5uD11cBG6qg9w0NENjTjMNZ5Asz0zxRo7t2twPQ84=
+github.com/Wifx/gonetworkmanager v0.3.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8=
 github.com/aws/aws-sdk-go v1.30.28 h1:SaPM7dlmp7h3Lj1nJ4jdzOkTdom08+g20k7AU5heZYg=
 github.com/aws/aws-sdk-go v1.30.28/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -64,6 +66,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/godbus/dbus/v5 v5.0.2/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

internal/distro/distro.go

@@ -29,6 +29,7 @@ var (
 	// initrd file paths
 	kernelCmdlinePath = "/proc/cmdline"
 	bootIDPath        = "/proc/sys/kernel/random/boot_id"
+	routeFilePath     = "/proc/net/route"
 	// initramfs directory containing distro-provided base config
 	systemConfigDir = "/usr/lib/ignition"
 
@@ -84,6 +85,7 @@ func DiskByPartUUIDDir() string { return diskByPartUUIDDir }
 
 func KernelCmdlinePath() string { return kernelCmdlinePath }
 func BootIDPath() string        { return bootIDPath }
+func RouteFilePath() string     { return routeFilePath }
 func SystemConfigDir() string   { return fromEnv("SYSTEM_CONFIG_DIR", systemConfigDir) }
 
 func GroupaddCmd() string { return groupaddCmd }

internal/networkmanager/networkmanager.go

@@ -0,0 +1,68 @@
+// Copyright 2021 Red Hat, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package networkmanager
+
+import (
+	"fmt"
+
+	"github.com/Wifx/gonetworkmanager"
+)
+
+type DHCPOptions map[string]string
+
+// GetDHCPOptions returns a map where keys are the interface names, for the
+// network interfaces where DHCP is enabled, and values the maps of DHCP options
+//
+// This uses the NetworkManager DBus API.
+func GetDHCPOptions() (map[string]DHCPOptions, error) {
+	// Create a new instance of gonetworkmanager
+	nm, err := gonetworkmanager.NewNetworkManager()
+	if err != nil {
+		return nil, err
+	}
+
+	// Get network devices
+	devices, err := nm.GetPropertyDevices()
+	if err != nil {
+		return nil, err
+	}
+
+	res := make(map[string]DHCPOptions)
+
+	for _, device := range devices {
+		interfaceName, err := device.GetPropertyInterface()
+		if err != nil {
+			return nil, err
+		}
+		dhcpd, err := device.GetPropertyDHCP4Config()
+		if err != nil {
+			return nil, err
+		}
+		if dhcpd == nil {
+			continue
+		}
+
+		dhcpo, err := dhcpd.GetPropertyOptions()
+		if err != nil {
+			return nil, err
+		}
+		res[interfaceName] = make(DHCPOptions)
+		for k, v := range dhcpo {
+			res[interfaceName][k] = fmt.Sprintf("%v", v)
+		}
+	}
+
+	return res, nil
+}

internal/providers/cloudstack/cloudstack.go

@@ -13,14 +13,16 @@
 // limitations under the License.
 
 // The CloudStack provider fetches configurations from the userdata available in
-// the config-drive.
+// both the config-drive as well as the network metadata service. Whichever
+// responds first is the config that is used.
 // NOTE: This provider is still EXPERIMENTAL.
 
 package cloudstack
 
 import (
 	"bufio"
 	"context"
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -34,6 +36,7 @@ import (
 	"github.com/coreos/ignition/v2/config/v3_4_experimental/types"
 	"github.com/coreos/ignition/v2/internal/distro"
 	"github.com/coreos/ignition/v2/internal/log"
+	"github.com/coreos/ignition/v2/internal/networkmanager"
 	"github.com/coreos/ignition/v2/internal/providers/util"
 	"github.com/coreos/ignition/v2/internal/resource"
 
@@ -43,7 +46,8 @@ import (
 
 const (
 	configDriveUserdataPath = "/cloudstack/userdata/user_data.txt"
-	LeaseRetryInterval      = 500 * time.Millisecond
+	retryInterval           = 500 * time.Millisecond
+	dataServerDNSName       = "data-server"
 )
 
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
@@ -87,7 +91,7 @@ func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
 	})
 
 	dispatch("metadata service", func() ([]byte, error) {
-		return fetchConfigFromMetadataService(f)
+		return fetchConfigFromMetadataService(ctx, f)
 	})
 
 Loop:
@@ -133,21 +137,90 @@ func findLease() (*os.File, error) {
 		return nil, fmt.Errorf("could not list interfaces: %v", err)
 	}
 
+	for _, iface := range ifaces {
+		lease, err := os.Open(fmt.Sprintf("/run/systemd/netif/leases/%d", iface.Index))
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, err
+		} else {
+			return lease, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no leases found")
+}
+
+func getVirtualRouterAddress(ctx context.Context, logger *log.Logger) (string, error) {
 	for {
-		for _, iface := range ifaces {
-			lease, err := os.Open(fmt.Sprintf("/run/systemd/netif/leases/%d", iface.Index))
-			if os.IsNotExist(err) {
-				continue
-			} else if err != nil {
-				return nil, err
-			} else {
-				return lease, nil
-			}
+		// Try "data-server" DNS entry first
+		if addr, err := getDataServerByDNS(); err != nil {
+			logger.Info("Could not find virtual router using DNS: %s", err.Error())
+			// continue with NetworkManager
+		} else {
+			logger.Info("Virtual router address found using DNS: %s", addr)
+			return addr, nil
+		}
+
+		// Then use NetworkManager to get the server option via DHCP
+		if addr, err := getNetworkManagerDHCPServerOption(); err != nil {
+			logger.Info("Could not find virtual router using NetworkManager DHCP server option: %s", err.Error())
+			// continue with networkd
+		} else {
+			logger.Info("Virtual router address found using NetworkManager DHCP server option: %s", addr)
+			return addr, nil
+		}
+
+		// Then try networkd
+		if addr, err := getDHCPServerAddress(); err != nil {
+			logger.Info("Could not find server address in DHCP networkd leases: %s", err.Error())
+			// continue with default gateway
+		} else {
+			logger.Info("Virtual router address found using DHCP networkd leases: %s", addr)
+			return addr, nil
+		}
+
+		// Fallback on default gateway
+		if addr, err := getDefaultGateway(); err != nil {
+			logger.Info("Could not find default gateway: %s", err.Error())
+		} else {
+			logger.Info("Fallback on default gateway: %s", addr)
+			return addr, nil
+		}
+
+		select {
+		case <-time.After(retryInterval):
+		case <-ctx.Done():
+			return "", ctx.Err()
 		}
+	}
+}
 
-		fmt.Printf("No leases found. Waiting...")
-		time.Sleep(LeaseRetryInterval)
+func getDataServerByDNS() (string, error) {
+	addrs, err := net.LookupHost(dataServerDNSName)
+	if err != nil {
+		return "", fmt.Errorf("could not execute DNS lookup: %v", err)
+	}
+
+	for _, addr := range addrs {
+		return addr, nil
+	}
+	return "", fmt.Errorf("DNS Entry %s not found", dataServerDNSName)
+}
+
+func getNetworkManagerDHCPServerOption() (string, error) {
+	options, err := networkmanager.GetDHCPOptions()
+	if err != nil {
+		return "", err
 	}
+	for _, netIface := range options {
+		for k, v := range netIface {
+			if k == "dhcp_server_identifier" {
+				return v, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("no DHCP option dhcp_server_identifier in NetworkManager")
 }
 
 func getDHCPServerAddress() (string, error) {
@@ -174,6 +247,59 @@ func getDHCPServerAddress() (string, error) {
 	return address, nil
 }
 
+func getDefaultGateway() (string, error) {
+	file, err := os.Open(distro.RouteFilePath())
+	if err != nil {
+		return "", fmt.Errorf("cannot read routes: %v", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// Ignore headers
+		if strings.HasPrefix(line, "Iface") {
+			continue
+		}
+
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			return "", fmt.Errorf("cannot parse route files")
+		}
+		if fields[1] == "00000000" {
+			// destination is "0.0.0.0", so the gateway is the default gateway
+			gw, err := parseIP(fields[2])
+			if err != nil {
+				return "", fmt.Errorf("cannot parse route files: %v", err)
+			}
+			return gw, nil
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return "", fmt.Errorf("cannot parse route files: %v", err)
+	}
+
+	return "", fmt.Errorf("default gateway not found")
+}
+
+// parseIP takes the reverse hex IP address string from rooute
+// file and converts it to dotted decimal IPv4 format.
+func parseIP(str string) (string, error) {
+	if str == "" {
+		return "", fmt.Errorf("input is empty")
+	}
+	bytes, err := hex.DecodeString(str)
+	if err != nil {
+		return "", err
+	}
+	if len(bytes) != 4 {
+		return "", fmt.Errorf("invalid IPv4 address %s", str)
+	}
+	return net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0]).String(), nil
+}
+
 func fetchConfigFromDevice(logger *log.Logger, ctx context.Context, label string) ([]byte, error) {
 	for !labelExists(label) {
 		logger.Debug("config drive (%q) not found. Waiting...", label)
@@ -214,19 +340,19 @@ func fetchConfigFromDevice(logger *log.Logger, ctx context.Context, label string
 	return ioutil.ReadFile(filepath.Join(mnt, configDriveUserdataPath))
 }
 
-func fetchConfigFromMetadataService(f *resource.Fetcher) ([]byte, error) {
-	addr, err := getDHCPServerAddress()
+func fetchConfigFromMetadataService(ctx context.Context, f *resource.Fetcher) ([]byte, error) {
+	addr, err := getVirtualRouterAddress(ctx, f.Logger)
 	if err != nil {
 		return nil, err
 	}
 
-	metadataServiceUrl := url.URL{
+	metadataServiceURL := url.URL{
 		Scheme: "http",
 		Host:   addr,
 		Path:   "/latest/user-data",
 	}
 
-	res, err := f.FetchToBuffer(metadataServiceUrl, resource.FetchOptions{})
+	res, err := f.FetchToBuffer(metadataServiceURL, resource.FetchOptions{})
 
 	// the metadata server exists but doesn't contain any actual metadata,
 	// assume that there is no config specified