Reserve Client Resources + Config Validation

client/client.go

@@ -157,6 +157,9 @@ func NewClient(cfg *config.Config) (*Client, error) {
 		return nil, fmt.Errorf("driver setup failed: %v", err)
 	}
 
+	// Setup the reserved resources
+	c.reservePorts()
+
 	// Set up the known servers list
 	c.SetServers(c.config.Servers)
 
@@ -537,6 +540,9 @@ func (c *Client) setupNode() error {
 	if node.Resources == nil {
 		node.Resources = &structs.Resources{}
 	}
+	if node.Reserved == nil {
+		node.Reserved = &structs.Resources{}
+	}
 	if node.Datacenter == "" {
 		node.Datacenter = "dc1"
 	}
@@ -550,6 +556,49 @@ func (c *Client) setupNode() error {
 	return nil
 }
 
+// reservePorts is used to reserve ports on the fingerprinted network devices.
+func (c *Client) reservePorts() {
+	c.configLock.RLock()
+	defer c.configLock.RUnlock()
+	global := c.config.GloballyReservedPorts
+	if len(global) == 0 {
+		return
+	}
+
+	node := c.config.Node
+	networks := node.Resources.Networks
+	reservedIndex := make(map[string]*structs.NetworkResource, len(networks))
+	for _, resNet := range node.Reserved.Networks {
+		reservedIndex[resNet.IP] = resNet
+	}
+
+	// Go through each network device and reserve ports on it.
+	for _, net := range networks {
+		res, ok := reservedIndex[net.IP]
+		if !ok {
+			res = net.Copy()
+			reservedIndex[net.IP] = res
+		}
+
+		for _, portVal := range global {
+			p := structs.Port{Value: portVal}
+			res.ReservedPorts = append(res.ReservedPorts, p)
+		}
+	}
+
+	// Clear the reserved networks.
+	if node.Reserved == nil {
+		node.Reserved = new(structs.Resources)
+	} else {
+		node.Reserved.Networks = nil
+	}
+
+	// Restore the reserved networks
+	for _, net := range reservedIndex {
+		node.Reserved.Networks = append(node.Reserved.Networks, net)
+	}
+}
+
 // fingerprint is used to fingerprint the client and setup the node
 func (c *Client) fingerprint() error {
 	whitelist := c.config.ReadStringListToMap("fingerprint.whitelist")

client/config/config.go

@@ -58,13 +58,17 @@ type Config struct {
 	Node *structs.Node
 
 	// ClientMaxPort is the upper range of the ports that the client uses for
-	// communicating with plugin subsystems
+	// communicating with plugin subsystems over loopback
 	ClientMaxPort uint
 
 	// ClientMinPort is the lower range of the ports that the client uses for
-	// communicating with plugin subsystems
+	// communicating with plugin subsystems over loopback
 	ClientMinPort uint
 
+	// GloballyReservedPorts are ports that are reserved across all network
+	// devices and IPs.
+	GloballyReservedPorts []int
+
 	// Options provides arbitrary key-value configuration for nomad internals,
 	// like fingerprinters and drivers. The format is:
 	//

command/agent/agent.go

@@ -224,6 +224,19 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
 	}
 	conf.Node.HTTPAddr = httpAddr
 	conf.Version = a.config.Version
+
+	// Reserve resources on the node.
+	r := conf.Node.Reserved
+	if r == nil {
+		r = new(structs.Resources)
+		conf.Node.Reserved = r
+	}
+	r.CPU = a.config.Client.Reserved.CPU
+	r.MemoryMB = a.config.Client.Reserved.MemoryMB
+	r.DiskMB = a.config.Client.Reserved.DiskMB
+	r.IOPS = a.config.Client.Reserved.IOPS
+	conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts
+
 	return conf, nil
 }
 

command/agent/config-test-fixtures/basic.hcl

@@ -0,0 +1,83 @@
+region = "foobar"
+datacenter = "dc2"
+name = "my-web"
+data_dir = "/tmp/nomad"
+log_level = "ERR"
+bind_addr = "192.168.0.1"
+enable_debug = true
+ports {
+	http = 1234
+	rpc = 2345
+	serf = 3456
+}
+addresses {
+	http = "127.0.0.1"
+	rpc = "127.0.0.2"
+	serf = "127.0.0.3"
+}
+advertise {
+	rpc = "127.0.0.3"
+	serf = "127.0.0.4"
+}
+client {
+	enabled = true
+	state_dir = "/tmp/client-state"
+	alloc_dir = "/tmp/alloc"
+	servers = ["a.b.c:80", "127.0.0.1:1234"]
+	node_class = "linux-medium-64bit"
+	meta {
+		foo = "bar"
+		baz = "zip"
+	}
+	options {
+		foo = "bar"
+		baz = "zip"
+	}
+	network_interface = "eth0"
+	network_speed = 100
+	reserved {
+		cpu = 10
+		memory = 10
+		disk = 10
+		iops = 10
+		reserved_ports = "1,100,10-12"
+	}
+	client_min_port = 1000
+	client_max_port = 2000
+    max_kill_timeout = "10s"
+}
+server {
+	enabled = true
+	bootstrap_expect = 5
+	data_dir = "/tmp/data"
+	protocol_version = 3
+	num_schedulers = 2
+	enabled_schedulers = ["test"]
+	node_gc_threshold = "12h"
+	heartbeat_grace   = "30s"
+	retry_join = [ "1.1.1.1", "2.2.2.2" ]
+	start_join = [ "1.1.1.1", "2.2.2.2" ]
+	retry_max = 3
+	retry_interval = "15s"
+	rejoin_after_leave = true
+}
+telemetry {
+	statsite_address = "127.0.0.1:1234"
+	statsd_address = "127.0.0.1:2345"
+	disable_hostname = true
+}
+leave_on_interrupt = true
+leave_on_terminate = true
+enable_syslog = true
+syslog_facility = "LOCAL1"
+disable_update_check = true
+disable_anonymous_signature = true
+atlas {
+	infrastructure = "armon/test"
+	token = "abcd"
+	join = true
+	endpoint = "127.0.0.1:1234"
+}
+http_api_response_headers {
+	Access-Control-Allow-Origin = "*"
+}