Support configurable dynamic port range

.changelog/11167.md

@@ -0,0 +1,3 @@
+```release-note:improvement
+client: Allow configuring minimum and maximum host ports used for dynamic ports
+```

api/nodes.go

@@ -554,6 +554,9 @@ type NodeResources struct {
 	Disk     NodeDiskResources
 	Networks []*NetworkResource
 	Devices  []*NodeDeviceResource
+
+	MinDynamicPort int
+	MaxDynamicPort int
 }
 
 type NodeCpuResources struct {

api/nodes_test.go

@@ -168,6 +168,9 @@ func TestNodes_Info(t *testing.T) {
 			result.ID, result.Datacenter)
 	}
 
+	require.Equal(t, 20000, result.NodeResources.MinDynamicPort)
+	require.Equal(t, 32000, result.NodeResources.MaxDynamicPort)
+
 	// Check that the StatusUpdatedAt field is being populated correctly
 	if result.StatusUpdatedAt < startTime {
 		t.Fatalf("start time: %v, status updated: %v", startTime, result.StatusUpdatedAt)

client/client.go

@@ -643,6 +643,18 @@ func (c *Client) init() error {
 
 	c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir)
 
+	reserved := "<none>"
+	if c.config.Node != nil && c.config.Node.ReservedResources != nil {
+		// Node should always be non-nil due to initialization in the
+		// agent package, but don't risk a panic just for a long line.
+		reserved = c.config.Node.ReservedResources.Networks.ReservedHostPorts
+	}
+	c.logger.Info("using dynamic ports",
+		"min", c.config.MinDynamicPort,
+		"max", c.config.MaxDynamicPort,
+		"reserved", reserved,
+	)
+
 	// Ensure cgroups are created on linux platform
 	if runtime.GOOS == "linux" && c.cpusetManager != nil {
 		err := c.cpusetManager.Init()
@@ -1385,6 +1397,8 @@ func (c *Client) setupNode() error {
 	}
 	if node.NodeResources == nil {
 		node.NodeResources = &structs.NodeResources{}
+		node.NodeResources.MinDynamicPort = c.config.MinDynamicPort
+		node.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
 	}
 	if node.ReservedResources == nil {
 		node.ReservedResources = &structs.NodeReservedResources{}
@@ -1496,6 +1510,14 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp
 			c.config.Node.NodeResources.Merge(response.NodeResources)
 			nodeHasChanged = true
 		}
+
+		response.NodeResources.MinDynamicPort = c.config.MinDynamicPort
+		response.NodeResources.MaxDynamicPort = c.config.MaxDynamicPort
+		if c.config.Node.NodeResources.MinDynamicPort != response.NodeResources.MinDynamicPort ||
+			c.config.Node.NodeResources.MaxDynamicPort != response.NodeResources.MaxDynamicPort {
+			nodeHasChanged = true
+		}
+
 	}
 
 	if nodeHasChanged {

client/client_test.go

@@ -737,11 +737,15 @@ func TestClient_Init(t *testing.T) {
 	defer os.RemoveAll(dir)
 	allocDir := filepath.Join(dir, "alloc")
 
+	config := config.DefaultConfig()
+	config.AllocDir = allocDir
+	config.StateDBFactory = cstate.GetStateDBFactory(true)
+
+	// Node is always initialized in agent.go:convertClientConfig()
+	config.Node = mock.Node()
+
 	client := &Client{
-		config: &config.Config{
-			AllocDir:       allocDir,
-			StateDBFactory: cstate.GetStateDBFactory(true),
-		},
+		config: config,
 		logger: testlog.HCLogger(t),
 	}
 

client/config/config.go

@@ -137,6 +137,12 @@ type Config struct {
 	// communicating with plugin subsystems over loopback
 	ClientMinPort uint
 
+	// MaxDynamicPort is the largest dynamic port generated
+	MaxDynamicPort int
+
+	// MinDynamicPort is the smallest dynamic port generated
+	MinDynamicPort int
+
 	// A mapping of directories on the host OS to attempt to embed inside each
 	// task's chroot.
 	ChrootEnv map[string]string
@@ -337,6 +343,8 @@ func DefaultConfig() *Config {
 		CNIInterfacePrefix: "eth",
 		HostNetworks:       map[string]*structs.ClientHostNetworkConfig{},
 		CgroupParent:       cgutil.DefaultCgroupParent,
+		MaxDynamicPort:     structs.DefaultMinDynamicPort,
+		MinDynamicPort:     structs.DefaultMaxDynamicPort,
 	}
 }
 

command/agent/agent.go

@@ -589,6 +589,8 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	}
 	conf.ClientMaxPort = uint(agentConfig.Client.ClientMaxPort)
 	conf.ClientMinPort = uint(agentConfig.Client.ClientMinPort)
+	conf.MaxDynamicPort = agentConfig.Client.MaxDynamicPort
+	conf.MinDynamicPort = agentConfig.Client.MinDynamicPort
 	conf.DisableRemoteExec = agentConfig.Client.DisableRemoteExec
 	if agentConfig.Client.TemplateConfig.FunctionBlacklist != nil {
 		conf.TemplateConfig.FunctionDenylist = agentConfig.Client.TemplateConfig.FunctionBlacklist

command/agent/command.go

@@ -372,6 +372,12 @@ func (c *Command) isValidConfig(config, cmdConfig *Config) bool {
 		return false
 	}
 
+	if config.Client.MinDynamicPort < 0 || config.Client.MaxDynamicPort > 65535 ||
+		config.Client.MinDynamicPort >= config.Client.MaxDynamicPort {
+		c.Ui.Error("Invalid dynamic port range")
+		return false
+	}
+
 	if !config.DevMode {
 		// Ensure that we have the directories we need to run.
 		if config.Server.Enabled && config.DataDir == "" {

command/agent/config.go

@@ -233,6 +233,14 @@ type ClientConfig struct {
 	// communicating with plugin subsystems
 	ClientMinPort int `hcl:"client_min_port"`
 
+	// MaxDynamicPort is the upper range of the dynamic ports that the client
+	// uses for allocations
+	MaxDynamicPort int `hcl:"max_dynamic_port"`
+
+	// MinDynamicPort is the lower range of the dynamic ports that the client
+	// uses for allocations
+	MinDynamicPort int `hcl:"min_dynamic_port"`
+
 	// Reserved is used to reserve resources from being used by Nomad. This can
 	// be used to target a certain utilization or to prevent Nomad from using a
 	// particular set of ports.
@@ -917,6 +925,8 @@ func DefaultConfig() *Config {
 			MaxKillTimeout:        "30s",
 			ClientMinPort:         14000,
 			ClientMaxPort:         14512,
+			MinDynamicPort:        20000,
+			MaxDynamicPort:        32000,
 			Reserved:              &Resources{},
 			GCInterval:            1 * time.Minute,
 			GCParallelDestroys:    2,
@@ -1598,6 +1608,12 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.ClientMinPort != 0 {
 		result.ClientMinPort = b.ClientMinPort
 	}
+	if b.MaxDynamicPort != 0 {
+		result.MaxDynamicPort = b.MaxDynamicPort
+	}
+	if b.MinDynamicPort != 0 {
+		result.MinDynamicPort = b.MinDynamicPort
+	}
 	if result.Reserved == nil && b.Reserved != nil {
 		reserved := *b.Reserved
 		result.Reserved = &reserved

command/agent/config_test.go

@@ -109,6 +109,8 @@ func TestConfig_Merge(t *testing.T) {
 			},
 			NetworkSpeed:      100,
 			CpuCompute:        100,
+			MinDynamicPort:    10001,
+			MaxDynamicPort:    10002,
 			MemoryMB:          100,
 			MaxKillTimeout:    "20s",
 			ClientMaxPort:     19996,
@@ -292,6 +294,8 @@ func TestConfig_Merge(t *testing.T) {
 			ClientMinPort:     22000,
 			NetworkSpeed:      105,
 			CpuCompute:        105,
+			MinDynamicPort:    10002,
+			MaxDynamicPort:    10003,
 			MemoryMB:          105,
 			MaxKillTimeout:    "50s",
 			DisableRemoteExec: false,

nomad/structs/network.go

@@ -8,11 +8,13 @@ import (
 )
 
 const (
-	// MinDynamicPort is the smallest dynamic port generated
-	MinDynamicPort = 20000
+	// DefaultMinDynamicPort is the smallest dynamic port generated by
+	// default
+	DefaultMinDynamicPort = 20000
 
-	// MaxDynamicPort is the largest dynamic port generated
-	MaxDynamicPort = 32000
+	// DefaultMaxDynamicPort is the largest dynamic port generated by
+	// default
+	DefaultMaxDynamicPort = 32000
 
 	// maxRandPortAttempts is the maximum number of attempt
 	// to assign a random port
@@ -39,6 +41,9 @@ type NetworkIndex struct {
 	AvailBandwidth map[string]int                  // Bandwidth by device
 	UsedPorts      map[string]Bitmap               // Ports by IP
 	UsedBandwidth  map[string]int                  // Bandwidth by device
+
+	MinDynamicPort int // The smallest dynamic port generated
+	MaxDynamicPort int // The largest dynamic port generated
 }
 
 // NewNetworkIndex is used to construct a new network index
@@ -48,6 +53,8 @@ func NewNetworkIndex() *NetworkIndex {
 		AvailBandwidth: make(map[string]int),
 		UsedPorts:      make(map[string]Bitmap),
 		UsedBandwidth:  make(map[string]int),
+		MinDynamicPort: DefaultMinDynamicPort,
+		MaxDynamicPort: DefaultMaxDynamicPort,
 	}
 }
 
@@ -136,6 +143,14 @@ func (idx *NetworkIndex) SetNode(node *Node) (collide bool) {
 		}
 	}
 
+	if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 {
+		idx.MinDynamicPort = node.NodeResources.MinDynamicPort
+	}
+
+	if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 {
+		idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort
+	}
+
 	return
 }
 
@@ -368,10 +383,10 @@ func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, erro
 			// lower memory usage.
 			var dynPorts []int
 			// TODO: its more efficient to find multiple dynamic ports at once
-			dynPorts, addrErr = getDynamicPortsStochastic(used, reservedIdx[port.HostNetwork], 1)
+			dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
 			if addrErr != nil {
 				// Fall back to the precise method if the random sampling failed.
-				dynPorts, addrErr = getDynamicPortsPrecise(used, reservedIdx[port.HostNetwork], 1)
+				dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
 				if addrErr != nil {
 					continue
 				}
@@ -450,13 +465,13 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
 		// lower memory usage.
 		var dynPorts []int
 		var dynErr error
-		dynPorts, dynErr = getDynamicPortsStochastic(used, ask.ReservedPorts, len(ask.DynamicPorts))
+		dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
 		if dynErr == nil {
 			goto BUILD_OFFER
 		}
 
 		// Fall back to the precise method if the random sampling failed.
-		dynPorts, dynErr = getDynamicPortsPrecise(used, ask.ReservedPorts, len(ask.DynamicPorts))
+		dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
 		if dynErr != nil {
 			err = dynErr
 			return
@@ -485,7 +500,7 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour
 // no ports have been allocated yet, the network ask and returns a set of unused
 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
 // means the ask can not be satisfied as the method does a precise search.
-func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int, error) {
+func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) {
 	// Create a copy of the used ports and apply the new reserves
 	var usedSet Bitmap
 	var err error
@@ -506,7 +521,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
 	}
 
 	// Get the indexes of the unset
-	availablePorts := usedSet.IndexesInRange(false, MinDynamicPort, MaxDynamicPort)
+	availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort))
 
 	// Randomize the amount we need
 	if len(availablePorts) < numDyn {
@@ -527,7 +542,7 @@ func getDynamicPortsPrecise(nodeUsed Bitmap, reserved []Port, numDyn int) ([]int
 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
 // does not mean the ask can not be satisfied as the method has a fixed amount
 // of random probes and if these fail, the search is aborted.
-func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int) ([]int, error) {
+func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) {
 	var reserved, dynamic []int
 	for _, port := range reservedPorts {
 		reserved = append(reserved, port.Value)
@@ -541,7 +556,7 @@ func getDynamicPortsStochastic(nodeUsed Bitmap, reservedPorts []Port, count int)
 			return nil, fmt.Errorf("stochastic dynamic port selection failed")
 		}
 
-		randPort := MinDynamicPort + rand.Intn(MaxDynamicPort-MinDynamicPort)
+		randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort)
 		if nodeUsed != nil && nodeUsed.Check(uint(randPort)) {
 			goto PICK
 		}

nomad/structs/network_test.go

@@ -323,7 +323,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 		},
 		ReservedResources: &NodeReservedResources{
 			Networks: NodeReservedNetworkResources{
-				ReservedHostPorts: fmt.Sprintf("%d-%d", MinDynamicPort, MaxDynamicPort-1),
+				ReservedHostPorts: fmt.Sprintf("%d-%d", idx.MinDynamicPort, idx.MaxDynamicPort-1),
 			},
 		},
 	}
@@ -346,8 +346,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be one dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+	if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
 	}
 }
 
@@ -646,7 +646,7 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 			},
 		},
 	}
-	for i := MinDynamicPort; i < MaxDynamicPort; i++ {
+	for i := idx.MinDynamicPort; i < idx.MaxDynamicPort; i++ {
 		n.Reserved.Networks[0].ReservedPorts = append(n.Reserved.Networks[0].ReservedPorts, Port{Value: i})
 	}
 
@@ -669,8 +669,8 @@ func TestNetworkIndex_AssignNetwork_Dynamic_Contention_Old(t *testing.T) {
 	if len(offer.DynamicPorts) != 1 {
 		t.Fatalf("There should be three dynamic ports")
 	}
-	if p := offer.DynamicPorts[0].Value; p != MaxDynamicPort {
-		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, MaxDynamicPort)
+	if p := offer.DynamicPorts[0].Value; p != idx.MaxDynamicPort {
+		t.Fatalf("Dynamic Port: should have been assigned %d; got %d", p, idx.MaxDynamicPort)
 	}
 }
 

nomad/structs/structs.go

@@ -2801,6 +2801,9 @@ type NodeResources struct {
 	Networks     Networks
 	NodeNetworks []*NodeNetworkResource
 	Devices      []*NodeDeviceResource
+
+	MinDynamicPort int
+	MaxDynamicPort int
 }
 
 func (n *NodeResources) Copy() *NodeResources {

website/content/docs/configuration/client.mdx

@@ -65,6 +65,14 @@ client {
 - `memory_total_mb` `(int:0)` - Specifies an override for the total memory. If set,
   this value overrides any detected memory.
 
+- `min_dynamic_port` `(int:20000)` - Specifies the minimum dynamic port to be
+  assigned. Individual ports and ranges of ports may be excluded from dynamic
+  port assignment via [`reserved`](#reserved-parameters) parameters.
+
+- `max_dynamic_port` `(int:32000)` - Specifies the maximum dynamic port to be
+  assigned. Individual ports and ranges of ports may be excluded from dynamic
+  port assignment via [`reserved`](#reserved-parameters) parameters.
+
 - `node_class` `(string: "")` - Specifies an arbitrary string used to logically
   group client nodes by user-defined class. This can be used during job
   placement as a filter.