From bdf2555b385f547cbf613d1636eac103a1c58ce8 Mon Sep 17 00:00:00 2001
From: Jasmine Dahilig <jasmine@hashicorp.com>
Date: Fri, 4 Jun 2021 12:38:46 -0700
Subject: [PATCH] deployment query rate limit (#10706)

---
 CHANGELOG.md            |  1 +
 command/agent/agent.go  | 10 ++++++++++
 command/agent/config.go |  8 ++++++++
 nomad/config.go         |  6 ++++++
 nomad/server.go         |  2 +-
 5 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 61c16a9b2da7..6eef86ca8cb2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ IMPROVEMENTS:
 * client/fingerprint: Consul fingerprinter probes for additional enterprise and connect related attributes [[GH-10699](https://github.com/hashicorp/nomad/pull/10699)]
 * consul/connect: Only schedule connect tasks on nodes where connect is enabled in Consul [[GH-10702](https://github.com/hashicorp/nomad/pull/10702)]
 * csi: Validate that `volume` blocks for CSI volumes include the required `attachment_mode` and `access_mode` fields. [[GH-10651](https://github.com/hashicorp/nomad/issues/10651)]
+* server: Make deployment rate limiting configurable for high volume loads [[GH-10706](https://github.com/hashicorp/nomad/pull/10706)]
 
 BUG FIXES:
 * api: Fixed event stream connection initialization when there are no events to send [[GH-10637](https://github.com/hashicorp/nomad/issues/10637)]
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 18cd812c3b2f..ff20dc99ab44 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -29,6 +29,7 @@ import (
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad"
+	"github.com/hashicorp/nomad/nomad/deploymentwatcher"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/raft"
@@ -420,6 +421,15 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		conf.RPCMaxConnsPerClient = limit
 	}
 
+	// Set deployment rate limit
+	if rate := agentConfig.Server.DeploymentQueryRateLimit; rate == 0 {
+		conf.DeploymentQueryRateLimit = deploymentwatcher.LimitStateQueriesPerSecond
+	} else if rate > 0 {
+		conf.DeploymentQueryRateLimit = rate
+	} else {
+		return nil, fmt.Errorf("deploy_query_rate_limit must be greater than 0")
+	}
+
 	// Add Enterprise license configs
 	conf.LicenseEnv = agentConfig.Server.LicenseEnv
 	conf.LicensePath = agentConfig.Server.LicensePath
diff --git a/command/agent/config.go b/command/agent/config.go
index c973c573831b..02fdb55880b4 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -516,6 +516,10 @@ type ServerConfig struct {
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
 
 	Search *Search `hcl:"search"`
+
+	// DeploymentQueryRateLimit is in queries per second and is used by the
+	// DeploymentWatcher to throttle the amount of simultaneously deployments
+	DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"`
 }
 
 // Search is used in servers to configure search API options.
@@ -1522,6 +1526,10 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.DefaultSchedulerConfig = &c
 	}
 
+	if b.DeploymentQueryRateLimit != 0 {
+		result.DeploymentQueryRateLimit = b.DeploymentQueryRateLimit
+	}
+
 	if b.Search != nil {
 		result.Search = &Search{FuzzyEnabled: b.Search.FuzzyEnabled}
 		if b.Search.LimitQuery > 0 {
diff --git a/nomad/config.go b/nomad/config.go
index 05db46fb383f..8c1f38a91177 100644
--- a/nomad/config.go
+++ b/nomad/config.go
@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/memberlist"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/deploymentwatcher"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/scheduler"
@@ -363,6 +364,10 @@ type Config struct {
 	// AgentShutdown is used to call agent.Shutdown from the context of a Server
 	// It is used primarily for licensing
 	AgentShutdown func() error
+
+	// DeploymentQueryRateLimit is in queries per second and is used by the
+	// DeploymentWatcher to throttle the amount of simultaneously deployments
+	DeploymentQueryRateLimit float64
 }
 
 // CheckVersion is used to check if the ProtocolVersion is valid
@@ -448,6 +453,7 @@ func DefaultConfig() *Config {
 				ServiceSchedulerEnabled: false,
 			},
 		},
+		DeploymentQueryRateLimit: deploymentwatcher.LimitStateQueriesPerSecond,
 	}
 
 	// Enable all known schedulers by default
diff --git a/nomad/server.go b/nomad/server.go
index 4c55b3c8f622..91d42f29e6c4 100644
--- a/nomad/server.go
+++ b/nomad/server.go
@@ -1021,7 +1021,7 @@ func (s *Server) setupDeploymentWatcher() error {
 		raftShim,
 		s.staticEndpoints.Deployment,
 		s.staticEndpoints.Job,
-		deploymentwatcher.LimitStateQueriesPerSecond,
+		s.config.DeploymentQueryRateLimit,
 		deploymentwatcher.CrossDeploymentUpdateBatchDuration,
 	)