From c74ab42419fd44fb9f3dfdaf3c0514063489ec9d Mon Sep 17 00:00:00 2001
From: Jasmine Dahilig <jasmine@hashicorp.com>
Date: Fri, 4 Jun 2021 12:38:46 -0700
Subject: [PATCH] deployment query rate limit (#10706)

---
 CHANGELOG.md            |  4 ++++
 command/agent/agent.go  | 10 ++++++++++
 command/agent/config.go |  8 ++++++++
 nomad/config.go         |  6 ++++++
 nomad/server.go         |  2 +-
 5 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72cc4e87c4d4..a3b22ac1ac23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.0.2-backport (Unreleased)
+
+* server: Make deployment rate limiting configurable for high volume loads [[GH-10706](https://github.com/hashicorp/nomad/pull/10706)]
+
 ## 1.0.2 (January 14, 2020)
 
 IMPROVEMENTS:
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 5207451bb4f9..e8a06e8aac44 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -27,6 +27,7 @@ import (
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/nomad"
+	"github.com/hashicorp/nomad/nomad/deploymentwatcher"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/raft"
@@ -418,6 +419,15 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		conf.RPCMaxConnsPerClient = limit
 	}
 
+	// Set deployment rate limit
+	if rate := agentConfig.Server.DeploymentQueryRateLimit; rate == 0 {
+		conf.DeploymentQueryRateLimit = deploymentwatcher.LimitStateQueriesPerSecond
+	} else if rate > 0 {
+		conf.DeploymentQueryRateLimit = rate
+	} else {
+		return nil, fmt.Errorf("deploy_query_rate_limit must be greater than 0")
+	}
+
 	return conf, nil
 }
 
diff --git a/command/agent/config.go b/command/agent/config.go
index b9d108e2f5ea..5162500cd392 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -496,6 +496,10 @@ type ServerConfig struct {
 
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
+
+	// DeploymentQueryRateLimit is in queries per second and is used by the
+	// DeploymentWatcher to throttle the amount of simultaneously deployments
+	DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"`
 }
 
 // ServerJoin is used in both clients and servers to bootstrap connections to
@@ -1424,6 +1428,10 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.DefaultSchedulerConfig = &c
 	}
 
+	if b.DeploymentQueryRateLimit != 0 {
+		result.DeploymentQueryRateLimit = b.DeploymentQueryRateLimit
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)
 
diff --git a/nomad/config.go b/nomad/config.go
index 08e4f562f5c3..339242a6a2a9 100644
--- a/nomad/config.go
+++ b/nomad/config.go
@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/memberlist"
 	"github.com/hashicorp/nomad/helper/pluginutils/loader"
 	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/nomad/deploymentwatcher"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/scheduler"
@@ -354,6 +355,10 @@ type Config struct {
 	// AgentShutdown is used to call agent.Shutdown from the context of a Server
 	// It is used primarily for licensing
 	AgentShutdown func() error
+
+	// DeploymentQueryRateLimit is in queries per second and is used by the
+	// DeploymentWatcher to throttle the amount of simultaneously deployments
+	DeploymentQueryRateLimit float64
 }
 
 // CheckVersion is used to check if the ProtocolVersion is valid
@@ -438,6 +443,7 @@ func DefaultConfig() *Config {
 				ServiceSchedulerEnabled: false,
 			},
 		},
+		DeploymentQueryRateLimit: deploymentwatcher.LimitStateQueriesPerSecond,
 	}
 
 	// Enable all known schedulers by default
diff --git a/nomad/server.go b/nomad/server.go
index 05735fe4ab39..5255c322355d 100644
--- a/nomad/server.go
+++ b/nomad/server.go
@@ -1017,7 +1017,7 @@ func (s *Server) setupDeploymentWatcher() error {
 		raftShim,
 		s.staticEndpoints.Deployment,
 		s.staticEndpoints.Job,
-		deploymentwatcher.LimitStateQueriesPerSecond,
+		s.config.DeploymentQueryRateLimit,
 		deploymentwatcher.CrossDeploymentUpdateBatchDuration,
 	)