From 57db4bcce6f3d4450526ea8b27c74db0d189da3e Mon Sep 17 00:00:00 2001 From: James Phillips Date: Wed, 24 Aug 2016 17:33:53 -0700 Subject: [PATCH 1/6] Adds performance tuning capability for Raft, detuned defaults, and supplemental docs. --- command/agent/agent.go | 5 ++ command/agent/agent_test.go | 41 +++++++++ command/agent/config.go | 18 +++- command/agent/config_test.go | 14 +++ consul/config.go | 19 ++++ consul/server.go | 11 +-- testutil/server.go | 47 ++++++---- .../source/docs/agent/options.html.markdown | 18 ++++ .../source/docs/agent/telemetry.html.markdown | 4 +- .../docs/guides/dns-cache.html.markdown | 2 + .../docs/guides/performance.html.markdown | 86 +++++++++++++++++++ .../docs/upgrade-specific.html.markdown | 21 +++++ website/source/layouts/docs.erb | 4 + 13 files changed, 263 insertions(+), 27 deletions(-) create mode 100644 website/source/docs/guides/performance.html.markdown diff --git a/command/agent/agent.go b/command/agent/agent.go index 1c41bddb4683..6284a25f020b 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -261,6 +261,11 @@ func (a *Agent) consulConfig() *consul.Config { // Apply dev mode base.DevMode = a.config.DevMode + // Apply performance factors + if a.config.Performance.RaftMultiplier > 0 { + base.ScaleRaft(a.config.Performance.RaftMultiplier) + } + // Override with our config if a.config.Datacenter != "" { base.Datacenter = a.config.Datacenter diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index af82a00e8fe6..b4fc74a2ca68 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/consul/consul" "github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/consul/testutil" + "github.com/hashicorp/raft" ) const ( @@ -191,6 +192,46 @@ func TestAgent_CheckAdvertiseAddrsSettings(t *testing.T) { } } +func TestAgent_CheckPerformanceSettings(t *testing.T) { + // Try a default config. + { + c := nextConfig() + c.ConsulConfig = nil + dir, agent := makeAgent(t, c) + defer os.RemoveAll(dir) + defer agent.Shutdown() + + raftMult := time.Duration(consul.DefaultRaftMultiplier) + r := agent.consulConfig().RaftConfig + def := raft.DefaultConfig() + if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout || + r.ElectionTimeout != raftMult*def.ElectionTimeout || + r.CommitTimeout != raftMult*def.CommitTimeout || + r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout { + t.Fatalf("bad: %#v", *r) + } + } + + // Try a multiplier. + { + c := nextConfig() + c.Performance.RaftMultiplier = 99 + dir, agent := makeAgent(t, c) + defer os.RemoveAll(dir) + defer agent.Shutdown() + + const raftMult time.Duration = 99 + r := agent.consulConfig().RaftConfig + def := raft.DefaultConfig() + if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout || + r.ElectionTimeout != raftMult*def.ElectionTimeout || + r.CommitTimeout != raftMult*def.CommitTimeout || + r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout { + t.Fatalf("bad: %#v", *r) + } + } +} + func TestAgent_ReconnectConfigSettings(t *testing.T) { c := nextConfig() func() { diff --git a/command/agent/config.go b/command/agent/config.go index 3675e100f8f8..92534d7ba960 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -111,6 +111,14 @@ type DNSConfig struct { DisableCompression bool `mapstructure:"disable_compression"` } +// Performance is used to tune the performance of Consul's subsystems. +type Performance struct { + // RaftMultiplier is an integer multiplier used to scale Raft timing + // parameters: HeartbeatTimeout, ElectionTimeout, CommitTimeout, and + // LeaderLeaseTimeout. + RaftMultiplier uint `mapstructure:"raft_multiplier"` +} + // Telemetry is the telemetry configuration for the server type Telemetry struct { // StatsiteAddr is the address of a statsite instance. If provided, @@ -205,10 +213,13 @@ type Telemetry struct { // Some of this is configurable as CLI flags, but most must // be set using a configuration file. type Config struct { - // DevMode enables a fast-path mode of opertaion to bring up an in-memory + // DevMode enables a fast-path mode of operation to bring up an in-memory // server with minimal configuration. Useful for developing Consul. DevMode bool `mapstructure:"-"` + // Performance is used to tune the performance of Consul's subsystems. + Performance Performance `mapstructure:"performance"` + // Bootstrap is used to bring up the first Consul server, and // permits that node to elect itself leader Bootstrap bool `mapstructure:"bootstrap"` @@ -1085,6 +1096,11 @@ func DecodeCheckDefinition(raw interface{}) (*CheckDefinition, error) { func MergeConfig(a, b *Config) *Config { var result Config = *a + // Propagate non-default performance settings + if b.Performance.RaftMultiplier > 0 { + result.Performance.RaftMultiplier = b.Performance.RaftMultiplier + } + // Copy the strings if they're set if b.Bootstrap { result.Bootstrap = true diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 3b4de35ed08e..34c9b47206aa 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -957,6 +957,17 @@ func TestDecodeConfig_invalidKeys(t *testing.T) { } } +func TestDecodeConfig_Performance(t *testing.T) { + input := `{"performance": { "raft_multiplier": 3 }}` + config, err := DecodeConfig(bytes.NewReader([]byte(input))) + if err != nil { + t.Fatalf("err: %s", err) + } + if config.Performance.RaftMultiplier != 3 { + t.Fatalf("bad: multiplier isn't set: %#v", config) + } +} + func TestDecodeConfig_Services(t *testing.T) { input := `{ "services": [ @@ -1382,6 +1393,9 @@ func TestMergeConfig(t *testing.T) { } b := &Config{ + Performance: Performance{ + RaftMultiplier: 99, + }, Bootstrap: true, BootstrapExpect: 3, Datacenter: "dc2", diff --git a/consul/config.go b/consul/config.go index 5bde0951d9fc..0115b171e152 100644 --- a/consul/config.go +++ b/consul/config.go @@ -17,6 +17,10 @@ const ( DefaultDC = "dc1" DefaultLANSerfPort = 8301 DefaultWANSerfPort = 8302 + + // See docs/guides/performance.html for information on how this value + // was obtained. + DefaultRaftMultiplier uint = 5 ) var ( @@ -333,6 +337,7 @@ func DefaultConfig() *Config { // Enable interoperability with unversioned Raft library, and don't // start using new ID-based features yet. conf.RaftConfig.ProtocolVersion = 1 + conf.ScaleRaft(DefaultRaftMultiplier) // Disable shutdown on removal conf.RaftConfig.ShutdownOnRemove = false @@ -340,6 +345,20 @@ func DefaultConfig() *Config { return conf } +// ScaleRaft sets the config to have Raft timing parameters scaled by the given +// performance multiplier. This is done in an idempotent way so it's not tricky +// to call this when composing configurations and potentially calling this +// multiple times on the same structure. +func (c *Config) ScaleRaft(raftMultRaw uint) { + raftMult := time.Duration(raftMultRaw) + + def := raft.DefaultConfig() + c.RaftConfig.HeartbeatTimeout = raftMult * def.HeartbeatTimeout + c.RaftConfig.ElectionTimeout = raftMult * def.ElectionTimeout + c.RaftConfig.CommitTimeout = raftMult * def.CommitTimeout + c.RaftConfig.LeaderLeaseTimeout = raftMult * def.LeaderLeaseTimeout +} + func (c *Config) tlsConfig() *tlsutil.Config { tlsConf := &tlsutil.Config{ VerifyIncoming: c.VerifyIncoming, diff --git a/consul/server.go b/consul/server.go index ab240ce45f93..94f158ed056b 100644 --- a/consul/server.go +++ b/consul/server.go @@ -820,11 +820,12 @@ func (s *Server) Stats() map[string]map[string]string { s.remoteLock.RUnlock() stats := map[string]map[string]string{ "consul": map[string]string{ - "server": "true", - "leader": fmt.Sprintf("%v", s.IsLeader()), - "leader_addr": string(s.raft.Leader()), - "bootstrap": fmt.Sprintf("%v", s.config.Bootstrap), - "known_datacenters": toString(uint64(numKnownDCs)), + "server": "true", + "leader": fmt.Sprintf("%v", s.IsLeader()), + "leader_addr": string(s.raft.Leader()), + "bootstrap": fmt.Sprintf("%v", s.config.Bootstrap), + "known_datacenters": toString(uint64(numKnownDCs)), + "leader_lease_timeout": fmt.Sprintf("%v", s.config.RaftConfig.LeaderLeaseTimeout), }, "raft": s.raft.Stats(), "serf_lan": s.serfLAN.Stats(), diff --git a/testutil/server.go b/testutil/server.go index 8a88a8b1c787..aad60e3866b6 100644 --- a/testutil/server.go +++ b/testutil/server.go @@ -32,6 +32,11 @@ import ( // offset is used to atomically increment the port numbers. var offset uint64 +// TestPerformanceConfig configures the performance parameters. +type TestPerformanceConfig struct { + RaftMultiplier uint `json:"raft_multiplier,omitempty"` +} + // TestPortConfig configures the various ports used for services // provided by the Consul server. type TestPortConfig struct { @@ -51,20 +56,21 @@ type TestAddressConfig struct { // TestServerConfig is the main server configuration struct. type TestServerConfig struct { - NodeName string `json:"node_name"` - Bootstrap bool `json:"bootstrap,omitempty"` - Server bool `json:"server,omitempty"` - DataDir string `json:"data_dir,omitempty"` - Datacenter string `json:"datacenter,omitempty"` - DisableCheckpoint bool `json:"disable_update_check"` - LogLevel string `json:"log_level,omitempty"` - Bind string `json:"bind_addr,omitempty"` - Addresses *TestAddressConfig `json:"addresses,omitempty"` - Ports *TestPortConfig `json:"ports,omitempty"` - ACLMasterToken string `json:"acl_master_token,omitempty"` - ACLDatacenter string `json:"acl_datacenter,omitempty"` - ACLDefaultPolicy string `json:"acl_default_policy,omitempty"` - Stdout, Stderr io.Writer `json:"-"` + NodeName string `json:"node_name"` + Performance *TestPerformanceConfig `json:"performance,omitempty"` + Bootstrap bool `json:"bootstrap,omitempty"` + Server bool `json:"server,omitempty"` + DataDir string `json:"data_dir,omitempty"` + Datacenter string `json:"datacenter,omitempty"` + DisableCheckpoint bool `json:"disable_update_check"` + LogLevel string `json:"log_level,omitempty"` + Bind string `json:"bind_addr,omitempty"` + Addresses *TestAddressConfig `json:"addresses,omitempty"` + Ports *TestPortConfig `json:"ports,omitempty"` + ACLMasterToken string `json:"acl_master_token,omitempty"` + ACLDatacenter string `json:"acl_datacenter,omitempty"` + ACLDefaultPolicy string `json:"acl_default_policy,omitempty"` + Stdout, Stderr io.Writer `json:"-"` } // ServerConfigCallback is a function interface which can be @@ -79,11 +85,14 @@ func defaultServerConfig() *TestServerConfig { return &TestServerConfig{ NodeName: fmt.Sprintf("node%d", idx), DisableCheckpoint: true, - Bootstrap: true, - Server: true, - LogLevel: "debug", - Bind: "127.0.0.1", - Addresses: &TestAddressConfig{}, + Performance: &TestPerformanceConfig{ + RaftMultiplier: 1, + }, + Bootstrap: true, + Server: true, + LogLevel: "debug", + Bind: "127.0.0.1", + Addresses: &TestAddressConfig{}, Ports: &TestPortConfig{ DNS: 20000 + idx, HTTP: 21000 + idx, diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index 952ec54d1f87..208fb38b3333 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -576,6 +576,24 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass * `node_name` Equivalent to the [`-node` command-line flag](#_node). +* `performance` Available in Consul 0.7 and + later, this is a nested object that allows tuning the performance of different subsystems in + Consul. See the [Server Performance](/docs/guides/performance.html) guide for more details. The + following parameters are available: + * `raft_multiplier` - An integer + multiplier used by Consul servers to scale key Raft timing parameters. Tuning this affects + the time it takes Consul to detect leader failures and to perform leader elections, at the + expense of requiring more network and CPU resources for better performance.

A value + of 0, the default, means that Consul will use a lower-performance timing that's suitable for + [minimal Consul servers](/docs/guides/performance.html#minumum), currently equivalent to + setting this to a value of 5 (this default may be changed in future versions of Consul, + depending if the target minimum server profile changes). Above 0, higher values imply lower + levels of performance. Setting this to a value of 1 will configure Raft to its + highest-performance mode, equivalent to the default timing of Consul prior to 0.7, and is + recommended for [production Consul servers](/docs/guides/performance.html#production). See + the note on [last contact](/docs/guides/performance.html#last-contact) timing for more + details on tuning this parameter. + * `ports` This is a nested object that allows setting the bind ports for the following keys: * `dns` - The DNS server, -1 to disable. Default 8600. diff --git a/website/source/docs/agent/telemetry.html.markdown b/website/source/docs/agent/telemetry.html.markdown index 1072609574a7..c662ae0d00be 100644 --- a/website/source/docs/agent/telemetry.html.markdown +++ b/website/source/docs/agent/telemetry.html.markdown @@ -129,8 +129,8 @@ These metrics are used to monitor the health of the Consul servers. timer - `consul.raft.leader.lastContact` - This measures the time that a Consul server was last contacted by the leader (will be zero on the leader itself). This is a general indicator of latency in the Raft subsystem, and gives a general indicator of how far behind [stale](/docs/agent/http.html#consistency) queries will be. + `consul.raft.leader.lastContact` + This will only be emitted by the Raft leader and measures the time since the leader was last able to contact the follower nodes when checking its leader lease. It can be used as a measure for how stable the Raft timing is and how close the leader is to timing out its lease.

The lease timeout is 500 ms times the [`raft_multiplier` configuration](/docs/agent/options.html#raft_multiplier), so this telemetry value should not be getting close to that configured value, otherwise the Raft timing is marginal and might need to be tuned, or more powerful servers might be needed. See the [Server Performance](/docs/guides/performance.html) guide for more details. ms timer diff --git a/website/source/docs/guides/dns-cache.html.markdown b/website/source/docs/guides/dns-cache.html.markdown index dbeffe355411..87f0b3283c28 100644 --- a/website/source/docs/guides/dns-cache.html.markdown +++ b/website/source/docs/guides/dns-cache.html.markdown @@ -20,6 +20,7 @@ for each lookup and can potentially exhaust the query throughput of a cluster. For this reason, Consul provides a number of tuning parameters that can customize how DNS queries are handled. + ## Stale Reads Stale reads can be used to reduce latency and increase the throughput @@ -60,6 +61,7 @@ client and Consul and set the cache values appropriately. In many cases "appropriately" simply is turning negative response caching off to get the best recovery time when a service becomes available again. + ## TTL Values TTL values can be set to allow DNS results to be cached downstream of Consul. Higher diff --git a/website/source/docs/guides/performance.html.markdown b/website/source/docs/guides/performance.html.markdown new file mode 100644 index 000000000000..dbc6a2965505 --- /dev/null +++ b/website/source/docs/guides/performance.html.markdown @@ -0,0 +1,86 @@ +--- +layout: "docs" +page_title: "Server Performance" +sidebar_current: "docs-guides-performance" +description: |- + Consul requires different amounts of compute resources, depending on cluster size and expected workload. This guide provides guidance on choosing compute resources. +--- + +# Server Performance + +Since Consul servers run a [consensus protocol](/docs/internals/consensus.html) to +process all write operations and are contacted on nearly all read operations, server +performance is critical for overall throughput and health of a Consul cluster. Servers +are generally I/O bound for writes because the underlying Raft log store performs a sync +to disk every time an entry is appended. Servers are generally CPU bound for reads since +reads work from a fully in-memory data store that is optimized for concurrent access. + + +## Minimum Server Requirements + +In Consul 0.7, the default server [performance parameters](/docs/agent/options.html#performance) +were tuned to allow Consul to run reliably (but relatively slowly) on a server cluster of three +[AWS t2.micro](https://aws.amazon.com/ec2/instance-types/) instances. These thresholds +were determined empirically using a leader instance that was under sufficient read, write, +and network load to cause it to permanently be at zero CPU credits, forcing it to the baseline +performance mode for that instance type. Real-world workloads typically have more bursts of +activity, so this is a conservative and pessimistic tuning strategy. + +This default was chosen based on feedback from users, many of whom wanted a low cost way +to run small production or development clusters with low cost compute resources, at the +expense of some performance in leader failure detection and leader election times. + +The default performance configuration is equivalent to this: + +```javascript +{ + "performance": { + "raft_multiplier": 5 + } +} +``` + + +## Production Server Requirements + +When running Consul 0.7 and later in production, it is recommended to configure the server +[performance parameters](/docs/agent/options.html#performance) back to Consul's original +high-performance settings. This will let Consul servers detect a failed leader and complete +leader elections much more quickly than the default configuration which extends key Raft +timeouts by a factor of 5, so it quite slow during these events. + +The high performance configuration is simple and looks like this: + +```javascript +{ + "performance": { + "raft_multiplier": 1 + } +} +``` + +It's best to benchmark with a realistic workload when choosing a production server for Consul. +Here are some general recommendations: + +* For write-heavy workloads, disk speed on the servers is key for performance. Use SSDs or +another fast disk technology for the best write throughput. + +* Spurious leader elections can be caused by networking issues between +the servers or lack of CPU. Users in cloud environments often bump their servers up to the next +instance class with improved networking and CPU until leader elections stabilize, and in Consul +0.7 or later the [performance parameters](/docs/agent/options.html#performance) configuration +now gives you the option to trade off performance instead of upsizing servers. You can use the +[`consul.raft.leader.lastContact` telemetry](/docs/agent/telemetry.html#last-contact) to help +observe how the Raft timing is performing and decide if de-tuning Raft performance or adding +more powerful servers might be needed. + +* For DNS-heavy workloads, configuring all Consul agents in a cluster with the +[`allow_stale`](/docs/agent/options.html#allow_stale) configuration option will allow reads to +scale across all Consul servers, not just the leader. See [Stale Reads](/docs/guides/dns-cache.html#stale) +in the [DNS Caching](/docs/guides/dns-cache.html) guide for more details. It's also good to set +reasonable, non-zero [DNS TTL values](/docs/guides/dns-cache.html#ttl) if your clients will +respect them. + +* In other applications that perform high volumes of reads against Consul, consider using the +[stale consistency mode](/docs/agent/http.html#consistency) available to allow reads to scale +across all the servers and not just be forwarded to the leader. diff --git a/website/source/docs/upgrade-specific.html.markdown b/website/source/docs/upgrade-specific.html.markdown index b0d9e9abca89..069caa36f573 100644 --- a/website/source/docs/upgrade-specific.html.markdown +++ b/website/source/docs/upgrade-specific.html.markdown @@ -19,6 +19,27 @@ standard upgrade flow. Consul version 0.7 is a very large release with many important changes. Changes to be aware of during an upgrade are categorized below. +#### Performance Tuning and New Defaults + +Consul 0.7 introduced support for tuning Raft performance using a new +[performance configuration block](/docs/agent/options.html#performance). Also, +the default Raft timing is set to a lower-performance mode suitable for +[minimal Consul servers](/docs/guides/performance.html#minumum). + +To continue to use the high-performance settings that were the default prior to +Consul 0.7 (recommended for production servers), add the following configuration +to all Consul servers when upgrading: + +```javascript +{ + "performance": { + "raft_multiplier": 1 + } +} +``` + +See the [Server Performance](/docs/guides/performance.html) guide for more details. + #### Default Configuration Changes The default behavior of [`skip_leave_on_interrupt`](/docs/agent/options.html#skip_leave_on_interrupt) diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 113e0647a328..26917cfed6a8 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -228,6 +228,10 @@ Atlas Integration + > + Server Performance + + > Adding/Removing Servers From c432aa540dc743e71d25e4f34caa55bc82167172 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Wed, 24 Aug 2016 22:10:59 -0700 Subject: [PATCH 2/6] Tweaks wording in performance guide. --- .../source/docs/guides/performance.html.markdown | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/website/source/docs/guides/performance.html.markdown b/website/source/docs/guides/performance.html.markdown index dbc6a2965505..19137be7b656 100644 --- a/website/source/docs/guides/performance.html.markdown +++ b/website/source/docs/guides/performance.html.markdown @@ -66,13 +66,13 @@ Here are some general recommendations: another fast disk technology for the best write throughput. * Spurious leader elections can be caused by networking issues between -the servers or lack of CPU. Users in cloud environments often bump their servers up to the next -instance class with improved networking and CPU until leader elections stabilize, and in Consul -0.7 or later the [performance parameters](/docs/agent/options.html#performance) configuration -now gives you the option to trade off performance instead of upsizing servers. You can use the -[`consul.raft.leader.lastContact` telemetry](/docs/agent/telemetry.html#last-contact) to help -observe how the Raft timing is performing and decide if de-tuning Raft performance or adding -more powerful servers might be needed. +the servers or insufficient CPU resources. Users in cloud environments often bump their servers +up to the next instance class with improved networking and CPU until leader elections stabilize, +and in Consul 0.7 or later the [performance parameters](/docs/agent/options.html#performance) +configuration now gives you tools to trade off performance instead of upsizing servers. You can +use the [`consul.raft.leader.lastContact` telemetry](/docs/agent/telemetry.html#last-contact) +to observe how the Raft timing is performing and guide the decision to de-tune Raft performance +or add more powerful servers. * For DNS-heavy workloads, configuring all Consul agents in a cluster with the [`allow_stale`](/docs/agent/options.html#allow_stale) configuration option will allow reads to From 679b3c0c6be94ed28f2bbdf4767a6c7c9e747215 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Wed, 24 Aug 2016 23:35:28 -0700 Subject: [PATCH 3/6] Increases RPC hold timeout for new default timing. Rather than scale this we just bump it up a bit. It'll be on the edge in the lower-performance default mode, and will have plenty of margin in the high-performance mode. This seems like a reasonable compromise to keep the logic here simple vs. scaling, and seems inline with the expectations of the different modes of operation. --- consul/config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/consul/config.go b/consul/config.go index 0115b171e152..53f3cf300429 100644 --- a/consul/config.go +++ b/consul/config.go @@ -318,8 +318,11 @@ func DefaultConfig() *Config { CoordinateUpdateBatchSize: 128, CoordinateUpdateMaxBatches: 5, - // Hold an RPC for up to 5 seconds by default - RPCHoldTimeout: 5 * time.Second, + // This holds RPCs during leader elections. For the default Raft + // config the election timeout is 5 seconds, so we set this a + // bit longer to try to cover that period. This should be more + // than enough when running in the high performance mode. + RPCHoldTimeout: 7 * time.Second, } // Increase our reap interval to 3 days instead of 24h. From 2822334bce7d0536e2d593664159eb8940e56ec2 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Thu, 25 Aug 2016 15:05:40 -0700 Subject: [PATCH 4/6] Stops scaling the commit timeout. --- command/agent/agent_test.go | 2 -- command/agent/config.go | 3 +-- consul/config.go | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index b4fc74a2ca68..c676a80929d6 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -206,7 +206,6 @@ func TestAgent_CheckPerformanceSettings(t *testing.T) { def := raft.DefaultConfig() if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout || r.ElectionTimeout != raftMult*def.ElectionTimeout || - r.CommitTimeout != raftMult*def.CommitTimeout || r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout { t.Fatalf("bad: %#v", *r) } @@ -225,7 +224,6 @@ func TestAgent_CheckPerformanceSettings(t *testing.T) { def := raft.DefaultConfig() if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout || r.ElectionTimeout != raftMult*def.ElectionTimeout || - r.CommitTimeout != raftMult*def.CommitTimeout || r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout { t.Fatalf("bad: %#v", *r) } diff --git a/command/agent/config.go b/command/agent/config.go index 92534d7ba960..ae1138ec8c97 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -114,8 +114,7 @@ type DNSConfig struct { // Performance is used to tune the performance of Consul's subsystems. type Performance struct { // RaftMultiplier is an integer multiplier used to scale Raft timing - // parameters: HeartbeatTimeout, ElectionTimeout, CommitTimeout, and - // LeaderLeaseTimeout. + // parameters: HeartbeatTimeout, ElectionTimeout, and LeaderLeaseTimeout. RaftMultiplier uint `mapstructure:"raft_multiplier"` } diff --git a/consul/config.go b/consul/config.go index 53f3cf300429..eb7e2947c5cd 100644 --- a/consul/config.go +++ b/consul/config.go @@ -358,7 +358,6 @@ func (c *Config) ScaleRaft(raftMultRaw uint) { def := raft.DefaultConfig() c.RaftConfig.HeartbeatTimeout = raftMult * def.HeartbeatTimeout c.RaftConfig.ElectionTimeout = raftMult * def.ElectionTimeout - c.RaftConfig.CommitTimeout = raftMult * def.CommitTimeout c.RaftConfig.LeaderLeaseTimeout = raftMult * def.LeaderLeaseTimeout } From 17b70c7efdba5821b5877183ea4d6f344a472eb9 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Thu, 25 Aug 2016 15:36:05 -0700 Subject: [PATCH 5/6] Adds a max raft multiplier and tweaks documentation. --- command/agent/config.go | 5 ++++ command/agent/config_test.go | 6 +++++ consul/config.go | 9 +++++-- .../source/docs/agent/options.html.markdown | 25 ++++++++++--------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/command/agent/config.go b/command/agent/config.go index ae1138ec8c97..96a707154854 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -942,6 +942,11 @@ func DecodeConfig(r io.Reader) (*Config, error) { result.AdvertiseAddrs.RPC = addr } + // Enforce the max Raft multiplier. + if result.Performance.RaftMultiplier > consul.MaxRaftMultiplier { + return nil, fmt.Errorf("Performance.RaftMultiplier must be <= %d", consul.MaxRaftMultiplier) + } + return &result, nil } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 34c9b47206aa..ed174a9f8287 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -966,6 +966,12 @@ func TestDecodeConfig_Performance(t *testing.T) { if config.Performance.RaftMultiplier != 3 { t.Fatalf("bad: multiplier isn't set: %#v", config) } + + input = `{"performance": { "raft_multiplier": 11 }}` + config, err = DecodeConfig(bytes.NewReader([]byte(input))) + if err == nil || !strings.Contains(err.Error(), "Performance.RaftMultiplier must be <=") { + t.Fatalf("bad: %v", err) + } } func TestDecodeConfig_Services(t *testing.T) { diff --git a/consul/config.go b/consul/config.go index eb7e2947c5cd..0e094f305bfa 100644 --- a/consul/config.go +++ b/consul/config.go @@ -18,9 +18,14 @@ const ( DefaultLANSerfPort = 8301 DefaultWANSerfPort = 8302 - // See docs/guides/performance.html for information on how this value - // was obtained. + // DefaultRaftMultiplier is used as a baseline Raft configuration that + // will be reliable on a very basic server. See docs/guides/performance.html + // for information on how this value was obtained. DefaultRaftMultiplier uint = 5 + + // MaxRaftMultiplier is a fairly arbitrary upper bound that limits the + // amount of performance detuning that's possible. + MaxRaftMultiplier uint = 10 ) var ( diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index 208fb38b3333..7d439cd357a9 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -581,18 +581,19 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass Consul. See the [Server Performance](/docs/guides/performance.html) guide for more details. The following parameters are available: * `raft_multiplier` - An integer - multiplier used by Consul servers to scale key Raft timing parameters. Tuning this affects - the time it takes Consul to detect leader failures and to perform leader elections, at the - expense of requiring more network and CPU resources for better performance.

A value - of 0, the default, means that Consul will use a lower-performance timing that's suitable for - [minimal Consul servers](/docs/guides/performance.html#minumum), currently equivalent to - setting this to a value of 5 (this default may be changed in future versions of Consul, - depending if the target minimum server profile changes). Above 0, higher values imply lower - levels of performance. Setting this to a value of 1 will configure Raft to its - highest-performance mode, equivalent to the default timing of Consul prior to 0.7, and is - recommended for [production Consul servers](/docs/guides/performance.html#production). See - the note on [last contact](/docs/guides/performance.html#last-contact) timing for more - details on tuning this parameter. + multiplier used by Consul servers to scale key Raft timing parameters. Omitting this value + or setting it to 0 uses default timing described below. Lower values are used to tighten + timing and increase sensitivity while higher values relax timings and reduce sensitivity. + Tuning this affects the time it takes Consul to detect leader failures and to perform + leader elections, at the expense of requiring more network and CPU resources for better + performance.

By default, Consul will use a lower-performance timing that's suitable + for [minimal Consul servers](/docs/guides/performance.html#minumum), currently equivalent + to setting this to a value of 5 (this default may be changed in future versions of Consul, + depending if the target minimum server profile changes). Setting this to a value of 1 will + configure Raft to its highest-performance mode, equivalent to the default timing of Consul + prior to 0.7, and is recommended for [production Consul servers](/docs/guides/performance.html#production). + See the note on [last contact](/docs/guides/performance.html#last-contact) timing for more + details on tuning this parameter. The maximum allowed value is 10. * `ports` This is a nested object that allows setting the bind ports for the following keys: From 80d1d88590f4956e5ee494424a4ebb26d959d641 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Thu, 25 Aug 2016 15:39:19 -0700 Subject: [PATCH 6/6] Removes leader_lease_timeout from stats. --- consul/server.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/consul/server.go b/consul/server.go index 94f158ed056b..ab240ce45f93 100644 --- a/consul/server.go +++ b/consul/server.go @@ -820,12 +820,11 @@ func (s *Server) Stats() map[string]map[string]string { s.remoteLock.RUnlock() stats := map[string]map[string]string{ "consul": map[string]string{ - "server": "true", - "leader": fmt.Sprintf("%v", s.IsLeader()), - "leader_addr": string(s.raft.Leader()), - "bootstrap": fmt.Sprintf("%v", s.config.Bootstrap), - "known_datacenters": toString(uint64(numKnownDCs)), - "leader_lease_timeout": fmt.Sprintf("%v", s.config.RaftConfig.LeaderLeaseTimeout), + "server": "true", + "leader": fmt.Sprintf("%v", s.IsLeader()), + "leader_addr": string(s.raft.Leader()), + "bootstrap": fmt.Sprintf("%v", s.config.Bootstrap), + "known_datacenters": toString(uint64(numKnownDCs)), }, "raft": s.raft.Stats(), "serf_lan": s.serfLAN.Stats(),