diff --git a/command/agent/agent.go b/command/agent/agent.go index b89d2d5a324b..c125be6ebef2 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -447,6 +447,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } } + // Set the raft bolt parameters + if bolt := agentConfig.Server.RaftBoltConfig; bolt != nil { + conf.RaftBoltNoFreelistSync = bolt.NoFreelistSync + } + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index e3a6afbc08f0..aad4f730ed6f 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -514,11 +514,26 @@ type ServerConfig struct { // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` + // Search configures UI search features. Search *Search `hcl:"search"` // DeploymentQueryRateLimit is in queries per second and is used by the // DeploymentWatcher to throttle the amount of simultaneously deployments DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"` + + // RaftBoltConfig configures boltdb as used by raft. + RaftBoltConfig *RaftBoltConfig `hcl:"raft_boltdb"` +} + +// RaftBoltConfig is used in servers to configure parameters of the boltdb +// used for raft consensus. +type RaftBoltConfig struct { + // NoFreelistSync toggles whether the underlying raft storage should sync its + // freelist to disk within the bolt .db file. When disabled, IO performance + // will be improved but at the expense of longer startup times. + // + // Default: false. + NoFreelistSync bool `hcl:"no_freelist_sync"` } // Search is used in servers to configure search API options. @@ -1597,6 +1612,12 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig { } } + if b.RaftBoltConfig != nil { + result.RaftBoltConfig = &RaftBoltConfig{ + NoFreelistSync: b.RaftBoltConfig.NoFreelistSync, + } + } + // Add the schedulers result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...) diff --git a/go.mod b/go.mod index 3e8528261897..94c788d593a5 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,8 @@ require ( github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/nomad/api v0.0.0-20200529203653-c4416b26d3eb github.com/hashicorp/raft v1.3.5 - github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea + github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea + github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.5 github.com/hashicorp/vault/api v1.0.5-0.20200805123347-1ef507638af6 github.com/hashicorp/vault/sdk v0.2.0 @@ -114,6 +115,7 @@ require ( github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/zclconf/go-cty v1.8.0 github.com/zclconf/go-cty-yaml v1.0.2 + go.etcd.io/bbolt v1.3.5 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f diff --git a/go.sum b/go.sum index 637a1e348a2e..4d83ddeefba3 100644 --- a/go.sum +++ b/go.sum @@ -764,12 +764,16 @@ github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1 github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.5 h1:93YBXmHWW2MuyMZfMxN1PsAnPXAt+hBfG0S0ZrZxRrY= github.com/hashicorp/raft v1.3.5/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= +github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw= +github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.4/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= @@ -1258,6 +1262,7 @@ github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd5 github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= diff --git a/nomad/config.go b/nomad/config.go index 3ffe823f56c0..2495d749885b 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -361,6 +361,9 @@ type Config struct { // SearchConfig provides knobs for Search API. SearchConfig *structs.SearchConfig + // RaftBoltNoFreelistSync configures whether freelist syncing is enabled. + RaftBoltNoFreelistSync bool + // AgentShutdown is used to call agent.Shutdown from the context of a Server // It is used primarily for licensing AgentShutdown func() error diff --git a/nomad/server.go b/nomad/server.go index d9f607aaa172..ad0ea6fd1c25 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -40,8 +40,9 @@ import ( "github.com/hashicorp/nomad/nomad/volumewatcher" "github.com/hashicorp/nomad/scheduler" "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" "github.com/hashicorp/serf/serf" + "go.etcd.io/bbolt" ) const ( @@ -1226,6 +1227,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // setupRaft is used to setup and initialize Raft func (s *Server) setupRaft() error { + // If we have an unclean exit then attempt to close the Raft store. defer func() { if s.raft == nil && s.raftStore != nil { @@ -1286,13 +1288,20 @@ func (s *Server) setupRaft() error { return err } - // Create the BoltDB backend - store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db")) - if err != nil { - return err - } + // Create the BoltDB backend, with NoFreelistSync option + store, err := raftboltdb.New(raftboltdb.Options{ + Path: filepath.Join(path, "raft.db"), + NoSync: false, // fsync each log write + BoltOptions: &bbolt.Options{ + NoFreelistSync: s.config.RaftBoltNoFreelistSync, + }, + }) s.raftStore = store stable = store + s.logger.Info("setting up raft bolt store", "no_freelist_sync", s.config.RaftBoltNoFreelistSync) + + // Start publishing bboltdb metrics + go store.RunMetrics(s.shutdownCtx, 0) // Wrap the store in a LogCache to improve performance cacheStore, err := raft.NewLogCache(raftLogCacheSize, store) diff --git a/website/content/docs/configuration/server.mdx b/website/content/docs/configuration/server.mdx index 751562d18864..68bc8e7b7ab3 100644 --- a/website/content/docs/configuration/server.mdx +++ b/website/content/docs/configuration/server.mdx @@ -161,6 +161,13 @@ server { required as the agent internally knows the latest version, but may be useful in some upgrade scenarios. +- `raft_boltdb` - This is a nested object that allows configuring options for + Raft's BoltDB based log store. + - `no_freelist_sync` - Setting this to `true` will disable syncing the BoltDB + freelist to disk within the `raft.db` file. Not syncing the freelist to disk + will reduce disk IO required for write operations at the expense of longer + server startup times. + - `raft_protocol` `(int: 3)` - Specifies the Raft protocol version to use when communicating with other Nomad servers. This affects available Autopilot features and is typically not required as the agent internally knows the