diff --git a/.changelog/12107.txt b/.changelog/12107.txt new file mode 100644 index 000000000000..a29cab2cfab5 --- /dev/null +++ b/.changelog/12107.txt @@ -0,0 +1,20 @@ +```release-note:improvement +deps: Update hashicorp/raft-boltdb to v2.2.0 +``` + +```release-note:improvement +agent: Switch from boltdb/bolt to go.etcd.io/bbolt +``` + +```release-note:improvement +core: Enable configuring raft boltdb freelist sync behavior +``` + +```release-note:improvement +metrics: Emit metrics regarding raft boltdb operations +``` + +```release-note:breaking-change +agent: The state database on both clients and servers will automatically migrate its underlying database on startup. Downgrading to a previous version of an agent after upgrading it to Nomad 1.3 is not supported. +``` + diff --git a/.golangci.yml b/.golangci.yml index 6be0b4d43cea..41a924c93885 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -60,6 +60,7 @@ linters-settings: list-type: blacklist packages: - github.com/hashicorp/consul/command/flags + - github.com/boltdb/bolt gocritic: disabled-checks: - commentFormatting diff --git a/client/state/state_database.go b/client/state/state_database.go index 5a7fa674900f..be81fd53ecb2 100644 --- a/client/state/state_database.go +++ b/client/state/state_database.go @@ -6,8 +6,6 @@ import ( "path/filepath" "time" - "github.com/boltdb/bolt" - hclog "github.com/hashicorp/go-hclog" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" dmstate "github.com/hashicorp/nomad/client/devicemanager/state" @@ -15,6 +13,7 @@ import ( driverstate "github.com/hashicorp/nomad/client/pluginmanager/drivermanager/state" "github.com/hashicorp/nomad/helper/boltdd" "github.com/hashicorp/nomad/nomad/structs" + "go.etcd.io/bbolt" ) /* @@ -139,11 +138,11 @@ func NewBoltStateDB(logger hclog.Logger, stateDir string) (StateDB, error) { firstRun := fi == nil // Timeout to force failure when accessing a data dir that is already in use - timeout := &bolt.Options{Timeout: 5 * time.Second} + timeout := &bbolt.Options{Timeout: 5 * time.Second} // Create or open the boltdb state database db, err := boltdd.Open(fn, 0600, timeout) - if err == bolt.ErrTimeout { + if err == bbolt.ErrTimeout { return nil, fmt.Errorf("timed out while opening database, is another Nomad process accessing data_dir %s?", stateDir) } else if err != nil { return nil, fmt.Errorf("failed to create state database: %v", err) diff --git a/client/state/upgrade.go b/client/state/upgrade.go index 63ccc88b338a..31f8c3bef6de 100644 --- a/client/state/upgrade.go +++ b/client/state/upgrade.go @@ -6,20 +6,20 @@ import ( "fmt" "os" - "github.com/boltdb/bolt" hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/nomad/client/dynamicplugins" "github.com/hashicorp/nomad/helper/boltdd" "github.com/hashicorp/nomad/nomad/structs" + "go.etcd.io/bbolt" ) // NeedsUpgrade returns true if the BoltDB needs upgrading or false if it is // already up to date. -func NeedsUpgrade(bdb *bolt.DB) (upgradeTo09, upgradeTo13 bool, err error) { +func NeedsUpgrade(bdb *bbolt.DB) (upgradeTo09, upgradeTo13 bool, err error) { upgradeTo09 = true upgradeTo13 = true - err = bdb.View(func(tx *bolt.Tx) error { + err = bdb.View(func(tx *bbolt.Tx) error { b := tx.Bucket(metaBucketName) if b == nil { // No meta bucket; upgrade @@ -53,7 +53,7 @@ func NeedsUpgrade(bdb *bolt.DB) (upgradeTo09, upgradeTo13 bool, err error) { // addMeta adds version metadata to BoltDB to mark it as upgraded and // should be run at the end of the upgrade transaction. -func addMeta(tx *bolt.Tx) error { +func addMeta(tx *bbolt.Tx) error { // Create the meta bucket if it doesn't exist bkt, err := tx.CreateBucketIfNotExists(metaBucketName) if err != nil { @@ -64,13 +64,13 @@ func addMeta(tx *bolt.Tx) error { // backupDB backs up the existing state database prior to upgrade overwriting // previous backups. -func backupDB(bdb *bolt.DB, dst string) error { +func backupDB(bdb *bbolt.DB, dst string) error { fd, err := os.Create(dst) if err != nil { return err } - return bdb.View(func(tx *bolt.Tx) error { + return bdb.View(func(tx *bbolt.Tx) error { if _, err := tx.WriteTo(fd); err != nil { fd.Close() return err @@ -145,7 +145,7 @@ func UpgradeAllocs(logger hclog.Logger, tx *boltdd.Tx) error { } // upgradeAllocBucket upgrades an alloc bucket. -func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bolt.Bucket, allocID string) error { +func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bbolt.Bucket, allocID string) error { allocFound := false taskBuckets := [][]byte{} cur := bkt.Cursor() @@ -253,7 +253,7 @@ func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bolt.Bucket, al // upgradeTaskBucket iterates over keys in a task bucket, deleting invalid keys // and returning the 0.8 version of the state. -func upgradeTaskBucket(logger hclog.Logger, bkt *bolt.Bucket) (*taskRunnerState08, error) { +func upgradeTaskBucket(logger hclog.Logger, bkt *bbolt.Bucket) (*taskRunnerState08, error) { simpleFound := false var trState taskRunnerState08 diff --git a/client/state/upgrade_int_test.go b/client/state/upgrade_int_test.go index 540b3f33b7bb..96df3fbadce7 100644 --- a/client/state/upgrade_int_test.go +++ b/client/state/upgrade_int_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" - "github.com/boltdb/bolt" "github.com/hashicorp/nomad/client/allocrunner" "github.com/hashicorp/nomad/client/allocwatcher" clientconfig "github.com/hashicorp/nomad/client/config" @@ -27,6 +26,7 @@ import ( pstructs "github.com/hashicorp/nomad/plugins/shared/structs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) // TestBoltStateDB_Upgrade_Ok asserts upgading an old state db does not error @@ -140,7 +140,7 @@ func TestBoltStateDB_UpgradeOld_Ok(t *testing.T) { defer db.Close() // Simply opening old files should *not* alter them - db.DB().BoltDB().View(func(tx *bolt.Tx) error { + db.DB().BoltDB().View(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte("meta")) if b == nil { return fmt.Errorf("meta bucket should exist") diff --git a/client/state/upgrade_test.go b/client/state/upgrade_test.go index 1bf36273fbf3..88cf6d112b3a 100644 --- a/client/state/upgrade_test.go +++ b/client/state/upgrade_test.go @@ -7,18 +7,18 @@ import ( "path/filepath" "testing" - "github.com/boltdb/bolt" "github.com/hashicorp/nomad/helper/boltdd" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) -func setupBoltDB(t *testing.T) (*bolt.DB, func()) { +func setupBoltDB(t *testing.T) (*bbolt.DB, func()) { dir, err := ioutil.TempDir("", "nomadtest") require.NoError(t, err) - db, err := bolt.Open(filepath.Join(dir, "state.db"), 0666, nil) + db, err := bbolt.Open(filepath.Join(dir, "state.db"), 0666, nil) if err != nil { os.RemoveAll(dir) require.NoError(t, err) @@ -54,7 +54,7 @@ func TestUpgrade_NeedsUpgrade_Old(t *testing.T) { // Create the allocations bucket which exists in both the old and 0.9 // schemas - require.NoError(t, db.Update(func(tx *bolt.Tx) error { + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { _, err := tx.CreateBucket(allocationsBucketName) return err })) @@ -91,7 +91,7 @@ func TestUpgrade_NeedsUpgrade_Error(t *testing.T) { db, cleanup := setupBoltDB(t) defer cleanup() - require.NoError(t, db.Update(func(tx *bolt.Tx) error { + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { bkt, err := tx.CreateBucketIfNotExists(metaBucketName) require.NoError(t, err) @@ -160,7 +160,7 @@ func TestUpgrade_upgradeTaskBucket_InvalidEntries(t *testing.T) { taskName := []byte("fake-task") // Insert unexpected bucket, unexpected key, and missing simple-all - require.NoError(t, db.Update(func(tx *bolt.Tx) error { + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { bkt, err := tx.CreateBucket(taskName) if err != nil { return err @@ -174,7 +174,7 @@ func TestUpgrade_upgradeTaskBucket_InvalidEntries(t *testing.T) { return bkt.Put([]byte("unexepectedKey"), []byte{'x'}) })) - require.NoError(t, db.Update(func(tx *bolt.Tx) error { + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { bkt := tx.Bucket(taskName) // upgradeTaskBucket should fail diff --git a/command/agent/agent.go b/command/agent/agent.go index 13dd19dc5bb1..61688bf13cf6 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -444,6 +444,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } } + // Set the raft bolt parameters + if bolt := agentConfig.Server.RaftBoltConfig; bolt != nil { + conf.RaftBoltNoFreelistSync = bolt.NoFreelistSync + } + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index fd48bd10e2da..1fb9b93c3c14 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -516,11 +516,26 @@ type ServerConfig struct { // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` + // Search configures UI search features. Search *Search `hcl:"search"` // DeploymentQueryRateLimit is in queries per second and is used by the // DeploymentWatcher to throttle the amount of simultaneously deployments DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"` + + // RaftBoltConfig configures boltdb as used by raft. + RaftBoltConfig *RaftBoltConfig `hcl:"raft_boltdb"` +} + +// RaftBoltConfig is used in servers to configure parameters of the boltdb +// used for raft consensus. +type RaftBoltConfig struct { + // NoFreelistSync toggles whether the underlying raft storage should sync its + // freelist to disk within the bolt .db file. When disabled, IO performance + // will be improved but at the expense of longer startup times. + // + // Default: false. + NoFreelistSync bool `hcl:"no_freelist_sync"` } // Search is used in servers to configure search API options. @@ -1599,6 +1614,12 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig { } } + if b.RaftBoltConfig != nil { + result.RaftBoltConfig = &RaftBoltConfig{ + NoFreelistSync: b.RaftBoltConfig.NoFreelistSync, + } + } + // Add the schedulers result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...) diff --git a/command/raft_tools/state.go b/command/raft_tools/state.go index 2078aa61eaff..90891fea86e3 100644 --- a/command/raft_tools/state.go +++ b/command/raft_tools/state.go @@ -3,7 +3,7 @@ package rafttools import ( "fmt" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" ) func RaftState(p string) (store *raftboltdb.BoltStore, firstIdx uint64, lastIdx uint64, err error) { diff --git a/go.mod b/go.mod index 3e8528261897..2b6e21658916 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/go-metrics v0.3.10 github.com/aws/aws-sdk-go v1.42.27 - github.com/boltdb/bolt v1.3.1 github.com/container-storage-interface/spec v1.4.0 github.com/containerd/go-cni v1.1.1 github.com/containernetworking/cni v1.0.1 @@ -77,7 +76,7 @@ require ( github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/nomad/api v0.0.0-20200529203653-c4416b26d3eb github.com/hashicorp/raft v1.3.5 - github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea + github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.5 github.com/hashicorp/vault/api v1.0.5-0.20200805123347-1ef507638af6 github.com/hashicorp/vault/sdk v0.2.0 @@ -114,6 +113,7 @@ require ( github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/zclconf/go-cty v1.8.0 github.com/zclconf/go-cty-yaml v1.0.2 + go.etcd.io/bbolt v1.3.5 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f @@ -154,6 +154,7 @@ require ( github.com/bgentry/speakeasy v0.1.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bmatcuk/doublestar v1.1.5 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/checkpoint-restore/go-criu/v5 v5.0.0 // indirect diff --git a/go.sum b/go.sum index 637a1e348a2e..4d83ddeefba3 100644 --- a/go.sum +++ b/go.sum @@ -764,12 +764,16 @@ github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1 github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.5 h1:93YBXmHWW2MuyMZfMxN1PsAnPXAt+hBfG0S0ZrZxRrY= github.com/hashicorp/raft v1.3.5/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= +github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw= +github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.4/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= @@ -1258,6 +1262,7 @@ github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd5 github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= diff --git a/helper/boltdd/boltdd.go b/helper/boltdd/boltdd.go index ea337047de7f..7c29385d3a21 100644 --- a/helper/boltdd/boltdd.go +++ b/helper/boltdd/boltdd.go @@ -1,5 +1,5 @@ -// BOLTdd contains a wrapper around BoltDB to deduplicate writes and encode -// values using mgspack. (dd stands for DeDuplicate) +// Package boltdd contains a wrapper around BBoltDB to deduplicate writes and encode +// values using mgspack. (dd stands for de-duplicate) package boltdd import ( @@ -8,9 +8,9 @@ import ( "os" "sync" - "github.com/boltdb/bolt" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/nomad/nomad/structs" + "go.etcd.io/bbolt" "golang.org/x/crypto/blake2b" ) @@ -37,19 +37,19 @@ func IsErrNotFound(e error) bool { return ok } -// DB wraps an underlying bolt.DB to create write deduplicating buckets and +// DB wraps an underlying bolt.DB to create write de-duplicating buckets and // msgpack encoded values. type DB struct { rootBuckets map[string]*bucketMeta rootBucketsLock sync.Mutex - bdb *bolt.DB + boltDB *bbolt.DB } -// Open a bolt.DB and wrap it in a write-deduplicating msgpack-encoding +// Open a bolt.DB and wrap it in a write-de-duplicating msgpack-encoding // implementation. -func Open(path string, mode os.FileMode, options *bolt.Options) (*DB, error) { - bdb, err := bolt.Open(path, mode, options) +func Open(path string, mode os.FileMode, options *bbolt.Options) (*DB, error) { + bdb, err := bbolt.Open(path, mode, options) if err != nil { return nil, err } @@ -57,15 +57,15 @@ func Open(path string, mode os.FileMode, options *bolt.Options) (*DB, error) { return New(bdb), nil } -// New deduplicating wrapper for the given boltdb. -func New(bdb *bolt.DB) *DB { +// New de-duplicating wrapper for the given bboltdb. +func New(bdb *bbolt.DB) *DB { return &DB{ rootBuckets: make(map[string]*bucketMeta), - bdb: bdb, + boltDB: bdb, } } -func (db *DB) bucket(btx *bolt.Tx, name []byte) *Bucket { +func (db *DB) bucket(btx *bbolt.Tx, name []byte) *Bucket { bb := btx.Bucket(name) if bb == nil { return nil @@ -87,7 +87,7 @@ func (db *DB) bucket(btx *bolt.Tx, name []byte) *Bucket { return newBucket(b, bb) } -func (db *DB) createBucket(btx *bolt.Tx, name []byte) (*Bucket, error) { +func (db *DB) createBucket(btx *bbolt.Tx, name []byte) (*Bucket, error) { bb, err := btx.CreateBucket(name) if err != nil { return nil, err @@ -99,7 +99,7 @@ func (db *DB) createBucket(btx *bolt.Tx, name []byte) (*Bucket, error) { // While creating a bucket on a closed db would error, we must recheck // after acquiring the lock to avoid races. if db.isClosed() { - return nil, bolt.ErrDatabaseNotOpen + return nil, bbolt.ErrDatabaseNotOpen } // Always create a new Bucket since CreateBucket above fails if the @@ -110,7 +110,7 @@ func (db *DB) createBucket(btx *bolt.Tx, name []byte) (*Bucket, error) { return newBucket(b, bb), nil } -func (db *DB) createBucketIfNotExists(btx *bolt.Tx, name []byte) (*Bucket, error) { +func (db *DB) createBucketIfNotExists(btx *bbolt.Tx, name []byte) (*Bucket, error) { bb, err := btx.CreateBucketIfNotExists(name) if err != nil { return nil, err @@ -122,7 +122,7 @@ func (db *DB) createBucketIfNotExists(btx *bolt.Tx, name []byte) (*Bucket, error // While creating a bucket on a closed db would error, we must recheck // after acquiring the lock to avoid races. if db.isClosed() { - return nil, bolt.ErrDatabaseNotOpen + return nil, bbolt.ErrDatabaseNotOpen } b, ok := db.rootBuckets[string(name)] @@ -135,21 +135,21 @@ func (db *DB) createBucketIfNotExists(btx *bolt.Tx, name []byte) (*Bucket, error } func (db *DB) Update(fn func(*Tx) error) error { - return db.bdb.Update(func(btx *bolt.Tx) error { + return db.boltDB.Update(func(btx *bbolt.Tx) error { tx := newTx(db, btx) return fn(tx) }) } func (db *DB) Batch(fn func(*Tx) error) error { - return db.bdb.Batch(func(btx *bolt.Tx) error { + return db.boltDB.Batch(func(btx *bbolt.Tx) error { tx := newTx(db, btx) return fn(tx) }) } func (db *DB) View(fn func(*Tx) error) error { - return db.bdb.View(func(btx *bolt.Tx) error { + return db.boltDB.View(func(btx *bbolt.Tx) error { tx := newTx(db, btx) return fn(tx) }) @@ -167,20 +167,20 @@ func (db *DB) Close() error { db.rootBucketsLock.Lock() db.rootBuckets = nil db.rootBucketsLock.Unlock() - return db.bdb.Close() + return db.boltDB.Close() } // BoltDB returns the underlying bolt.DB. -func (db *DB) BoltDB() *bolt.DB { - return db.bdb +func (db *DB) BoltDB() *bbolt.DB { + return db.boltDB } type Tx struct { db *DB - btx *bolt.Tx + btx *bbolt.Tx } -func newTx(db *DB, btx *bolt.Tx) *Tx { +func newTx(db *DB, btx *bbolt.Tx) *Tx { return &Tx{ db: db, btx: btx, @@ -208,7 +208,7 @@ func (tx *Tx) Writable() bool { } // BoltTx returns the underlying bolt.Tx. -func (tx *Tx) BoltTx() *bolt.Tx { +func (tx *Tx) BoltTx() *bbolt.Tx { return tx.btx } @@ -290,12 +290,12 @@ func (bm *bucketMeta) getOrCreateBucket(name []byte) *bucketMeta { type Bucket struct { bm *bucketMeta - boltBucket *bolt.Bucket + boltBucket *bbolt.Bucket } // newBucket creates a new view into a bucket backed by a boltdb // transaction. -func newBucket(b *bucketMeta, bb *bolt.Bucket) *Bucket { +func newBucket(b *bucketMeta, bb *bbolt.Bucket) *Bucket { return &Bucket{ bm: b, boltBucket: bb, @@ -408,7 +408,7 @@ func (b *Bucket) CreateBucketIfNotExists(name []byte) (*Bucket, error) { func (b *Bucket) DeleteBucket(name []byte) error { // Delete the bucket from the underlying boltdb err := b.boltBucket.DeleteBucket(name) - if err == bolt.ErrBucketNotFound { + if err == bbolt.ErrBucketNotFound { err = nil } @@ -419,6 +419,6 @@ func (b *Bucket) DeleteBucket(name []byte) error { // BoltBucket returns the internal bolt.Bucket for this Bucket. Only valid // for the duration of the current transaction. -func (b *Bucket) BoltBucket() *bolt.Bucket { +func (b *Bucket) BoltBucket() *bbolt.Bucket { return b.boltBucket } diff --git a/helper/boltdd/boltdd_test.go b/helper/boltdd/boltdd_test.go index 8425c746b333..19c4e6ec2e02 100644 --- a/helper/boltdd/boltdd_test.go +++ b/helper/boltdd/boltdd_test.go @@ -8,11 +8,11 @@ import ( "path/filepath" "testing" - "github.com/boltdb/bolt" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) type testingT interface { @@ -66,12 +66,12 @@ func TestDB_Close(t *testing.T) { require.Equal(t, db.Update(func(tx *Tx) error { _, err := tx.CreateBucketIfNotExists([]byte("foo")) return err - }), bolt.ErrDatabaseNotOpen) + }), bbolt.ErrDatabaseNotOpen) require.Equal(t, db.Update(func(tx *Tx) error { _, err := tx.CreateBucket([]byte("foo")) return err - }), bolt.ErrDatabaseNotOpen) + }), bbolt.ErrDatabaseNotOpen) } func TestBucket_Create(t *testing.T) { diff --git a/helper/raftutil/fsm.go b/helper/raftutil/fsm.go index 2d27f797298b..26539215cb2b 100644 --- a/helper/raftutil/fsm.go +++ b/helper/raftutil/fsm.go @@ -11,7 +11,7 @@ import ( "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" ) var ErrNoMoreLogs = fmt.Errorf("no more logs") diff --git a/helper/raftutil/state.go b/helper/raftutil/state.go index 68c26b5c0fcc..4ed6381b3362 100644 --- a/helper/raftutil/state.go +++ b/helper/raftutil/state.go @@ -9,11 +9,11 @@ import ( "strings" "time" - "github.com/boltdb/bolt" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" + "go.etcd.io/bbolt" ) var ( @@ -24,7 +24,7 @@ var ( func RaftStateInfo(p string) (store *raftboltdb.BoltStore, firstIdx uint64, lastIdx uint64, err error) { opts := raftboltdb.Options{ Path: p, - BoltOptions: &bolt.Options{ + BoltOptions: &bbolt.Options{ ReadOnly: true, Timeout: 1 * time.Second, }, diff --git a/helper/raftutil/state_test.go b/helper/raftutil/state_test.go index ea8075ca31a4..3d653fbc98e5 100644 --- a/helper/raftutil/state_test.go +++ b/helper/raftutil/state_test.go @@ -4,7 +4,7 @@ import ( "path/filepath" "testing" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/nomad/config.go b/nomad/config.go index 696080dfc22b..2396900c109d 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -339,6 +339,9 @@ type Config struct { // SearchConfig provides knobs for Search API. SearchConfig *structs.SearchConfig + // RaftBoltNoFreelistSync configures whether freelist syncing is enabled. + RaftBoltNoFreelistSync bool + // AgentShutdown is used to call agent.Shutdown from the context of a Server // It is used primarily for licensing AgentShutdown func() error diff --git a/nomad/server.go b/nomad/server.go index e7fce5ef85f5..5e3d2eb51ac2 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -40,8 +40,9 @@ import ( "github.com/hashicorp/nomad/nomad/volumewatcher" "github.com/hashicorp/nomad/scheduler" "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" "github.com/hashicorp/serf/serf" + "go.etcd.io/bbolt" ) const ( @@ -1222,6 +1223,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // setupRaft is used to setup and initialize Raft func (s *Server) setupRaft() error { + // If we have an unclean exit then attempt to close the Raft store. defer func() { if s.raft == nil && s.raftStore != nil { @@ -1282,13 +1284,23 @@ func (s *Server) setupRaft() error { return err } - // Create the BoltDB backend - store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db")) - if err != nil { - return err + // Create the BoltDB backend, with NoFreelistSync option + store, raftErr := raftboltdb.New(raftboltdb.Options{ + Path: filepath.Join(path, "raft.db"), + NoSync: false, // fsync each log write + BoltOptions: &bbolt.Options{ + NoFreelistSync: s.config.RaftBoltNoFreelistSync, + }, + }) + if raftErr != nil { + return raftErr } s.raftStore = store stable = store + s.logger.Info("setting up raft bolt store", "no_freelist_sync", s.config.RaftBoltNoFreelistSync) + + // Start publishing bboltdb metrics + go store.RunMetrics(s.shutdownCtx, 0) // Wrap the store in a LogCache to improve performance cacheStore, err := raft.NewLogCache(raftLogCacheSize, store) diff --git a/website/content/docs/configuration/server.mdx b/website/content/docs/configuration/server.mdx index ad293be7b7d2..7872913a0640 100644 --- a/website/content/docs/configuration/server.mdx +++ b/website/content/docs/configuration/server.mdx @@ -156,6 +156,13 @@ server { disallow this server from making any scheduling decisions. This defaults to the number of CPU cores. +- `raft_boltdb` - This is a nested object that allows configuring options for + Raft's BoltDB based log store. + - `no_freelist_sync` - Setting this to `true` will disable syncing the BoltDB + freelist to disk within the `raft.db` file. Not syncing the freelist to disk + will reduce disk IO required for write operations at the expense of longer + server startup times. + - `raft_protocol` `(int: 3)` - Specifies the Raft protocol version to use when communicating with other Nomad servers. This affects available Autopilot features and is typically not required as the agent internally knows the diff --git a/website/content/docs/operations/metrics-reference.mdx b/website/content/docs/operations/metrics-reference.mdx index 61311a25e28f..be88e831d58e 100644 --- a/website/content/docs/operations/metrics-reference.mdx +++ b/website/content/docs/operations/metrics-reference.mdx @@ -449,6 +449,32 @@ those listed in [Key Metrics](#key-metrics) above. | `nomad.scheduler.allocs.rescheduled.wait_until` | Time that a rescheduled allocation will be delayed | Float | Gauge | alloc_id, job, namespace, task_group, follow_up_eval_id | | `nomad.state.snapshotIndex` | Current snapshot index | Integer | Gauge | host | +## Raft BoltDB Metrics + +Raft database metrics are emitted by the `raft-boltdb` library. + +| Metric | Description | Unit | Type | +| ----------------------------------------- | ----------------------------------------- | ----------- | ------- | +| `nomad.raft.boltdb.numFreePages` | Number of free pages | Integer | Gauge | +| `nomad.raft.boltdb.numPendingPages` | Number of pending pages | Integer | Gauge | +| `nomad.raft.boltdb.freePageBytes` | Number of free page bytes | Integer | Gauge | +| `nomad.raft.boltdb.freelistBytes` | Number of freelist bytes | Integer | Gauge | +| `nomad.raft.boltdb.totalReadTxn` | Count of total read transactions | Integer | Counter | +| `nomad.raft.boltdb.openReadTxn` | Number of current open read transactions | Integer | Gauge | +| `nomad.raft.boltdb.txstats.pageCount` | Number of pages in use | Integer | Gauge | +| `nomad.raft.boltdb.txstats.pageAlloc` | Number of page allocations | Integer | Gauge | +| `nomad.raft.boltdb.txstats.cursorCount` | Count of total database cursors | Integer | Counter | +| `nomad.raft.boltdb.txstats.nodeCount` | Count of total database nodes | Integer | Counter | +| `nomad.raft.boltdb.txstats.nodeDeref` | Count of total database node dereferences | Integer | Counter | +| `nomad.raft.boltdb.txstats.rebalance` | Count of total rebalance operations | Integer | Counter | +| `nomad.raft.boltdb.txstats.rebalanceTime` | Sample of rebalance operation times | Nanoseconds | Summary | +| `nomad.raft.boltdb.txstats.split` | Count of total split operations | Integer | Counter | +| `nomad.raft.boltdb.txstats.spill` | Count of total spill operations | Integer | Counter | +| `nomad.raft.boltdb.txstats.spillTime` | Sample of spill operation times | Nanoseconds | Summary | +| `nomad.raft.boltdb.txstats.write` | Count of total write operations | Integer | Counter | +| `nomad.raft.boltdb.txstats.writeTime` | Sample of write operation times | Nanoseconds | Summary | [tagged-metrics]: /docs/telemetry/metrics#tagged-metrics [s_port_plan_failure]: /s/port-plan-failure + + diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 16473d896d70..baa5703bdc17 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -51,6 +51,28 @@ The volume staging directory for new CSI plugin tasks will now be mounted to the task's `NOMAD_TASK_DIR` instead of the `csi_plugin.mount_config`. +#### Server Raft Database + +The server raft database in `raft.db` will be automatically migrated to a new +underlying implementation provided by `go.etcd.io/bbolt`. Downgrading to a previous +version of the server after upgrading it to Nomad 1.3 is not supported. Like with +any Nomad upgrade it is recommended to take a snapshot of your database prior to +upgrading in case a downgrade becomes necessary. + +The new database implementation enables a new server configuration option for +controlling the underlying freelist-sync behavior. Clusters experiencing extreme +disk IO on servers may want to consider disabling freelist-sync to reduce load. +The tradeoff is longer server startup times, as the database must be completely +scanned to re-build the freelist from scratch. + +```hcl +server { + raft_boltdb { + no_freelist_sync = true + } +} +``` + ## Nomad 1.2.6, 1.1.12, and 1.0.18 #### ACL requirement for the job parse endpoint