hashicorp · shoenig · Feb 24, 2022 · Feb 16, 2022 · Feb 23, 2022 · Feb 23, 2022
diff --git a/.changelog/12107.txt b/.changelog/12107.txt
@@ -0,0 +1,20 @@
+```release-note:improvement
+deps: Update hashicorp/raft-boltdb to v2.2.0
+```
+
+```release-note:improvement
+agent: Switch from boltdb/bolt to go.etcd.io/bbolt
+```
+
+```release-note:improvement
+core: Enable configuring raft boltdb freelist sync behavior
+```
+
+```release-note:improvement
+metrics: Emit metrics regarding raft boltdb operations
+```
+
+```release-note:breaking-change
+agent: The state database on both clients and servers will automatically migrate its underlying database on startup. Downgrading to a previous version of an agent after upgrading it to Nomad 1.3 is not supported.
+```
+
diff --git a/.golangci.yml b/.golangci.yml
@@ -60,6 +60,7 @@ linters-settings:
     list-type: blacklist
     packages:
       - github.com/hashicorp/consul/command/flags
+      - github.com/boltdb/bolt
   gocritic:
     disabled-checks:
       - commentFormatting

diff --git a/client/state/state_database.go b/client/state/state_database.go
@@ -6,15 +6,14 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/boltdb/bolt"
-
 	hclog "github.com/hashicorp/go-hclog"
 	trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state"
 	dmstate "github.com/hashicorp/nomad/client/devicemanager/state"
 	"github.com/hashicorp/nomad/client/dynamicplugins"
 	driverstate "github.com/hashicorp/nomad/client/pluginmanager/drivermanager/state"
 	"github.com/hashicorp/nomad/helper/boltdd"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"go.etcd.io/bbolt"
 )
 
 /*
@@ -139,11 +138,11 @@ func NewBoltStateDB(logger hclog.Logger, stateDir string) (StateDB, error) {
 	firstRun := fi == nil
 
 	// Timeout to force failure when accessing a data dir that is already in use
-	timeout := &bolt.Options{Timeout: 5 * time.Second}
+	timeout := &bbolt.Options{Timeout: 5 * time.Second}
 
 	// Create or open the boltdb state database
 	db, err := boltdd.Open(fn, 0600, timeout)
-	if err == bolt.ErrTimeout {
+	if err == bbolt.ErrTimeout {
 		return nil, fmt.Errorf("timed out while opening database, is another Nomad process accessing data_dir %s?", stateDir)
 	} else if err != nil {
 		return nil, fmt.Errorf("failed to create state database: %v", err)

diff --git a/client/state/upgrade.go b/client/state/upgrade.go
@@ -6,20 +6,20 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/boltdb/bolt"
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-msgpack/codec"
 	"github.com/hashicorp/nomad/client/dynamicplugins"
 	"github.com/hashicorp/nomad/helper/boltdd"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"go.etcd.io/bbolt"
 )
 
 // NeedsUpgrade returns true if the BoltDB needs upgrading or false if it is
 // already up to date.
-func NeedsUpgrade(bdb *bolt.DB) (upgradeTo09, upgradeTo13 bool, err error) {
+func NeedsUpgrade(bdb *bbolt.DB) (upgradeTo09, upgradeTo13 bool, err error) {
 	upgradeTo09 = true
 	upgradeTo13 = true
-	err = bdb.View(func(tx *bolt.Tx) error {
+	err = bdb.View(func(tx *bbolt.Tx) error {
 		b := tx.Bucket(metaBucketName)
 		if b == nil {
 			// No meta bucket; upgrade
@@ -53,7 +53,7 @@ func NeedsUpgrade(bdb *bolt.DB) (upgradeTo09, upgradeTo13 bool, err error) {
 
 // addMeta adds version metadata to BoltDB to mark it as upgraded and
 // should be run at the end of the upgrade transaction.
-func addMeta(tx *bolt.Tx) error {
+func addMeta(tx *bbolt.Tx) error {
 	// Create the meta bucket if it doesn't exist
 	bkt, err := tx.CreateBucketIfNotExists(metaBucketName)
 	if err != nil {
@@ -64,13 +64,13 @@ func addMeta(tx *bolt.Tx) error {
 
 // backupDB backs up the existing state database prior to upgrade overwriting
 // previous backups.
-func backupDB(bdb *bolt.DB, dst string) error {
+func backupDB(bdb *bbolt.DB, dst string) error {
 	fd, err := os.Create(dst)
 	if err != nil {
 		return err
 	}
 
-	return bdb.View(func(tx *bolt.Tx) error {
+	return bdb.View(func(tx *bbolt.Tx) error {
 		if _, err := tx.WriteTo(fd); err != nil {
 			fd.Close()
 			return err
@@ -145,7 +145,7 @@ func UpgradeAllocs(logger hclog.Logger, tx *boltdd.Tx) error {
 }
 
 // upgradeAllocBucket upgrades an alloc bucket.
-func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bolt.Bucket, allocID string) error {
+func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bbolt.Bucket, allocID string) error {
 	allocFound := false
 	taskBuckets := [][]byte{}
 	cur := bkt.Cursor()
@@ -253,7 +253,7 @@ func upgradeAllocBucket(logger hclog.Logger, tx *boltdd.Tx, bkt *bolt.Bucket, al
 
 // upgradeTaskBucket iterates over keys in a task bucket, deleting invalid keys
 // and returning the 0.8 version of the state.
-func upgradeTaskBucket(logger hclog.Logger, bkt *bolt.Bucket) (*taskRunnerState08, error) {
+func upgradeTaskBucket(logger hclog.Logger, bkt *bbolt.Bucket) (*taskRunnerState08, error) {
 	simpleFound := false
 	var trState taskRunnerState08
 

diff --git a/client/state/upgrade_int_test.go b/client/state/upgrade_int_test.go
@@ -11,7 +11,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/boltdb/bolt"
 	"github.com/hashicorp/nomad/client/allocrunner"
 	"github.com/hashicorp/nomad/client/allocwatcher"
 	clientconfig "github.com/hashicorp/nomad/client/config"
@@ -27,6 +26,7 @@ import (
 	pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.etcd.io/bbolt"
 )
 
 // TestBoltStateDB_Upgrade_Ok asserts upgading an old state db does not error
@@ -140,7 +140,7 @@ func TestBoltStateDB_UpgradeOld_Ok(t *testing.T) {
 		defer db.Close()
 
 		// Simply opening old files should *not* alter them
-		db.DB().BoltDB().View(func(tx *bolt.Tx) error {
+		db.DB().BoltDB().View(func(tx *bbolt.Tx) error {
 			b := tx.Bucket([]byte("meta"))
 			if b == nil {
 				return fmt.Errorf("meta bucket should exist")

diff --git a/client/state/upgrade_test.go b/client/state/upgrade_test.go
@@ -7,18 +7,18 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/boltdb/bolt"
 	"github.com/hashicorp/nomad/helper/boltdd"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/stretchr/testify/require"
+	"go.etcd.io/bbolt"
 )
 
-func setupBoltDB(t *testing.T) (*bolt.DB, func()) {
+func setupBoltDB(t *testing.T) (*bbolt.DB, func()) {
 	dir, err := ioutil.TempDir("", "nomadtest")
 	require.NoError(t, err)
 
-	db, err := bolt.Open(filepath.Join(dir, "state.db"), 0666, nil)
+	db, err := bbolt.Open(filepath.Join(dir, "state.db"), 0666, nil)
 	if err != nil {
 		os.RemoveAll(dir)
 		require.NoError(t, err)
@@ -54,7 +54,7 @@ func TestUpgrade_NeedsUpgrade_Old(t *testing.T) {
 
 	// Create the allocations bucket which exists in both the old and 0.9
 	// schemas
-	require.NoError(t, db.Update(func(tx *bolt.Tx) error {
+	require.NoError(t, db.Update(func(tx *bbolt.Tx) error {
 		_, err := tx.CreateBucket(allocationsBucketName)
 		return err
 	}))
@@ -91,7 +91,7 @@ func TestUpgrade_NeedsUpgrade_Error(t *testing.T) {
 			db, cleanup := setupBoltDB(t)
 			defer cleanup()
 
-			require.NoError(t, db.Update(func(tx *bolt.Tx) error {
+			require.NoError(t, db.Update(func(tx *bbolt.Tx) error {
 				bkt, err := tx.CreateBucketIfNotExists(metaBucketName)
 				require.NoError(t, err)
 
@@ -160,7 +160,7 @@ func TestUpgrade_upgradeTaskBucket_InvalidEntries(t *testing.T) {
 	taskName := []byte("fake-task")
 
 	// Insert unexpected bucket, unexpected key, and missing simple-all
-	require.NoError(t, db.Update(func(tx *bolt.Tx) error {
+	require.NoError(t, db.Update(func(tx *bbolt.Tx) error {
 		bkt, err := tx.CreateBucket(taskName)
 		if err != nil {
 			return err
@@ -174,7 +174,7 @@ func TestUpgrade_upgradeTaskBucket_InvalidEntries(t *testing.T) {
 		return bkt.Put([]byte("unexepectedKey"), []byte{'x'})
 	}))
 
-	require.NoError(t, db.Update(func(tx *bolt.Tx) error {
+	require.NoError(t, db.Update(func(tx *bbolt.Tx) error {
 		bkt := tx.Bucket(taskName)
 
 		// upgradeTaskBucket should fail

diff --git a/command/agent/agent.go b/command/agent/agent.go
@@ -444,6 +444,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		}
 	}
 
+	// Set the raft bolt parameters
+	if bolt := agentConfig.Server.RaftBoltConfig; bolt != nil {
+		conf.RaftBoltNoFreelistSync = bolt.NoFreelistSync
+	}
+
 	return conf, nil
 }
 

diff --git a/command/agent/config.go b/command/agent/config.go
@@ -516,11 +516,26 @@ type ServerConfig struct {
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
 
+	// Search configures UI search features.
 	Search *Search `hcl:"search"`
 
 	// DeploymentQueryRateLimit is in queries per second and is used by the
 	// DeploymentWatcher to throttle the amount of simultaneously deployments
 	DeploymentQueryRateLimit float64 `hcl:"deploy_query_rate_limit"`
+
+	// RaftBoltConfig configures boltdb as used by raft.
+	RaftBoltConfig *RaftBoltConfig `hcl:"raft_boltdb"`
+}
+
+// RaftBoltConfig is used in servers to configure parameters of the boltdb
+// used for raft consensus.
+type RaftBoltConfig struct {
+	// NoFreelistSync toggles whether the underlying raft storage should sync its
+	// freelist to disk within the bolt .db file. When disabled, IO performance
+	// will be improved but at the expense of longer startup times.
+	//
+	// Default: false.
+	NoFreelistSync bool `hcl:"no_freelist_sync"`
 }
 
 // Search is used in servers to configure search API options.
@@ -1599,6 +1614,12 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		}
 	}
 
+	if b.RaftBoltConfig != nil {
+		result.RaftBoltConfig = &RaftBoltConfig{
+			NoFreelistSync: b.RaftBoltConfig.NoFreelistSync,
+		}
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)
 

diff --git a/command/raft_tools/state.go b/command/raft_tools/state.go
@@ -3,7 +3,7 @@ package rafttools
 import (
 	"fmt"
 
-	raftboltdb "github.com/hashicorp/raft-boltdb"
+	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
 )
 
 func RaftState(p string) (store *raftboltdb.BoltStore, firstIdx uint64, lastIdx uint64, err error) {

diff --git a/go.mod b/go.mod
@@ -19,7 +19,6 @@ require (
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e
 	github.com/armon/go-metrics v0.3.10
 	github.com/aws/aws-sdk-go v1.42.27
-	github.com/boltdb/bolt v1.3.1
 	github.com/container-storage-interface/spec v1.4.0
 	github.com/containerd/go-cni v1.1.1
 	github.com/containernetworking/cni v1.0.1
@@ -77,7 +76,7 @@ require (
 	github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69
 	github.com/hashicorp/nomad/api v0.0.0-20200529203653-c4416b26d3eb
 	github.com/hashicorp/raft v1.3.5
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
+	github.com/hashicorp/raft-boltdb/v2 v2.2.0
 	github.com/hashicorp/serf v0.9.5
 	github.com/hashicorp/vault/api v1.0.5-0.20200805123347-1ef507638af6
 	github.com/hashicorp/vault/sdk v0.2.0
@@ -114,6 +113,7 @@ require (
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
 	github.com/zclconf/go-cty v1.8.0
 	github.com/zclconf/go-cty-yaml v1.0.2
+	go.etcd.io/bbolt v1.3.5
 	go.uber.org/goleak v1.1.12
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
@@ -154,6 +154,7 @@ require (
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/bmatcuk/doublestar v1.1.5 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/checkpoint-restore/go-criu/v5 v5.0.0 // indirect

diff --git a/go.sum b/go.sum
@@ -764,12 +764,16 @@ github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1
 github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE=
 github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q=
+github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
 github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
 github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
 github.com/hashicorp/raft v1.3.5 h1:93YBXmHWW2MuyMZfMxN1PsAnPXAt+hBfG0S0ZrZxRrY=
 github.com/hashicorp/raft v1.3.5/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4=
 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
+github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU=
+github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I=
+github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw=
+github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.4/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
@@ -1258,6 +1262,7 @@ github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd5
 github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=