fix panic from keyring raft entries being written during upgrade

nomad/encrypter_test.go

@@ -7,7 +7,9 @@ import (
 	"testing"
 	"time"
 
+	version "github.com/hashicorp/go-version"
 	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/nomad/ci"
@@ -312,3 +314,25 @@ func TestEncrypter_SignVerify(t *testing.T) {
 	require.Equal(t, alloc.JobID, got.JobID)
 	require.Equal(t, "web", got.TaskName)
 }
+
+func TestEncrypter_VersionCheck(t *testing.T) {
+
+	validVersions := []string{"1.4.0", "1.4.1", "1.5.2", "1.4.0-dev", "1.4.0-beta.1"}
+	invalidVersions := []string{"1.3.5", "1.3.5-dev"}
+
+	for _, v := range validVersions {
+		memberVersion := version.Must(version.NewVersion(v))
+
+		must.True(t, memberVersion.Core().GreaterThanOrEqual(minVersionKeyring),
+			must.Sprintf("expected %v >= 1.4.0", v),
+		)
+	}
+
+	for _, v := range invalidVersions {
+		memberVersion := version.Must(version.NewVersion(v))
+		must.True(t, memberVersion.Core().LessThan(minVersionKeyring),
+			must.Sprintf("expected %v < 1.4.0", v),
+		)
+	}
+
+}

nomad/leader.go

@@ -294,10 +294,7 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error {
 	schedulerConfig := s.getOrCreateSchedulerConfig()
 
 	// Create the first root key if it doesn't already exist
-	err := s.initializeKeyring()
-	if err != nil {
-		return err
-	}
+	go s.initializeKeyring(stopCh)
 
 	// Initialize the ClusterID
 	_, _ = s.ClusterID()
@@ -1966,12 +1963,13 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
 	return config
 }
 
+var minVersionKeyring = version.Must(version.NewVersion("1.4.0"))
+
 // initializeKeyring creates the first root key if the leader doesn't
 // already have one. The metadata will be replicated via raft and then
 // the followers will get the key material from their own key
 // replication.
-func (s *Server) initializeKeyring() error {
-
+func (s *Server) initializeKeyring(stopCh <-chan struct{}) error {
 	store := s.fsm.State()
 	keyMeta, err := store.GetActiveRootKeyMeta(nil)
 	if err != nil {
@@ -1981,6 +1979,35 @@ func (s *Server) initializeKeyring() error {
 		return nil
 	}
 
+	s.logger.Named("core").Trace("verifying cluster is ready to initialize keyring")
+
+	versionCheck := func() bool {
+		members := s.serf.Members()
+		for _, member := range members {
+			build := member.Tags["build"]
+			memberVersion := version.Must(version.NewVersion(build))
+			if memberVersion.Core().LessThan(minVersionKeyring) {
+				return false
+			}
+		}
+		return true
+	}
+
+	for {
+		select {
+		case <-stopCh:
+			return nil
+		default:
+		}
+		if versionCheck() {
+			break
+		}
+	}
+	// we might have lost leadershuip during the version check
+	if !s.IsLeader() {
+		return nil
+	}
+
 	s.logger.Named("core").Trace("initializing keyring")
 
 	rootKey, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM)