bootstrap keyring (#13124)

When a server becomes leader, it will check if there are any keys in
the state store, and create one if there is not. The key metadata will
be replicated via raft to all followers, who will then get the key
material via key replication (not implemented in this changeset).

api/keyring_test.go

@@ -27,8 +27,7 @@ func TestKeyring_CRUD(t *testing.T) {
 	keys, qm, err := kr.List(&QueryOptions{WaitIndex: key.CreateIndex})
 	require.NoError(t, err)
 	assertQueryMeta(t, qm)
-	// TODO: there'll be 2 keys here once we get bootstrapping done
-	require.Len(t, keys, 1)
+	require.Len(t, keys, 2)
 
 	// Write a new active key, forcing a rotation
 	id := "fd77c376-9785-4c80-8e62-4ec3ab5f8b9a"
@@ -57,8 +56,12 @@ func TestKeyring_CRUD(t *testing.T) {
 	keys, qm, err = kr.List(&QueryOptions{WaitIndex: key.CreateIndex})
 	require.NoError(t, err)
 	assertQueryMeta(t, qm)
-	// TODO: there'll be 2 keys here once we get bootstrapping done
-	require.Len(t, keys, 1)
-	require.Equal(t, id, keys[0].KeyID)
-
+	require.Len(t, keys, 2)
+	for _, key := range keys {
+		if key.KeyID == id {
+			require.True(t, key.Active, "new key should be active")
+		} else {
+			require.False(t, key.Active, "initial key should be inactive")
+		}
+	}
 }

command/agent/keyring_endpoint_test.go

@@ -32,6 +32,7 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		rotateResp := obj.(structs.KeyringRotateRootKeyResponse)
 		require.NotNil(t, rotateResp.Key)
 		require.True(t, rotateResp.Key.Active)
+		newID1 := rotateResp.Key.KeyID
 
 		// List
 
@@ -40,8 +41,14 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		obj, err = s.Server.KeyringRequest(respW, req)
 		require.NoError(t, err)
 		listResp := obj.([]*structs.RootKeyMeta)
-		require.Len(t, listResp, 1)
-		require.True(t, listResp[0].Active)
+		require.Len(t, listResp, 2)
+		for _, key := range listResp {
+			if key.KeyID == newID1 {
+				require.True(t, key.Active, "new key should be active")
+			} else {
+				require.False(t, key.Active, "initial key should be inactive")
+			}
+		}
 
 		// Update
 
@@ -51,12 +58,12 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		encodedKey := make([]byte, base64.StdEncoding.EncodedLen(32))
 		base64.StdEncoding.Encode(encodedKey, keyBuf)
 
-		newID := uuid.Generate()
+		newID2 := uuid.Generate()
 
 		key := &api.RootKey{
 			Meta: &api.RootKeyMeta{
 				Active:           true,
-				KeyID:            newID,
+				KeyID:            newID2,
 				Algorithm:        api.EncryptionAlgorithm(keyMeta.Algorithm),
 				EncryptionsCount: 500,
 			},
@@ -84,9 +91,15 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		obj, err = s.Server.KeyringRequest(respW, req)
 		require.NoError(t, err)
 		listResp = obj.([]*structs.RootKeyMeta)
-		require.Len(t, listResp, 1)
-		require.True(t, listResp[0].Active)
-		require.Equal(t, newID, listResp[0].KeyID)
-
+		require.Len(t, listResp, 2)
+
+		for _, key := range listResp {
+			require.NotEqual(t, newID1, key.KeyID)
+			if key.KeyID == newID2 {
+				require.True(t, key.Active, "new key should be active")
+			} else {
+				require.False(t, key.Active, "initial key should be inactive")
+			}
+		}
 	})
 }

nomad/encrypter_test.go

@@ -57,9 +57,19 @@ func TestEncrypter_Restore(t *testing.T) {
 	defer shutdown()
 	testutil.WaitForLeader(t, srv.RPC)
 	codec := rpcClient(t, srv)
-
 	nodeID := srv.GetConfig().NodeID
 
+	// Verify we have a bootstrap key
+
+	listReq := &structs.KeyringListRootKeyMetaRequest{
+		QueryOptions: structs.QueryOptions{
+			Region: "global",
+		},
+	}
+	var listResp structs.KeyringListRootKeyMetaResponse
+	msgpackrpc.CallWithCodec(codec, "Keyring.List", listReq, &listResp)
+	require.Len(t, listResp.Keys, 1)
+
 	// Send a few key rotations to add keys
 
 	rotateReq := &structs.KeyringRotateRootKeyRequest{
@@ -89,15 +99,9 @@ func TestEncrypter_Restore(t *testing.T) {
 
 	// Verify we've restored all the keys from the old keystore
 
-	listReq := &structs.KeyringListRootKeyMetaRequest{
-		QueryOptions: structs.QueryOptions{
-			Region: "global",
-		},
-	}
-	var listResp structs.KeyringListRootKeyMetaResponse
 	err := msgpackrpc.CallWithCodec(codec, "Keyring.List", listReq, &listResp)
 	require.NoError(t, err)
-	require.Len(t, listResp.Keys, 4)
+	require.Len(t, listResp.Keys, 5) // 4 new + the bootstrap key
 
 	for _, keyMeta := range listResp.Keys {
 

nomad/keyring_endpoint_test.go

@@ -85,7 +85,7 @@ func TestKeyringEndpoint_CRUD(t *testing.T) {
 	// wait for the blocking query to complete and check the response
 	wg.Wait()
 	require.Greater(t, listResp.Index, getResp.Index)
-	require.Len(t, listResp.Keys, 1)
+	require.Len(t, listResp.Keys, 2) // bootstrap + new one
 
 	// Delete the key and verify that it's gone
 
@@ -117,7 +117,7 @@ func TestKeyringEndpoint_CRUD(t *testing.T) {
 	err = msgpackrpc.CallWithCodec(codec, "Keyring.List", listReq, &listResp)
 	require.NoError(t, err)
 	require.Greater(t, listResp.Index, getResp.Index)
-	require.Len(t, listResp.Keys, 0)
+	require.Len(t, listResp.Keys, 1) // just the bootstrap key
 }
 
 // TestKeyringEndpoint_validateUpdate exercises all the various
@@ -215,7 +215,6 @@ func TestKeyringEndpoint_Rotate(t *testing.T) {
 	// Setup an existing key
 	key, err := structs.NewRootKey(structs.EncryptionAlgorithmXChaCha20)
 	require.NoError(t, err)
-	id := key.Meta.KeyID
 	key.Meta.Active = true
 
 	updateReq := &structs.KeyringUpdateRootKeyRequest{
@@ -245,6 +244,8 @@ func TestKeyringEndpoint_Rotate(t *testing.T) {
 	require.NoError(t, err)
 	require.NotEqual(t, updateResp.Index, rotateResp.Index)
 
+	newID := rotateResp.Key.KeyID
+
 	// Verify we have a new key and the old one is inactive
 
 	listReq := &structs.KeyringListRootKeyMetaRequest{
@@ -257,15 +258,13 @@ func TestKeyringEndpoint_Rotate(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Greater(t, listResp.Index, updateResp.Index)
-	require.Len(t, listResp.Keys, 2)
+	require.Len(t, listResp.Keys, 3) // bootstrap + old + new
 
-	var newID string
 	for _, keyMeta := range listResp.Keys {
-		if keyMeta.KeyID == id {
-			require.False(t, keyMeta.Active, "expected old key to be inactive")
+		if keyMeta.KeyID != newID {
+			require.False(t, keyMeta.Active, "expected old keys to be inactive")
 		} else {
 			require.True(t, keyMeta.Active, "expected new key to be inactive")
-			newID = keyMeta.KeyID
 		}
 	}
 

nomad/leader.go

@@ -292,6 +292,12 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error {
 	// Initialize scheduler configuration
 	s.getOrCreateSchedulerConfig()
 
+	// Create the first root key if it doesn't already exist
+	err := s.initializeKeyring()
+	if err != nil {
+		return err
+	}
+
 	// Initialize the ClusterID
 	_, _ = s.ClusterID()
 	// todo: use cluster ID for stuff, later!
@@ -1678,6 +1684,46 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
 	return config
 }
 
+// initializeKeyring creates the first root key if the leader doesn't
+// already have one. The metadata will be replicated via raft and then
+// the followers will get the key material from their own key
+// replication.
+func (s *Server) initializeKeyring() error {
+
+	store := s.fsm.State()
+	keyMeta, err := store.GetActiveRootKeyMeta(nil)
+	if err != nil {
+		return err
+	}
+	if keyMeta != nil {
+		return nil
+	}
+
+	s.logger.Named("core").Trace("initializing keyring")
+
+	// TODO: algorithm should be set from config
+	rootKey, err := structs.NewRootKey(structs.EncryptionAlgorithmXChaCha20)
+	rootKey.Meta.Active = true
+	if err != nil {
+		return fmt.Errorf("could not initialize keyring: %v", err)
+	}
+
+	err = s.staticEndpoints.Keyring.encrypter.AddKey(rootKey)
+	if err != nil {
+		return fmt.Errorf("could not add initial key to keyring: %v", err)
+	}
+
+	if _, _, err = s.raftApply(structs.RootKeyMetaUpsertRequestType,
+		structs.KeyringUpdateRootKeyMetaRequest{
+			RootKeyMeta: rootKey.Meta,
+		}); err != nil {
+		return fmt.Errorf("could not initialize keyring: %v", err)
+	}
+
+	s.logger.Named("core").Info("initialized keyring", "id", rootKey.Meta.KeyID)
+	return nil
+}
+
 func (s *Server) generateClusterID() (string, error) {
 	if !ServersMeetMinimumVersion(s.Members(), minClusterIDVersion, false) {
 		s.logger.Named("core").Warn("cannot initialize cluster ID until all servers are above minimum version", "min_version", minClusterIDVersion)

nomad/server.go

@@ -1172,6 +1172,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) error {
 		s.staticEndpoints.Search = &Search{srv: s, logger: s.logger.Named("search")}
 		s.staticEndpoints.Namespace = &Namespace{srv: s}
 		s.staticEndpoints.SecureVariables = &SecureVariables{srv: s, logger: s.logger.Named("secure_variables"), encrypter: encrypter}
+		s.staticEndpoints.Keyring = &Keyring{srv: s, logger: s.logger.Named("keyring"), encrypter: encrypter}
 
 		s.staticEndpoints.Enterprise = NewEnterpriseEndpoints(s)
 
@@ -1229,7 +1230,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) error {
 	node := &Node{srv: s, ctx: ctx, logger: s.logger.Named("client")}
 	plan := &Plan{srv: s, ctx: ctx, logger: s.logger.Named("plan")}
 	serviceReg := &ServiceRegistration{srv: s, ctx: ctx}
-	keyringReg := &Keyring{srv: s, logger: s.logger.Named("keyring"), encrypter: encrypter}
+	keyringReg := &Keyring{srv: s, ctx: ctx, logger: s.logger.Named("keyring"), encrypter: encrypter}
 
 	// Register the dynamic endpoints
 	server.Register(alloc)