core job for secure variables re-key (#13440)

When the `Full` flag is passed for key rotation, we kick off a core
job to decrypt and re-encrypt all the secure variables so that they
use the new key.

api/keyring.go

@@ -33,14 +33,24 @@ type RootKey struct {
 
 // RootKeyMeta is the metadata used to refer to a RootKey.
 type RootKeyMeta struct {
-	Active      bool
 	KeyID       string // UUID
 	Algorithm   EncryptionAlgorithm
 	CreateTime  time.Time
 	CreateIndex uint64
 	ModifyIndex uint64
+	State       RootKeyState
 }
 
+// RootKeyState enum describes the lifecycle of a root key.
+type RootKeyState string
+
+const (
+	RootKeyStateInactive   RootKeyState = "inactive"
+	RootKeyStateActive                  = "active"
+	RootKeyStateRekeying                = "rekeying"
+	RootKeyStateDeprecated              = "deprecated"
+)
+
 // List lists all the keyring metadata
 func (k *Keyring) List(q *QueryOptions) ([]*RootKeyMeta, *QueryMeta, error) {
 	var resp []*RootKeyMeta

api/keyring_test.go

@@ -39,7 +39,7 @@ func TestKeyring_CRUD(t *testing.T) {
 		Key: encodedKey,
 		Meta: &RootKeyMeta{
 			KeyID:     id,
-			Active:    true,
+			State:     RootKeyStateActive,
 			Algorithm: EncryptionAlgorithmAES256GCM,
 		}}, nil)
 	require.NoError(t, err)
@@ -57,9 +57,11 @@ func TestKeyring_CRUD(t *testing.T) {
 	require.Len(t, keys, 2)
 	for _, key := range keys {
 		if key.KeyID == id {
-			require.True(t, key.Active, "new key should be active")
+			require.Equal(t, RootKeyState(RootKeyStateActive),
+				key.State, "new key should be active")
 		} else {
-			require.False(t, key.Active, "initial key should be inactive")
+			require.Equal(t, RootKeyState(RootKeyStateInactive),
+				key.State, "initial key should be inactive")
 		}
 	}
 }

command/agent/keyring_endpoint.go

@@ -104,9 +104,9 @@ func (s *HTTPServer) keyringUpsertRequest(resp http.ResponseWriter, req *http.Re
 		RootKey: &structs.RootKey{
 			Key: decodedKey,
 			Meta: &structs.RootKeyMeta{
-				Active:    key.Meta.Active,
 				KeyID:     key.Meta.KeyID,
 				Algorithm: structs.EncryptionAlgorithm(key.Meta.Algorithm),
+				State:     structs.RootKeyState(key.Meta.State),
 			},
 		},
 	}

command/agent/keyring_endpoint_test.go

@@ -31,7 +31,7 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		require.NotZero(t, respW.HeaderMap.Get("X-Nomad-Index"))
 		rotateResp := obj.(structs.KeyringRotateRootKeyResponse)
 		require.NotNil(t, rotateResp.Key)
-		require.True(t, rotateResp.Key.Active)
+		require.True(t, rotateResp.Key.Active())
 		newID1 := rotateResp.Key.KeyID
 
 		// List
@@ -44,9 +44,9 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		require.Len(t, listResp, 2)
 		for _, key := range listResp {
 			if key.KeyID == newID1 {
-				require.True(t, key.Active, "new key should be active")
+				require.True(t, key.Active(), "new key should be active")
 			} else {
-				require.False(t, key.Active, "initial key should be inactive")
+				require.False(t, key.Active(), "initial key should be inactive")
 			}
 		}
 
@@ -61,7 +61,7 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 
 		key := &api.RootKey{
 			Meta: &api.RootKeyMeta{
-				Active:    true,
+				State:     api.RootKeyStateActive,
 				KeyID:     newID2,
 				Algorithm: api.EncryptionAlgorithm(keyMeta.Algorithm),
 			},
@@ -94,9 +94,9 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		for _, key := range listResp {
 			require.NotEqual(t, newID1, key.KeyID)
 			if key.KeyID == newID2 {
-				require.True(t, key.Active, "new key should be active")
+				require.True(t, key.Active(), "new key should be active")
 			} else {
-				require.False(t, key.Active, "initial key should be inactive")
+				require.False(t, key.Active(), "initial key should be inactive")
 			}
 		}
 	})

command/operator_secure_variables_keyring.go

@@ -76,11 +76,11 @@ func renderSecureVariablesKeysResponse(keys []*api.RootKeyMeta, verbose bool) st
 		length = 8
 	}
 	out := make([]string, len(keys)+1)
-	out[0] = "Key|Active|Create Time"
+	out[0] = "Key|State|Create Time"
 	i := 1
 	for _, k := range keys {
 		out[i] = fmt.Sprintf("%s|%v|%s",
-			k.KeyID[:length], k.Active, formatTime(k.CreateTime))
+			k.KeyID[:length], k.State, formatTime(k.CreateTime))
 		i = i + 1
 	}
 	return formatList(out)

nomad/config.go

@@ -208,6 +208,10 @@ type Config struct {
 	// before it's rotated
 	RootKeyRotationThreshold time.Duration
 
+	// SecureVariablesRekeyInterval is how often we dispatch a job to
+	// rekey any variables associated with a key in the Rekeying state
+	SecureVariablesRekeyInterval time.Duration
+
 	// EvalNackTimeout controls how long we allow a sub-scheduler to
 	// work on an evaluation before we consider it failed and Nack it.
 	// This allows that evaluation to be handed to another sub-scheduler
@@ -400,6 +404,7 @@ func DefaultConfig() *Config {
 		RootKeyGCInterval:                10 * time.Minute,
 		RootKeyGCThreshold:               1 * time.Hour,
 		RootKeyRotationThreshold:         720 * time.Hour, // 30 days
+		SecureVariablesRekeyInterval:     10 * time.Minute,
 		EvalNackTimeout:                  60 * time.Second,
 		EvalDeliveryLimit:                3,
 		EvalNackInitialReenqueueDelay:    1 * time.Second,

nomad/core_sched.go

@@ -1,6 +1,7 @@
 package nomad
 
 import (
+	"encoding/json"
 	"fmt"
 	"math"
 	"strings"
@@ -53,6 +54,8 @@ func (c *CoreScheduler) Process(eval *structs.Evaluation) error {
 		return c.expiredOneTimeTokenGC(eval)
 	case structs.CoreJobRootKeyRotateOrGC:
 		return c.rootKeyRotateOrGC(eval)
+	case structs.CoreJobSecureVariablesRekey:
+		return c.secureVariablesRekey(eval)
 	case structs.CoreJobForceGC:
 		return c.forceGC(eval)
 	default:
@@ -817,7 +820,7 @@ func (c *CoreScheduler) rootKeyRotateOrGC(eval *structs.Evaluation) error {
 			break
 		}
 		keyMeta := raw.(*structs.RootKeyMeta)
-		if keyMeta.Active {
+		if keyMeta.Active() {
 			continue // never GC the active key
 		}
 		if keyMeta.CreateIndex > oldThreshold {
@@ -885,6 +888,118 @@ func (c *CoreScheduler) rootKeyRotation(eval *structs.Evaluation) (bool, error)
 	return true, nil
 }
 
+// secureVariablesReKey is optionally run after rotating the active
+// root key. It iterates over all the variables for the keys in the
+// re-keying state, decrypts them, and re-encrypts them in batches
+// with the currently active key. This job does not GC the keys, which
+// is handled in the normal periodic GC job.
+func (c *CoreScheduler) secureVariablesRekey(eval *structs.Evaluation) error {
+
+	ws := memdb.NewWatchSet()
+	iter, err := c.snap.RootKeyMetas(ws)
+	if err != nil {
+		return err
+	}
+
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+		keyMeta := raw.(*structs.RootKeyMeta)
+		if !keyMeta.Rekeying() {
+			continue
+		}
+		varIter, err := c.snap.GetSecureVariablesByKeyID(ws, keyMeta.KeyID)
+		if err != nil {
+			return err
+		}
+		err = c.batchRotateVariables(varIter, eval)
+		if err != nil {
+			return err
+		}
+
+		// we've now rotated all this key's variables, so set its state
+		keyMeta = keyMeta.Copy()
+		keyMeta.SetDeprecated()
+
+		key, err := c.srv.encrypter.GetKey(keyMeta.KeyID)
+		if err != nil {
+			return err
+		}
+		req := &structs.KeyringUpdateRootKeyRequest{
+			RootKey: &structs.RootKey{
+				Meta: keyMeta,
+				Key:  key,
+			},
+			Rekey: false,
+			WriteRequest: structs.WriteRequest{
+				Region:    c.srv.config.Region,
+				AuthToken: eval.LeaderACL},
+		}
+		if err := c.srv.RPC("Keyring.Update",
+			req, &structs.KeyringUpdateRootKeyResponse{}); err != nil {
+			c.logger.Error("root key update failed", "error", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// rootKeyFullRotatePerKey runs over an iterator of secure variables
+// and decrypts them, and then sends them back as batches to be
+// re-encrypted with the currently active key.
+func (c *CoreScheduler) batchRotateVariables(iter memdb.ResultIterator, eval *structs.Evaluation) error {
+
+	upsertFn := func(variables []*structs.SecureVariableDecrypted) error {
+		if len(variables) == 0 {
+			return nil
+		}
+		args := &structs.SecureVariablesUpsertRequest{
+			Data: variables,
+			WriteRequest: structs.WriteRequest{
+				Region:    c.srv.config.Region,
+				AuthToken: eval.LeaderACL,
+			},
+		}
+		reply := &structs.SecureVariablesUpsertResponse{}
+		return c.srv.RPC("SecureVariables.Upsert", args, reply)
+	}
+
+	variables := []*structs.SecureVariableDecrypted{}
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+		ev := raw.(*structs.SecureVariableEncrypted)
+		cleartext, err := c.srv.encrypter.Decrypt(ev.Data, ev.KeyID)
+		if err != nil {
+			return err
+		}
+		dv := &structs.SecureVariableDecrypted{
+			SecureVariableMetadata: ev.SecureVariableMetadata,
+		}
+		dv.Items = make(map[string]string)
+		err = json.Unmarshal(cleartext, &dv.Items)
+		if err != nil {
+			return err
+		}
+		variables = append(variables, dv)
+		if len(variables) == 20 {
+			err := upsertFn(variables)
+			if err != nil {
+				return err
+			}
+			variables = []*structs.SecureVariableDecrypted{}
+		}
+	}
+
+	// ensure we submit any partial batch
+	return upsertFn(variables)
+}
+
 // getThreshold returns the index threshold for determining whether an
 // object is old enough to GC
 func (c *CoreScheduler) getThreshold(eval *structs.Evaluation, objectName, configName string, configThreshold time.Duration) uint64 {