diff --git a/api/keyring.go b/api/keyring.go index 6b7f5f9d492f..9f5ec9b6195c 100644 --- a/api/keyring.go +++ b/api/keyring.go @@ -33,14 +33,24 @@ type RootKey struct { // RootKeyMeta is the metadata used to refer to a RootKey. type RootKeyMeta struct { - Active bool KeyID string // UUID Algorithm EncryptionAlgorithm CreateTime time.Time CreateIndex uint64 ModifyIndex uint64 + State RootKeyState } +// RootKeyState enum describes the lifecycle of a root key. +type RootKeyState string + +const ( + RootKeyStateInactive RootKeyState = "inactive" + RootKeyStateActive = "active" + RootKeyStateRekeying = "rekeying" + RootKeyStateDeprecated = "deprecated" +) + // List lists all the keyring metadata func (k *Keyring) List(q *QueryOptions) ([]*RootKeyMeta, *QueryMeta, error) { var resp []*RootKeyMeta diff --git a/api/keyring_test.go b/api/keyring_test.go index 6b6520241956..041a50ca0336 100644 --- a/api/keyring_test.go +++ b/api/keyring_test.go @@ -39,7 +39,7 @@ func TestKeyring_CRUD(t *testing.T) { Key: encodedKey, Meta: &RootKeyMeta{ KeyID: id, - Active: true, + State: RootKeyStateActive, Algorithm: EncryptionAlgorithmAES256GCM, }}, nil) require.NoError(t, err) @@ -57,9 +57,11 @@ func TestKeyring_CRUD(t *testing.T) { require.Len(t, keys, 2) for _, key := range keys { if key.KeyID == id { - require.True(t, key.Active, "new key should be active") + require.Equal(t, RootKeyState(RootKeyStateActive), + key.State, "new key should be active") } else { - require.False(t, key.Active, "initial key should be inactive") + require.Equal(t, RootKeyState(RootKeyStateInactive), + key.State, "initial key should be inactive") } } } diff --git a/command/agent/keyring_endpoint.go b/command/agent/keyring_endpoint.go index ecbdd991a907..c67ac7201344 100644 --- a/command/agent/keyring_endpoint.go +++ b/command/agent/keyring_endpoint.go @@ -104,9 +104,9 @@ func (s *HTTPServer) keyringUpsertRequest(resp http.ResponseWriter, req *http.Re RootKey: &structs.RootKey{ Key: decodedKey, Meta: &structs.RootKeyMeta{ - Active: key.Meta.Active, KeyID: key.Meta.KeyID, Algorithm: structs.EncryptionAlgorithm(key.Meta.Algorithm), + State: structs.RootKeyState(key.Meta.State), }, }, } diff --git a/command/agent/keyring_endpoint_test.go b/command/agent/keyring_endpoint_test.go index ef2ed6af2aa5..9cb116071a93 100644 --- a/command/agent/keyring_endpoint_test.go +++ b/command/agent/keyring_endpoint_test.go @@ -31,7 +31,7 @@ func TestHTTP_Keyring_CRUD(t *testing.T) { require.NotZero(t, respW.HeaderMap.Get("X-Nomad-Index")) rotateResp := obj.(structs.KeyringRotateRootKeyResponse) require.NotNil(t, rotateResp.Key) - require.True(t, rotateResp.Key.Active) + require.True(t, rotateResp.Key.Active()) newID1 := rotateResp.Key.KeyID // List @@ -44,9 +44,9 @@ func TestHTTP_Keyring_CRUD(t *testing.T) { require.Len(t, listResp, 2) for _, key := range listResp { if key.KeyID == newID1 { - require.True(t, key.Active, "new key should be active") + require.True(t, key.Active(), "new key should be active") } else { - require.False(t, key.Active, "initial key should be inactive") + require.False(t, key.Active(), "initial key should be inactive") } } @@ -61,7 +61,7 @@ func TestHTTP_Keyring_CRUD(t *testing.T) { key := &api.RootKey{ Meta: &api.RootKeyMeta{ - Active: true, + State: api.RootKeyStateActive, KeyID: newID2, Algorithm: api.EncryptionAlgorithm(keyMeta.Algorithm), }, @@ -94,9 +94,9 @@ func TestHTTP_Keyring_CRUD(t *testing.T) { for _, key := range listResp { require.NotEqual(t, newID1, key.KeyID) if key.KeyID == newID2 { - require.True(t, key.Active, "new key should be active") + require.True(t, key.Active(), "new key should be active") } else { - require.False(t, key.Active, "initial key should be inactive") + require.False(t, key.Active(), "initial key should be inactive") } } }) diff --git a/command/operator_secure_variables_keyring.go b/command/operator_secure_variables_keyring.go index a2a948cec795..90d7bce03852 100644 --- a/command/operator_secure_variables_keyring.go +++ b/command/operator_secure_variables_keyring.go @@ -76,11 +76,11 @@ func renderSecureVariablesKeysResponse(keys []*api.RootKeyMeta, verbose bool) st length = 8 } out := make([]string, len(keys)+1) - out[0] = "Key|Active|Create Time" + out[0] = "Key|State|Create Time" i := 1 for _, k := range keys { out[i] = fmt.Sprintf("%s|%v|%s", - k.KeyID[:length], k.Active, formatTime(k.CreateTime)) + k.KeyID[:length], k.State, formatTime(k.CreateTime)) i = i + 1 } return formatList(out) diff --git a/nomad/config.go b/nomad/config.go index 35d0ebb5672d..7c67f1b380ff 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -208,6 +208,10 @@ type Config struct { // before it's rotated RootKeyRotationThreshold time.Duration + // SecureVariablesRekeyInterval is how often we dispatch a job to + // rekey any variables associated with a key in the Rekeying state + SecureVariablesRekeyInterval time.Duration + // EvalNackTimeout controls how long we allow a sub-scheduler to // work on an evaluation before we consider it failed and Nack it. // This allows that evaluation to be handed to another sub-scheduler @@ -400,6 +404,7 @@ func DefaultConfig() *Config { RootKeyGCInterval: 10 * time.Minute, RootKeyGCThreshold: 1 * time.Hour, RootKeyRotationThreshold: 720 * time.Hour, // 30 days + SecureVariablesRekeyInterval: 10 * time.Minute, EvalNackTimeout: 60 * time.Second, EvalDeliveryLimit: 3, EvalNackInitialReenqueueDelay: 1 * time.Second, diff --git a/nomad/core_sched.go b/nomad/core_sched.go index 87661d59dc05..f57b123372e7 100644 --- a/nomad/core_sched.go +++ b/nomad/core_sched.go @@ -1,6 +1,7 @@ package nomad import ( + "encoding/json" "fmt" "math" "strings" @@ -53,6 +54,8 @@ func (c *CoreScheduler) Process(eval *structs.Evaluation) error { return c.expiredOneTimeTokenGC(eval) case structs.CoreJobRootKeyRotateOrGC: return c.rootKeyRotateOrGC(eval) + case structs.CoreJobSecureVariablesRekey: + return c.secureVariablesRekey(eval) case structs.CoreJobForceGC: return c.forceGC(eval) default: @@ -817,7 +820,7 @@ func (c *CoreScheduler) rootKeyRotateOrGC(eval *structs.Evaluation) error { break } keyMeta := raw.(*structs.RootKeyMeta) - if keyMeta.Active { + if keyMeta.Active() { continue // never GC the active key } if keyMeta.CreateIndex > oldThreshold { @@ -885,6 +888,118 @@ func (c *CoreScheduler) rootKeyRotation(eval *structs.Evaluation) (bool, error) return true, nil } +// secureVariablesReKey is optionally run after rotating the active +// root key. It iterates over all the variables for the keys in the +// re-keying state, decrypts them, and re-encrypts them in batches +// with the currently active key. This job does not GC the keys, which +// is handled in the normal periodic GC job. +func (c *CoreScheduler) secureVariablesRekey(eval *structs.Evaluation) error { + + ws := memdb.NewWatchSet() + iter, err := c.snap.RootKeyMetas(ws) + if err != nil { + return err + } + + for { + raw := iter.Next() + if raw == nil { + break + } + keyMeta := raw.(*structs.RootKeyMeta) + if !keyMeta.Rekeying() { + continue + } + varIter, err := c.snap.GetSecureVariablesByKeyID(ws, keyMeta.KeyID) + if err != nil { + return err + } + err = c.batchRotateVariables(varIter, eval) + if err != nil { + return err + } + + // we've now rotated all this key's variables, so set its state + keyMeta = keyMeta.Copy() + keyMeta.SetDeprecated() + + key, err := c.srv.encrypter.GetKey(keyMeta.KeyID) + if err != nil { + return err + } + req := &structs.KeyringUpdateRootKeyRequest{ + RootKey: &structs.RootKey{ + Meta: keyMeta, + Key: key, + }, + Rekey: false, + WriteRequest: structs.WriteRequest{ + Region: c.srv.config.Region, + AuthToken: eval.LeaderACL}, + } + if err := c.srv.RPC("Keyring.Update", + req, &structs.KeyringUpdateRootKeyResponse{}); err != nil { + c.logger.Error("root key update failed", "error", err) + return err + } + } + + return nil +} + +// rootKeyFullRotatePerKey runs over an iterator of secure variables +// and decrypts them, and then sends them back as batches to be +// re-encrypted with the currently active key. +func (c *CoreScheduler) batchRotateVariables(iter memdb.ResultIterator, eval *structs.Evaluation) error { + + upsertFn := func(variables []*structs.SecureVariableDecrypted) error { + if len(variables) == 0 { + return nil + } + args := &structs.SecureVariablesUpsertRequest{ + Data: variables, + WriteRequest: structs.WriteRequest{ + Region: c.srv.config.Region, + AuthToken: eval.LeaderACL, + }, + } + reply := &structs.SecureVariablesUpsertResponse{} + return c.srv.RPC("SecureVariables.Upsert", args, reply) + } + + variables := []*structs.SecureVariableDecrypted{} + for { + raw := iter.Next() + if raw == nil { + break + } + ev := raw.(*structs.SecureVariableEncrypted) + cleartext, err := c.srv.encrypter.Decrypt(ev.Data, ev.KeyID) + if err != nil { + return err + } + dv := &structs.SecureVariableDecrypted{ + SecureVariableMetadata: ev.SecureVariableMetadata, + } + dv.Items = make(map[string]string) + err = json.Unmarshal(cleartext, &dv.Items) + if err != nil { + return err + } + variables = append(variables, dv) + if len(variables) == 20 { + err := upsertFn(variables) + if err != nil { + return err + } + variables = []*structs.SecureVariableDecrypted{} + } + } + + // ensure we submit any partial batch + return upsertFn(variables) +} + // getThreshold returns the index threshold for determining whether an // object is old enough to GC func (c *CoreScheduler) getThreshold(eval *structs.Evaluation, objectName, configName string, configThreshold time.Duration) uint64 { diff --git a/nomad/core_sched_test.go b/nomad/core_sched_test.go index d12c09ae3efd..82017ff94cef 100644 --- a/nomad/core_sched_test.go +++ b/nomad/core_sched_test.go @@ -2472,13 +2472,13 @@ func TestCoreScheduler_RootKeyGC(t *testing.T) { // insert an "old" and inactive key key1 := structs.NewRootKeyMeta() - key1.Active = false - require.NoError(t, store.UpsertRootKeyMeta(500, key1)) + key1.SetInactive() + require.NoError(t, store.UpsertRootKeyMeta(500, key1, false)) // insert an "old" and inactive key with a variable that's using it key2 := structs.NewRootKeyMeta() - key2.Active = false - require.NoError(t, store.UpsertRootKeyMeta(600, key2)) + key2.SetInactive() + require.NoError(t, store.UpsertRootKeyMeta(600, key2, false)) variable := mock.SecureVariableEncrypted() variable.KeyID = key2.KeyID @@ -2493,8 +2493,8 @@ func TestCoreScheduler_RootKeyGC(t *testing.T) { // insert an "old" key that's newer than oldest alloc key3 := structs.NewRootKeyMeta() - key3.Active = false - require.NoError(t, store.UpsertRootKeyMeta(750, key3)) + key3.SetInactive() + require.NoError(t, store.UpsertRootKeyMeta(750, key3, false)) // insert a time table index before the last key tt := srv.fsm.TimeTable() @@ -2502,8 +2502,8 @@ func TestCoreScheduler_RootKeyGC(t *testing.T) { // insert a "new" but inactive key key4 := structs.NewRootKeyMeta() - key4.Active = false - require.NoError(t, store.UpsertRootKeyMeta(1500, key4)) + key4.SetInactive() + require.NoError(t, store.UpsertRootKeyMeta(1500, key4, false)) // run the core job snap, err := store.Snapshot() @@ -2535,6 +2535,88 @@ func TestCoreScheduler_RootKeyGC(t *testing.T) { require.NotNil(t, key, "new key should not have been GCd") } +// TestCoreScheduler_SecureVariablesRekey exercises secure variables rekeying +func TestCoreScheduler_SecureVariablesRekey(t *testing.T) { + ci.Parallel(t) + + srv, cleanup := TestServer(t, nil) + defer cleanup() + testutil.WaitForLeader(t, srv.RPC) + + store := srv.fsm.State() + key0, err := store.GetActiveRootKeyMeta(nil) + require.NotNil(t, key0, "expected keyring to be bootstapped") + require.NoError(t, err) + + req := &structs.SecureVariablesUpsertRequest{ + Data: []*structs.SecureVariableDecrypted{ + mock.SecureVariable(), + mock.SecureVariable(), + mock.SecureVariable(), + }, + WriteRequest: structs.WriteRequest{ + Region: srv.config.Region, + }, + } + resp := &structs.SecureVariablesUpsertResponse{} + require.NoError(t, srv.RPC("SecureVariables.Upsert", req, resp)) + + rotateReq := &structs.KeyringRotateRootKeyRequest{ + WriteRequest: structs.WriteRequest{ + Region: srv.config.Region, + }, + } + var rotateResp structs.KeyringRotateRootKeyResponse + require.NoError(t, srv.RPC("Keyring.Rotate", rotateReq, &rotateResp)) + + req2 := &structs.SecureVariablesUpsertRequest{ + Data: []*structs.SecureVariableDecrypted{ + mock.SecureVariable(), + mock.SecureVariable(), + mock.SecureVariable(), + }, + WriteRequest: structs.WriteRequest{ + Region: srv.config.Region, + }, + } + require.NoError(t, srv.RPC("SecureVariables.Upsert", req2, resp)) + + rotateReq.Full = true + require.NoError(t, srv.RPC("Keyring.Rotate", rotateReq, &rotateResp)) + newKeyID := rotateResp.Key.KeyID + + require.Eventually(t, func() bool { + ws := memdb.NewWatchSet() + iter, err := store.SecureVariables(ws) + require.NoError(t, err) + for { + raw := iter.Next() + if raw == nil { + break + } + variable := raw.(*structs.SecureVariableEncrypted) + if variable.KeyID != newKeyID { + return false + } + } + return true + }, time.Second*5, 100*time.Millisecond, + "secure variable rekey should be complete") + + iter, err := store.RootKeyMetas(memdb.NewWatchSet()) + require.NoError(t, err) + for { + raw := iter.Next() + if raw == nil { + break + } + keyMeta := raw.(*structs.RootKeyMeta) + if keyMeta.KeyID != newKeyID { + require.True(t, keyMeta.Deprecated()) + } + } +} + func TestCoreScheduler_FailLoop(t *testing.T) { ci.Parallel(t) diff --git a/nomad/encrypter.go b/nomad/encrypter.go index ff77e6df0dec..4a0027e80ac3 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -348,7 +348,7 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { } meta := &structs.RootKeyMeta{ - Active: storedKey.Meta.Active, + State: storedKey.Meta.State, KeyID: storedKey.Meta.KeyID, Algorithm: storedKey.Meta.Algorithm, CreateTime: storedKey.Meta.CreateTime, diff --git a/nomad/fsm.go b/nomad/fsm.go index 0d80f85871c2..0b4cc1305fd3 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -2074,7 +2074,7 @@ func (n *nomadFSM) applyRootKeyMetaUpsert(msgType structs.MessageType, buf []byt panic(fmt.Errorf("failed to decode request: %v", err)) } - if err := n.state.UpsertRootKeyMeta(index, req.RootKeyMeta); err != nil { + if err := n.state.UpsertRootKeyMeta(index, req.RootKeyMeta, req.Rekey); err != nil { n.logger.Error("UpsertRootKeyMeta failed", "error", err) return err } diff --git a/nomad/keyring_endpoint.go b/nomad/keyring_endpoint.go index fc9c29477ab4..df0e3f0002a1 100644 --- a/nomad/keyring_endpoint.go +++ b/nomad/keyring_endpoint.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" ) @@ -33,9 +34,6 @@ func (k *Keyring) Rotate(args *structs.KeyringRotateRootKeyRequest, reply *struc return structs.ErrPermissionDenied } - if args.Full { - // TODO: implement full key rotation via a core job - } if args.Algorithm == "" { args.Algorithm = structs.EncryptionAlgorithmAES256GCM } @@ -45,7 +43,7 @@ func (k *Keyring) Rotate(args *structs.KeyringRotateRootKeyRequest, reply *struc return err } - rootKey.Meta.Active = true + rootKey.Meta.SetActive() // make sure it's been added to the local keystore before we write // it to raft, so that followers don't try to Get a key that @@ -58,6 +56,7 @@ func (k *Keyring) Rotate(args *structs.KeyringRotateRootKeyRequest, reply *struc // Update metadata via Raft so followers can retrieve this key req := structs.KeyringUpdateRootKeyMetaRequest{ RootKeyMeta: rootKey.Meta, + Rekey: args.Full, WriteRequest: args.WriteRequest, } out, index, err := k.srv.raftApply(structs.RootKeyMetaUpsertRequestType, req) @@ -69,6 +68,24 @@ func (k *Keyring) Rotate(args *structs.KeyringRotateRootKeyRequest, reply *struc } reply.Key = rootKey.Meta reply.Index = index + + if args.Full { + // like most core jobs, we don't commit this to raft b/c it's not + // going to be periodically recreated and the ACL is from this leader + eval := &structs.Evaluation{ + ID: uuid.Generate(), + Namespace: "-", + Priority: structs.CoreJobPriority, + Type: structs.JobTypeCore, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: structs.CoreJobSecureVariablesRekey, + Status: structs.EvalStatusPending, + ModifyIndex: index, + LeaderACL: k.srv.getLeaderAcl(), + } + k.srv.evalBroker.Enqueue(eval) + } + return nil } @@ -284,7 +301,7 @@ func (k *Keyring) Delete(args *structs.KeyringDeleteRootKeyRequest, reply *struc if keyMeta == nil { return nil // safe to bail out early } - if keyMeta.Active { + if keyMeta.Active() { return fmt.Errorf("active root key cannot be deleted - call rotate first") } diff --git a/nomad/keyring_endpoint_test.go b/nomad/keyring_endpoint_test.go index 15e226539ca2..f9af1434e17f 100644 --- a/nomad/keyring_endpoint_test.go +++ b/nomad/keyring_endpoint_test.go @@ -29,7 +29,7 @@ func TestKeyringEndpoint_CRUD(t *testing.T) { key, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) require.NoError(t, err) id := key.Meta.KeyID - key.Meta.Active = true + key.Meta.SetActive() updateReq := &structs.KeyringUpdateRootKeyRequest{ RootKey: key, @@ -104,7 +104,7 @@ func TestKeyringEndpoint_CRUD(t *testing.T) { require.EqualError(t, err, "active root key cannot be deleted - call rotate first") // set inactive - updateReq.RootKey.Meta.Active = false + updateReq.RootKey.Meta.SetInactive() err = msgpackrpc.CallWithCodec(codec, "Keyring.Update", updateReq, &updateResp) require.NoError(t, err) @@ -137,7 +137,7 @@ func TestKeyringEndpoint_InvalidUpdates(t *testing.T) { key, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) require.NoError(t, err) id := key.Meta.KeyID - key.Meta.Active = true + key.Meta.SetActive() updateReq := &structs.KeyringUpdateRootKeyRequest{ RootKey: key, @@ -171,14 +171,24 @@ func TestKeyringEndpoint_InvalidUpdates(t *testing.T) { KeyID: id, Algorithm: structs.EncryptionAlgorithmAES256GCM, }}, + expectedErrMsg: "root key state \"\" is invalid", + }, + { + key: &structs.RootKey{Meta: &structs.RootKeyMeta{ + KeyID: id, + Algorithm: structs.EncryptionAlgorithmAES256GCM, + State: structs.RootKeyStateActive, + }}, expectedErrMsg: "root key material is required", }, + { key: &structs.RootKey{ Key: []byte{0x01}, Meta: &structs.RootKeyMeta{ KeyID: id, Algorithm: "whatever", + State: structs.RootKeyStateActive, }}, expectedErrMsg: "root key algorithm cannot be changed after a key is created", }, @@ -216,7 +226,7 @@ func TestKeyringEndpoint_Rotate(t *testing.T) { // Setup an existing key key, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) require.NoError(t, err) - key.Meta.Active = true + key.Meta.SetActive() updateReq := &structs.KeyringUpdateRootKeyRequest{ RootKey: key, @@ -263,9 +273,9 @@ func TestKeyringEndpoint_Rotate(t *testing.T) { for _, keyMeta := range listResp.Keys { if keyMeta.KeyID != newID { - require.False(t, keyMeta.Active, "expected old keys to be inactive") + require.False(t, keyMeta.Active(), "expected old keys to be inactive") } else { - require.True(t, keyMeta.Active, "expected new key to be inactive") + require.True(t, keyMeta.Active(), "expected new key to be inactive") } } diff --git a/nomad/leader.go b/nomad/leader.go index 2904d9d88de3..6a71aced4666 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -769,6 +769,8 @@ func (s *Server) schedulePeriodic(stopCh chan struct{}) { defer oneTimeTokenGC.Stop() rootKeyGC := time.NewTicker(s.config.RootKeyGCInterval) defer rootKeyGC.Stop() + secureVariablesRekey := time.NewTicker(s.config.SecureVariablesRekeyInterval) + defer secureVariablesRekey.Stop() // getLatest grabs the latest index from the state store. It returns true if // the index was retrieved successfully. @@ -821,6 +823,11 @@ func (s *Server) schedulePeriodic(stopCh chan struct{}) { if index, ok := getLatest(); ok { s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobRootKeyRotateOrGC, index)) } + case <-secureVariablesRekey.C: + if index, ok := getLatest(); ok { + s.evalBroker.Enqueue(s.coreJobEval(structs.CoreJobSecureVariablesRekey, index)) + } + case <-stopCh: return } @@ -1710,7 +1717,7 @@ func (s *Server) initializeKeyring() error { s.logger.Named("core").Trace("initializing keyring") rootKey, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) - rootKey.Meta.Active = true + rootKey.Meta.SetActive() if err != nil { return fmt.Errorf("could not initialize keyring: %v", err) } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 139f2972a93e..0aae2d2a29b4 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -6629,7 +6629,7 @@ func getPreemptedAllocDesiredDescription(preemptedByAllocID string) string { } // UpsertRootKeyMeta saves root key meta or updates it in-place. -func (s *StateStore) UpsertRootKeyMeta(index uint64, rootKeyMeta *structs.RootKeyMeta) error { +func (s *StateStore) UpsertRootKeyMeta(index uint64, rootKeyMeta *structs.RootKeyMeta, rekey bool) error { txn := s.db.WriteTxn(index) defer txn.Abort() @@ -6645,14 +6645,18 @@ func (s *StateStore) UpsertRootKeyMeta(index uint64, rootKeyMeta *structs.RootKe existing := raw.(*structs.RootKeyMeta) rootKeyMeta.CreateIndex = existing.CreateIndex rootKeyMeta.CreateTime = existing.CreateTime - isRotation = !existing.Active && rootKeyMeta.Active + isRotation = !existing.Active() && rootKeyMeta.Active() } else { rootKeyMeta.CreateIndex = index rootKeyMeta.CreateTime = time.Now() - isRotation = rootKeyMeta.Active + isRotation = rootKeyMeta.Active() } rootKeyMeta.ModifyIndex = index + if rekey && !isRotation { + return fmt.Errorf("cannot rekey without setting the new key active") + } + // if the upsert is for a newly-active key, we need to set all the // other keys as inactive in the same transaction. if isRotation { @@ -6666,13 +6670,32 @@ func (s *StateStore) UpsertRootKeyMeta(index uint64, rootKeyMeta *structs.RootKe break } key := raw.(*structs.RootKeyMeta) - if key.Active { - key.Active = false + modified := false + + switch key.State { + case structs.RootKeyStateInactive: + if rekey { + key.SetRekeying() + modified = true + } + case structs.RootKeyStateActive: + if rekey { + key.SetRekeying() + } else { + key.SetInactive() + } + modified = true + case structs.RootKeyStateRekeying, structs.RootKeyStateDeprecated: + // nothing to do + } + + if modified { key.ModifyIndex = index if err := txn.Insert(TableRootKeyMeta, key); err != nil { return err } } + } } @@ -6758,7 +6781,7 @@ func (s *StateStore) GetActiveRootKeyMeta(ws memdb.WatchSet) (*structs.RootKeyMe break } key := raw.(*structs.RootKeyMeta) - if key.Active { + if key.Active() { return key, nil } } diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 512f0336f278..5f3c9f4e17cb 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -9920,10 +9920,10 @@ func TestStateStore_RootKeyMetaData_CRUD(t *testing.T) { key := structs.NewRootKeyMeta() keyIDs = append(keyIDs, key.KeyID) if i == 0 { - key.Active = true + key.SetActive() } index++ - require.NoError(t, store.UpsertRootKeyMeta(index, key)) + require.NoError(t, store.UpsertRootKeyMeta(index, key, false)) } // retrieve the active key @@ -9937,9 +9937,9 @@ func TestStateStore_RootKeyMetaData_CRUD(t *testing.T) { require.NotNil(t, inactiveKey) oldCreateIndex := inactiveKey.CreateIndex newlyActiveKey := inactiveKey.Copy() - newlyActiveKey.Active = true + newlyActiveKey.SetActive() index++ - require.NoError(t, store.UpsertRootKeyMeta(index, newlyActiveKey)) + require.NoError(t, store.UpsertRootKeyMeta(index, newlyActiveKey, false)) iter, err := store.RootKeyMetas(nil) require.NoError(t, err) @@ -9950,10 +9950,10 @@ func TestStateStore_RootKeyMetaData_CRUD(t *testing.T) { } key := raw.(*structs.RootKeyMeta) if key.KeyID == newlyActiveKey.KeyID { - require.True(t, key.Active, "expected updated key to be active") + require.True(t, key.Active(), "expected updated key to be active") require.Equal(t, oldCreateIndex, key.CreateIndex) } else { - require.False(t, key.Active, "expected other keys to be inactive") + require.False(t, key.Active(), "expected other keys to be inactive") } } @@ -9971,7 +9971,7 @@ func TestStateStore_RootKeyMetaData_CRUD(t *testing.T) { } key := raw.(*structs.RootKeyMeta) require.NotEqual(t, keyIDs[1], key.KeyID) - require.False(t, key.Active, "expected remaining keys to be inactive") + require.False(t, key.Active(), "expected remaining keys to be inactive") found++ } require.Equal(t, 2, found, "expected only 2 keys remaining") diff --git a/nomad/structs/secure_variables.go b/nomad/structs/secure_variables.go index 9ea5fe81600e..dbc11f040de5 100644 --- a/nomad/structs/secure_variables.go +++ b/nomad/structs/secure_variables.go @@ -291,19 +291,30 @@ func NewRootKey(algorithm EncryptionAlgorithm) (*RootKey, error) { // RootKeyMeta is the metadata used to refer to a RootKey. It is // stored in raft. type RootKeyMeta struct { - Active bool KeyID string // UUID Algorithm EncryptionAlgorithm CreateTime time.Time CreateIndex uint64 ModifyIndex uint64 + State RootKeyState } +// RootKeyState enum describes the lifecycle of a root key. +type RootKeyState string + +const ( + RootKeyStateInactive RootKeyState = "inactive" + RootKeyStateActive = "active" + RootKeyStateRekeying = "rekeying" + RootKeyStateDeprecated = "deprecated" +) + // NewRootKeyMeta returns a new RootKeyMeta with default values func NewRootKeyMeta() *RootKeyMeta { return &RootKeyMeta{ KeyID: uuid.Generate(), Algorithm: EncryptionAlgorithmAES256GCM, + State: RootKeyStateInactive, CreateTime: time.Now(), } } @@ -316,7 +327,41 @@ type RootKeyMetaStub struct { KeyID string Algorithm EncryptionAlgorithm CreateTime time.Time - Active bool + State RootKeyState +} + +// Active indicates his key is the one currently being used for +// crypto operations (at most one key can be Active) +func (rkm *RootKeyMeta) Active() bool { + return rkm.State == RootKeyStateActive +} + +func (rkm *RootKeyMeta) SetActive() { + rkm.State = RootKeyStateActive +} + +// Rekeying indicates that variables encrypted with this key should be +// rekeyed +func (rkm *RootKeyMeta) Rekeying() bool { + return rkm.State == RootKeyStateRekeying +} + +func (rkm *RootKeyMeta) SetRekeying() { + rkm.State = RootKeyStateRekeying +} + +func (rkm *RootKeyMeta) SetInactive() { + rkm.State = RootKeyStateInactive +} + +// Deprecated indicates that variables encrypted with this key +// have been rekeyed +func (rkm *RootKeyMeta) Deprecated() bool { + return rkm.State == RootKeyStateDeprecated +} + +func (rkm *RootKeyMeta) SetDeprecated() { + rkm.State = RootKeyStateDeprecated } func (rkm *RootKeyMeta) Stub() *RootKeyMetaStub { @@ -327,7 +372,7 @@ func (rkm *RootKeyMeta) Stub() *RootKeyMetaStub { KeyID: rkm.KeyID, Algorithm: rkm.Algorithm, CreateTime: rkm.CreateTime, - Active: rkm.Active, + State: rkm.State, } } @@ -349,6 +394,12 @@ func (rkm *RootKeyMeta) Validate() error { if rkm.Algorithm == "" { return fmt.Errorf("root key algorithm is required") } + switch rkm.State { + case RootKeyStateInactive, RootKeyStateActive, + RootKeyStateRekeying, RootKeyStateDeprecated: + default: + return fmt.Errorf("root key state %q is invalid", rkm.State) + } return nil } @@ -388,6 +439,7 @@ type KeyringListRootKeyMetaResponse struct { // (see below) type KeyringUpdateRootKeyRequest struct { RootKey *RootKey + Rekey bool WriteRequest } @@ -412,6 +464,7 @@ type KeyringGetRootKeyResponse struct { // metadata to the FSM without including the key material type KeyringUpdateRootKeyMetaRequest struct { RootKeyMeta *RootKeyMeta + Rekey bool WriteRequest } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 485d6c9d5b49..031436d7c73d 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -10791,6 +10791,11 @@ const ( // garbage collection of unused encryption keys. CoreJobRootKeyRotateOrGC = "root-key-rotate-gc" + // CoreJobSecureVariablesRekey is used to fully rotate the + // encryption keys for secure variables by decrypting all secure + // variables and re-encrypting them with the active key + CoreJobSecureVariablesRekey = "secure-variables-rekey" + // CoreJobForceGC is used to force garbage collection of all GCable objects. CoreJobForceGC = "force-gc" )