Skip to content

Commit

Permalink
kmsv2: KDF based nonce extension
Browse files Browse the repository at this point in the history
Signed-off-by: Monis Khan <mok@microsoft.com>
  • Loading branch information
enj committed Jul 21, 2023
1 parent 31d662e commit bf49c72
Show file tree
Hide file tree
Showing 19 changed files with 1,686 additions and 372 deletions.
9 changes: 9 additions & 0 deletions staging/src/k8s.io/apiserver/pkg/features/kube_features.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ const (
// Enables KMS v2 API for encryption at rest.
KMSv2 featuregate.Feature = "KMSv2"

// owner: @enj
// kep: https://kep.k8s.io/3299
// beta: v1.28
//
// Enables the use of derived encryption keys with KMS v2.
KMSv2KDF featuregate.Feature = "KMSv2KDF"

// owner: @jiahuif
// kep: https://kep.k8s.io/2887
// alpha: v1.23
Expand Down Expand Up @@ -251,6 +258,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS

KMSv2: {Default: true, PreRelease: featuregate.Beta},

KMSv2KDF: {Default: false, PreRelease: featuregate.Beta}, // default and lock to true in 1.29, remove in 1.31

OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},

OpenAPIV3: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import (
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
Expand All @@ -63,13 +64,13 @@ const (
kmsTransformerPrefixV2 = "k8s:enc:kms:v2:"

// these constants relate to how the KMS v2 plugin status poll logic
// and the DEK generation logic behave. In particular, the positive
// and the DEK/seed generation logic behave. In particular, the positive
// interval and max TTL are closely related as the difference between
// these values defines the worst case window in which the write DEK
// these values defines the worst case window in which the write DEK/seed
// could expire due to the plugin going into an error state. The
// worst case window divided by the negative interval defines the
// minimum amount of times the server will attempt to return to a
// healthy state before the DEK expires and writes begin to fail.
// healthy state before the DEK/seed expires and writes begin to fail.
//
// For now, these values are kept small and hardcoded to support being
// able to perform a "passive" storage migration while tolerating some
Expand All @@ -82,13 +83,13 @@ const (
// At that point, they are guaranteed to either migrate to the new key
// or get errors during the migration.
//
// If the API server coasted forever on the last DEK, they would need
// If the API server coasted forever on the last DEK/seed, they would need
// to actively check if it had observed the new key ID before starting
// a migration - otherwise it could keep using the old DEK and their
// a migration - otherwise it could keep using the old DEK/seed and their
// storage migration would not do what they thought it did.
kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
kmsv2PluginHealthzNegativeInterval = 10 * time.Second
kmsv2PluginWriteDEKMaxTTL = 3 * time.Minute
kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute

kmsPluginHealthzNegativeTTL = 3 * time.Second
kmsPluginHealthzPositiveTTL = 20 * time.Second
Expand Down Expand Up @@ -332,8 +333,8 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
return nil
}

// rotateDEKOnKeyIDChange tries to rotate to a new DEK if the key ID returned by Status does not match the
// current state. If a successful rotation is performed, the new DEK and keyID overwrite the existing state.
// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
// current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
// On any failure during rotation (including mismatch between status and encrypt calls), the current state is
// preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
// If the key ID returned by Status matches the current state, the expiration of the current state is extended
Expand All @@ -346,32 +347,38 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey

// allow reads indefinitely in all cases
// allow writes indefinitely as long as there is no error
// allow writes for only up to kmsv2PluginWriteDEKMaxTTL from now when there are errors
// we start the timer before we make the network call because kmsv2PluginWriteDEKMaxTTL is meant to be the upper bound
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKMaxTTL)

// state is valid and status keyID is unchanged from when we generated this DEK so there is no need to rotate it
// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)

// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
// this gate can only change during tests, but the check is cheap enough to always make
// this allows us to easily exercise both modes without restarting the API server
// TODO integration test that this dynamically takes effect
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED

// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
// just move the expiration of the current state forward by the reuse interval
if errState == nil && state.KeyID == statusKeyID {
// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
state.ExpirationTimestamp = expirationTimestamp
h.state.Store(&state)
return nil
}

transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)

if resp == nil {
resp = &kmsservice.EncryptResponse{} // avoid nil panics
if encObject == nil {
encObject = &kmstypes.EncryptedObject{} // avoid nil panics
}

// happy path, should be the common case
// TODO maybe add success metrics?
if errGen == nil && resp.KeyID == statusKeyID {
if errGen == nil && encObject.KeyID == statusKeyID {
h.state.Store(&envelopekmsv2.State{
Transformer: transformer,
EncryptedDEK: resp.Ciphertext,
KeyID: resp.KeyID,
Annotations: resp.Annotations,
EncryptedObject: *encObject,
UID: uid,
ExpirationTimestamp: expirationTimestamp,
CacheKey: cacheKey,
Expand All @@ -384,17 +391,18 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
if klogV6.Enabled() {
klogV6.InfoS("successfully rotated DEK",
"uid", uid,
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(resp.KeyID),
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.KeyID),
"useSeed", useSeed,
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
)
}
return nil
}
}

return fmt.Errorf("failed to rotate DEK uid=%q, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
uid, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(resp.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
}

// getCurrentState returns the latest state from the last status and encrypt calls.
Expand All @@ -407,12 +415,13 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
}

if len(state.EncryptedDEK) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty EncryptedDEK")
encryptedObjectCopy := state.EncryptedObject
if len(encryptedObjectCopy.EncryptedData) != 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
}

if len(state.KeyID) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty keyID")
encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
}

if state.ExpirationTimestamp.IsZero() {
Expand Down Expand Up @@ -772,7 +781,7 @@ func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName st

// make sure that the plugin's key ID is reasonably up-to-date
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKMaxTTL
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
go wait.PollUntilWithContext(
ctx,
kmsv2PluginHealthzPositiveInterval,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
Expand Down Expand Up @@ -606,7 +607,7 @@ func TestKMSPluginHealthz(t *testing.T) {
ttl: 3 * time.Second,
}
keyID := "1"
kmsv2Probe.state.Store(&envelopekmsv2.State{KeyID: keyID})
kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}})

testCases := []struct {
desc string
Expand Down Expand Up @@ -1711,6 +1712,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
name string
service *testKMSv2EnvelopeService
state envelopekmsv2.State
useSeed bool
statusKeyID string
wantState envelopekmsv2.State
wantEncryptCalls int
Expand All @@ -1723,13 +1725,13 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: envelopekmsv2.State{},
statusKeyID: "1",
wantState: envelopekmsv2.State{
KeyID: "1",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "1"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" newKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" oldKeyIDHash="" expirationTimestamp="%s"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" oldKeyIDHash="" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
Expand All @@ -1740,20 +1742,38 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "2", now),
statusKeyID: "2",
wantState: envelopekmsv2.State{
KeyID: "2",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 0,
wantLogs: nil,
wantErr: "",
},
{
name: "happy path, with previous state, useSeed=true",
service: &testKMSv2EnvelopeService{keyID: "2"},
state: validState(t, "2", now),
useSeed: true,
statusKeyID: "2",
wantState: envelopekmsv2.State{
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=true newKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" oldKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
},
{
name: "previous state expired but key ID matches",
service: &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called
state: validState(t, "3", now.Add(-time.Hour)),
statusKeyID: "3",
wantState: envelopekmsv2.State{
KeyID: "3",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "3"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 0,
Expand All @@ -1766,13 +1786,13 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "3", now.Add(-time.Hour)),
statusKeyID: "4",
wantState: envelopekmsv2.State{
KeyID: "4",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"},
ExpirationTimestamp: now.Add(3 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" newKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a" oldKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" expirationTimestamp="%s"`,
fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a" oldKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" expirationTimestamp="%s"`,
now.Add(3*time.Minute).Format(time.RFC3339)),
},
wantErr: "",
Expand All @@ -1783,14 +1803,14 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "4", now.Add(7*time.Minute)),
statusKeyID: "5",
wantState: envelopekmsv2.State{
KeyID: "4",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"},
ExpirationTimestamp: now.Add(7 * time.Minute),
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=<nil>, errGen=failed to encrypt DEK, error: broken, statusKeyIDHash="sha256:ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", ` +
`encryptKeyIDHash="", stateKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a", expirationTimestamp=` + now.Add(7*time.Minute).Format(time.RFC3339),
},
Expand All @@ -1804,7 +1824,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=got unexpected nil transformer, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` +
`should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", ` +
`encryptKeyIDHash="", stateKeyIDHash="", expirationTimestamp=` + (time.Time{}).Format(time.RFC3339),
Expand All @@ -1815,21 +1835,23 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
state: validState(t, "2", now),
statusKeyID: "3",
wantState: envelopekmsv2.State{
KeyID: "2",
EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"},
ExpirationTimestamp: now,
},
wantEncryptCalls: 1,
wantLogs: []string{
`"encrypting content using envelope service" uid="panda"`,
},
wantErr: `failed to rotate DEK uid="panda", ` +
wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` +
`errState=<nil>, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` +
`should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce", ` +
`encryptKeyIDHash="", stateKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", expirationTimestamp=` + now.Format(time.RFC3339),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2KDF, tt.useSeed)()

var buf bytes.Buffer
klog.SetOutput(&buf)

Expand All @@ -1850,14 +1872,29 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
t.Errorf("log mismatch (-want +got):\n%s", diff)
}

ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID", "CacheKey")
ignoredFields := sets.NewString("Transformer", "EncryptedObject.EncryptedDEKSource", "UID", "CacheKey")

gotState := *h.state.Load()

if diff := cmp.Diff(tt.wantState, *h.state.Load(),
if diff := cmp.Diff(tt.wantState, gotState,
cmp.FilterPath(func(path cmp.Path) bool { return ignoredFields.Has(path.String()) }, cmp.Ignore()),
); len(diff) > 0 {
t.Errorf("state mismatch (-want +got):\n%s", diff)
}

if len(cmp.Diff(tt.wantState, gotState)) > 0 { // we only need to run this check when the state changes
validCiphertext := len(gotState.EncryptedObject.EncryptedDEKSource) > 0
if tt.useSeed {
validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
} else {
validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
}
if !validCiphertext {
t.Errorf("invalid ciphertext with useSeed=%v, encryptedDEKSourceLen=%d, encryptedDEKSourceType=%d", tt.useSeed,
len(gotState.EncryptedObject.EncryptedDEKSource), gotState.EncryptedObject.EncryptedDEKSourceType)
}
}

if tt.wantEncryptCalls != tt.service.encryptCalls {
t.Errorf("want %d encryptCalls, got %d", tt.wantEncryptCalls, tt.service.encryptCalls)
}
Expand Down Expand Up @@ -1900,15 +1937,15 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
func validState(t *testing.T, keyID string, exp time.Time) envelopekmsv2.State {
t.Helper()

transformer, resp, cacheKey, err := envelopekmsv2.GenerateTransformer(testContext(t), "", &testKMSv2EnvelopeService{keyID: keyID})
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF) // match the current default behavior

transformer, encObject, cacheKey, err := envelopekmsv2.GenerateTransformer(testContext(t), "", &testKMSv2EnvelopeService{keyID: keyID}, useSeed)
if err != nil {
t.Fatal(err)
}
return envelopekmsv2.State{
Transformer: transformer,
EncryptedDEK: resp.Ciphertext,
KeyID: resp.KeyID,
Annotations: resp.Annotations,
EncryptedObject: *encObject,
ExpirationTimestamp: exp,
CacheKey: cacheKey,
}
Expand Down
Loading

0 comments on commit bf49c72

Please sign in to comment.