Skip to content

Commit

Permalink
cert-rotation: allow specifying multiple target certs in CertRotation…
Browse files Browse the repository at this point in the history
…Controller

Instead of defining several controllers managing the same signer/CA
bundle pair and different target certs the same controller can accept
a list of target certs to create.
  • Loading branch information
vrutkovs committed Apr 22, 2024
1 parent f1541d6 commit d10d787
Show file tree
Hide file tree
Showing 2 changed files with 331 additions and 15 deletions.
83 changes: 68 additions & 15 deletions pkg/operator/certrotation/client_cert_rotation_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"time"

operatorv1 "github.com/openshift/api/operator/v1"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"

"github.com/openshift/library-go/pkg/controller/factory"
Expand Down Expand Up @@ -65,9 +67,8 @@ type CertRotationController struct {
RotatedSigningCASecret RotatedSigningCASecret
// CABundleConfigMap maintains a CA bundle config map, by adding new CA certs coming from rotatedSigningCASecret, and by removing expired old ones.
CABundleConfigMap CABundleConfigMap
// RotatedSelfSignedCertKeySecret rotates a key and cert signed by a signing CA and stores it in a secret.
RotatedSelfSignedCertKeySecret RotatedSelfSignedCertKeySecret

// RotatedTargetSecrets contains a list of key and cert signed by a signing CA to rotate.
RotatedTargetSecrets []RotatedSelfSignedCertKeySecret
// Plumbing:
StatusReporter StatusReporter
}
Expand All @@ -81,11 +82,11 @@ func NewCertRotationController(
reporter StatusReporter,
) factory.Controller {
c := &CertRotationController{
Name: name,
RotatedSigningCASecret: rotatedSigningCASecret,
CABundleConfigMap: caBundleConfigMap,
RotatedSelfSignedCertKeySecret: rotatedSelfSignedCertKeySecret,
StatusReporter: reporter,
Name: name,
RotatedSigningCASecret: rotatedSigningCASecret,
CABundleConfigMap: caBundleConfigMap,
RotatedTargetSecrets: []RotatedSelfSignedCertKeySecret{rotatedSelfSignedCertKeySecret},
StatusReporter: reporter,
}
return factory.New().
ResyncEvery(time.Minute).
Expand All @@ -101,6 +102,42 @@ func NewCertRotationController(
ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name))
}

func NewCertRotationControllerMultipleTargets(
name string,
rotatedSigningCASecret RotatedSigningCASecret,
caBundleConfigMap CABundleConfigMap,
rotatedTargetSecrets []RotatedSelfSignedCertKeySecret,
recorder events.Recorder,
reporter StatusReporter,
) factory.Controller {
informers := sets.New[factory.Informer](
rotatedSigningCASecret.Informer.Informer(),
caBundleConfigMap.Informer.Informer(),
)

for _, target := range rotatedTargetSecrets {
informers = informers.Insert(target.Informer.Informer())
}

c := &CertRotationController{
Name: name,
RotatedSigningCASecret: rotatedSigningCASecret,
CABundleConfigMap: caBundleConfigMap,
RotatedTargetSecrets: rotatedTargetSecrets,
StatusReporter: reporter,
}
return factory.New().
ResyncEvery(time.Minute).
WithSync(c.Sync).
WithInformers(
informers.UnsortedList()...,
).
WithPostStartHooks(
c.targetCertRecheckerPostRunHook,
).
ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name))
}

func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
syncErr := c.SyncWorker(ctx)

Expand Down Expand Up @@ -132,24 +169,40 @@ func (c CertRotationController) SyncWorker(ctx context.Context) error {
return err
}

if _, err := c.RotatedSelfSignedCertKeySecret.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil {
return err
var errs []error
for _, secret := range c.RotatedTargetSecrets {
if _, err := secret.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return errors.NewAggregate(errs)
}

return nil
}

func (c CertRotationController) targetCertRecheckerPostRunHook(ctx context.Context, syncCtx factory.SyncContext) error {
// If we have a need to force rechecking the cert, use this channel to do it.
refresher, ok := c.RotatedSelfSignedCertKeySecret.CertCreator.(TargetCertRechecker)
if !ok {
return nil
var targetRefreshers []<-chan struct{}
for _, target := range c.RotatedTargetSecrets {
if refresher, ok := target.CertCreator.(TargetCertRechecker); ok {
targetRefreshers = append(targetRefreshers, refresher.RecheckChannel())
}
}
targetRefresh := refresher.RecheckChannel()
aggregateTargetRefresher := make(chan struct{})
for _, ch := range targetRefreshers {
go func(c <-chan struct{}) {
for msg := range c {
aggregateTargetRefresher <- msg
}
}(ch)
}

go wait.Until(func() {
for {
select {
case <-targetRefresh:
case <-aggregateTargetRefresher:
syncCtx.Queue().Add(factory.DefaultQueueKey)
case <-ctx.Done():
return
Expand Down
263 changes: 263 additions & 0 deletions pkg/operator/certrotation/client_cert_rotation_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
package certrotation

import (
"context"
"testing"
"time"

"github.com/davecgh/go-spew/spew"
"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/events/eventstesting"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/informers"
kubefake "k8s.io/client-go/kubernetes/fake"
corev1listers "k8s.io/client-go/listers/core/v1"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
)

type fakeStatusReporter struct{}

func (f *fakeStatusReporter) Report(ctx context.Context, controllerName string, syncErr error) (updated bool, updateErr error) {
return false, nil
}

func TestCertRotationController(t *testing.T) {
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
client := kubefake.NewSimpleClientset()
controllerCtx, cancel := context.WithCancel(context.Background())

ns, signerName, caName, targetName := "ns", "test-signer", "test-ca", "test-target"
eventRecorder := events.NewInMemoryRecorder("test")
additionalAnnotations := AdditionalAnnotations{
JiraComponent: "test",
}
owner := &metav1.OwnerReference{
Name: "operator",
}

informerFactory := informers.NewSharedInformerFactoryWithOptions(client, 1*time.Minute, informers.WithNamespace(ns))

signerSecret := RotatedSigningCASecret{
Namespace: ns,
Name: signerName,
Validity: 24 * time.Hour,
Refresh: 12 * time.Hour,
Client: client.CoreV1(),
Lister: corev1listers.NewSecretLister(indexer),
Informer: informerFactory.Core().V1().Secrets(),
EventRecorder: eventRecorder,
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
UseSecretUpdateOnly: true,
}
caBundleConfigMap := CABundleConfigMap{
Namespace: ns,
Name: caName,
Client: client.CoreV1(),
Lister: corev1listers.NewConfigMapLister(indexer),
EventRecorder: eventRecorder,
Informer: informerFactory.Core().V1().ConfigMaps(),
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
}
targetSecret := RotatedSelfSignedCertKeySecret{
Name: targetName,
Namespace: ns,
Validity: 24 * time.Hour,
Refresh: 12 * time.Hour,
CertCreator: &ServingRotation{
Hostnames: func() []string { return []string{"foo", "bar"} },
},
Client: client.CoreV1(),
Informer: informerFactory.Core().V1().Secrets(),
Lister: corev1listers.NewSecretLister(indexer),
EventRecorder: eventRecorder,
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
UseSecretUpdateOnly: true,
}

c := NewCertRotationController("operator", signerSecret, caBundleConfigMap, targetSecret, eventRecorder, &fakeStatusReporter{})

time.AfterFunc(1*time.Second, func() {
cancel()
})
c.Run(controllerCtx, 1)

syncCtx := factory.NewSyncContext("test", eventstesting.NewTestingEventRecorder(t))
err := c.Sync(controllerCtx, syncCtx)
if err != nil {
t.Errorf("sync error: %v", err)
}

actions := client.Actions()
if len(actions) != 6 {
t.Fatal(spew.Sdump(actions))
}

if !actions[0].Matches("get", "secrets") {
t.Error(actions[0])
}
if !actions[1].Matches("create", "secrets") {
t.Error(actions[1])
}
if !actions[2].Matches("get", "configmaps") {
t.Error(actions[2])
}
if !actions[3].Matches("create", "configmaps") {
t.Error(actions[3])
}
if !actions[4].Matches("get", "secrets") {
t.Error(actions[4])
}
if !actions[5].Matches("create", "secrets") {
t.Error(actions[5])
}
actualSignerSecret := actions[1].(clienttesting.CreateAction).GetObject().(*corev1.Secret)
if actualSignerSecret.Name != signerName {
t.Errorf("expected signer secret name to be %s, got %s", signerName, actualSignerSecret.Name)
}
actualCABundleConfigMap := actions[3].(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap)
if actualCABundleConfigMap.Name != caName {
t.Errorf("expected CA bundle configmap name to be %s, got %s", signerName, actualCABundleConfigMap.Name)
}
actualTargetSecret := actions[5].(clienttesting.CreateAction).GetObject().(*corev1.Secret)
if actualTargetSecret.Name != targetName {
t.Errorf("expected target secret name to be %s, got %s", signerName, actualTargetSecret.Name)
}
}

func TestCertRotationControllerMultipleTargets(t *testing.T) {
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
client := kubefake.NewSimpleClientset()
controllerCtx, cancel := context.WithCancel(context.Background())

ns, signerName, caName, targetFirstName, targetSecondName := "ns", "test-signer", "test-ca", "test-target-one", "test-target-two"
eventRecorder := events.NewInMemoryRecorder("test")
additionalAnnotations := AdditionalAnnotations{
JiraComponent: "test",
}
owner := &metav1.OwnerReference{
Name: "operator",
}

informerFactory := informers.NewSharedInformerFactoryWithOptions(client, 1*time.Minute, informers.WithNamespace(ns))

signerSecret := RotatedSigningCASecret{
Namespace: ns,
Name: signerName,
Validity: 24 * time.Hour,
Refresh: 12 * time.Hour,
Client: client.CoreV1(),
Lister: corev1listers.NewSecretLister(indexer),
Informer: informerFactory.Core().V1().Secrets(),
EventRecorder: eventRecorder,
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
UseSecretUpdateOnly: true,
}
caBundleConfigMap := CABundleConfigMap{
Namespace: ns,
Name: caName,
Client: client.CoreV1(),
Lister: corev1listers.NewConfigMapLister(indexer),
EventRecorder: eventRecorder,
Informer: informerFactory.Core().V1().ConfigMaps(),
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
}
targetFirstSecret := RotatedSelfSignedCertKeySecret{
Name: targetFirstName,
Namespace: ns,
Validity: 24 * time.Hour,
Refresh: 12 * time.Hour,
CertCreator: &ServingRotation{
Hostnames: func() []string { return []string{"foo", "bar"} },
},
Client: client.CoreV1(),
Informer: informerFactory.Core().V1().Secrets(),
Lister: corev1listers.NewSecretLister(indexer),
EventRecorder: eventRecorder,
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
UseSecretUpdateOnly: true,
}
targetSecondSecret := RotatedSelfSignedCertKeySecret{
Name: targetSecondName,
Namespace: ns,
Validity: 24 * time.Hour,
Refresh: 12 * time.Hour,
CertCreator: &ServingRotation{
Hostnames: func() []string { return []string{"foo", "bar"} },
},
Client: client.CoreV1(),
Informer: informerFactory.Core().V1().Secrets(),
Lister: corev1listers.NewSecretLister(indexer),
EventRecorder: eventRecorder,
AdditionalAnnotations: additionalAnnotations,
Owner: owner,
UseSecretUpdateOnly: true,
}

c := NewCertRotationControllerMultipleTargets("operator", signerSecret, caBundleConfigMap, []RotatedSelfSignedCertKeySecret{targetFirstSecret, targetSecondSecret}, eventRecorder, &fakeStatusReporter{})

time.AfterFunc(1*time.Second, func() {
cancel()
})
c.Run(controllerCtx, 1)

syncCtx := factory.NewSyncContext("test", eventstesting.NewTestingEventRecorder(t))
err := c.Sync(controllerCtx, syncCtx)
if err != nil {
t.Errorf("sync error: %v", err)
}

actions := client.Actions()
if len(actions) != 8 {
t.Fatal(spew.Sdump(actions))
}

if !actions[0].Matches("get", "secrets") {
t.Error(actions[0])
}
if !actions[1].Matches("create", "secrets") {
t.Error(actions[1])
}
if !actions[2].Matches("get", "configmaps") {
t.Error(actions[2])
}
if !actions[3].Matches("create", "configmaps") {
t.Error(actions[3])
}
if !actions[4].Matches("get", "secrets") {
t.Error(actions[4])
}
if !actions[5].Matches("create", "secrets") {
t.Error(actions[5])
}
if !actions[6].Matches("get", "secrets") {
t.Error(actions[6])
}
if !actions[7].Matches("create", "secrets") {
t.Error(actions[7])
}
actualSignerSecret := actions[1].(clienttesting.CreateAction).GetObject().(*corev1.Secret)
if actualSignerSecret.Name != signerName {
t.Errorf("expected signer secret name to be %s, got %s", signerName, actualSignerSecret.Name)
}
actualCABundleConfigMap := actions[3].(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap)
if actualCABundleConfigMap.Name != caName {
t.Errorf("expected CA bundle configmap name to be %s, got %s", signerName, actualCABundleConfigMap.Name)
}
actualFirstTargetSecret := actions[5].(clienttesting.CreateAction).GetObject().(*corev1.Secret)
if actualFirstTargetSecret.Name != targetFirstName {
t.Errorf("expected first target secret name to be %s, got %s", signerName, actualFirstTargetSecret.Name)
}
actualSecondTargetSecret := actions[7].(clienttesting.CreateAction).GetObject().(*corev1.Secret)
if actualSecondTargetSecret.Name != targetSecondName {
t.Errorf("expected second target secret name to be %s, got %s", signerName, actualFirstTargetSecret.Name)
}
}

0 comments on commit d10d787

Please sign in to comment.