diff --git a/README.md b/README.md index 496b48e..9449d49 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Basereconciler is an attempt to create a reconciler that can be imported and use At the moment basereconciler can perform the following tasks: * **Get the custom resource and perform some common tasks on it**: + * Management of initialization logic: custom initialization functions can be passed to perform initialization tasks on the custom resource. Initialization can be done persisting changes in the API server (use reconciler.WithInitializationFunc) or without persisting them (reconciler.WithInMemoryInitializationFunc). * Management of resource finalizer: some custom resources required more complex finalization logic. For this to happen a finalizer must be in place. Basereconciler can keep this finalizer in place and remove it when necessary during resource finalization. * Management of finalization logic: it checks if the resource is being finalized and executed the finalization logic passed to it if that is the case. When all finalization logic is completed it removes the finalizer on the custom resource. * **Reconcile resources owned by the custom resource**: basereconciler can keep the owned resources of a custom resource in it's desired state. It works for any resource type, and only requires that the user configures how each specific resource type has to be configured. The resource reconciler only works in "update mode" right now, so any operation to transition a given resource from its live state to its desired state will be an Update. We might add a "patch mode" in the future. @@ -85,6 +86,11 @@ func (r *GuestbookReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // a finalization function that will casuse a log line to show when the resource is being deleted. guestbook := &webappv1.Guestbook{} result := r.ManageResourceLifecycle(ctx, req, guestbook, + reconciler.WithInitializationFunc( + func(context.Context, client.Client) error { + logger.Info("initializing resource") + return nil + }), reconciler.WithFinalizer("guestbook-finalizer"), reconciler.WithFinalizationFunc( func(context.Context, client.Client) error { @@ -140,13 +146,12 @@ func (r *GuestbookReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } func (r *GuestbookReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&webappv1.Guestbook{}). - // add the watches for the specific resource types that the - // custom resource owns to watch for changes on those - Owns(&appsv1.Deployment{}). - Owns(&corev1.Service{}). - Complete(r) + // SetupWithDynamicTypeWatches will configure the controller to dynamically + // watch for any resource type that the controller owns. + return reconciler.SetupWithDynamicTypeWatches(r, + ctrl.NewControllerManagedBy(mgr). + For(&webappv1.Guestbook{}), + ) } ``` diff --git a/config/global.go b/config/global.go index 0b508b9..8cc6311 100644 --- a/config/global.go +++ b/config/global.go @@ -17,10 +17,12 @@ type ReconcileConfigForGVK struct { var config = struct { annotationsDomain string resourcePruner bool + dynamicWatches bool defaultResourceReconcileConfig map[string]ReconcileConfigForGVK }{ annotationsDomain: "basereconciler.3cale.net", resourcePruner: true, + dynamicWatches: true, defaultResourceReconcileConfig: map[string]ReconcileConfigForGVK{ "*": { EnsureProperties: []string{ @@ -56,6 +58,17 @@ func DisableResourcePruner() { config.resourcePruner = false } // IsResourcePrunerEnabled returs a boolean indicating wheter the resource pruner is enabled or not. func IsResourcePrunerEnabled() bool { return config.resourcePruner } +// EnableDynamicWatches enables controller dynamic watches. Dynamic watches keep track of the resource +// types that the controller owns and dynamically adds watches to the controller for those. +func EnableDynamicWatches() { config.dynamicWatches = true } + +// DisableDynamicWatches disables controller dynamic watches. Dynamic watches keep track of the resource +// types that the controller owns and dynamically adds watches to the controller for those. +func DisableDynamicWatches() { config.dynamicWatches = false } + +// AreDynamicWatchesEnabled returs a boolean indicating wheter the dynamic watches are enabled or not. +func AreDynamicWatchesEnabled() bool { return config.dynamicWatches } + // GetDefaultReconcileConfigForGVK returns the default configuration that instructs basereconciler how to reconcile // a given kubernetes GVK (GroupVersionKind). This default config will be used if the "resource.Template" object (see // the resource package) does not specify a configuration itself. diff --git a/mutators/mutators.go b/mutators/mutators.go index 840204c..8fae6c6 100644 --- a/mutators/mutators.go +++ b/mutators/mutators.go @@ -50,7 +50,7 @@ func SetDeploymentReplicas(enforce bool) resource.TemplateMutationFunction { // SetServiceLiveValues retrieves some live values of the Service spec from the Kubernetes // API to avoid overwriting them. These values are typically set the by the kube-controller-manager -// (in some rare occasions the user might explicitely set them) and should not be modified by the +// (in some rare occasions the user might explicitly set them) and should not be modified by the // reconciler. The fields that this function keeps in sync with the live state are: // - spec.clusterIP // - spec.ClisterIPs diff --git a/reconciler/pruner.go b/reconciler/pruner.go index 57550b6..9a4d704 100644 --- a/reconciler/pruner.go +++ b/reconciler/pruner.go @@ -3,16 +3,13 @@ package reconciler import ( "context" "fmt" - "reflect" "strconv" - "sync" "github.com/3scale-ops/basereconciler/config" "github.com/3scale-ops/basereconciler/util" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -72,18 +69,3 @@ func isPrunerEnabled(owner client.Object) bool { } return prune && config.IsResourcePrunerEnabled() } - -type typeTracker struct { - seenTypes []schema.GroupVersionKind - mu sync.Mutex -} - -func (tt *typeTracker) trackType(gvk schema.GroupVersionKind) { - if !util.ContainsBy(tt.seenTypes, func(x schema.GroupVersionKind) bool { - return reflect.DeepEqual(x, gvk) - }) { - tt.mu.Lock() - defer tt.mu.Unlock() - tt.seenTypes = append(tt.seenTypes, gvk) - } -} diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 20ff957..e68b375 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/3scale-ops/basereconciler/config" "github.com/3scale-ops/basereconciler/resource" "github.com/3scale-ops/basereconciler/util" "github.com/go-logr/logr" @@ -252,7 +253,7 @@ func (r *Reconciler) isInitialized(ctx context.Context, obj client.Object, fns [ // inMemoryInitialization can be used to perform initializarion on the resource that is not // persisted in the API storage. This can be used to perform initialization on the resource without // writing it to the API to avoid surfacing it uo to the user. This approach is a bit more -// gitops firendly as it avoids modifying the resource, but it doesn't provide any information +// gitops friendly as it avoids modifying the resource, but it doesn't provide any information // to the user on the initialization being used for reconciliation. func (r *Reconciler) inMemoryInitialization(ctx context.Context, obj client.Object, fns []inMemoryinitializationFunction) error { for _, fn := range fns { @@ -285,9 +286,10 @@ func (r *Reconciler) finalize(ctx context.Context, fns []finalizationFunction, l // - Each template is added to the list of managed resources if resource.CreateOrUpdate returns with no error // - If the resource pruner is enabled any resource owned by the custom resource not present in the list of managed // resources is deleted. The resource pruner must be enabled in the global config (see package config) and also not -// explicitely disabled in the resource by the '/prune: true/false' annotation. +// explicitly disabled in the resource by the '/prune: true/false' annotation. func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.Object, list []resource.TemplateInterface) Result { managedResources := []corev1.ObjectReference{} + requeue := false for _, template := range list { ref, err := resource.CreateOrUpdate(ctx, r.Client, r.Scheme, owner, template) @@ -296,7 +298,13 @@ func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.O } if ref != nil { managedResources = append(managedResources, *ref) - r.typeTracker.trackType(schema.FromAPIVersionAndKind(ref.APIVersion, ref.Kind)) + gvk := schema.FromAPIVersionAndKind(ref.APIVersion, ref.Kind) + if changed := r.typeTracker.trackType(gvk); changed && config.AreDynamicWatchesEnabled() { + r.watchOwned(gvk, owner) + // requeue so we make sure we haven't lost any events related to the owned resource + // while the watch was not still up + requeue = true + } } } @@ -307,7 +315,11 @@ func (r *Reconciler) ReconcileOwnedResources(ctx context.Context, owner client.O } } - return Result{Action: ContinueAction} + if requeue { + return Result{Action: ReturnAndRequeueAction} + } else { + return Result{Action: ContinueAction} + } } // FilteredEventHandler returns an EventHandler for the specific client.ObjectList diff --git a/reconciler/reconciler_test.go b/reconciler/reconciler_test.go index 3760300..fc0c36e 100644 --- a/reconciler/reconciler_test.go +++ b/reconciler/reconciler_test.go @@ -15,12 +15,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) func TestResult_ShouldReturn(t *testing.T) { @@ -396,11 +400,23 @@ func TestReconciler_ManageResourceLifecycle(t *testing.T) { } } +type testController struct { + reconcile.Reconciler +} + +func (c *testController) Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error { + return nil +} +func (c *testController) Start(ctx context.Context) error { return nil } +func (c *testController) GetLogger() logr.Logger { return logr.Discard() } + func TestReconciler_ReconcileOwnedResources(t *testing.T) { + type fields struct { - Client client.Client - Log logr.Logger - Scheme *runtime.Scheme + Client client.Client + Log logr.Logger + Scheme *runtime.Scheme + SeenTypes []schema.GroupVersionKind } type args struct { owner client.Object @@ -415,16 +431,51 @@ func TestReconciler_ReconcileOwnedResources(t *testing.T) { { name: "Creates owned resources", fields: fields{ - Client: fake.NewClientBuilder().Build(), + Client: fake.NewClientBuilder().Build(), + Log: logr.Discard(), + Scheme: scheme.Scheme, + SeenTypes: []schema.GroupVersionKind{}, + }, + args: args{ + owner: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}}, + list: []resource.TemplateInterface{ + resource.NewTemplateFromObjectFunction[*corev1.Service]( + func() *corev1.Service { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}} + }), + resource.NewTemplateFromObjectFunction[*corev1.ConfigMap]( + func() *corev1.ConfigMap { + return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "ns"}} + }), + }, + }, + want: Result{ + Action: ReturnAndRequeueAction, + RequeueAfter: 0, + Error: nil, + }, + }, + { + name: "Updates owned resources and does not add new watches", + fields: fields{ + Client: fake.NewClientBuilder().WithObjects( + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm", Namespace: "ns"}}, + ).Build(), Log: logr.Discard(), Scheme: scheme.Scheme, + SeenTypes: []schema.GroupVersionKind{ + {Group: "", Version: "v1", Kind: "Service"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + }, }, args: args{ owner: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "owner", Namespace: "ns"}}, list: []resource.TemplateInterface{ resource.NewTemplateFromObjectFunction[*corev1.Service]( func() *corev1.Service { - return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns"}} + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "ns", Labels: map[string]string{"key": "value"}}} }), resource.NewTemplateFromObjectFunction[*corev1.ConfigMap]( func() *corev1.ConfigMap { @@ -444,6 +495,10 @@ func TestReconciler_ReconcileOwnedResources(t *testing.T) { r := &Reconciler{ Client: tt.fields.Client, Scheme: tt.fields.Scheme, + typeTracker: typeTracker{ + seenTypes: tt.fields.SeenTypes, + ctrl: &testController{}, + }, } got := r.ReconcileOwnedResources(context.TODO(), tt.args.owner, tt.args.list) if diff := cmp.Diff(got, tt.want); len(diff) > 0 { diff --git a/reconciler/tracker.go b/reconciler/tracker.go new file mode 100644 index 0000000..0354454 --- /dev/null +++ b/reconciler/tracker.go @@ -0,0 +1,85 @@ +package reconciler + +import ( + "reflect" + "sync" + + "github.com/3scale-ops/basereconciler/util" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// ReconcilerWithTypeTracker is a reconciler with a TypeTracker +// The type tracker is used by the "resource pruner" and "dynamic watches" +// features +type ReconcilerWithTypeTracker interface { + reconcile.Reconciler + BuildTypeTracker(ctrl controller.Controller) +} + +// SetupWithDynamicTypeWatches is a helper to build a controller that can watch resource +// types dynamically. It is typically used within the "SetupWithManager" function. +// Example usage: +// +// func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { +// return reconciler.SetupWithDynamicTypeWatches(r, +// ctrl.NewControllerManagedBy(mgr). +// For(&v1alpha1.Test{}). +// // add any other watches here +// Watches(...}.Watches(...), +// ) +// } +func SetupWithDynamicTypeWatches(r ReconcilerWithTypeTracker, bldr *builder.Builder) error { + ctrl, err := bldr.Build(r) + if err != nil { + return err + } + r.BuildTypeTracker(ctrl) + return nil +} + +type typeTracker struct { + seenTypes []schema.GroupVersionKind + ctrl controller.Controller + mu sync.Mutex +} + +func (tt *typeTracker) trackType(gvk schema.GroupVersionKind) bool { + if !util.ContainsBy(tt.seenTypes, func(x schema.GroupVersionKind) bool { + return reflect.DeepEqual(x, gvk) + }) { + tt.mu.Lock() + defer tt.mu.Unlock() + tt.seenTypes = append(tt.seenTypes, gvk) + return true + } + return false +} + +func (r *Reconciler) watchOwned(gvk schema.GroupVersionKind, owner client.Object) error { + o, err := util.NewObjectFromGVK(gvk, r.Scheme) + if err != nil { + return err + } + r.typeTracker.mu.Lock() + defer r.typeTracker.mu.Unlock() + err = r.typeTracker.ctrl.Watch(&source.Kind{Type: o}, &handler.EnqueueRequestForOwner{OwnerType: owner, IsController: true}) + if err != nil { + return err + } + return nil +} + +// BuildTypeTracker passes the controller to the reconciler so watches +// can be added dynamically +func (r *Reconciler) BuildTypeTracker(ctrl controller.Controller) { + r.typeTracker = typeTracker{ + seenTypes: []schema.GroupVersionKind{}, + ctrl: ctrl, + } +} diff --git a/resource/create_or_update.go b/resource/create_or_update.go index 3bac2dc..27bf850 100644 --- a/resource/create_or_update.go +++ b/resource/create_or_update.go @@ -30,7 +30,7 @@ import ( // - owner: the object that owns the resource. Used to set the OwnerReference in the resource // - template: the struct that describes how the resource needs to be reconciled. It must implement // the TemplateInterface interface. When template.GetEnsureProperties is not set or an empty list, this -// function will lookup for configuration in the global configuration (see pacakge config). +// function will lookup for configuration in the global configuration (see package config). func CreateOrUpdate(ctx context.Context, cl client.Client, scheme *runtime.Scheme, owner client.Object, template TemplateInterface) (*corev1.ObjectReference, error) { diff --git a/resource/doc.go b/resource/doc.go new file mode 100644 index 0000000..84b5759 --- /dev/null +++ b/resource/doc.go @@ -0,0 +1,3 @@ +// Package resource contains types and methods to reconcile controller owned resources +// It is generalized to work with any GroupVersionKind. +package resource diff --git a/resource/template.go b/resource/template.go index 1b720b3..038e9cc 100644 --- a/resource/template.go +++ b/resource/template.go @@ -37,7 +37,7 @@ type Template[T client.Object] struct { // TemplateBuilder has been invoked, to perform mutations on the object that require // access to a kubernetes API server. TemplateMutations []TemplateMutationFunction - // IsEnabled specifies whether the resourse described by this Template should + // IsEnabled specifies whether the resource described by this Template should // exist or not. IsEnabled bool // EnsureProperties are the properties from the desired object that should be enforced diff --git a/test/doc.go b/test/doc.go new file mode 100644 index 0000000..721a709 --- /dev/null +++ b/test/doc.go @@ -0,0 +1,2 @@ +// Package test contains a test controller with its test suite +package test diff --git a/test/test_controller.go b/test/test_controller.go index e8683cb..e857f30 100644 --- a/test/test_controller.go +++ b/test/test_controller.go @@ -162,20 +162,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.Test{}). - Owns(&appsv1.Deployment{}). - Owns(&corev1.Service{}). - Owns(&policyv1.PodDisruptionBudget{}). - Owns(&autoscalingv2.HorizontalPodAutoscaler{}). - Watches(&source.Kind{Type: &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret"}}}, - r.FilteredEventHandler( - &v1alpha1.TestList{}, - func(event, o client.Object) bool { - return event.GetName() == "secret" - }, - r.Log)). - Complete(r) + return reconciler.SetupWithDynamicTypeWatches(r, + ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.Test{}). + Watches(&source.Kind{Type: &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret"}}}, + r.FilteredEventHandler( + &v1alpha1.TestList{}, + func(event, o client.Object) bool { + return event.GetName() == "secret" + }, + r.Log)), + ) } func deployment(namespace string) resource.TemplateBuilderFunction[*appsv1.Deployment] { diff --git a/test/test_controller_suite_test.go b/test/test_controller_suite_test.go index f840eda..fe2ed90 100644 --- a/test/test_controller_suite_test.go +++ b/test/test_controller_suite_test.go @@ -134,6 +134,31 @@ var _ = Describe("Test controller", func() { k8sClient.Delete(context.Background(), instance, client.PropagationPolicy(metav1.DeletePropagationForeground)) }) + It("watches for changes in the owned resources and avoids drifts", func() { + dep := resources[0].(*appsv1.Deployment) + t := dep.GetCreationTimestamp() + GinkgoWriter.Printf("[debug] Creation timestamp: %v\n", t) + // ensure some time passes so the creation timestamps are different + time.Sleep(1 * time.Second) + + By("deleting the owned Deployment") + err := k8sClient.Delete(context.Background(), dep) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + err := k8sClient.Get( + context.Background(), + types.NamespacedName{Name: "deployment", Namespace: namespace}, + dep, + ) + if err != nil { + return false + } + GinkgoWriter.Printf("[debug] Creation timestamp: %v\n", dep.GetCreationTimestamp()) + return dep.GetCreationTimestamp().After(t.Time) + }, timeout, poll).Should(BeTrue()) + }) + It("triggers a Deployment rollout on Secret contents change", func() { dep := resources[0].(*appsv1.Deployment) diff --git a/util/doc.go b/util/doc.go new file mode 100644 index 0000000..550f706 --- /dev/null +++ b/util/doc.go @@ -0,0 +1,2 @@ +// Package util contains utility functions used by other packages +package util