From 6dbf5ed65d7f2ea8e51228c91e51b8854e3451ef Mon Sep 17 00:00:00 2001 From: Joaquim Moreno Prusi Date: Mon, 4 Sep 2023 15:51:15 +0200 Subject: [PATCH] Feature-gated Builtin healtcheck Adds a "builtin" healthcheck function. It returns true if all resources are healthy with nil error. It returns false if any of the resources are not healthy, the error contains the GVK + resource name, and the error message of each unhealthy resource. The current list of supported resources is: - Deployments - StatefulSets - DaemonSets - ReplicaSets - Pods - APIServices - CustomResourceDefinitions If the resource is not supported, it is assumed to be healthy. Signed-off-by: Joaquim Moreno Prusi --- go.mod | 1 + go.sum | 2 + .../bundledeployment/bundledeployment.go | 95 ++- internal/healthchecks/builtin.go | 206 +++++ internal/healthchecks/builtin_test.go | 760 ++++++++++++++++++ manifests/base/core/resources/deployment.yaml | 5 +- test/e2e/helm_provisioner_test.go | 1 - test/e2e/plain_provisioner_test.go | 25 +- 8 files changed, 1043 insertions(+), 52 deletions(-) create mode 100644 internal/healthchecks/builtin.go create mode 100644 internal/healthchecks/builtin_test.go diff --git a/go.mod b/go.mod index 031c941c..02987a94 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( k8s.io/cli-runtime v0.26.1 k8s.io/client-go v0.26.1 k8s.io/component-base v0.26.1 + k8s.io/kube-aggregator v0.26.1 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 sigs.k8s.io/controller-runtime v0.14.4 sigs.k8s.io/yaml v1.3.0 diff --git a/go.sum b/go.sum index 667c5ae1..4c4e2957 100644 --- a/go.sum +++ b/go.sum @@ -1696,6 +1696,8 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.26.1 h1:TqDWwuaUJpyhWGWw4JrXR8ZAAaHa9qrsXxR41aR3igw= +k8s.io/kube-aggregator v0.26.1/go.mod h1:E6dnKoQ6f4eFl8QQXHxTASZKXBX6+XcjROWl7GRltl4= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= diff --git a/internal/controllers/bundledeployment/bundledeployment.go b/internal/controllers/bundledeployment/bundledeployment.go index 94c80224..39bdb2ec 100644 --- a/internal/controllers/bundledeployment/bundledeployment.go +++ b/internal/controllers/bundledeployment/bundledeployment.go @@ -35,9 +35,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" rukpakv1alpha1 "github.com/operator-framework/rukpak/api/v1alpha1" + "github.com/operator-framework/rukpak/internal/healthchecks" helmpredicate "github.com/operator-framework/rukpak/internal/helm-operator-plugins/predicate" "github.com/operator-framework/rukpak/internal/storage" "github.com/operator-framework/rukpak/internal/util" + "github.com/operator-framework/rukpak/pkg/features" ) /* @@ -263,12 +265,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep cl, err := c.acg.ActionClientFor(bd) bd.SetNamespace("") if err != nil { - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonErrorGettingClient, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonErrorGettingClient, err.Error()) return ctrl.Result{}, err } @@ -281,12 +278,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep rel, state, err := c.getReleaseState(cl, bd, chrt, values, post) if err != nil { - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonErrorGettingReleaseState, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonErrorGettingReleaseState, err.Error()) return ctrl.Result{}, err } @@ -306,12 +298,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep if isResourceNotFoundErr(err) { err = errRequiredResourceNotFound{err} } - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonInstallFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonInstallFailed, err.Error()) return ctrl.Result{}, err } case stateNeedsUpgrade: @@ -326,12 +313,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep if isResourceNotFoundErr(err) { err = errRequiredResourceNotFound{err} } - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonUpgradeFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonUpgradeFailed, err.Error()) return ctrl.Result{}, err } case stateUnchanged: @@ -339,12 +321,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep if isResourceNotFoundErr(err) { err = errRequiredResourceNotFound{err} } - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonReconcileFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonReconcileFailed, err.Error()) return ctrl.Result{}, err } default: @@ -353,24 +330,14 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep relObjects, err := util.ManifestObjects(strings.NewReader(rel.Manifest), fmt.Sprintf("%s-release-manifest", rel.Name)) if err != nil { - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonCreateDynamicWatchFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonCreateDynamicWatchFailed, err.Error()) return ctrl.Result{}, err } for _, obj := range relObjects { uMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) if err != nil { - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonCreateDynamicWatchFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonCreateDynamicWatchFailed, err.Error()) return ctrl.Result{}, err } @@ -391,12 +358,7 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep } return nil }(); err != nil { - meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ - Type: rukpakv1alpha1.TypeInstalled, - Status: metav1.ConditionFalse, - Reason: rukpakv1alpha1.ReasonCreateDynamicWatchFailed, - Message: err.Error(), - }) + setInstalledAndHealthyFalse(bd, rukpakv1alpha1.ReasonCreateDynamicWatchFailed, err.Error()) return ctrl.Result{}, err } } @@ -408,6 +370,23 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep }) bd.Status.ActiveBundle = bundle.GetName() + if features.RukpakFeatureGate.Enabled(features.BundleDeploymentHealth) { + if err = healthchecks.AreObjectsHealthy(ctx, c.cl, relObjects); err != nil { + meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ + Type: rukpakv1alpha1.TypeHealthy, + Status: metav1.ConditionFalse, + Reason: rukpakv1alpha1.ReasonUnhealthy, + Message: err.Error(), + }) + return ctrl.Result{}, err + } + meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ + Type: rukpakv1alpha1.TypeHealthy, + Status: metav1.ConditionTrue, + Reason: rukpakv1alpha1.ReasonHealthy, + Message: "BundleDeployment is healthy", + }) + } if err := c.reconcileOldBundles(ctx, bundle, allBundles); err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete old bundles: %v", err) } @@ -415,6 +394,26 @@ func (c *controller) reconcile(ctx context.Context, bd *rukpakv1alpha1.BundleDep return ctrl.Result{}, nil } +// setInstalledAndHealthyFalse sets the Installed and if the feature gate is enabled, the Healthy conditions to False, +// and allows to set the Installed condition reason and message. +func setInstalledAndHealthyFalse(bd *rukpakv1alpha1.BundleDeployment, installedConditionReason, installedConditionMessage string) { + meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ + Type: rukpakv1alpha1.TypeInstalled, + Status: metav1.ConditionFalse, + Reason: installedConditionReason, + Message: installedConditionMessage, + }) + + if features.RukpakFeatureGate.Enabled(features.BundleDeploymentHealth) { + meta.SetStatusCondition(&bd.Status.Conditions, metav1.Condition{ + Type: rukpakv1alpha1.TypeHealthy, + Status: metav1.ConditionFalse, + Reason: rukpakv1alpha1.ReasonInstallationStatusFalse, + Message: "Installed condition is false", + }) + } +} + // reconcileOldBundles is responsible for garbage collecting any Bundles // that no longer match the desired Bundle template. func (c *controller) reconcileOldBundles(ctx context.Context, currBundle *rukpakv1alpha1.Bundle, allBundles *rukpakv1alpha1.BundleList) error { diff --git a/internal/healthchecks/builtin.go b/internal/healthchecks/builtin.go new file mode 100644 index 00000000..e31b781a --- /dev/null +++ b/internal/healthchecks/builtin.go @@ -0,0 +1,206 @@ +package healthchecks + +import ( + "context" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AreObjectsHealthy checks if the given resources are healthy. +// It returns a nil error if all the resources are healthy, if any resource is not healthy, the error will +// contain the GVK + namespace/resourceName and the error message of each unhealthy resource. +// +// The current list of supported resources is: +// - Deployments +// - StatefulSets +// - DaemonSets +// - ReplicaSets +// - Pods +// - APIServices +// - CustomResourceDefinitions +// +// If the resource is not supported, it is assumed to be healthy. +func AreObjectsHealthy(ctx context.Context, client client.Client, objects []client.Object) error { + var gvkErrors []error + + for _, object := range objects { + objectKey := types.NamespacedName{ + Name: object.GetName(), + Namespace: object.GetNamespace(), + } + + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(object.GetObjectKind().GroupVersionKind()) + if err := client.Get(ctx, objectKey, u); err != nil { + gvkErrors = appendResourceError(gvkErrors, object, err.Error()) + continue + } + + switch u.GroupVersionKind() { + case appsv1.SchemeGroupVersion.WithKind("Deployment"): + // Check if the deployment is available. + obj := &appsv1.Deployment{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + conditionExists := false + for _, condition := range obj.Status.Conditions { + if condition.Type == appsv1.DeploymentAvailable { + if condition.Status != "True" { + gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) + } + conditionExists = true + break + } + } + if conditionExists { + continue + } + gvkErrors = appendResourceError(gvkErrors, obj, "DeploymentAvailable condition not found") + case appsv1.SchemeGroupVersion.WithKind("StatefulSet"): + // This logic has been adapted from the helm codebase. + obj := &appsv1.StatefulSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + // This logic has been adapted from the helm codebase. + // - https://github.com/helm/helm/blob/e7bb860d9a32e8739c944b8e7b7f7031d752411a/pkg/kube/ready.go#L357-L410 + + // If the statefulset is not using the RollingUpdate strategy, we assume it's healthy. + if obj.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType { + continue + } + if obj.Status.ObservedGeneration < obj.Generation { + gvkErrors = appendResourceError(gvkErrors, obj, "StatefulSet is not ready (update has not yet been observed)") + } + + var partition int + var replicas = 1 + if obj.Spec.UpdateStrategy.RollingUpdate != nil && obj.Spec.UpdateStrategy.RollingUpdate.Partition != nil { + partition = int(*obj.Spec.UpdateStrategy.RollingUpdate.Partition) + } + if obj.Spec.Replicas != nil { + replicas = int(*obj.Spec.Replicas) + } + expectedReplicas := replicas - partition + + if obj.Status.UpdatedReplicas < int32(expectedReplicas) { + gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", expectedReplicas, obj.Status.UpdatedReplicas)) + continue + } + if int(obj.Status.ReadyReplicas) != replicas { + gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected %d replicas, got %d)", replicas, obj.Status.ReadyReplicas)) + continue + } + if partition == 0 && obj.Status.CurrentRevision != obj.Status.UpdateRevision { + gvkErrors = appendResourceError(gvkErrors, obj, fmt.Sprintf("StatefulSet is not ready (expected revision %s, got %s)", obj.Status.CurrentRevision, obj.Status.UpdateRevision)) + continue + } + case appsv1.SchemeGroupVersion.WithKind("DaemonSet"): + // Check if the daemonset is ready. + obj := &appsv1.DaemonSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + if obj.Status.NumberAvailable != obj.Status.DesiredNumberScheduled { + gvkErrors = appendResourceError(gvkErrors, obj, "DaemonSet is not ready") + } + case appsv1.SchemeGroupVersion.WithKind("ReplicaSet"): + // Check if the replicaset is ready. + obj := &appsv1.ReplicaSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + if obj.Status.AvailableReplicas != obj.Status.Replicas { + gvkErrors = appendResourceError(gvkErrors, obj, "ReplicaSet is not ready") + } + case corev1.SchemeGroupVersion.WithKind("Pod"): + // Check if the pod is running or succeeded. + obj := &corev1.Pod{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + if obj.Status.Phase != corev1.PodRunning && obj.Status.Phase != corev1.PodSucceeded { + gvkErrors = appendResourceError(gvkErrors, obj, "Pod is not Running or Succeeded") + } + case apiregistrationv1.SchemeGroupVersion.WithKind("APIService"): + // Check if the APIService is available. + obj := &apiregistrationv1.APIService{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + conditionExists := false + for _, condition := range obj.Status.Conditions { + if condition.Type == apiregistrationv1.Available { + if condition.Status != "True" { + gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) + } + conditionExists = true + break + } + } + if conditionExists { + continue + } + // If we are here we didn't find the "Available" condition, so we assume the APIService is non healthy. + gvkErrors = appendResourceError(gvkErrors, obj, "Available condition not found") + case apiextensionsv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"): + // Check if the CRD is established. + obj := &apiextensionsv1.CustomResourceDefinition{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + gvkErrors = appendResourceError(gvkErrors, obj, err.Error()) + continue + } + conditionExists := false + for _, condition := range obj.Status.Conditions { + if condition.Type == apiextensionsv1.Established { + if condition.Status != "True" { + gvkErrors = appendResourceError(gvkErrors, obj, condition.Message) + } + conditionExists = true + break + } + } + if conditionExists { + continue + } + gvkErrors = appendResourceError(gvkErrors, obj, "Established condition not found") + default: + // If we don't know how to check the health of the object, we assume it's healthy. + continue + } + } + + return errors.Join(gvkErrors...) +} + +// toErrKey returns a string that identifies a resource based on its GVK and namespace/name. This key is used +// to identify the resource in the error message. +func toErrKey(resource client.Object) string { + // If the resource is namespaced, include the namespace in the key. + if resource.GetNamespace() != "" { + return fmt.Sprintf("(%s)(%s/%s)", resource.GetObjectKind().GroupVersionKind().String(), resource.GetNamespace(), resource.GetName()) + } + + return fmt.Sprintf("(%s)(%s)", resource.GetObjectKind().GroupVersionKind().String(), resource.GetName()) +} + +// appendResourceError appends a new error to the given slice of errors and returns it. +func appendResourceError(gvkErrors []error, resource client.Object, message string) []error { + return append(gvkErrors, errors.New(toErrKey(resource)+": "+message)) +} diff --git a/internal/healthchecks/builtin_test.go b/internal/healthchecks/builtin_test.go new file mode 100644 index 00000000..edac3a02 --- /dev/null +++ b/internal/healthchecks/builtin_test.go @@ -0,0 +1,760 @@ +package healthchecks + +import ( + "context" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestAreObjectsHealthy(t *testing.T) { + for _, tt := range []struct { + name string + resources []client.Object + expectedErr bool + }{ + { + name: "Return true, all resources are healthy", + resources: []client.Object{ + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDeployment", + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: "True", + }, + }, + }, + }, + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: 1, + Replicas: 1, + }, + }, + &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDaemonSet", + }, + Status: appsv1.DaemonSetStatus{ + NumberAvailable: 1, + DesiredNumberScheduled: 1, + }, + }, + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyReplicatSet", + }, + Status: appsv1.ReplicaSetStatus{ + AvailableReplicas: 1, + Replicas: 1, + }, + }, + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyPod", + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: "True", + }, + }, + Phase: corev1.PodRunning, + }, + }, + &apiregistrationv1.APIService{ + TypeMeta: metav1.TypeMeta{ + Kind: "APIService", + APIVersion: "apiregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyAPIService", + }, + Status: apiregistrationv1.APIServiceStatus{ + Conditions: []apiregistrationv1.APIServiceCondition{ + { + Type: apiregistrationv1.Available, + Status: "True", + }, + }, + }, + }, + &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyCustomResourceDefinition", + }, + Status: apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: "True", + Message: "CustomResourceDefinition is established", + }, + }, + }, + }, + }, + expectedErr: false, + }, + { + name: "multiple resources are healthy, only one is not, return error", + resources: []client.Object{ + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDeployment", + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: "True", + }, + }, + }, + }, + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: 1, + Replicas: 1, + }, + }, + &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDaemonSet", + }, + Status: appsv1.DaemonSetStatus{ + NumberAvailable: 0, + DesiredNumberScheduled: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "All resources are unhealthy, return error", + resources: []client.Object{ + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDeployment", + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: "False", + Message: "Something went wrong", + }, + }, + }, + }, + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: 0, + Replicas: 1, + }, + }, + &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDaemonSet", + }, + Status: appsv1.DaemonSetStatus{ + NumberAvailable: 0, + DesiredNumberScheduled: 1, + }, + }, + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyReplicatSet", + }, + Status: appsv1.ReplicaSetStatus{ + AvailableReplicas: 0, + Replicas: 1, + }, + }, + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyPod", + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: "False", + }, + }, + }, + }, + &apiregistrationv1.APIService{ + TypeMeta: metav1.TypeMeta{ + Kind: "APIService", + APIVersion: "apiregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyAPIService", + }, + Status: apiregistrationv1.APIServiceStatus{ + Conditions: []apiregistrationv1.APIServiceCondition{ + { + Type: apiregistrationv1.Available, + Status: "False", + Message: "Something went wrong", + }, + }, + }, + }, + &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyCustomResourceDefinition", + }, + Status: apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.Established, + Status: "False", + Message: "CustomResourceDefinition is not established", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "unknown resource, no error", + resources: []client.Object{ + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyService", + }, + }, + }, + expectedErr: false, + }, + { + name: "Pod: valid resource with no conditions doesn't return error", + resources: []client.Object{ + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyPod", + }, + Status: corev1.PodStatus{ + Conditions: nil, + Phase: corev1.PodRunning, + }, + }, + }, + expectedErr: false, + }, + { + name: "APIService: resource with no conditions, return error", + resources: []client.Object{ + &apiregistrationv1.APIService{ + TypeMeta: metav1.TypeMeta{ + Kind: "APIService", + APIVersion: "apiregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyAPIService", + }, + Status: apiregistrationv1.APIServiceStatus{ + Conditions: nil, + }, + }, + }, + expectedErr: true, + }, + { + name: "CustomResourceDefinition: resource with no conditions return error", + resources: []client.Object{ + &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyCustomResourceDefinition", + }, + Status: apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: nil, + }, + }, + }, + expectedErr: true, + }, + { + name: "Deployment: resource with no conditions return error", + resources: []client.Object{ + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDeployment", + }, + Status: appsv1.DeploymentStatus{ + Conditions: nil, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet: valid resource with no conditions, doesn't return error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + Generation: 1, + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: func() *int32 { i := int32(1); return &i }(), + }, + Status: appsv1.StatefulSetStatus{ + Conditions: nil, + ObservedGeneration: 1, + ReadyReplicas: 1, + Replicas: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "DaemonSet: valid resource with no conditions, doesn't return error", + resources: []client.Object{ + &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDaemonSet", + }, + Status: appsv1.DaemonSetStatus{ + DesiredNumberScheduled: 1, + NumberAvailable: 1, + Conditions: nil, + }, + }, + }, + expectedErr: false, + }, + { + name: "ReplicaSet: valid resource with no conditions, doesn't return error", + resources: []client.Object{ + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyReplicaSet", + }, + Status: appsv1.ReplicaSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + Conditions: nil, + }, + }, + }, + expectedErr: false, + }, + { + name: "APIService: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + + &apiregistrationv1.APIService{ + TypeMeta: metav1.TypeMeta{ + Kind: "APIService", + APIVersion: "apiregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyAPIService", + }, + Status: apiregistrationv1.APIServiceStatus{ + Conditions: []apiregistrationv1.APIServiceCondition{ + { + Type: "testing", + Status: "True", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "CustomResourceDefinition: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: "CustomResourceDefinition", + APIVersion: "apiextensions.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyCustomResourceDefinition", + }, + Status: apiextensionsv1.CustomResourceDefinitionStatus{ + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ + { + Type: apiextensionsv1.NamesAccepted, + Status: "True", + Message: "CustomResourceDefinition names have been accepted", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "Deployment: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyDeployment", + }, + Status: appsv1.DeploymentStatus{ + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentProgressing, + Status: "True", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + Generation: 1, + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: func() *int32 { i := int32(1); return &i }(), + }, + Status: appsv1.StatefulSetStatus{ + Conditions: []appsv1.StatefulSetCondition{}, + ObservedGeneration: 1, + ReadyReplicas: 1, + Replicas: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "Pod: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyPod", + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: "True", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "APIService: resource with conditions but not the one we are looking for, return error", + resources: []client.Object{ + &apiregistrationv1.APIService{ + TypeMeta: metav1.TypeMeta{ + Kind: "APIService", + APIVersion: "apiregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyAPIService", + }, + Status: apiregistrationv1.APIServiceStatus{ + Conditions: []apiregistrationv1.APIServiceCondition{ + { + Type: "testing", + Status: "True", + }, + }, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet is not ready as observedGeneration doesn't match, return error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + Generation: 2, + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: func() *int32 { i := int32(1); return &i }(), + }, + Status: appsv1.StatefulSetStatus{ + ObservedGeneration: 1, + ReadyReplicas: 1, + Replicas: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet is not ready as replicas are not ready, return error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + Generation: 2, + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: func() *int32 { i := int32(1); return &i }(), + }, + Status: appsv1.StatefulSetStatus{ + ObservedGeneration: 2, + ReadyReplicas: 0, + UpdatedReplicas: 1, + Replicas: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet is not ready as Revisions don't match, return error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + Generation: 2, + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Replicas: func() *int32 { i := int32(1); return &i }(), + }, + Status: appsv1.StatefulSetStatus{ + CurrentRevision: "revision2", + UpdateRevision: "revision1", + ObservedGeneration: 2, + ReadyReplicas: 1, + UpdatedReplicas: 1, + Replicas: 1, + }, + }, + }, + expectedErr: true, + }, + { + name: "StatefulSet is not using the RollingUpdate strategy, return no error", + resources: []client.Object{ + &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "MyStatefulSet", + }, + Spec: appsv1.StatefulSetSpec{ + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.OnDeleteStatefulSetStrategyType, + }, + }, + Status: appsv1.StatefulSetStatus{}, + }, + }, + expectedErr: false, + }, + } { + ctx := context.Background() + client := fakeClient{} + t.Run(tt.name, func(t *testing.T) { + // Instantiate a fake client. + client.setResources(tt.resources) + err := AreObjectsHealthy(ctx, client, tt.resources) + if (err != nil) != tt.expectedErr { + t.Errorf("AreRelObjectsHealthy() testName=%q error = %v, expectedErr %v", tt.name, err, tt.expectedErr) + } + }) + } +} + +// Fake client for testing, implementing the client.Client interface. +type fakeClient struct { + client.Client + resources []client.Object +} + +// setResources is used to populate the fake client with the resources we want to test. +func (f *fakeClient) setResources(resources []client.Object) { + f.resources = resources +} + +// Get is a fake implementation of the client.Client.Get method, the generic healthcheck only requires the Get method. +func (f fakeClient) Get(_ context.Context, objectKey types.NamespacedName, obj client.Object, _ ...client.GetOption) error { + for _, resource := range f.resources { + if resource.GetNamespace() == objectKey.Namespace && resource.GetName() == objectKey.Name && resource.GetObjectKind().GroupVersionKind() == obj.GetObjectKind().GroupVersionKind() { + // copy the resource into the obj, obj is unstructured.Unstructured + // and resource is a typed object, so we need to convert it. + u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(resource) + if err != nil { + return err + } + return runtime.DefaultUnstructuredConverter.FromUnstructured(u, obj) + } + } + return errors.New("resource not found") +} + +func (f fakeClient) Scheme() *runtime.Scheme { + s := runtime.NewScheme() + utilruntime.Must(apiregistrationv1.AddToScheme(s)) + utilruntime.Must(apiextensionsv1.AddToScheme(s)) + utilruntime.Must(appsv1.AddToScheme(s)) + utilruntime.Must(corev1.AddToScheme(s)) + return s +} diff --git a/manifests/base/core/resources/deployment.yaml b/manifests/base/core/resources/deployment.yaml index a6f2514e..898f6a3a 100644 --- a/manifests/base/core/resources/deployment.yaml +++ b/manifests/base/core/resources/deployment.yaml @@ -27,7 +27,7 @@ spec: securityContext: allowPrivilegeEscalation: false capabilities: - drop: [ "ALL" ] + drop: ["ALL"] image: quay.io/brancz/kube-rbac-proxy:v0.12.0 args: - "--secure-listen-address=0.0.0.0:8443" @@ -48,7 +48,7 @@ spec: securityContext: allowPrivilegeEscalation: false capabilities: - drop: [ "ALL" ] + drop: ["ALL"] image: quay.io/operator-framework/rukpak:devel imagePullPolicy: IfNotPresent command: ["/core"] @@ -59,6 +59,7 @@ spec: - "--upload-storage-dir=/var/cache/uploads" - "--http-bind-address=127.0.0.1:8080" - "--http-external-address=https://$(CORE_SERVICE_NAME).$(CORE_SERVICE_NAMESPACE).svc" + - "--feature-gates=BundleDeploymentHealth=true" ports: - containerPort: 8080 volumeMounts: diff --git a/test/e2e/helm_provisioner_test.go b/test/e2e/helm_provisioner_test.go index 4e88147a..9ad94ebe 100644 --- a/test/e2e/helm_provisioner_test.go +++ b/test/e2e/helm_provisioner_test.go @@ -567,7 +567,6 @@ var _ = Describe("helm provisioner bundledeployment", func() { WithTransform(func(c *appsv1.DeploymentCondition) string { return c.Reason }, Equal("MinimumReplicasAvailable")), WithTransform(func(c *appsv1.DeploymentCondition) string { return c.Message }, ContainSubstring("Deployment has minimum availability.")), )) - }) }) }) diff --git a/test/e2e/plain_provisioner_test.go b/test/e2e/plain_provisioner_test.go index bb16a110..5d9e1373 100644 --- a/test/e2e/plain_provisioner_test.go +++ b/test/e2e/plain_provisioner_test.go @@ -1426,6 +1426,18 @@ var _ = Describe("plain provisioner bundledeployment", func() { WithTransform(func(c *metav1.Condition) string { return c.Reason }, Equal(rukpakv1alpha1.ReasonInstallationSucceeded)), WithTransform(func(c *metav1.Condition) string { return c.Message }, ContainSubstring("Instantiated bundle")), )) + By("waiting until the BD reports a healthy condition") + Eventually(func() (*metav1.Condition, error) { + if err := c.Get(ctx, client.ObjectKeyFromObject(bd), bd); err != nil { + return nil, err + } + return meta.FindStatusCondition(bd.Status.Conditions, rukpakv1alpha1.TypeHealthy), nil + }).Should(And( + Not(BeNil()), + WithTransform(func(c *metav1.Condition) string { return c.Type }, Equal(rukpakv1alpha1.TypeHealthy)), + WithTransform(func(c *metav1.Condition) metav1.ConditionStatus { return c.Status }, Equal(metav1.ConditionTrue)), + WithTransform(func(c *metav1.Condition) string { return c.Reason }, Equal(rukpakv1alpha1.ReasonHealthy)), + )) }) AfterEach(func() { By("deleting the testing BD resource") @@ -1602,7 +1614,6 @@ var _ = Describe("plain provisioner bundledeployment", func() { WithTransform(func(c *metav1.Condition) string { return c.Reason }, Equal(rukpakv1alpha1.ReasonInstallationSucceeded)), WithTransform(func(c *metav1.Condition) string { return c.Message }, ContainSubstring("Instantiated bundle")), )) - By("verifying that the old Bundle no longer exists") Eventually(func() error { return c.Get(ctx, client.ObjectKeyFromObject(originalBundle), &rukpakv1alpha1.Bundle{}) @@ -1735,6 +1746,18 @@ var _ = Describe("plain provisioner bundledeployment", func() { ContainSubstring(`no matches for kind "OperatorGroup" in version "operators.coreos.com/v1"`), )), )) + By("waiting until the BD reports a non healthy condition") + Eventually(func() (*metav1.Condition, error) { + if err := c.Get(ctx, client.ObjectKeyFromObject(bd), bd); err != nil { + return nil, err + } + return meta.FindStatusCondition(bd.Status.Conditions, rukpakv1alpha1.TypeHealthy), nil + }).Should(And( + Not(BeNil()), + WithTransform(func(c *metav1.Condition) string { return c.Type }, Equal(rukpakv1alpha1.TypeHealthy)), + WithTransform(func(c *metav1.Condition) metav1.ConditionStatus { return c.Status }, Equal(metav1.ConditionFalse)), + WithTransform(func(c *metav1.Condition) string { return c.Reason }, Equal(rukpakv1alpha1.ReasonInstallationStatusFalse)), + )) }) })