From 39cebf7be944eaaf1eaed0133f90d730f56a5d12 Mon Sep 17 00:00:00 2001 From: Andy Lo-A-Foe Date: Mon, 29 Jul 2024 12:20:03 +0200 Subject: [PATCH] chore: add stack dump for better debugging Signed-off-by: Andy Lo-A-Foe --- cmd/controlplane/api.go | 5 +++++ cmd/controlplane/controller.go | 8 ++++++++ cmd/controlplane/garbage_collector.go | 5 +++++ cmd/controlplane/management_controller.go | 5 +++++ cmd/controlplane/webhooks.go | 5 +++++ .../controller/argocd/api/v1alpha1/application_types.go | 6 +++--- internal/controller/promotions/promotions.go | 5 +++-- 7 files changed, 34 insertions(+), 5 deletions(-) diff --git a/cmd/controlplane/api.go b/cmd/controlplane/api.go index 9637a9dbe..f883c5ced 100644 --- a/cmd/controlplane/api.go +++ b/cmd/controlplane/api.go @@ -12,8 +12,10 @@ import ( kubescheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlcfg "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/metrics/server" kargoapi "github.com/akuity/kargo/api/v1alpha1" @@ -168,6 +170,9 @@ func (o *apiOptions) setupAPIClient(ctx context.Context) (*rest.Config, client.C }, }, }, + Controller: ctrlcfg.Controller{ + RecoverPanic: ptr.To(true), + }, }) if err != nil { return nil, nil, nil, fmt.Errorf("error initializing Kargo API manager: %w", err) diff --git a/cmd/controlplane/controller.go b/cmd/controlplane/controller.go index 47d6084c2..e8afb49ae 100644 --- a/cmd/controlplane/controller.go +++ b/cmd/controlplane/controller.go @@ -9,9 +9,11 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -194,6 +196,9 @@ func (o *controllerOptions) setupKargoManager( BindAddress: "0", }, Cache: cacheOpts, + Controller: config.Controller{ + RecoverPanic: ptr.To(true), + }, }, ) return mgr, stagesReconcilerCfg, err @@ -262,6 +267,9 @@ func (o *controllerOptions) setupArgoCDManager(ctx context.Context) (manager.Man BindAddress: "0", }, Cache: cacheOpts, + Controller: config.Controller{ + RecoverPanic: ptr.To(true), + }, }, ) } diff --git a/cmd/controlplane/garbage_collector.go b/cmd/controlplane/garbage_collector.go index a8b17c6ed..756283ea5 100644 --- a/cmd/controlplane/garbage_collector.go +++ b/cmd/controlplane/garbage_collector.go @@ -8,7 +8,9 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -105,6 +107,9 @@ func (o *garbageCollectorOptions) setupManager(ctx context.Context) (manager.Man Metrics: server.Options{ BindAddress: "0", }, + Controller: config.Controller{ + RecoverPanic: ptr.To(true), + }, }, ) if err != nil { diff --git a/cmd/controlplane/management_controller.go b/cmd/controlplane/management_controller.go index 3cba0531e..566349ebe 100644 --- a/cmd/controlplane/management_controller.go +++ b/cmd/controlplane/management_controller.go @@ -8,7 +8,9 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -118,6 +120,9 @@ func (o *managementControllerOptions) setupManager(ctx context.Context) (manager Metrics: server.Options{ BindAddress: "0", }, + Controller: config.Controller{ + RecoverPanic: ptr.To(true), + }, }, ) } diff --git a/cmd/controlplane/webhooks.go b/cmd/controlplane/webhooks.go index d0b10fde6..ab8291cef 100644 --- a/cmd/controlplane/webhooks.go +++ b/cmd/controlplane/webhooks.go @@ -9,7 +9,9 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -100,6 +102,9 @@ func (o *webhooksServerOptions) run(ctx context.Context) error { Metrics: server.Options{ BindAddress: "0", }, + Controller: config.Controller{ + RecoverPanic: ptr.To(true), + }, }, ) if err != nil { diff --git a/internal/controller/argocd/api/v1alpha1/application_types.go b/internal/controller/argocd/api/v1alpha1/application_types.go index 0155315c0..9808ab8e8 100644 --- a/internal/controller/argocd/api/v1alpha1/application_types.go +++ b/internal/controller/argocd/api/v1alpha1/application_types.go @@ -176,7 +176,7 @@ type OperationState struct { } type SyncOperationResult struct { - Revision string `json:"revision,omitempty"` - Source ApplicationSource `json:"source,omitempty"` - Sources ApplicationSources `json:"sources,omitempty"` + Revision string `json:"revision,omitempty"` + Source ApplicationSource `json:"source,omitempty"` + Sources ApplicationSources `json:"sources,omitempty"` } diff --git a/internal/controller/promotions/promotions.go b/internal/controller/promotions/promotions.go index 8ed8ddde9..74aa87041 100644 --- a/internal/controller/promotions/promotions.go +++ b/internal/controller/promotions/promotions.go @@ -3,6 +3,7 @@ package promotions import ( "context" "fmt" + "runtime/debug" "slices" "strconv" "sync" @@ -344,9 +345,9 @@ func (r *reconciler) Reconcile( defer func() { if err := recover(); err != nil { if theErr, ok := err.(error); ok { - logger.Error(theErr, "Promotion panic") + logger.Error(theErr, "Promotion panic", "stack", string(debug.Stack())) } else { - logger.Error(nil, "Promotion panic") + logger.Error(nil, "Promotion panic", "stack", string(debug.Stack())) } newStatus.Phase = kargoapi.PromotionPhaseErrored newStatus.Message = fmt.Sprintf("%v", err)