Debug/memory increase

cmd/kubernetes-cronhpa-controller/main.go

@@ -26,7 +26,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 )
 
 var (

go.mod

@@ -1,21 +1,70 @@
 module github.com/AliyunContainerService/kubernetes-cronhpa-controller
 
-go 1.14
+go 1.19
 
 require (
-	github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6 // indirect
-	github.com/go-logr/zapr v0.2.0 // indirect; indrect
-	github.com/googleapis/gnostic v0.5.1 // indirect
 	github.com/gorilla/mux v1.8.0
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.14.0
 	github.com/ringtail/go-cron v1.0.1-0.20201027122514-cfb21c105f50
 	github.com/satori/go.uuid v1.2.0
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
-	k8s.io/api v0.19.0
-	k8s.io/apiextensions-apiserver v0.19.0
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v0.19.0
-	k8s.io/klog/v2 v2.2.0
-	sigs.k8s.io/controller-runtime v0.6.2
+	k8s.io/api v0.24.0
+	k8s.io/apiextensions-apiserver v0.24.0
+	k8s.io/apimachinery v0.24.0
+	k8s.io/client-go v0.24.0
+	k8s.io/klog/v2 v2.90.1
+	sigs.k8s.io/controller-runtime v0.12.1
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/zapr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.14 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.4.0 // indirect
+	github.com/onsi/gomega v1.23.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.29.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/component-base v0.24.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
+	k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

multi_arch_Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.14.2 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.4 as builder
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG BUILDARCH

pkg/apis/autoscaling/v1beta1/register.go

@@ -23,7 +23,7 @@ package v1beta1
 
 import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
 )
 
 var (

pkg/controller/cronexecutor.go

@@ -49,11 +49,11 @@ func (ce *CronHPAExecutor) FindJob(job CronJob) (bool, FailedFindJobReason) {
 	entries := ce.Engine.Entries()
 	for _, e := range entries {
 		if e.Job.ID() == job.ID() {
-			// clean up out of date jobs when it reach maxOutOfDateTimeout
+			// clean up out of date jobs when it reached maxOutOfDateTimeout
 			if e.Next.Add(maxOutOfDateTimeout).After(time.Now()) {
 				return true, ""
 			}
-			log.Warningf("The job %s is out of date and need to be clean up.", job.Name())
+			log.Warningf("The job %s(job id %s) in cronhpa %s namespace %s is out of date.", job.Name(), job.ID(), job.CronHPAMeta().Name, job.CronHPAMeta().Namespace)
 			return false, JobTimeOut
 		}
 	}

pkg/controller/cronhorizontalpodautoscaler_controller.go

@@ -70,15 +70,16 @@ type ReconcileCronHorizontalPodAutoscaler struct {
 // Automatically generate RBAC rules to allow the Controller to read and write Deployments
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=autoscaling.alibabacloud.com,resources=cronhorizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
-func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
 	// Fetch the CronHorizontalPodAutoscaler instance
 	log.Infof("Start to handle cronHPA %s in %s namespace", request.Name, request.Namespace)
 	instance := &autoscalingv1beta1.CronHorizontalPodAutoscaler{}
-	err := r.Get(context.TODO(), request.NamespacedName, instance)
+	err := r.Get(context, request.NamespacedName, instance)
 	if err != nil {
 		if errors.IsNotFound(err) {
 			// Object not found, return.  Created objects are automatically garbage collected.
 			// For additional cleanup logic use finalizers.
+			log.Infof("GC start for: cronHPA %s in %s namespace is not found", request.Name, request.Namespace)
 			go r.CronManager.GC()
 			return reconcile.Result{}, nil
 		}
@@ -95,7 +96,7 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
 		for _, cJob := range conditions {
 			err := r.CronManager.delete(cJob.JobId)
 			if err != nil {
-				log.Errorf("Failed to delete job %s,because of %v", cJob.Name, err)
+				log.Errorf("Failed to delete job %s in cronHPA %s namespace %s, because of %v", cJob.Name, instance.Name, instance.Namespace, err)
 			}
 		}
 		// update scaleTargetRef and excludeDates
@@ -113,25 +114,27 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
 						if cJob.JobId != "" {
 							err := r.CronManager.delete(cJob.JobId)
 							if err != nil {
-								log.Errorf("Failed to delete expired job %s,because of %v", cJob.Name, err)
+								log.Errorf("Failed to delete expired job %s in cronHPA %s namespace %s,because of %v", cJob.Name, instance.Name, instance.Namespace, err)
 							}
 						}
 						continue
 					}
+					// if nothing changed
 					skip = true
 				}
 			}
 
+			// need remove this condition because this is not job spec
 			if !skip {
 				if cJob.JobId != "" {
 					err := r.CronManager.delete(cJob.JobId)
 					if err != nil {
-						log.Errorf("Failed to delete expired job %s,because of %v", cJob.Name, err)
+						log.Errorf("Failed to delete expired job %s in cronHPA %s namespace %s, because of %v", cJob.Name, instance.Name, instance.Namespace, err)
 					}
 				}
 			}
 
-			// need remove this condition because this is not job spec
+			// if job nothing changed then append to left conditions
 			if skip {
 				leftConditions = append(leftConditions, cJob)
 			}
@@ -156,7 +159,8 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
 
 		if err != nil {
 			jobCondition.State = v1beta1.Failed
-			jobCondition.Message = fmt.Sprintf("Failed to create cron hpa job %s,because of %v", job.Name, err)
+			jobCondition.Message = fmt.Sprintf("Failed to create cron hpa job %s in %s namespace %s,because of %v",
+				job.Name, instance.Name, instance.Namespace, err)
 			log.Errorf("Failed to create cron hpa job %s,because of %v", job.Name, err)
 		} else {
 			name := job.Name
@@ -168,7 +172,8 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
 				if runOnce(job) && (c.State == v1beta1.Succeed || c.State == v1beta1.Failed) {
 					err := r.CronManager.delete(jobId)
 					if err != nil {
-						log.Errorf("cron hpa %s(%s) has ran once but fail to exit,because of %v", name, jobId, err)
+						log.Errorf("cron hpa runonce job %s(%s) in %s namespace %s has ran once but fail to exit,because of %v",
+							name, jobId, instance.Name, instance.Namespace, err)
 					}
 					continue
 				}
@@ -190,11 +195,11 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
 		noNeedUpdateStatus = false
 		instance.Status.Conditions = updateConditions(instance.Status.Conditions, jobCondition)
 	}
-	// conditions doesn't changed and no need to update.
+	// conditions are not changed and no need to update.
 	if !noNeedUpdateStatus || len(leftConditions) != len(conditions) {
-		err := r.Update(context.Background(), instance)
+		err := r.Update(context, instance)
 		if err != nil {
-			log.Errorf("Failed to update cron hpa %s status,because of %v", instance.Name, err)
+			log.Errorf("Failed to update cron hpa %s in namespace %s status, because of %v", instance.Name, instance.Namespace, err)
 		}
 	}
 

pkg/controller/cronjob.go

@@ -147,7 +147,7 @@ func (ch *CronJobHPA) ScaleHPA() (msg string, err error) {
 
 	targetGV, err := schema.ParseGroupVersion(targetRef.APIVersion)
 	if err != nil {
-		return "", fmt.Errorf("Failed to get TargetGroup of HPA %s,because of %v", hpa.Name, err)
+		return "", fmt.Errorf("Failed to get TargetGroup of HPA %s in namespace %s ,because of %v", hpa.Name, hpa.Namespace, err)
 	}
 
 	targetGK := schema.GroupKind{
@@ -207,7 +207,8 @@ func (ch *CronJobHPA) ScaleHPA() (msg string, err error) {
 
 	if hpa.Status.CurrentReplicas >= ch.DesiredSize {
 		// skip change replicas and exit
-		return fmt.Sprintf("Skip scale replicas because HPA %s current replicas:%d >= desired replicas:%d.", hpa.Name, scale.Spec.Replicas, ch.DesiredSize), nil
+		return fmt.Sprintf("Skip scale replicas because HPA %s in namespace %s current replicas:%d >= desired replicas:%d.",
+			hpa.Name, hpa.Namespace, scale.Spec.Replicas, ch.DesiredSize), nil
 	}
 
 	msg = fmt.Sprintf("current replicas:%d, desired replicas:%d.", scale.Spec.Replicas, ch.DesiredSize)
@@ -239,7 +240,7 @@ func (ch *CronJobHPA) ScalePlainRef() (msg string, err error) {
 		scale, err = ch.scaler.Scales(ch.TargetRef.RefNamespace).Get(context.Background(), targetGR, ch.TargetRef.RefName, v1.GetOptions{})
 		if err == nil {
 			found = true
-			log.Infof("%s %s in namespace %s has been scaled successfully. job: %s replicas: %d", ch.TargetRef.RefKind, ch.TargetRef.RefName, ch.TargetRef.RefNamespace, ch.Name(), ch.DesiredSize)
+			log.Infof("%s %s in namespace %s has been scaled successfully. job: %s replicas: %d id: %s", ch.TargetRef.RefKind, ch.TargetRef.RefName, ch.TargetRef.RefNamespace, ch.Name(), ch.DesiredSize, ch.ID())
 			break
 		}
 	}