Skip to content

Commit

Permalink
Debug/memory increase (#125)
Browse files Browse the repository at this point in the history
* add cronhpa name&namespace in log and change to job queue to *sync.map
Co-authored-by: hexi.ghx <hexi.ghx@alibaba-inc.com>
  • Loading branch information
IrisIris committed Apr 26, 2023
1 parent b1a762f commit 2f290b2
Show file tree
Hide file tree
Showing 2,538 changed files with 311,858 additions and 55,956 deletions.
2 changes: 1 addition & 1 deletion cmd/kubernetes-cronhpa-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
_ "net/http/pprof"
"os"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
)

var (
Expand Down
75 changes: 62 additions & 13 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,21 +1,70 @@
module github.com/AliyunContainerService/kubernetes-cronhpa-controller

go 1.14
go 1.19

require (
github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6 // indirect
github.com/go-logr/zapr v0.2.0 // indirect; indrect
github.com/googleapis/gnostic v0.5.1 // indirect
github.com/gorilla/mux v1.8.0
github.com/prometheus/client_golang v1.7.1
github.com/prometheus/client_golang v1.14.0
github.com/ringtail/go-cron v1.0.1-0.20201027122514-cfb21c105f50
github.com/satori/go.uuid v1.2.0
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
k8s.io/api v0.19.0
k8s.io/apiextensions-apiserver v0.19.0
k8s.io/apimachinery v0.19.0
k8s.io/client-go v0.19.0
k8s.io/klog/v2 v2.2.0
sigs.k8s.io/controller-runtime v0.6.2
k8s.io/api v0.24.0
k8s.io/apiextensions-apiserver v0.24.0
k8s.io/apimachinery v0.24.0
k8s.io/client-go v0.24.0
k8s.io/klog/v2 v2.90.1
sigs.k8s.io/controller-runtime v0.12.1
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/zapr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/imdario/mergo v0.3.14 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo/v2 v2.4.0 // indirect
github.com/onsi/gomega v1.23.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.29.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/component-base v0.24.0 // indirect
k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
868 changes: 603 additions & 265 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion multi_arch_Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Build the manager binary
FROM --platform=$BUILDPLATFORM golang:1.14.2 as builder
FROM --platform=$BUILDPLATFORM golang:1.19.4 as builder
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG BUILDARCH
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/autoscaling/v1beta1/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ package v1beta1

import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/runtime/scheme"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)

var (
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/cronexecutor.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ func (ce *CronHPAExecutor) FindJob(job CronJob) (bool, FailedFindJobReason) {
entries := ce.Engine.Entries()
for _, e := range entries {
if e.Job.ID() == job.ID() {
// clean up out of date jobs when it reach maxOutOfDateTimeout
// clean up out of date jobs when it reached maxOutOfDateTimeout
if e.Next.Add(maxOutOfDateTimeout).After(time.Now()) {
return true, ""
}
log.Warningf("The job %s is out of date and need to be clean up.", job.Name())
log.Warningf("The job %s(job id %s) in cronhpa %s namespace %s is out of date.", job.Name(), job.ID(), job.CronHPAMeta().Name, job.CronHPAMeta().Namespace)
return false, JobTimeOut
}
}
Expand Down
27 changes: 16 additions & 11 deletions pkg/controller/cronhorizontalpodautoscaler_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,16 @@ type ReconcileCronHorizontalPodAutoscaler struct {
// Automatically generate RBAC rules to allow the Controller to read and write Deployments
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=autoscaling.alibabacloud.com,resources=cronhorizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Request) (reconcile.Result, error) {
func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
// Fetch the CronHorizontalPodAutoscaler instance
log.Infof("Start to handle cronHPA %s in %s namespace", request.Name, request.Namespace)
instance := &autoscalingv1beta1.CronHorizontalPodAutoscaler{}
err := r.Get(context.TODO(), request.NamespacedName, instance)
err := r.Get(context, request.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
// Object not found, return. Created objects are automatically garbage collected.
// For additional cleanup logic use finalizers.
log.Infof("GC start for: cronHPA %s in %s namespace is not found", request.Name, request.Namespace)
go r.CronManager.GC()
return reconcile.Result{}, nil
}
Expand All @@ -95,7 +96,7 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
for _, cJob := range conditions {
err := r.CronManager.delete(cJob.JobId)
if err != nil {
log.Errorf("Failed to delete job %s,because of %v", cJob.Name, err)
log.Errorf("Failed to delete job %s in cronHPA %s namespace %s, because of %v", cJob.Name, instance.Name, instance.Namespace, err)
}
}
// update scaleTargetRef and excludeDates
Expand All @@ -113,25 +114,27 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
if cJob.JobId != "" {
err := r.CronManager.delete(cJob.JobId)
if err != nil {
log.Errorf("Failed to delete expired job %s,because of %v", cJob.Name, err)
log.Errorf("Failed to delete expired job %s in cronHPA %s namespace %s,because of %v", cJob.Name, instance.Name, instance.Namespace, err)
}
}
continue
}
// if nothing changed
skip = true
}
}

// need remove this condition because this is not job spec
if !skip {
if cJob.JobId != "" {
err := r.CronManager.delete(cJob.JobId)
if err != nil {
log.Errorf("Failed to delete expired job %s,because of %v", cJob.Name, err)
log.Errorf("Failed to delete expired job %s in cronHPA %s namespace %s, because of %v", cJob.Name, instance.Name, instance.Namespace, err)
}
}
}

// need remove this condition because this is not job spec
// if job nothing changed then append to left conditions
if skip {
leftConditions = append(leftConditions, cJob)
}
Expand All @@ -156,7 +159,8 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque

if err != nil {
jobCondition.State = v1beta1.Failed
jobCondition.Message = fmt.Sprintf("Failed to create cron hpa job %s,because of %v", job.Name, err)
jobCondition.Message = fmt.Sprintf("Failed to create cron hpa job %s in %s namespace %s,because of %v",
job.Name, instance.Name, instance.Namespace, err)
log.Errorf("Failed to create cron hpa job %s,because of %v", job.Name, err)
} else {
name := job.Name
Expand All @@ -168,7 +172,8 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
if runOnce(job) && (c.State == v1beta1.Succeed || c.State == v1beta1.Failed) {
err := r.CronManager.delete(jobId)
if err != nil {
log.Errorf("cron hpa %s(%s) has ran once but fail to exit,because of %v", name, jobId, err)
log.Errorf("cron hpa runonce job %s(%s) in %s namespace %s has ran once but fail to exit,because of %v",
name, jobId, instance.Name, instance.Namespace, err)
}
continue
}
Expand All @@ -190,11 +195,11 @@ func (r *ReconcileCronHorizontalPodAutoscaler) Reconcile(request reconcile.Reque
noNeedUpdateStatus = false
instance.Status.Conditions = updateConditions(instance.Status.Conditions, jobCondition)
}
// conditions doesn't changed and no need to update.
// conditions are not changed and no need to update.
if !noNeedUpdateStatus || len(leftConditions) != len(conditions) {
err := r.Update(context.Background(), instance)
err := r.Update(context, instance)
if err != nil {
log.Errorf("Failed to update cron hpa %s status,because of %v", instance.Name, err)
log.Errorf("Failed to update cron hpa %s in namespace %s status, because of %v", instance.Name, instance.Namespace, err)
}
}

Expand Down
7 changes: 4 additions & 3 deletions pkg/controller/cronjob.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (ch *CronJobHPA) ScaleHPA() (msg string, err error) {

targetGV, err := schema.ParseGroupVersion(targetRef.APIVersion)
if err != nil {
return "", fmt.Errorf("Failed to get TargetGroup of HPA %s,because of %v", hpa.Name, err)
return "", fmt.Errorf("Failed to get TargetGroup of HPA %s in namespace %s ,because of %v", hpa.Name, hpa.Namespace, err)
}

targetGK := schema.GroupKind{
Expand Down Expand Up @@ -207,7 +207,8 @@ func (ch *CronJobHPA) ScaleHPA() (msg string, err error) {

if hpa.Status.CurrentReplicas >= ch.DesiredSize {
// skip change replicas and exit
return fmt.Sprintf("Skip scale replicas because HPA %s current replicas:%d >= desired replicas:%d.", hpa.Name, scale.Spec.Replicas, ch.DesiredSize), nil
return fmt.Sprintf("Skip scale replicas because HPA %s in namespace %s current replicas:%d >= desired replicas:%d.",
hpa.Name, hpa.Namespace, scale.Spec.Replicas, ch.DesiredSize), nil
}

msg = fmt.Sprintf("current replicas:%d, desired replicas:%d.", scale.Spec.Replicas, ch.DesiredSize)
Expand Down Expand Up @@ -239,7 +240,7 @@ func (ch *CronJobHPA) ScalePlainRef() (msg string, err error) {
scale, err = ch.scaler.Scales(ch.TargetRef.RefNamespace).Get(context.Background(), targetGR, ch.TargetRef.RefName, v1.GetOptions{})
if err == nil {
found = true
log.Infof("%s %s in namespace %s has been scaled successfully. job: %s replicas: %d", ch.TargetRef.RefKind, ch.TargetRef.RefName, ch.TargetRef.RefNamespace, ch.Name(), ch.DesiredSize)
log.Infof("%s %s in namespace %s has been scaled successfully. job: %s replicas: %d id: %s", ch.TargetRef.RefKind, ch.TargetRef.RefName, ch.TargetRef.RefNamespace, ch.Name(), ch.DesiredSize, ch.ID())
break
}
}
Expand Down
Loading

0 comments on commit 2f290b2

Please sign in to comment.