From d382a7c571bcb1923a4c368f2fcac0516ca4f1e4 Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Sun, 8 Nov 2020 15:04:01 +0800 Subject: [PATCH] feat(controllers): make resyncPeriod configurable (#1519) (#1521) --- Makefile | 6 +++--- pkg/workflow/common/const.go | 4 ---- pkg/workflow/controller/config.go | 16 ++++++++++++++++ .../controller/controllers/exucution_cluster.go | 5 +++-- pkg/workflow/controller/controllers/pod.go | 5 +++-- .../controller/controllers/workflow_trigger.go | 5 +++-- .../controller/controllers/workflowrun.go | 5 +++-- .../controller/handlers/workflowrun/handler.go | 4 ++-- pkg/workflow/workflowrun/limits.go | 4 ++-- 9 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index c8c723b7d..bf3547b1b 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ ROOT := github.com/caicloud/cyclone # Target binaries. You can build multiple binaries for a single project. -TARGETS := server workflow/controller workflow/coordinator cicd/cd toolbox/fstream -IMAGES := server workflow/controller workflow/coordinator resolver/git resolver/svn resolver/image watcher cicd/cd cicd/sonarqube toolbox +TARGETS ?= server workflow/controller workflow/coordinator cicd/cd toolbox/fstream +IMAGES ?= server workflow/controller workflow/coordinator resolver/git resolver/svn resolver/image watcher cicd/cd cicd/sonarqube toolbox BASE_IMAGES := base/alpine base/openjdk # Container image prefix and suffix added to targets. @@ -236,4 +236,4 @@ clean: clean-generated: -rm -rf ./pkg/k8s/informers -rm -rf ./pkg/k8s/clientset - -rm -rf ./pkg/k8s/listers \ No newline at end of file + -rm -rf ./pkg/k8s/listers diff --git a/pkg/workflow/common/const.go b/pkg/workflow/common/const.go index 237da6013..da16bee67 100644 --- a/pkg/workflow/common/const.go +++ b/pkg/workflow/common/const.go @@ -2,16 +2,12 @@ package common import ( "fmt" - "time" ) // ContainerState represents container state. type ContainerState string const ( - // ResyncPeriod defines resync period for controllers - ResyncPeriod = time.Minute * 3 - // EnvStagePodName is an environment which represents pod name. EnvStagePodName = "POD_NAME" // EnvStageInfo is an environment which represents stage information. diff --git a/pkg/workflow/controller/config.go b/pkg/workflow/controller/config.go index eee35eefd..f2c2dc31a 100644 --- a/pkg/workflow/controller/config.go +++ b/pkg/workflow/controller/config.go @@ -52,6 +52,8 @@ type WorkflowControllerConfig struct { NotificationURL string `json:"notification_url"` // DindSettings is settings for Docker in Docker DindSettings DindSettings `json:"dind"` + // ResyncPeriodSeconds defines resync period in seconds for controllers + ResyncPeriodSeconds time.Duration `json:"resync_period_seconds"` } // LoggingConfig configures logging @@ -133,11 +135,13 @@ func LoadConfig(cm *corev1.ConfigMap) error { return err } + defaultValues(&Config) if !validate(&Config) { return fmt.Errorf("validate config failed") } InitLogger(&Config.Logging) + log.Info("ResyncPeriod is %s", Config.ResyncPeriodSeconds*time.Second) return nil } @@ -146,10 +150,22 @@ func validate(config *WorkflowControllerConfig) bool { if config.ExecutionContext.PVC == "" { log.Warn("PVC not configured, resources won't be shared among stages and artifacts unsupported.") } + if config.ResyncPeriodSeconds < 0 { + log.Errorf("Invalid ResyncPeriodSeconds: %d", config.ResyncPeriodSeconds) + return false + } return true } +// defaultValues give the config some default value if they are not set. +func defaultValues(config *WorkflowControllerConfig) { + if config.ResyncPeriodSeconds == 0 { + config.ResyncPeriodSeconds = 180 + log.Info("ResyncPeriodSeconds not configured, will use default value '180'") + } +} + // ImagePullPolicy determines image pull policy based on environment variable DEVELOP_MODE // This pull policy will be used in image resolver containers and coordinator containers. func ImagePullPolicy() corev1.PullPolicy { diff --git a/pkg/workflow/controller/controllers/exucution_cluster.go b/pkg/workflow/controller/controllers/exucution_cluster.go index bdff66fa3..a87773b09 100644 --- a/pkg/workflow/controller/controllers/exucution_cluster.go +++ b/pkg/workflow/controller/controllers/exucution_cluster.go @@ -2,13 +2,14 @@ package controllers import ( "reflect" + "time" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "github.com/caicloud/cyclone/pkg/k8s/clientset" "github.com/caicloud/cyclone/pkg/k8s/informers" - "github.com/caicloud/cyclone/pkg/workflow/common" + "github.com/caicloud/cyclone/pkg/workflow/controller" "github.com/caicloud/cyclone/pkg/workflow/controller/handlers/executioncluster" ) @@ -17,7 +18,7 @@ func NewExecutionClusterController(client clientset.Interface) *Controller { queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) factory := informers.NewSharedInformerFactoryWithOptions( client, - common.ResyncPeriod, + controller.Config.ResyncPeriodSeconds*time.Second, ) informer := factory.Cyclone().V1alpha1().ExecutionClusters().Informer() diff --git a/pkg/workflow/controller/controllers/pod.go b/pkg/workflow/controller/controllers/pod.go index 6013f4975..c581eb394 100644 --- a/pkg/workflow/controller/controllers/pod.go +++ b/pkg/workflow/controller/controllers/pod.go @@ -2,6 +2,7 @@ package controllers import ( "reflect" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" @@ -11,7 +12,7 @@ import ( "github.com/caicloud/cyclone/pkg/k8s/clientset" "github.com/caicloud/cyclone/pkg/meta" - "github.com/caicloud/cyclone/pkg/workflow/common" + "github.com/caicloud/cyclone/pkg/workflow/controller" "github.com/caicloud/cyclone/pkg/workflow/controller/handlers/pod" ) @@ -20,7 +21,7 @@ func NewPodController(clusterClient kubernetes.Interface, client clientset.Inter queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) factory := informers.NewSharedInformerFactoryWithOptions( clusterClient, - common.ResyncPeriod, + controller.Config.ResyncPeriodSeconds*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = meta.CyclonePodSelector() }), diff --git a/pkg/workflow/controller/controllers/workflow_trigger.go b/pkg/workflow/controller/controllers/workflow_trigger.go index 3767ea20d..c6c7ae8ab 100644 --- a/pkg/workflow/controller/controllers/workflow_trigger.go +++ b/pkg/workflow/controller/controllers/workflow_trigger.go @@ -2,6 +2,7 @@ package controllers import ( "reflect" + "time" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -11,7 +12,7 @@ import ( "github.com/caicloud/cyclone/pkg/k8s/clientset" "github.com/caicloud/cyclone/pkg/k8s/informers" "github.com/caicloud/cyclone/pkg/meta" - "github.com/caicloud/cyclone/pkg/workflow/common" + "github.com/caicloud/cyclone/pkg/workflow/controller" "github.com/caicloud/cyclone/pkg/workflow/controller/handlers/workflowtrigger" ) @@ -20,7 +21,7 @@ func NewWorkflowTriggerController(client clientset.Interface) *Controller { queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) factory := informers.NewSharedInformerFactoryWithOptions( client, - common.ResyncPeriod, + controller.Config.ResyncPeriodSeconds*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = meta.WorkflowTriggerSelector() }), diff --git a/pkg/workflow/controller/controllers/workflowrun.go b/pkg/workflow/controller/controllers/workflowrun.go index 59d0ece7c..b6f908d1c 100644 --- a/pkg/workflow/controller/controllers/workflowrun.go +++ b/pkg/workflow/controller/controllers/workflowrun.go @@ -1,6 +1,8 @@ package controllers import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -8,7 +10,6 @@ import ( "github.com/caicloud/cyclone/pkg/k8s/clientset" "github.com/caicloud/cyclone/pkg/k8s/informers" "github.com/caicloud/cyclone/pkg/meta" - "github.com/caicloud/cyclone/pkg/workflow/common" "github.com/caicloud/cyclone/pkg/workflow/controller" handlers "github.com/caicloud/cyclone/pkg/workflow/controller/handlers/workflowrun" "github.com/caicloud/cyclone/pkg/workflow/workflowrun" @@ -19,7 +20,7 @@ func NewWorkflowRunController(client clientset.Interface) *Controller { queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) factory := informers.NewSharedInformerFactoryWithOptions( client, - common.ResyncPeriod, + controller.Config.ResyncPeriodSeconds*time.Second, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.LabelSelector = meta.WorkflowRunSelector() }), diff --git a/pkg/workflow/controller/handlers/workflowrun/handler.go b/pkg/workflow/controller/handlers/workflowrun/handler.go index 7f1e2c4ea..d0d0f84e1 100644 --- a/pkg/workflow/controller/handlers/workflowrun/handler.go +++ b/pkg/workflow/controller/handlers/workflowrun/handler.go @@ -67,7 +67,7 @@ func (h *Handler) ObjectCreated(obj interface{}) { if originWfr.Status.Overall.Phase == "" { switch h.ParallelismController.AttemptNew(originWfr.Namespace, originWfr.Spec.WorkflowRef.Name, originWfr.Name) { case workflowrun.AttemptActionQueued: - log.WithField("wfr", originWfr.Name).Infof("Too many WorkflowRun are running, stay pending in queue, will retry in %s", common.ResyncPeriod.String()) + log.WithField("wfr", originWfr.Name).Infof("Too many WorkflowRun are running, stay pending in queue, will retry in %d seconds", controller.Config.ResyncPeriodSeconds) return case workflowrun.AttemptActionFailed: if err := h.SetStatus(originWfr.Namespace, originWfr.Name, &v1alpha1.Status{ @@ -129,7 +129,7 @@ func (h *Handler) ObjectUpdated(old, new interface{}) { log.WithField("wfr", originWfr.Name).Info("Attempt to run WorkflowRun") switch h.ParallelismController.AttemptNew(originWfr.Namespace, originWfr.Spec.WorkflowRef.Name, originWfr.Name) { case workflowrun.AttemptActionQueued: - log.WithField("wfr", originWfr.Name).Infof("Too many WorkflowRun are running, stay pending in queue, will retry in %s", common.ResyncPeriod.String()) + log.WithField("wfr", originWfr.Name).Infof("Too many WorkflowRun are running, stay pending in queue, will retry in %d seconds", controller.Config.ResyncPeriodSeconds) return case workflowrun.AttemptActionFailed: if err := h.SetStatus(originWfr.Namespace, originWfr.Name, &v1alpha1.Status{ diff --git a/pkg/workflow/workflowrun/limits.go b/pkg/workflow/workflowrun/limits.go index ad3046a27..887828eae 100644 --- a/pkg/workflow/workflowrun/limits.go +++ b/pkg/workflow/workflowrun/limits.go @@ -11,7 +11,7 @@ import ( "github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1" "github.com/caicloud/cyclone/pkg/k8s/clientset" - "github.com/caicloud/cyclone/pkg/workflow/common" + "github.com/caicloud/cyclone/pkg/workflow/controller" ) // LimitedQueues manages WorkflowRun queue for each Workflow. Queue for each Workflow is limited to @@ -102,7 +102,7 @@ func scanQueue(q *LimitedSortedQueue) { // If the node's refresh time is old enough compared to the resync time // (5 minutes by default) of WorkflowRun Controller, it means the WorkflowRun // is actually removed from etcd somehow, so we will remove it also here. - if h.next.refresh.Add(common.ResyncPeriod * 2).Before(time.Now()) { + if h.next.refresh.Add(controller.Config.ResyncPeriodSeconds * time.Second * 2).Before(time.Now()) { log.WithField("wfr", h.next.wfr).Info("remove wfr with outdated refresh time from queue") h.next = h.next.next q.size--