feat(controller): add parameter to fine-tune max concurrent reconciles (

#288)

cmd/controllers/start.go

@@ -35,6 +35,7 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command {
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.FailureGracePeriod, "failure-grace-period", defaultFailureGracePeriod, "initial time before retry, goes exponential function of number failure. Must end with s, m or h.")
 	cmd.Flags().IntVar(&app.Config.Controller.TerraformMaxRetries, "terraform-max-retries", 5, "default number of retries for terraform actions (can be overriden in CRDs)")
+	cmd.Flags().IntVar(&app.Config.Controller.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "maximum number of concurrent reconciles")
 	cmd.Flags().BoolVar(&app.Config.Controller.LeaderElection.Enabled, "leader-election", true, "whether leader election is enabled or not, default to true")
 	cmd.Flags().StringVar(&app.Config.Controller.LeaderElection.ID, "leader-election-id", "6d185457.terraform.padok.cloud", "lease id used for leader election")
 	cmd.Flags().StringVar(&app.Config.Controller.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "address to bind the metrics server embedded in the controllers")

deploy/charts/burrito/values.yaml

@@ -18,6 +18,7 @@ config:
       # -- By default, the controller will only watch the tenants namespaces
       namespaces: []
       timers: {}
+      maxConcurrentReconciles: 1
       terraformMaxRetries: 3
       types: ["layer", "repository", "run", "pullrequest"]
       leaderElection:
@@ -46,7 +47,7 @@ config:
         mock: false # -- Use in-memory storage for testing - not intended for production use 
         gcs: {}
         azure: {}
-        aws: {}
+        s3: {}
       addr: ":8080"
 
     # Burrito server configuration

internal/burrito/config/config.go

@@ -59,18 +59,19 @@ type WebhookGitlabConfig struct {
 }
 
 type ControllerConfig struct {
-	MainNamespace          string               `mapstructure:"mainNamespace"`
-	Namespaces             []string             `mapstructure:"namespaces"`
-	Timers                 ControllerTimers     `mapstructure:"timers"`
-	TerraformMaxRetries    int                  `mapstructure:"terraformMaxRetries"`
-	Types                  []string             `mapstructure:"types"`
-	LeaderElection         LeaderElectionConfig `mapstructure:"leaderElection"`
-	MetricsBindAddress     string               `mapstructure:"metricsBindAddress"`
-	HealthProbeBindAddress string               `mapstructure:"healthProbeBindAddress"`
-	KubernetesWebhookPort  int                  `mapstructure:"kubernetesWebhookPort"`
-	GithubConfig           GithubConfig         `mapstructure:"githubConfig"`
-	GitlabConfig           GitlabConfig         `mapstructure:"gitlabConfig"`
-	RunParallelism         int                  `mapstructure:"runParallelism"`
+	MainNamespace           string               `mapstructure:"mainNamespace"`
+	Namespaces              []string             `mapstructure:"namespaces"`
+	Timers                  ControllerTimers     `mapstructure:"timers"`
+	TerraformMaxRetries     int                  `mapstructure:"terraformMaxRetries"`
+	Types                   []string             `mapstructure:"types"`
+	LeaderElection          LeaderElectionConfig `mapstructure:"leaderElection"`
+	MetricsBindAddress      string               `mapstructure:"metricsBindAddress"`
+	HealthProbeBindAddress  string               `mapstructure:"healthProbeBindAddress"`
+	KubernetesWebhookPort   int                  `mapstructure:"kubernetesWebhookPort"`
+	GithubConfig            GithubConfig         `mapstructure:"githubConfig"`
+	GitlabConfig            GitlabConfig         `mapstructure:"gitlabConfig"`
+	RunParallelism          int                  `mapstructure:"runParallelism"`
+	MaxConcurrentReconciles int                  `mapstructure:"maxConcurrentReconciles"`
 }
 
 type GithubConfig struct {
@@ -207,7 +208,8 @@ func bindEnvironmentVariables(v *viper.Viper, iface interface{}, parts ...string
 func TestConfig() *Config {
 	return &Config{
 		Controller: ControllerConfig{
-			TerraformMaxRetries: 5,
+			TerraformMaxRetries:     5,
+			MaxConcurrentReconciles: 1,
 			Timers: ControllerTimers{
 				DriftDetection:     20 * time.Minute,
 				WaitAction:         5 * time.Minute,

internal/burrito/config/config_test.go

@@ -59,8 +59,9 @@ func TestConfig_FromYamlFile(t *testing.T) {
 				WaitAction:         1 * time.Minute,
 				FailureGracePeriod: 15 * time.Second,
 			},
-			TerraformMaxRetries: 5,
-			Types:               []string{"layer", "repository", "run", "pullrequest"},
+			TerraformMaxRetries:     5,
+			MaxConcurrentReconciles: 1,
+			Types:                   []string{"layer", "repository", "run", "pullrequest"},
 			LeaderElection: config.LeaderElectionConfig{
 				Enabled: true,
 				ID:      "6d185457.terraform.padok.cloud",
@@ -136,6 +137,7 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
 	setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_ONERROR", "30s", &envVarList)
 	setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_WAITACTION", "30s", &envVarList)
 	setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_FAILUREGRACEPERIOD", "1m", &envVarList)
+	setEnvVar(t, "BURRITO_CONTROLLER_MAXCONCURRENTRECONCILES", "3", &envVarList)
 	setEnvVar(t, "BURRITO_CONTROLLER_TERRAFORMMAXRETRIES", "32", &envVarList)
 	setEnvVar(t, "BURRITO_CONTROLLER_LEADERELECTION_ID", "other-leader-id", &envVarList)
 	setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_APPID", "123456", &envVarList)
@@ -189,8 +191,9 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
 				WaitAction:         30 * time.Second,
 				FailureGracePeriod: 1 * time.Minute,
 			},
-			TerraformMaxRetries: 32,
-			Types:               []string{"layer", "repository"},
+			MaxConcurrentReconciles: 3,
+			TerraformMaxRetries:     32,
+			Types:                   []string{"layer", "repository"},
 			LeaderElection: config.LeaderElectionConfig{
 				Enabled: true,
 				ID:      "other-leader-id",

internal/burrito/config/testdata/test-config-1.yaml

@@ -19,6 +19,7 @@ controller:
     waitAction: 1m
     failureGracePeriod: 15s
   terraformMaxRetries: 5
+  maxConcurrentReconciles: 1
   types: ["layer", "repository", "run", "pullrequest"]
   leaderElection:
     enabled: true

internal/controllers/manager.go

@@ -120,6 +120,7 @@ func (c *Controllers) Exec() {
 				Client:   mgr.GetClient(),
 				Scheme:   mgr.GetScheme(),
 				Recorder: mgr.GetEventRecorderFor("Burrito"),
+				Config:   c.config,
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create repository controller: %s", err)
 			}

internal/controllers/terraformlayer/controller.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
@@ -202,6 +203,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.Clock = RealClock{}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&configv1alpha1.TerraformLayer{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
 		WithEventFilter(ignorePredicate()).
 		Complete(r)
 }

internal/controllers/terraformpullrequest/controller.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
@@ -111,6 +112,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.Providers = providers
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&configv1alpha1.TerraformPullRequest{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
 		WithEventFilter(ignorePredicate()).
 		Complete(r)
 }

internal/controllers/terraformrepository/controller.go

@@ -24,15 +24,18 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
+	"github.com/padok-team/burrito/internal/burrito/config"
 )
 
 // RepositoryReconciler reconciles a TerraformRepository object
 type Reconciler struct {
 	client.Client
 	Scheme   *runtime.Scheme
 	Recorder record.EventRecorder
+	Config   *config.Config
 }
 
 //+kubebuilder:rbac:groups=config.terraform.padok.cloud,resources=terraformrepositories,verbs=get;list;watch;create;update;patch;delete
@@ -60,5 +63,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&configv1alpha1.TerraformRepository{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
 		Complete(r)
 }

internal/controllers/terraformrun/controller.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
@@ -186,6 +187,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.Clock = RealClock{}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&configv1alpha1.TerraformRun{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
 		WithEventFilter(ignorePredicate()).
 		Complete(r)
 }