Added ComponentConfig API

PROJECT

@@ -38,4 +38,11 @@ resources:
   kind: ResourceFlavor
   path: sigs.k8s.io/kueue/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: x-k8s.io
+  group: config
+  kind: KueueConfiguration
+  path: sigs.k8s.io/kueue/apis/config/v1alpha1
+  version: v1alpha1
 version: "3"

apis/config/v1alpha1/configuration_types.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+)
+
+//+kubebuilder:object:root=true
+
+// Configuration is the Schema for the kueueconfigurations API
+type Configuration struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// ControllerManagerConfigurationSpec returns the configurations for controllers
+	cfg.ControllerManagerConfigurationSpec `json:",inline"`
+
+	// ManageJobsWithoutQueueName controls whether or not Kueue reconciles
+	// batch/v1.Jobs that don't set the annotation kueue.x-k8s.io/queue-name.
+	// If set to true, then those jobs will be suspended and never started unless
+	// they are assigned a queue and eventually admitted. This also applies to
+	// jobs created before starting the kueue controller.
+	// Defaults to false; therefore, those jobs are not managed and if they are created
+	// unsuspended, they will start immediately.
+	ManageJobsWithoutQueueName bool `json:"manageJobsWithoutQueueName"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Configuration{})
+}

apis/config/v1alpha1/groupversion_info.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1alpha1 contains API Schema definitions for the config v1alpha1 API group
+//+kubebuilder:object:generate=true
+//+groupName=config.x-k8s.io
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "config.kueue.x-k8s.io", Version: "v1alpha1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

config/default/kustomization.yaml

@@ -32,7 +32,7 @@ patchesStrategicMerge:
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
-#- manager_config_patch.yaml
+- manager_config_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

config/default/manager_auth_proxy_patch.yaml

@@ -22,7 +22,4 @@ spec:
           name: https
       - name: manager
         args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
         - "--zap-log-level=2"

config/default/manager_config_patch.yaml

@@ -10,6 +10,7 @@ spec:
       - name: manager
         args:
         - "--config=controller_manager_config.yaml"
+        - "--zap-log-level=2"
         volumeMounts:
         - name: manager-config
           mountPath: /controller_manager_config.yaml

config/manager/controller_manager_config.yaml

@@ -1,11 +1,12 @@
-apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
-kind: ControllerManagerConfig
+apiVersion: config.kueue.x-k8s.io/v1alpha1
+kind: Configuration
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: 127.0.0.1:8080
+  bindAddress: :8080
 webhook:
   port: 9443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
+#manageJobsWithoutQueueName: true

config/manager/kustomization.yaml

@@ -8,6 +8,7 @@ configMapGenerator:
 - files:
   - controller_manager_config.yaml
   name: manager-config
+
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:

main.go

@@ -17,7 +17,9 @@ limitations under the License.
 package main
 
 import (
+	"bytes"
 	"flag"
+	"fmt"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -26,12 +28,14 @@ import (
 
 	schedulingv1 "k8s.io/api/scheduling/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	configv1alpha1 "sigs.k8s.io/kueue/apis/config/v1alpha1"
 	kueuev1alpha1 "sigs.k8s.io/kueue/apis/core/v1alpha1"
 	"sigs.k8s.io/kueue/pkg/cache"
 	"sigs.k8s.io/kueue/pkg/constants"
@@ -52,32 +56,46 @@ func init() {
 	utilruntime.Must(schedulingv1.AddToScheme(scheme))
 
 	utilruntime.Must(kueuev1alpha1.AddToScheme(scheme))
+	utilruntime.Must(configv1alpha1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 
 func main() {
-	var metricsAddr string
-	var enableLeaderElection bool
-	var probeAddr string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
+	var configFile string
+	flag.StringVar(&configFile, "config", "",
+		"The controller will load its initial configuration from this file. "+
+			"Omit this flag to use the default configuration values. ")
+
 	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	options := ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
+		HealthProbeBindAddress: ":8081",
+		MetricsBindAddress:     ":8080",
 		Port:                   9443,
-		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "c1f6bfd2.kueue.x-k8s.io",
-	})
+	}
+	var err error
+	config := configv1alpha1.Configuration{}
+	if configFile != "" {
+		options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&config))
+		if err != nil {
+			setupLog.Error(err, "unable to load the config file")
+			os.Exit(1)
+		}
+		cfgStr, err := encodeConfig(&config)
+		if err != nil {
+			setupLog.Error(err, "unable to encode config file")
+			os.Exit(1)
+		}
+		setupLog.Info("Successfully loaded config file", "config", cfgStr)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
@@ -94,8 +112,11 @@ func main() {
 	if failedCtrl, err := core.SetupControllers(mgr, queues, cCache); err != nil {
 		setupLog.Error(err, "Unable to create controller", "controller", failedCtrl)
 	}
-	if err = job.NewReconciler(mgr.GetScheme(), mgr.GetClient(),
-		mgr.GetEventRecorderFor(constants.JobControllerName)).SetupWithManager(mgr); err != nil {
+	if err = job.NewReconciler(mgr.GetScheme(),
+		mgr.GetClient(),
+		mgr.GetEventRecorderFor(constants.JobControllerName),
+		job.WithManageJobsWithoutQueueName(config.ManageJobsWithoutQueueName),
+	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Job")
 		os.Exit(1)
 	}
@@ -125,3 +146,19 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func encodeConfig(cfg *configv1alpha1.Configuration) (string, error) {
+	codecs := serializer.NewCodecFactory(scheme)
+	const mediaType = runtime.ContentTypeYAML
+	info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return "", fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType)
+	}
+
+	encoder := codecs.EncoderForVersion(info.Serializer, configv1alpha1.GroupVersion)
+	buf := new(bytes.Buffer)
+	if err := encoder.Encode(cfg, buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}

pkg/controller/workload/job/job_controller.go

@@ -46,24 +46,24 @@ var (
 
 // JobReconciler reconciles a Job object
 type JobReconciler struct {
-	client                      client.Client
-	scheme                      *runtime.Scheme
-	record                      record.EventRecorder
-	processJobsWithoutQueueName bool
+	client                     client.Client
+	scheme                     *runtime.Scheme
+	record                     record.EventRecorder
+	manageJobsWithoutQueueName bool
 }
 
 type options struct {
-	processJobsWithoutQueueName bool
+	manageJobsWithoutQueueName bool
 }
 
 // Option configures the reconciler.
 type Option func(*options)
 
-// WithProcessJobsWithoutQueueName indicates if the controller should reconcile
+// WithManageJobsWithoutQueueName indicates if the controller should reconcile
 // jobs that don't set the queue name annotation.
-func WithProcessJobsWithoutQueueName(f bool) Option {
+func WithManageJobsWithoutQueueName(f bool) Option {
 	return func(o *options) {
-		o.processJobsWithoutQueueName = f
+		o.manageJobsWithoutQueueName = f
 	}
 }
 
@@ -81,10 +81,10 @@ func NewReconciler(
 	}
 
 	return &JobReconciler{
-		scheme:                      scheme,
-		client:                      client,
-		record:                      record,
-		processJobsWithoutQueueName: options.processJobsWithoutQueueName,
+		scheme:                     scheme,
+		client:                     client,
+		record:                     record,
+		manageJobsWithoutQueueName: options.manageJobsWithoutQueueName,
 	}
 }
 
@@ -133,7 +133,7 @@ func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 
 	log := ctrl.LoggerFrom(ctx).WithValues("job", klog.KObj(&job))
 	ctx = ctrl.LoggerInto(ctx, log)
-	if queueName(&job) == "" && !r.processJobsWithoutQueueName {
+	if queueName(&job) == "" && !r.manageJobsWithoutQueueName {
 		log.V(3).Info(fmt.Sprintf("%s annotation is not set, ignoring the job", constants.QueueAnnotation))
 		return ctrl.Result{}, nil
 	}

test/integration/controller/job/job_controller_test.go

@@ -64,7 +64,7 @@ var (
 var _ = ginkgo.Describe("Job controller", func() {
 	ginkgo.BeforeEach(func() {
 		fwk = &framework.Framework{
-			ManagerSetup: managerSetup(job.WithProcessJobsWithoutQueueName(true)),
+			ManagerSetup: managerSetup(job.WithManageJobsWithoutQueueName(true)),
 			CRDPath:      crdPath,
 		}
 		ctx, cfg, k8sClient = fwk.Setup()