From 28a413539e4d14311a2d5d58e7e145e0e1e56ac6 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Thu, 18 Jan 2024 16:02:23 +0100 Subject: [PATCH] operator: Replace deprecated ctrl-runtime cfg with custom package (#11678) Co-authored-by: Robert Jacob Co-authored-by: Robert Jacob --- .../apis/config/v1/projectconfig_types.go | 65 +++++++- .../apis/config/v1/zz_generated.deepcopy.go | 99 ++++++++++++ operator/config/docs/config.json | 26 +-- operator/docs/operator/feature-gates.md | 153 ------------------ operator/go.mod | 2 +- operator/internal/config/loader.go | 30 ++++ operator/internal/config/options.go | 94 +++++++++++ operator/main.go | 13 +- 8 files changed, 300 insertions(+), 182 deletions(-) create mode 100644 operator/internal/config/loader.go create mode 100644 operator/internal/config/options.go diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go index b6a80175266b..488f7b2cb64f 100644 --- a/operator/apis/config/v1/projectconfig_types.go +++ b/operator/apis/config/v1/projectconfig_types.go @@ -2,7 +2,7 @@ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + configv1alpha1 "k8s.io/component-base/config/v1alpha1" ) // BuiltInCertManagement is the configuration for the built-in facility to generate and rotate @@ -142,6 +142,67 @@ const ( TLSProfileModernType TLSProfileType = "Modern" ) +// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration. +type ControllerManagerConfigurationSpec struct { + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + // +optional + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + + // Metrics contains the controller metrics configuration + // +optional + Metrics ControllerMetrics `json:"metrics,omitempty"` + + // Health contains the controller health configuration + // +optional + Health ControllerHealth `json:"health,omitempty"` + + // Webhook contains the controllers webhook configuration + // +optional + Webhook ControllerWebhook `json:"webhook,omitempty"` +} + +// ControllerMetrics defines the metrics configs. +type ControllerMetrics struct { + // BindAddress is the TCP address that the controller should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // +optional + BindAddress string `json:"bindAddress,omitempty"` +} + +// ControllerHealth defines the health configs. +type ControllerHealth struct { + // HealthProbeBindAddress is the TCP address that the controller should bind to + // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. + // +optional + HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` +} + +// ControllerWebhook defines the webhook server for the controller. +type ControllerWebhook struct { + // Port is the port that the webhook server serves at. + // It is used to set webhook.Server.Port. + // +optional + Port *int `json:"port,omitempty"` +} + +//+kubebuilder:object:root=true + +// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API. +type ControllerManagerConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // ControllerManagerConfiguration returns the contfigurations for controllers + ControllerManagerConfigurationSpec `json:",inline"` +} + +// Complete returns the configuration for controller-runtime. +func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) { + return *c, nil +} + //+kubebuilder:object:root=true // ProjectConfig is the Schema for the projectconfigs API @@ -149,7 +210,7 @@ type ProjectConfig struct { metav1.TypeMeta `json:",inline"` // ControllerManagerConfigurationSpec returns the contfigurations for controllers - cfg.ControllerManagerConfigurationSpec `json:",inline"` + ControllerManagerConfigurationSpec `json:",inline"` Gates FeatureGates `json:"featureGates,omitempty"` } diff --git a/operator/apis/config/v1/zz_generated.deepcopy.go b/operator/apis/config/v1/zz_generated.deepcopy.go index ef20274e286e..f047818445aa 100644 --- a/operator/apis/config/v1/zz_generated.deepcopy.go +++ b/operator/apis/config/v1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1 import ( runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/config/v1alpha1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -23,6 +24,104 @@ func (in *BuiltInCertManagement) DeepCopy() *BuiltInCertManagement { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. +func (in *ControllerHealth) DeepCopy() *ControllerHealth { + if in == nil { + return nil + } + out := new(ControllerHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerManagerConfiguration) DeepCopyInto(out *ControllerManagerConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfiguration. +func (in *ControllerManagerConfiguration) DeepCopy() *ControllerManagerConfiguration { + if in == nil { + return nil + } + out := new(ControllerManagerConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ControllerManagerConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerManagerConfigurationSpec) DeepCopyInto(out *ControllerManagerConfigurationSpec) { + *out = *in + if in.LeaderElection != nil { + in, out := &in.LeaderElection, &out.LeaderElection + *out = new(v1alpha1.LeaderElectionConfiguration) + (*in).DeepCopyInto(*out) + } + out.Metrics = in.Metrics + out.Health = in.Health + in.Webhook.DeepCopyInto(&out.Webhook) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfigurationSpec. +func (in *ControllerManagerConfigurationSpec) DeepCopy() *ControllerManagerConfigurationSpec { + if in == nil { + return nil + } + out := new(ControllerManagerConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. +func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { + if in == nil { + return nil + } + out := new(ControllerMetrics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. +func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { + if in == nil { + return nil + } + out := new(ControllerWebhook) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureGates) DeepCopyInto(out *FeatureGates) { *out = *in diff --git a/operator/config/docs/config.json b/operator/config/docs/config.json index 3d912e857d2b..fb1b7d8a11b2 100644 --- a/operator/config/docs/config.json +++ b/operator/config/docs/config.json @@ -4,7 +4,13 @@ ], "hideTypePatterns": [ "ParseError$", - "List$" + "List$", + "ControllerHealth$", + "ControllerManagerConfiguration$", + "ControllerManagerConfigurationSpec$", + "ControllerMetrics$", + "ControllerWebhook$", + "ProjectConfig$" ], "externalPackages": [ { @@ -38,22 +44,6 @@ { "typeMatchPrefix": "^k8s\\.io/component-base/config/v1alpha1\\.LeaderElectionConfiguration$", "docsURLTemplate": "https://pkg.go.dev/k8s.io/component-base/config#LeaderElectionConfiguration" - }, - { - "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerConfigurationSpec$", - "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerConfigurationSpec" - }, - { - "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerMetrics$", - "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerMetrics" - }, - { - "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerHealth$", - "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerHealth" - }, - { - "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerWebhook$", - "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerWebhook" } ], "typeDisplayNamePrefixOverrides": { @@ -66,4 +56,4 @@ "github.com/grafana/loki/operator/apis/loki/config/v1": "Feature Gates" }, "markdownDisabled": false -} \ No newline at end of file +} diff --git a/operator/docs/operator/feature-gates.md b/operator/docs/operator/feature-gates.md index 7475dfa8a1cc..1d5c046be775 100644 --- a/operator/docs/operator/feature-gates.md +++ b/operator/docs/operator/feature-gates.md @@ -99,9 +99,6 @@ The refresh is applied to all LokiStack certificates at once.

## FeatureGates { #config-loki-grafana-com-v1-FeatureGates } -

-(Appears on:ProjectConfig) -

FeatureGates is the supported set of all operator feature gates.

@@ -415,156 +412,6 @@ bool -## ProjectConfig { #config-loki-grafana-com-v1-ProjectConfig } -
-

ProjectConfig is the Schema for the projectconfigs API

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-syncPeriod
- - -Kubernetes meta/v1.Duration - - -		 -(Optional) -

SyncPeriod determines the minimum frequency at which watched resources are -reconciled. A lower period will correct entropy more quickly, but reduce -responsiveness to change if there are many watched resources. Change this -value only if you know what you are doing. Defaults to 10 hours if unset. -there will a 10 percent jitter between the SyncPeriod of all controllers -so that all controllers will not send list requests simultaneously.

-
-leaderElection
- - -Kubernetes v1alpha1.LeaderElectionConfiguration - - -		 -(Optional) -

LeaderElection is the LeaderElection config to be used when configuring -the manager.Manager leader election

-
-cacheNamespace
- -string - -		 -(Optional) -

CacheNamespace if specified restricts the manager’s cache to watch objects in -the desired namespace Defaults to all namespaces

-

Note: If a namespace is specified, controllers can still Watch for a -cluster-scoped resource (e.g Node). For namespaced resources the cache -will only hold objects from the desired namespace.

-
-gracefulShutDown
- - -Kubernetes meta/v1.Duration - - -		 -

GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop. -To disable graceful shutdown, set to time.Duration(0) -To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1) -The graceful shutdown is skipped for safety reasons in case the leader election lease is lost.

-
-controller
- - -K8S Controller-runtime v1alpha1.ControllerConfigurationSpec - - -		 -(Optional) -

Controller contains global configuration options for controllers -registered within this manager.

-
-metrics
- - -K8S Controller-runtime v1alpha1.ControllerMetrics - - -		 -(Optional) -

Metrics contains the controller metrics configuration

-
-health
- - -K8S Controller-runtime v1alpha1.ControllerHealth - - -		 -(Optional) -

Health contains the controller health configuration

-
-webhook
- - -K8S Controller-runtime v1alpha1.ControllerWebhook - - -		 -(Optional) -

Webhook contains the controllers webhook configuration

-
-featureGates
- - -FeatureGates - - -		 -
- ## TLSProfileType { #config-loki-grafana-com-v1-TLSProfileType } (string alias)
diff --git a/operator/go.mod b/operator/go.mod index 4ffc3899d11c..0ee7c037f16c 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -23,6 +23,7 @@ require ( k8s.io/apimachinery v0.27.7 k8s.io/apiserver v0.27.7 k8s.io/client-go v0.27.7 + k8s.io/component-base v0.27.7 k8s.io/utils v0.0.0-20230505201702-9f6742963106 sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.3.0 @@ -150,7 +151,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect - k8s.io/component-base v0.27.7 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/operator/internal/config/loader.go b/operator/internal/config/loader.go new file mode 100644 index 000000000000..b5af090ddb88 --- /dev/null +++ b/operator/internal/config/loader.go @@ -0,0 +1,30 @@ +package config + +import ( + "errors" + "fmt" + "os" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" +) + +var errConfigFileLoading = errors.New("could not read file at path") + +func loadConfigFile(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, error) { + content, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("%w %s", errConfigFileLoading, configFile) + } + + codecs := serializer.NewCodecFactory(scheme) + + outConfig := &configv1.ProjectConfig{} + if err = runtime.DecodeInto(codecs.UniversalDecoder(), content, outConfig); err != nil { + return nil, fmt.Errorf("could not decode file into runtime.Object: %w", err) + } + + return outConfig, nil +} diff --git a/operator/internal/config/options.go b/operator/internal/config/options.go new file mode 100644 index 000000000000..8aeba0eb53b3 --- /dev/null +++ b/operator/internal/config/options.go @@ -0,0 +1,94 @@ +package config + +import ( + "fmt" + "reflect" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" +) + +// LoadConfig initializes the controller configuration, optionally overriding the defaults +// from a provided configuration file. +func LoadConfig(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, ctrl.Options, error) { + options := ctrl.Options{Scheme: scheme} + if configFile == "" { + return &configv1.ProjectConfig{}, options, nil + } + + ctrlCfg, err := loadConfigFile(scheme, configFile) + if err != nil { + return nil, options, fmt.Errorf("failed to parse controller manager config file: %w", err) + } + + options = mergeOptionsFromFile(options, ctrlCfg) + return ctrlCfg, options, nil +} + +func mergeOptionsFromFile(o manager.Options, cfg *configv1.ProjectConfig) manager.Options { + o = setLeaderElectionConfig(o, cfg.ControllerManagerConfigurationSpec) + + if o.MetricsBindAddress == "" && cfg.Metrics.BindAddress != "" { + o.MetricsBindAddress = cfg.Metrics.BindAddress + } + + if o.HealthProbeBindAddress == "" && cfg.Health.HealthProbeBindAddress != "" { + o.HealthProbeBindAddress = cfg.Health.HealthProbeBindAddress + } + + //nolint:staticcheck + if o.Port == 0 && cfg.Webhook.Port != nil { + o.Port = *cfg.Webhook.Port + } + + //nolint:staticcheck + if o.WebhookServer == nil { + o.WebhookServer = webhook.NewServer(webhook.Options{ + Port: o.Port, + }) + } + + return o +} + +func setLeaderElectionConfig(o manager.Options, obj configv1.ControllerManagerConfigurationSpec) manager.Options { + if obj.LeaderElection == nil { + // The source does not have any configuration; noop + return o + } + + if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil { + o.LeaderElection = *obj.LeaderElection.LeaderElect + } + + if o.LeaderElectionResourceLock == "" && obj.LeaderElection.ResourceLock != "" { + o.LeaderElectionResourceLock = obj.LeaderElection.ResourceLock + } + + if o.LeaderElectionNamespace == "" && obj.LeaderElection.ResourceNamespace != "" { + o.LeaderElectionNamespace = obj.LeaderElection.ResourceNamespace + } + + if o.LeaderElectionID == "" && obj.LeaderElection.ResourceName != "" { + o.LeaderElectionID = obj.LeaderElection.ResourceName + } + + if o.LeaseDuration == nil && !reflect.DeepEqual(obj.LeaderElection.LeaseDuration, metav1.Duration{}) { + o.LeaseDuration = &obj.LeaderElection.LeaseDuration.Duration + } + + if o.RenewDeadline == nil && !reflect.DeepEqual(obj.LeaderElection.RenewDeadline, metav1.Duration{}) { + o.RenewDeadline = &obj.LeaderElection.RenewDeadline.Duration + } + + if o.RetryPeriod == nil && !reflect.DeepEqual(obj.LeaderElection.RetryPeriod, metav1.Duration{}) { + o.RetryPeriod = &obj.LeaderElection.RetryPeriod.Duration + } + + return o +} diff --git a/operator/main.go b/operator/main.go index 6b101175407e..ffa16608707c 100644 --- a/operator/main.go +++ b/operator/main.go @@ -21,6 +21,7 @@ import ( lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" lokiv1beta1 "github.com/grafana/loki/operator/apis/loki/v1beta1" lokictrl "github.com/grafana/loki/operator/controllers/loki" + "github.com/grafana/loki/operator/internal/config" "github.com/grafana/loki/operator/internal/metrics" "github.com/grafana/loki/operator/internal/operator" "github.com/grafana/loki/operator/internal/validation" @@ -59,14 +60,10 @@ func main() { var err error - ctrlCfg := ctrlconfigv1.ProjectConfig{} - options := ctrl.Options{Scheme: scheme} - if configFile != "" { - options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&ctrlCfg)) //nolint:staticcheck - if err != nil { - logger.Error(err, "failed to parse controller manager config file") - os.Exit(1) - } + ctrlCfg, options, err := config.LoadConfig(scheme, configFile) + if err != nil { + logger.Error(err, "failed to load operator configuration") + os.Exit(1) } if ctrlCfg.Gates.LokiStackAlerts && !ctrlCfg.Gates.ServiceMonitors {