From 941405b693258bfb0d2dea5dda216af627aad099 Mon Sep 17 00:00:00 2001
From: Periklis Tsirakidis
Date: Thu, 18 Jan 2024 16:02:23 +0100
Subject: [PATCH] operator: Replace deprecated ctrl-runtime cfg with custom
package (#11678)
Co-authored-by: Robert Jacob
---
.../apis/config/v1/projectconfig_types.go | 65 +++++++-
.../apis/config/v1/zz_generated.deepcopy.go | 99 ++++++++++++
operator/config/docs/config.json | 26 +--
operator/docs/operator/feature-gates.md | 153 ------------------
operator/go.mod | 2 +-
operator/internal/config/loader.go | 30 ++++
operator/internal/config/options.go | 85 ++++++++++
operator/main.go | 13 +-
8 files changed, 291 insertions(+), 182 deletions(-)
create mode 100644 operator/internal/config/loader.go
create mode 100644 operator/internal/config/options.go
diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go
index 4896cc5eb15a..0943222754ae 100644
--- a/operator/apis/config/v1/projectconfig_types.go
+++ b/operator/apis/config/v1/projectconfig_types.go
@@ -2,7 +2,7 @@ package v1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+ configv1alpha1 "k8s.io/component-base/config/v1alpha1"
)
// BuiltInCertManagement is the configuration for the built-in facility to generate and rotate
@@ -139,6 +139,67 @@ const (
TLSProfileModernType TLSProfileType = "Modern"
)
+// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration.
+type ControllerManagerConfigurationSpec struct {
+ // LeaderElection is the LeaderElection config to be used when configuring
+ // the manager.Manager leader election
+ // +optional
+ LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+
+ // Metrics contains the controller metrics configuration
+ // +optional
+ Metrics ControllerMetrics `json:"metrics,omitempty"`
+
+ // Health contains the controller health configuration
+ // +optional
+ Health ControllerHealth `json:"health,omitempty"`
+
+ // Webhook contains the controllers webhook configuration
+ // +optional
+ Webhook ControllerWebhook `json:"webhook,omitempty"`
+}
+
+// ControllerMetrics defines the metrics configs.
+type ControllerMetrics struct {
+ // BindAddress is the TCP address that the controller should bind to
+ // for serving prometheus metrics.
+ // It can be set to "0" to disable the metrics serving.
+ // +optional
+ BindAddress string `json:"bindAddress,omitempty"`
+}
+
+// ControllerHealth defines the health configs.
+type ControllerHealth struct {
+ // HealthProbeBindAddress is the TCP address that the controller should bind to
+ // for serving health probes
+ // It can be set to "0" or "" to disable serving the health probe.
+ // +optional
+ HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"`
+}
+
+// ControllerWebhook defines the webhook server for the controller.
+type ControllerWebhook struct {
+ // Port is the port that the webhook server serves at.
+ // It is used to set webhook.Server.Port.
+ // +optional
+ Port *int `json:"port,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API.
+type ControllerManagerConfiguration struct {
+ metav1.TypeMeta `json:",inline"`
+
+ // ControllerManagerConfiguration returns the contfigurations for controllers
+ ControllerManagerConfigurationSpec `json:",inline"`
+}
+
+// Complete returns the configuration for controller-runtime.
+func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) {
+ return *c, nil
+}
+
//+kubebuilder:object:root=true
// ProjectConfig is the Schema for the projectconfigs API
@@ -146,7 +207,7 @@ type ProjectConfig struct {
metav1.TypeMeta `json:",inline"`
// ControllerManagerConfigurationSpec returns the contfigurations for controllers
- cfg.ControllerManagerConfigurationSpec `json:",inline"`
+ ControllerManagerConfigurationSpec `json:",inline"`
Gates FeatureGates `json:"featureGates,omitempty"`
}
diff --git a/operator/apis/config/v1/zz_generated.deepcopy.go b/operator/apis/config/v1/zz_generated.deepcopy.go
index c85446c21e0b..f383e41aa37d 100644
--- a/operator/apis/config/v1/zz_generated.deepcopy.go
+++ b/operator/apis/config/v1/zz_generated.deepcopy.go
@@ -7,6 +7,7 @@ package v1
import (
runtime "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/component-base/config/v1alpha1"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -24,6 +25,104 @@ func (in *BuiltInCertManagement) DeepCopy() *BuiltInCertManagement {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth.
+func (in *ControllerHealth) DeepCopy() *ControllerHealth {
+ if in == nil {
+ return nil
+ }
+ out := new(ControllerHealth)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerManagerConfiguration) DeepCopyInto(out *ControllerManagerConfiguration) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfiguration.
+func (in *ControllerManagerConfiguration) DeepCopy() *ControllerManagerConfiguration {
+ if in == nil {
+ return nil
+ }
+ out := new(ControllerManagerConfiguration)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ControllerManagerConfiguration) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerManagerConfigurationSpec) DeepCopyInto(out *ControllerManagerConfigurationSpec) {
+ *out = *in
+ if in.LeaderElection != nil {
+ in, out := &in.LeaderElection, &out.LeaderElection
+ *out = new(v1alpha1.LeaderElectionConfiguration)
+ (*in).DeepCopyInto(*out)
+ }
+ out.Metrics = in.Metrics
+ out.Health = in.Health
+ in.Webhook.DeepCopyInto(&out.Webhook)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfigurationSpec.
+func (in *ControllerManagerConfigurationSpec) DeepCopy() *ControllerManagerConfigurationSpec {
+ if in == nil {
+ return nil
+ }
+ out := new(ControllerManagerConfigurationSpec)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics.
+func (in *ControllerMetrics) DeepCopy() *ControllerMetrics {
+ if in == nil {
+ return nil
+ }
+ out := new(ControllerMetrics)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) {
+ *out = *in
+ if in.Port != nil {
+ in, out := &in.Port, &out.Port
+ *out = new(int)
+ **out = **in
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook.
+func (in *ControllerWebhook) DeepCopy() *ControllerWebhook {
+ if in == nil {
+ return nil
+ }
+ out := new(ControllerWebhook)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *FeatureGates) DeepCopyInto(out *FeatureGates) {
*out = *in
diff --git a/operator/config/docs/config.json b/operator/config/docs/config.json
index 3d912e857d2b..fb1b7d8a11b2 100644
--- a/operator/config/docs/config.json
+++ b/operator/config/docs/config.json
@@ -4,7 +4,13 @@
],
"hideTypePatterns": [
"ParseError$",
- "List$"
+ "List$",
+ "ControllerHealth$",
+ "ControllerManagerConfiguration$",
+ "ControllerManagerConfigurationSpec$",
+ "ControllerMetrics$",
+ "ControllerWebhook$",
+ "ProjectConfig$"
],
"externalPackages": [
{
@@ -38,22 +44,6 @@
{
"typeMatchPrefix": "^k8s\\.io/component-base/config/v1alpha1\\.LeaderElectionConfiguration$",
"docsURLTemplate": "https://pkg.go.dev/k8s.io/component-base/config#LeaderElectionConfiguration"
- },
- {
- "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerConfigurationSpec$",
- "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerConfigurationSpec"
- },
- {
- "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerMetrics$",
- "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerMetrics"
- },
- {
- "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerHealth$",
- "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerHealth"
- },
- {
- "typeMatchPrefix": "^sigs\\.k8s\\.io/controller-runtime/pkg/config/v1alpha1\\.ControllerWebhook$",
- "docsURLTemplate": "https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/config/v1alpha1#ControllerWebhook"
}
],
"typeDisplayNamePrefixOverrides": {
@@ -66,4 +56,4 @@
"github.com/grafana/loki/operator/apis/loki/config/v1": "Feature Gates"
},
"markdownDisabled": false
-}
\ No newline at end of file
+}
diff --git a/operator/docs/operator/feature-gates.md b/operator/docs/operator/feature-gates.md
index 2173bb82be27..71ef65798204 100644
--- a/operator/docs/operator/feature-gates.md
+++ b/operator/docs/operator/feature-gates.md
@@ -99,9 +99,6 @@ The refresh is applied to all LokiStack certificates at once.
## FeatureGates { #config-loki-grafana-com-v1-FeatureGates }
-
-(Appears on:ProjectConfig)
-
FeatureGates is the supported set of all operator feature gates.
@@ -404,156 +401,6 @@ More details:
-Kubernetes meta/v1.Duration
-
-
-
-
-(Optional)
- SyncPeriod determines the minimum frequency at which watched resources are
-reconciled. A lower period will correct entropy more quickly, but reduce
-responsiveness to change if there are many watched resources. Change this
-value only if you know what you are doing. Defaults to 10 hours if unset.
-there will a 10 percent jitter between the SyncPeriod of all controllers
-so that all controllers will not send list requests simultaneously.
- |
-
-
-
-leaderElection
-
-
-Kubernetes v1alpha1.LeaderElectionConfiguration
-
-
- |
-
-(Optional)
- LeaderElection is the LeaderElection config to be used when configuring
-the manager.Manager leader election
- |
-
-
-
-cacheNamespace
-
-string
-
- |
-
-(Optional)
- CacheNamespace if specified restricts the manager’s cache to watch objects in
-the desired namespace Defaults to all namespaces
-Note: If a namespace is specified, controllers can still Watch for a
-cluster-scoped resource (e.g Node). For namespaced resources the cache
-will only hold objects from the desired namespace.
- |
-
-
-
-gracefulShutDown
-
-
-Kubernetes meta/v1.Duration
-
-
- |
-
- GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop.
-To disable graceful shutdown, set to time.Duration(0)
-To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1)
-The graceful shutdown is skipped for safety reasons in case the leader election lease is lost.
- |
-
-
-
-controller
-
-
-K8S Controller-runtime v1alpha1.ControllerConfigurationSpec
-
-
- |
-
-(Optional)
- Controller contains global configuration options for controllers
-registered within this manager.
- |
-
-
-
-metrics
-
-
-K8S Controller-runtime v1alpha1.ControllerMetrics
-
-
- |
-
-(Optional)
- Metrics contains thw controller metrics configuration
- |
-
-
-
-health
-
-
-K8S Controller-runtime v1alpha1.ControllerHealth
-
-
- |
-
-(Optional)
- Health contains the controller health configuration
- |
-
-
-
-webhook
-
-
-K8S Controller-runtime v1alpha1.ControllerWebhook
-
-
- |
-
-(Optional)
- Webhook contains the controllers webhook configuration
- |
-
-
-
-featureGates
-
-
-FeatureGates
-
-
- |
-
- |
-
-
-
-
## TLSProfileType { #config-loki-grafana-com-v1-TLSProfileType }
(string
alias)
diff --git a/operator/go.mod b/operator/go.mod
index 411d8ee512d6..f4d0e67f2540 100644
--- a/operator/go.mod
+++ b/operator/go.mod
@@ -23,6 +23,7 @@ require (
k8s.io/apimachinery v0.26.10
k8s.io/apiserver v0.26.10
k8s.io/client-go v0.26.10
+ k8s.io/component-base v0.26.10
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
sigs.k8s.io/controller-runtime v0.14.7
sigs.k8s.io/yaml v1.3.0
@@ -150,7 +151,6 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.26.10 // indirect
- k8s.io/component-base v0.26.10 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
diff --git a/operator/internal/config/loader.go b/operator/internal/config/loader.go
new file mode 100644
index 000000000000..b5af090ddb88
--- /dev/null
+++ b/operator/internal/config/loader.go
@@ -0,0 +1,30 @@
+package config
+
+import (
+ "errors"
+ "fmt"
+ "os"
+
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/serializer"
+
+ configv1 "github.com/grafana/loki/operator/apis/config/v1"
+)
+
+var errConfigFileLoading = errors.New("could not read file at path")
+
+func loadConfigFile(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, error) {
+ content, err := os.ReadFile(configFile)
+ if err != nil {
+ return nil, fmt.Errorf("%w %s", errConfigFileLoading, configFile)
+ }
+
+ codecs := serializer.NewCodecFactory(scheme)
+
+ outConfig := &configv1.ProjectConfig{}
+ if err = runtime.DecodeInto(codecs.UniversalDecoder(), content, outConfig); err != nil {
+ return nil, fmt.Errorf("could not decode file into runtime.Object: %w", err)
+ }
+
+ return outConfig, nil
+}
diff --git a/operator/internal/config/options.go b/operator/internal/config/options.go
new file mode 100644
index 000000000000..3094e72ebab1
--- /dev/null
+++ b/operator/internal/config/options.go
@@ -0,0 +1,85 @@
+package config
+
+import (
+ "fmt"
+ "reflect"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+
+ configv1 "github.com/grafana/loki/operator/apis/config/v1"
+)
+
+// LoadConfig initializes the controller configuration, optionally overriding the defaults
+// from a provided configuration file.
+func LoadConfig(scheme *runtime.Scheme, configFile string) (*configv1.ProjectConfig, ctrl.Options, error) {
+ options := ctrl.Options{Scheme: scheme}
+ if configFile == "" {
+ return &configv1.ProjectConfig{}, options, nil
+ }
+
+ ctrlCfg, err := loadConfigFile(scheme, configFile)
+ if err != nil {
+ return nil, options, fmt.Errorf("failed to parse controller manager config file: %w", err)
+ }
+
+ options = mergeOptionsFromFile(options, ctrlCfg)
+ return ctrlCfg, options, nil
+}
+
+func mergeOptionsFromFile(o manager.Options, cfg *configv1.ProjectConfig) manager.Options {
+ o = setLeaderElectionConfig(o, cfg.ControllerManagerConfigurationSpec)
+
+ if o.MetricsBindAddress == "" && cfg.Metrics.BindAddress != "" {
+ o.MetricsBindAddress = cfg.Metrics.BindAddress
+ }
+
+ if o.HealthProbeBindAddress == "" && cfg.Health.HealthProbeBindAddress != "" {
+ o.HealthProbeBindAddress = cfg.Health.HealthProbeBindAddress
+ }
+
+ if o.Port == 0 && cfg.Webhook.Port != nil {
+ o.Port = *cfg.Webhook.Port
+ }
+
+ return o
+}
+
+func setLeaderElectionConfig(o manager.Options, obj configv1.ControllerManagerConfigurationSpec) manager.Options {
+ if obj.LeaderElection == nil {
+ // The source does not have any configuration; noop
+ return o
+ }
+
+ if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil {
+ o.LeaderElection = *obj.LeaderElection.LeaderElect
+ }
+
+ if o.LeaderElectionResourceLock == "" && obj.LeaderElection.ResourceLock != "" {
+ o.LeaderElectionResourceLock = obj.LeaderElection.ResourceLock
+ }
+
+ if o.LeaderElectionNamespace == "" && obj.LeaderElection.ResourceNamespace != "" {
+ o.LeaderElectionNamespace = obj.LeaderElection.ResourceNamespace
+ }
+
+ if o.LeaderElectionID == "" && obj.LeaderElection.ResourceName != "" {
+ o.LeaderElectionID = obj.LeaderElection.ResourceName
+ }
+
+ if o.LeaseDuration == nil && !reflect.DeepEqual(obj.LeaderElection.LeaseDuration, metav1.Duration{}) {
+ o.LeaseDuration = &obj.LeaderElection.LeaseDuration.Duration
+ }
+
+ if o.RenewDeadline == nil && !reflect.DeepEqual(obj.LeaderElection.RenewDeadline, metav1.Duration{}) {
+ o.RenewDeadline = &obj.LeaderElection.RenewDeadline.Duration
+ }
+
+ if o.RetryPeriod == nil && !reflect.DeepEqual(obj.LeaderElection.RetryPeriod, metav1.Duration{}) {
+ o.RetryPeriod = &obj.LeaderElection.RetryPeriod.Duration
+ }
+
+ return o
+}
diff --git a/operator/main.go b/operator/main.go
index 0b1304ad8684..db332562af8b 100644
--- a/operator/main.go
+++ b/operator/main.go
@@ -21,6 +21,7 @@ import (
lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
lokiv1beta1 "github.com/grafana/loki/operator/apis/loki/v1beta1"
lokictrl "github.com/grafana/loki/operator/controllers/loki"
+ "github.com/grafana/loki/operator/internal/config"
"github.com/grafana/loki/operator/internal/metrics"
"github.com/grafana/loki/operator/internal/validation"
"github.com/grafana/loki/operator/internal/validation/openshift"
@@ -58,14 +59,10 @@ func main() {
var err error
- ctrlCfg := ctrlconfigv1.ProjectConfig{}
- options := ctrl.Options{Scheme: scheme}
- if configFile != "" {
- options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&ctrlCfg))
- if err != nil {
- logger.Error(err, "failed to parse controller manager config file")
- os.Exit(1)
- }
+ ctrlCfg, options, err := config.LoadConfig(scheme, configFile)
+ if err != nil {
+ logger.Error(err, "failed to load operator configuration")
+ os.Exit(1)
}
if ctrlCfg.Gates.LokiStackAlerts && !ctrlCfg.Gates.ServiceMonitors {