From 0fceab521ad57f73ea92a832c079adf69dbb4f3a Mon Sep 17 00:00:00 2001 From: Lan Liang Date: Tue, 27 Aug 2024 12:37:21 +0000 Subject: [PATCH] Standardize the health probe and metrics arguments of scheduler. Signed-off-by: Lan Liang --- artifacts/deploy/karmada-scheduler.yaml | 3 +- cmd/scheduler/app/options/options.go | 20 ++++++ cmd/scheduler/app/options/validation.go | 21 +++--- cmd/scheduler/app/options/validation_test.go | 14 ---- cmd/scheduler/app/scheduler.go | 66 +++++++++++++++---- operator/pkg/controlplane/manifests.go | 4 +- .../cmdinit/kubernetes/deployments.go | 4 +- 7 files changed, 91 insertions(+), 41 deletions(-) diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index f863fba55a3a..5affd87b1172 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -36,7 +36,8 @@ spec: - /bin/karmada-scheduler - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - - --secure-port=10351 + - --metrics-bind-address=0.0.0.0:10351 + - --health-probe-bind-address=0.0.0.0:10351 - --enable-scheduler-estimator=true - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index ac994775253c..1923dd346bb2 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -52,10 +52,24 @@ type Options struct { KubeConfig string Master string // BindAddress is the IP address on which to listen for the --secure-port port. + // Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+. BindAddress string // SecurePort is the port that the server serves at. + // Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+. SecurePort int + // MetricsBindAddress is the TCP address that the controller should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // Defaults to ":10351". + MetricsBindAddress string + + // HealthProbeBindAddress is the TCP address that the controller should bind to + // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. + // Defaults to ":10351". + HealthProbeBindAddress string + // KubeAPIQPS is the QPS to use while talking with karmada-apiserver. KubeAPIQPS float32 // KubeAPIBurst is the burst to allow while talking with karmada-apiserver. @@ -139,6 +153,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server. Overrides any value in KubeConfig. Only required if out-of-cluster.") fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") fs.IntVar(&o.SecurePort, "secure-port", defaultPort, "The secure port on which to serve HTTPS.") + // nolint: errcheck + fs.MarkDeprecated("bind-address", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.") + // nolint: errcheck + fs.MarkDeprecated("secure-port", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.") + fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", "", "The TCP address that the server should bind to for serving prometheus metrics(e.g. 127.0.0.1:9000, :10351). It can be set to \"0\" to disable the metrics serving. Defaults to 0.0.0.0:10351.") + fs.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", "", "The TCP address that the server should bind to for serving health probes(e.g. 127.0.0.1:9000, :10351). It can be set to \"0\" to disable serving the health probe. Defaults to 0.0.0.0:10351.") fs.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver.") fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.") fs.BoolVar(&o.EnableSchedulerEstimator, "enable-scheduler-estimator", false, "Enable calling cluster scheduler estimator for adjusting replicas.") diff --git a/cmd/scheduler/app/options/validation.go b/cmd/scheduler/app/options/validation.go index 7612ff2e83ce..793ac542eb99 100644 --- a/cmd/scheduler/app/options/validation.go +++ b/cmd/scheduler/app/options/validation.go @@ -18,27 +18,30 @@ package options import ( "net" + "strconv" "k8s.io/apimachinery/pkg/util/validation/field" ) +// Complete ensures that options are valid and marshals them if necessary. +func (o *Options) Complete() error { + if len(o.HealthProbeBindAddress) == 0 { + o.HealthProbeBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort)) + } + if len(o.MetricsBindAddress) == 0 { + o.MetricsBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort)) + } + return nil +} + // Validate checks Options and return a slice of found errs. func (o *Options) Validate() field.ErrorList { errs := field.ErrorList{} newPath := field.NewPath("Options") - if net.ParseIP(o.BindAddress) == nil { - errs = append(errs, field.Invalid(newPath.Child("BindAddress"), o.BindAddress, "not a valid textual representation of an IP address")) - } - - if o.SecurePort < 0 || o.SecurePort > 65535 { - errs = append(errs, field.Invalid(newPath.Child("SecurePort"), o.SecurePort, "must be a valid port between 0 and 65535 inclusive")) - } - if o.SchedulerEstimatorPort < 0 || o.SchedulerEstimatorPort > 65535 { errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorPort"), o.SchedulerEstimatorPort, "must be a valid port between 0 and 65535 inclusive")) } - if o.SchedulerEstimatorTimeout.Duration < 0 { errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), o.SchedulerEstimatorTimeout, "must be greater than or equal to 0")) } diff --git a/cmd/scheduler/app/options/validation_test.go b/cmd/scheduler/app/options/validation_test.go index 42940a6e6a03..ee451959dc55 100644 --- a/cmd/scheduler/app/options/validation_test.go +++ b/cmd/scheduler/app/options/validation_test.go @@ -62,8 +62,6 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) { LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, }, - BindAddress: "127.0.0.1", - SecurePort: 9000, KubeAPIQPS: 40, KubeAPIBurst: 30, SchedulerName: "default-scheduler", @@ -81,18 +79,6 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) { opt Options expectedErrs field.ErrorList }{ - "invalid BindAddress": { - opt: New(func(option *Options) { - option.BindAddress = "127.0.0.1:8080" - }), - expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")}, - }, - "invalid SecurePort": { - opt: New(func(option *Options) { - option.SecurePort = 90000 - }), - expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, - }, "invalid SchedulerEstimatorPort": { opt: New(func(option *Options) { option.SchedulerEstimatorPort = 90000 diff --git a/cmd/scheduler/app/scheduler.go b/cmd/scheduler/app/scheduler.go index 9d4bad259136..c7d8899294a5 100644 --- a/cmd/scheduler/app/scheduler.go +++ b/cmd/scheduler/app/scheduler.go @@ -19,10 +19,8 @@ package app import ( "context" "fmt" - "net" "net/http" "os" - "strconv" "time" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -100,6 +98,10 @@ The scheduler determines which clusters are valid placements for each resource i constraints and available resources. The scheduler then ranks each valid cluster and binds the resource to the most suitable cluster.`, RunE: func(_ *cobra.Command, _ []string) error { + // complete options + if err := opts.Complete(); err != nil { + return err + } // validate options if errs := opts.Validate(); len(errs) != 0 { return errs.ToAggregate() @@ -139,7 +141,7 @@ the most suitable cluster.`, func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Option) error { klog.Infof("karmada-scheduler version: %s", version.Get()) - go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort))) + serveHealthzAndMetrics(opts.HealthProbeBindAddress, opts.MetricsBindAddress) profileflag.ListenAndServe(opts.ProfileOpts) @@ -225,26 +227,64 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt return nil } -func serveHealthzAndMetrics(address string) { +func serveHealthzAndMetrics(healthProbeBindAddress, metricsBindAddress string) { + if healthProbeBindAddress == metricsBindAddress { + if healthProbeBindAddress != "0" { + go serveCombined(healthProbeBindAddress) + } + } else { + if healthProbeBindAddress != "0" { + go serveHealthz(healthProbeBindAddress) + } + if metricsBindAddress != "0" { + go serveMetrics(metricsBindAddress) + } + } +} + +func serveCombined(address string) { mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) - }) + mux.HandleFunc("/healthz", healthzHandler) + mux.Handle("/metrics", metricsHandler()) + + serveHTTP(address, mux, "healthz and metrics") +} - mux.Handle("/metrics", promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{ +func serveHealthz(address string) { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", healthzHandler) + serveHTTP(address, mux, "healthz") +} + +func serveMetrics(address string) { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler()) + serveHTTP(address, mux, "metrics") +} + +func healthzHandler(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) +} + +func metricsHandler() http.Handler { + return promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{ ErrorHandling: promhttp.HTTPErrorOnError, - })) + }) +} - httpServer := http.Server{ +func serveHTTP(address string, handler http.Handler, name string) { + httpServer := &http.Server{ Addr: address, - Handler: mux, + Handler: handler, ReadHeaderTimeout: ReadHeaderTimeout, WriteTimeout: WriteTimeout, ReadTimeout: ReadTimeout, } + + klog.Infof("Starting %s server on %s", name, address) if err := httpServer.ListenAndServe(); err != nil { - klog.Errorf("Failed to serve healthz and metrics: %v", err) + klog.Errorf("Failed to serve %s on %s: %v", name, address, err) os.Exit(1) } } diff --git a/operator/pkg/controlplane/manifests.go b/operator/pkg/controlplane/manifests.go index fc67c4d2fa96..32f202ea6c33 100644 --- a/operator/pkg/controlplane/manifests.go +++ b/operator/pkg/controlplane/manifests.go @@ -184,8 +184,8 @@ spec: command: - /bin/karmada-scheduler - --kubeconfig=/etc/karmada/kubeconfig - - --bind-address=0.0.0.0 - - --secure-port=10351 + - --metrics-bind-address=0.0.0.0:10351 + - --health-probe-bind-address=0.0.0.0:10351 - --enable-scheduler-estimator=true - --leader-elect-resource-namespace={{ .SystemNamespace }} - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 3c112b5c28b9..2541288073c3 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -452,8 +452,8 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment Command: []string{ "/bin/karmada-scheduler", "--kubeconfig=/etc/kubeconfig", - "--bind-address=0.0.0.0", - "--secure-port=10351", + "--metrics-bind-address=0.0.0.0:10351", + "--health-probe-bind-address=0.0.0.0:10351", "--enable-scheduler-estimator=true", "--leader-elect=true", "--scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt",