From fc6f48e0c887711109cde1dfbbe8fadcdcf3a144 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Thu, 13 Jul 2023 08:36:16 +0200 Subject: [PATCH] Implement secure diagnostics (metrics, pprof, log level changes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- Makefile | 5 + bootstrap/kubeadm/config/manager/manager.yaml | 6 +- bootstrap/kubeadm/config/rbac/role.yaml | 12 ++ bootstrap/kubeadm/main.go | 77 ++++---- config/manager/manager.yaml | 6 +- config/rbac/role.yaml | 12 ++ .../kubeadm/config/manager/manager.yaml | 6 +- controlplane/kubeadm/config/rbac/role.yaml | 12 ++ controlplane/kubeadm/main.go | 78 ++++---- docs/book/src/SUMMARY.md | 1 + .../providers/migrations/v1.5-to-v1.6.md | 2 + docs/book/src/tasks/diagnostics.md | 183 ++++++++++++++++++ go.mod | 19 ++ go.sum | 81 ++++++++ hack/observability/parca/kustomization.yaml | 1 + hack/observability/parca/rbac-pprof.yaml | 22 +++ hack/observability/parca/values.yaml | 7 + hack/observability/prometheus/values.yaml | 7 + hack/tools/internal/tilt-prepare/main.go | 45 +---- main.go | 55 +++--- test/e2e/config/docker.yaml | 38 ++-- test/go.mod | 2 + test/go.sum | 3 + .../docker/config/manager/manager.yaml | 6 +- .../docker/config/rbac/role.yaml | 12 ++ test/infrastructure/docker/main.go | 58 +++--- .../inmemory/config/manager/manager.yaml | 6 +- .../inmemory/config/rbac/role.yaml | 12 ++ test/infrastructure/inmemory/main.go | 37 ++-- util/flags/diagnostics.go | 94 +++++++++ 30 files changed, 693 insertions(+), 212 deletions(-) create mode 100644 docs/book/src/tasks/diagnostics.md create mode 100644 hack/observability/parca/rbac-pprof.yaml create mode 100644 util/flags/diagnostics.go diff --git a/Makefile b/Makefile index d1d3befbe8d5..c5667bc5c586 100644 --- a/Makefile +++ b/Makefile @@ -278,6 +278,7 @@ generate-manifests: $(addprefix generate-manifests-,$(ALL_GENERATE_MODULES)) ## generate-manifests-core: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core $(MAKE) clean-generated-yaml SRC_DIRS="./config/crd/bases" $(CONTROLLER_GEN) \ + paths=./ \ paths=./api/... \ paths=./internal/controllers/... \ paths=./internal/webhooks/... \ @@ -304,6 +305,7 @@ generate-manifests-core: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e. generate-manifests-kubeadm-bootstrap: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. for kubeadm bootstrap $(MAKE) clean-generated-yaml SRC_DIRS="./bootstrap/kubeadm/config/crd/bases" $(CONTROLLER_GEN) \ + paths=./bootstrap/kubeadm \ paths=./bootstrap/kubeadm/api/... \ paths=./bootstrap/kubeadm/internal/controllers/... \ crd:crdVersions=v1 \ @@ -317,6 +319,7 @@ generate-manifests-kubeadm-bootstrap: $(CONTROLLER_GEN) ## Generate manifests e. generate-manifests-kubeadm-control-plane: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. for kubeadm control plane $(MAKE) clean-generated-yaml SRC_DIRS="./controlplane/kubeadm/config/crd/bases" $(CONTROLLER_GEN) \ + paths=./controlplane/kubeadm \ paths=./controlplane/kubeadm/api/... \ paths=./controlplane/kubeadm/internal/controllers/... \ paths=./controlplane/kubeadm/internal/webhooks/... \ @@ -331,6 +334,7 @@ generate-manifests-kubeadm-control-plane: $(CONTROLLER_GEN) ## Generate manifest generate-manifests-docker-infrastructure: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. for docker infrastructure provider $(MAKE) clean-generated-yaml SRC_DIRS="$(CAPD_DIR)/config/crd/bases" cd $(CAPD_DIR); $(CONTROLLER_GEN) \ + paths=./ \ paths=./api/... \ paths=./$(EXP_DIR)/api/... \ paths=./$(EXP_DIR)/internal/controllers/... \ @@ -346,6 +350,7 @@ generate-manifests-docker-infrastructure: $(CONTROLLER_GEN) ## Generate manifest generate-manifests-in-memory-infrastructure: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. for in-memory infrastructure provider $(MAKE) clean-generated-yaml SRC_DIRS="$(CAPIM_DIR)/config/crd/bases" cd $(CAPIM_DIR); $(CONTROLLER_GEN) \ + paths=./ \ paths=./api/... \ paths=./internal/controllers/... \ crd:crdVersions=v1 \ diff --git a/bootstrap/kubeadm/config/manager/manager.yaml b/bootstrap/kubeadm/config/manager/manager.yaml index d54386179159..d5e12b4572e8 100644 --- a/bootstrap/kubeadm/config/manager/manager.yaml +++ b/bootstrap/kubeadm/config/manager/manager.yaml @@ -20,7 +20,8 @@ spec: - /manager args: - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false}" - "--bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m}" image: controller:latest @@ -29,6 +30,9 @@ spec: - containerPort: 9440 name: healthz protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz diff --git a/bootstrap/kubeadm/config/rbac/role.yaml b/bootstrap/kubeadm/config/rbac/role.yaml index 7740337f293b..e97c00a68311 100644 --- a/bootstrap/kubeadm/config/rbac/role.yaml +++ b/bootstrap/kubeadm/config/rbac/role.yaml @@ -18,6 +18,18 @@ rules: - patch - update - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - bootstrap.cluster.x-k8s.io resources: diff --git a/bootstrap/kubeadm/main.go b/bootstrap/kubeadm/main.go index c92ba43e640c..47cbe1de6682 100644 --- a/bootstrap/kubeadm/main.go +++ b/bootstrap/kubeadm/main.go @@ -41,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -59,11 +58,33 @@ var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") controllerName = "cluster-api-kubeadm-bootstrap-manager" + + // flags. + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + watchFilterValue string + watchNamespace string + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + webhookPort int + webhookCertDir string + healthAddr string + tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() + // CABPK specific flags. + clusterConcurrency int + clusterCacheTrackerConcurrency int + kubeadmConfigConcurrency int + tokenTTL time.Duration ) func init() { - klog.InitFlags(nil) - _ = clientgoscheme.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) _ = expv1.AddToScheme(scheme) @@ -71,37 +92,10 @@ func init() { _ = bootstrapv1.AddToScheme(scheme) } -var ( - metricsBindAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - watchFilterValue string - watchNamespace string - profilerAddress string - enableContentionProfiling bool - clusterConcurrency int - clusterCacheTrackerConcurrency int - kubeadmConfigConcurrency int - syncPeriod time.Duration - restConfigQPS float32 - restConfigBurst int - webhookPort int - webhookCertDir string - healthAddr string - tokenTTL time.Duration - tlsOptions = flags.TLSOptions{} - logOptions = logs.NewOptions() -) - -// InitFlags initializes this manager's flags. +// InitFlags initializes the flags. func InitFlags(fs *pflag.FlagSet) { logsv1.AddFlags(logOptions, fs) - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -117,11 +111,14 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&watchNamespace, "namespace", "", "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") + fs.StringVar(&watchFilterValue, "watch-filter", "", + fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) + fs.StringVar(&profilerAddress, "profiler-address", "", "Bind address to expose the pprof profiler (e.g. localhost:6060)") fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, - "Enable block profiling, if profiler-address is set.") + "Enable block profiling") fs.IntVar(&clusterConcurrency, "cluster-concurrency", 10, "Number of clusters to process simultaneously") @@ -145,9 +142,6 @@ func InitFlags(fs *pflag.FlagSet) { fs.DurationVar(&tokenTTL, "bootstrap-token-ttl", kubeadmbootstrapcontrollers.DefaultTokenTTL, "The amount of time the bootstrap token will be valid") - fs.StringVar(&watchFilterValue, "watch-filter", "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) - fs.IntVar(&webhookPort, "webhook-port", 9443, "Webhook Server port") @@ -157,11 +151,16 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) flags.AddTLSOptions(fs, &tlsOptions) feature.MutableGates.AddFlag(fs) } +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + func main() { InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) @@ -192,6 +191,8 @@ func main() { os.Exit(1) } + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + var watchNamespaces map[string]cache.Config if watchNamespace != "" { watchNamespaces = map[string]cache.Config{ @@ -199,7 +200,7 @@ func main() { } } - if profilerAddress != "" && enableContentionProfiling { + if enableContentionProfiling { goruntime.SetBlockProfileRate(1) } @@ -216,9 +217,7 @@ func main() { LeaderElectionResourceLock: resourcelock.LeasesResourceLock, HealthProbeBindAddress: healthAddr, PprofBindAddress: profilerAddress, - Metrics: metricsserver.Options{ - BindAddress: metricsBindAddr, - }, + Metrics: diagnosticsOpts, Cache: cache.Options{ DefaultNamespaces: watchNamespaces, SyncPeriod: &syncPeriod, diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8a2cd9c3c23d..37c4ae3cbc1b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -21,7 +21,8 @@ spec: - /manager args: - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterResourceSet=${EXP_CLUSTER_RESOURCE_SET:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false},MachineSetPreflightChecks=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=false}" image: controller:latest name: manager @@ -42,6 +43,9 @@ spec: - containerPort: 9440 name: healthz protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c78f191ceff1..55346c81bf24 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -41,6 +41,18 @@ rules: - get - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - bootstrap.cluster.x-k8s.io - controlplane.cluster.x-k8s.io diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml index 34e85f86e86d..d79579b40b5f 100644 --- a/controlplane/kubeadm/config/manager/manager.yaml +++ b/controlplane/kubeadm/config/manager/manager.yaml @@ -20,7 +20,8 @@ spec: - /manager args: - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - "--feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false}" image: controller:latest name: manager @@ -41,6 +42,9 @@ spec: - containerPort: 9440 name: healthz protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz diff --git a/controlplane/kubeadm/config/rbac/role.yaml b/controlplane/kubeadm/config/rbac/role.yaml index ec2334e965fe..8bae8c0a19f5 100644 --- a/controlplane/kubeadm/config/rbac/role.yaml +++ b/controlplane/kubeadm/config/rbac/role.yaml @@ -12,6 +12,18 @@ rules: - get - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - bootstrap.cluster.x-k8s.io - controlplane.cluster.x-k8s.io diff --git a/controlplane/kubeadm/main.go b/controlplane/kubeadm/main.go index ada11277e0dd..736b5117b0d0 100644 --- a/controlplane/kubeadm/main.go +++ b/controlplane/kubeadm/main.go @@ -43,7 +43,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -65,9 +64,33 @@ var ( controllerName = "cluster-api-kubeadm-control-plane-manager" ) -func init() { - klog.InitFlags(nil) +var ( + // flags. + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + watchFilterValue string + watchNamespace string + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + webhookPort int + webhookCertDir string + healthAddr string + tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() + // KCP specific flags. + kubeadmControlPlaneConcurrency int + clusterCacheTrackerConcurrency int + etcdDialTimeout time.Duration + etcdCallTimeout time.Duration +) +func init() { _ = clientgoscheme.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) _ = controlplanev1alpha4.AddToScheme(scheme) @@ -76,37 +99,10 @@ func init() { _ = apiextensionsv1.AddToScheme(scheme) } -var ( - metricsBindAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - watchFilterValue string - watchNamespace string - profilerAddress string - enableContentionProfiling bool - kubeadmControlPlaneConcurrency int - clusterCacheTrackerConcurrency int - syncPeriod time.Duration - restConfigQPS float32 - restConfigBurst int - webhookPort int - webhookCertDir string - healthAddr string - etcdDialTimeout time.Duration - etcdCallTimeout time.Duration - tlsOptions = flags.TLSOptions{} - logOptions = logs.NewOptions() -) - // InitFlags initializes the flags. func InitFlags(fs *pflag.FlagSet) { logsv1.AddFlags(logOptions, fs) - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -122,11 +118,14 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&watchNamespace, "namespace", "", "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") + fs.StringVar(&watchFilterValue, "watch-filter", "", + fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) + fs.StringVar(&profilerAddress, "profiler-address", "", "Bind address to expose the pprof profiler (e.g. localhost:6060)") fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, - "Enable block profiling, if profiler-address is set.") + "Enable block profiling") fs.IntVar(&kubeadmControlPlaneConcurrency, "kubeadmcontrolplane-concurrency", 10, "Number of kubeadm control planes to process simultaneously") @@ -143,9 +142,6 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&restConfigBurst, "kube-api-burst", 30, "Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30") - fs.StringVar(&watchFilterValue, "watch-filter", "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) - fs.IntVar(&webhookPort, "webhook-port", 9443, "Webhook Server port") @@ -161,10 +157,16 @@ func InitFlags(fs *pflag.FlagSet) { fs.DurationVar(&etcdCallTimeout, "etcd-call-timeout-duration", etcd.DefaultCallTimeout, "Duration that the etcd client waits at most for read and write operations to etcd.") + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) flags.AddTLSOptions(fs, &tlsOptions) feature.MutableGates.AddFlag(fs) } + +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + func main() { InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) @@ -195,6 +197,8 @@ func main() { os.Exit(1) } + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + var watchNamespaces map[string]cache.Config if watchNamespace != "" { watchNamespaces = map[string]cache.Config{ @@ -202,7 +206,7 @@ func main() { } } - if profilerAddress != "" && enableContentionProfiling { + if enableContentionProfiling { goruntime.SetBlockProfileRate(1) } @@ -219,9 +223,7 @@ func main() { LeaderElectionResourceLock: resourcelock.LeasesResourceLock, HealthProbeBindAddress: healthAddr, PprofBindAddress: profilerAddress, - Metrics: metricsserver.Options{ - BindAddress: metricsBindAddr, - }, + Metrics: diagnosticsOpts, Cache: cache.Options{ DefaultNamespaces: watchNamespaces, SyncPeriod: &syncPeriod, diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index 4cd2d6133db2..1f4c594806b2 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -41,6 +41,7 @@ - [Ignition Bootstrap configuration](./tasks/experimental-features/ignition.md) - [Running multiple providers](./tasks/multiple-providers.md) - [Verification of Container Images](./tasks/verify-container-images.md) + - [Diagnostics](./tasks/diagnostics.md) - [Security Guidelines](./security/index.md) - [Pod Security Standards](./security/pod-security-standards.md) - [clusterctl CLI](./clusterctl/overview.md) diff --git a/docs/book/src/developer/providers/migrations/v1.5-to-v1.6.md b/docs/book/src/developer/providers/migrations/v1.5-to-v1.6.md index a464b924052c..49a09c9eadb0 100644 --- a/docs/book/src/developer/providers/migrations/v1.5-to-v1.6.md +++ b/docs/book/src/developer/providers/migrations/v1.5-to-v1.6.md @@ -38,3 +38,5 @@ maintainers of providers and consumers of our Go API. ### Suggested changes for providers - In order to reduce dependencies for API package consumers, CAPI has diverged from the default kubebuilder scheme builder. This new pattern may also be useful for reducing dependencies in provider API packages. For more information [see the implementers guide.](../implementers-guide/create_api.md#registering-apis-in-the-scheme) +- We deprecated the `--metrics-bind-addr` flag and introduced the new `--diagnostics-address` and `--insecure-diagnostic` flags. These flags allow exposing metrics, a pprof endpoint and + an endpoint to change log levels securely in production. It is recommended to implement the same changes in providers, please see [#9264](https://github.com/kubernetes-sigs/cluster-api/pull/9264) for more details. diff --git a/docs/book/src/tasks/diagnostics.md b/docs/book/src/tasks/diagnostics.md new file mode 100644 index 000000000000..713048441686 --- /dev/null +++ b/docs/book/src/tasks/diagnostics.md @@ -0,0 +1,183 @@ +# Diagnostics + +## Introduction + +With CAPI v1.6 we introduced new flags to allow serving metrics, the pprof endpoint and an endpoint to dynamically change log levels securely in production. + +This feature is enabled per default via: +```yaml + args: + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" +``` + +As soon as the feature is enabled the metrics endpoint is served via https and protected via authentication and authorization. This works the same way as +metrics in core Kubernetes components: [Metrics in Kubernetes](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/). + +To continue serving metrics via http the following configuration can be used: +```yaml + args: + - "--diagnostics-address=localhost:8080" + - "--insecure-diagnostics" +``` + +The same can be achieved via clusterctl: +```bash +export CAPI_DIAGNOSTICS_ADDRESS: "localhost:8080" +export CAPI_INSECURE_DIAGNOSTICS: "true" +clusterctl init ... +``` + +**Note**: If insecure serving is configured the pprof and log level endpoints are disabled for security reasons. + +## Scraping metrics + +A ServiceAccount token is now required to scrape metrics. The corresponding ServiceAccount needs permissions on the `/metrics` path. +This can be achieved e.g. by following the [Kubernetes documentation](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/). + +### via Prometheus + +With the Prometheus Helm chart it is as easy as using the following config for the Prometheus job scraping the Cluster API controllers: +```yaml + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # The diagnostics endpoint is using a self-signed certificate, so we don't verify it. + insecure_skip_verify: true +``` + +For more details please see our Prometheus development setup: [Prometheus](https://github.com/kubernetes-sigs/cluster-api/tree/main/hack/observability/prometheus) + +**Note**: The Prometheus Helm chart deploys the required ClusterRole out-of-the-box. + +### via kubectl + +First deploy the following RBAC configuration: +```yaml +cat << EOT | kubectl apply -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: default-metrics +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: default-metrics +subjects: +- kind: ServiceAccount + name: default + namespace: default +EOT +``` + +Then let's open a port-forward, create a ServiceAccount token and scrape the metrics: +```bash +# Terminal 1 +kubectl -n capi-system port-forward deployments/capi-controller-manager 8443 + +# Terminal 2 +TOKEN=$(kubectl create token default) +curl https://localhost:8443/metrics --header "Authorization: Bearer $TOKEN" -k +``` + +## Collecting profiles + +### via Parca + +Parca can be used to continuously scrape profiles from CAPI providers. For more details please see our Parca +development setup: [parca](https://github.com/kubernetes-sigs/cluster-api/tree/main/hack/observability/parca) + +### via kubectl + +First deploy the following RBAC configuration: +```yaml +cat << EOT | kubectl apply -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: default-pprof +rules: +- nonResourceURLs: + - "/debug/pprof/*" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default-pprof +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: default-pprof +subjects: +- kind: ServiceAccount + name: default + namespace: default +EOT +``` + +Then let's open a port-forward, create a ServiceAccount token and scrape the profile: +```bash +# Terminal 1 +kubectl -n capi-system port-forward deployments/capi-controller-manager 8443 + +# Terminal 2 +TOKEN=$(kubectl create token default) +curl "https://localhost:8443/debug/pprof/profile?seconds=10" --header "Authorization: Bearer $TOKEN" -k > ./profile.out +go tool pprof -http=:8080 ./profile.out +``` + +## Changing the log level + +### via kubectl + +First deploy the following RBAC configuration: +```yaml +cat << EOT | kubectl apply -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: default-loglevel +rules: +- nonResourceURLs: + - "/debug/flags/v" + verbs: + - put +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default-loglevel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: default-loglevel +subjects: +- kind: ServiceAccount + name: default + namespace: default +EOT +``` + +Then let's open a port-forward, create a ServiceAccount token and change the log level to `8`: +```bash +# Terminal 1 +kubectl -n capi-system port-forward deployments/capi-controller-manager 8443 + +# Terminal 2 +TOKEN=$(kubectl create token default) +curl "https://localhost:8443/debug/flags/v" --header "Authorization: Bearer $TOKEN" -X PUT -d '8' -k +``` diff --git a/go.mod b/go.mod index c2b4f4d960e4..01a4e4f558f7 100644 --- a/go.mod +++ b/go.mod @@ -54,12 +54,14 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudflare/circl v1.3.3 // indirect @@ -72,9 +74,11 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fvbommel/sortorder v1.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -91,6 +95,8 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -134,6 +140,16 @@ require ( github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect + go.opentelemetry.io/otel/metric v0.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect @@ -153,11 +169,14 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cli-runtime v0.28.1 // indirect k8s.io/component-helpers v0.28.1 // indirect + k8s.io/kms v0.28.1 // indirect k8s.io/metrics v0.28.1 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 // indirect diff --git a/go.sum b/go.sum index 019d42400beb..bd2a5648caad 100644 --- a/go.sum +++ b/go.sum @@ -17,12 +17,15 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -49,6 +52,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= @@ -62,6 +67,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -82,8 +88,11 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= @@ -98,6 +107,11 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= github.com/coredns/corefile-migration v1.0.21 h1:W/DCETrHDiFo0Wj03EyMkaQ9fwsmSgqTCQDHpceaSsE= @@ -130,6 +144,7 @@ github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m3 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -137,6 +152,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -149,6 +166,8 @@ github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwo github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= @@ -171,8 +190,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -192,7 +214,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -221,6 +246,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= @@ -246,6 +272,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= @@ -281,12 +308,19 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -449,6 +483,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -465,11 +500,13 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -512,12 +549,14 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= @@ -528,18 +567,44 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= +go.etcd.io/etcd/client/v2 v2.305.9 h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo= go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= +go.etcd.io/etcd/pkg/v3 v3.5.9 h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ= +go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= +go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 h1:sxoY9kG1s1WpSYNyzm24rlwH4lnRYFXUVVBmKMBfRgw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -659,6 +724,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -740,6 +806,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -862,6 +929,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -875,6 +943,7 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= @@ -894,9 +963,13 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.1 h1:36vzoa06ohIaveIgzr0qWDCImkKeVjvnNSV6uOmwnOw= google.golang.org/grpc v1.55.1/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -911,6 +984,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -924,10 +998,13 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -965,6 +1042,8 @@ k8s.io/component-helpers v0.28.1 h1:ts/vykhyUmPLhUl/hdLdf+a4BWA0giQ3f25HAIhl+RI= k8s.io/component-helpers v0.28.1/go.mod h1:rHFPj33uXNbgppg+ilmjJ4oR73prZQNRRmg+utVOAb0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.28.1 h1:QLNTIc0k7Yebkt9yobj9Y9qBoRCMB4dq+pFCxVXVBnY= +k8s.io/kms v0.28.1/go.mod h1:I2TwA8oerDRInHWWBOqSUzv1EJDC1+55FQKYkxaPxh0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/kubectl v0.28.1 h1:jAq4yKEqQL+fwkWcEsUWxhJ7uIRcOYQraJxx4SyAMTY= @@ -976,6 +1055,8 @@ k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/hack/observability/parca/kustomization.yaml b/hack/observability/parca/kustomization.yaml index 60ceb93b9b4c..b26ea57cfc28 100644 --- a/hack/observability/parca/kustomization.yaml +++ b/hack/observability/parca/kustomization.yaml @@ -1,5 +1,6 @@ resources: - ../namespace.yaml + - rbac-pprof.yaml helmCharts: - name: parca diff --git a/hack/observability/parca/rbac-pprof.yaml b/hack/observability/parca/rbac-pprof.yaml new file mode 100644 index 000000000000..572ab00c123e --- /dev/null +++ b/hack/observability/parca/rbac-pprof.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: parca-debug +rules: +- nonResourceURLs: + - "/debug/pprof/*" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: parca-debug +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: parca-debug +subjects: +- kind: ServiceAccount + name: parca + namespace: default diff --git a/hack/observability/parca/values.yaml b/hack/observability/parca/values.yaml index 30c3284732ee..65f37210d086 100644 --- a/hack/observability/parca/values.yaml +++ b/hack/observability/parca/values.yaml @@ -10,6 +10,13 @@ server: scrapeConfigs: - job_name: 'capi-providers' scrape_interval: 1m + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true kubernetes_sd_configs: - role: pod relabel_configs: diff --git a/hack/observability/prometheus/values.yaml b/hack/observability/prometheus/values.yaml index a28932f5c961..761fbe0577f7 100644 --- a/hack/observability/prometheus/values.yaml +++ b/hack/observability/prometheus/values.yaml @@ -27,6 +27,13 @@ extraScrapeConfigs: | # 15s is a bit often for production but helps to get metrics quicker for development. scrape_interval: 15s metrics_path: /metrics + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true kubernetes_sd_configs: - role: pod diff --git a/hack/tools/internal/tilt-prepare/main.go b/hack/tools/internal/tilt-prepare/main.go index ddd41ee9531a..9691b54eb115 100644 --- a/hack/tools/internal/tilt-prepare/main.go +++ b/hack/tools/internal/tilt-prepare/main.go @@ -36,7 +36,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -118,7 +117,6 @@ type tiltSettingsDebugConfig struct { Continue *bool `json:"continue"` Port *int `json:"port"` ProfilerPort *int `json:"profiler_port"` - MetricsPort *int `json:"metrics_port"` } type tiltSettingsExtraArgs []string @@ -228,9 +226,6 @@ func setTiltSettingsDefaults(ts *tiltSettings) { if p.ProfilerPort == nil { p.ProfilerPort = pointer.Int(0) } - if p.MetricsPort == nil { - p.MetricsPort = pointer.Int(0) - } ts.Debug[k] = p } @@ -821,54 +816,22 @@ func prepareWorkload(name, prefix, binaryName, containerName string, objs []unst args = debugArgs // Set annotation to point parca to the profiler port. - if d.ProfilerPort != nil && *d.ProfilerPort > 0 { - if deployment.Spec.Template.Annotations == nil { - deployment.Spec.Template.Annotations = map[string]string{} - } - deployment.Spec.Template.Annotations["parca.dev/port"] = "6060" + if deployment.Spec.Template.Annotations == nil { + deployment.Spec.Template.Annotations = map[string]string{} } + deployment.Spec.Template.Annotations["parca.dev/port"] = "8443" container.LivenessProbe = nil container.ReadinessProbe = nil } - // alter the controller deployment for working nicely with prometheus metrics scraping. Specifically, the - // metrics endpoint is set to listen on all interfaces instead of only localhost, and another port is added - // to the container to expose the metrics endpoint. - finalArgs := make([]string, 0, len(args)) - for _, a := range args { - if strings.HasPrefix(a, "--metrics-bind-addr=") { - finalArgs = append(finalArgs, "--metrics-bind-addr=0.0.0.0:8080") - continue - } - finalArgs = append(finalArgs, a) - } - // tilt-prepare should add the metrics port if and only if it's missing: - // as best practices with controller-runtime, it's named metrics and listening on port 8080. - if !containerHasMetricsPort(container.Ports) { - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "metrics", - ContainerPort: 8080, - Protocol: corev1.ProtocolTCP, - }) - } container.Command = cmd - container.Args = finalArgs + container.Args = args deployment.Spec.Template.Spec.Containers[j] = container } }) } -func containerHasMetricsPort(ports []corev1.ContainerPort) bool { - for _, port := range ports { - if port.ContainerPort == 8080 { - return true - } - } - - return false -} - type updateDeploymentFunction func(deployment *appsv1.Deployment) // updateDeployment passes all (typed) deployments to the updateDeploymentFunction diff --git a/main.go b/main.go index 1e3ce3d974d2..9537201dd9fa 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -79,15 +78,24 @@ var ( controllerName = "cluster-api-controller-manager" // flags. - metricsBindAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - watchNamespace string - watchFilterValue string - profilerAddress string - enableContentionProfiling bool + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + watchFilterValue string + watchNamespace string + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + webhookPort int + webhookCertDir string + healthAddr string + tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() + // core Cluster API specific flags. clusterTopologyConcurrency int clusterCacheTrackerConcurrency int clusterClassConcurrency int @@ -99,15 +107,7 @@ var ( machinePoolConcurrency int clusterResourceSetConcurrency int machineHealthCheckConcurrency int - syncPeriod time.Duration - restConfigQPS float32 - restConfigBurst int nodeDrainClientTimeout time.Duration - webhookPort int - webhookCertDir string - healthAddr string - tlsOptions = flags.TLSOptions{} - logOptions = logs.NewOptions() ) func init() { @@ -135,9 +135,6 @@ func init() { func InitFlags(fs *pflag.FlagSet) { logsv1.AddFlags(logOptions, fs) - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -160,7 +157,7 @@ func InitFlags(fs *pflag.FlagSet) { "Bind address to expose the pprof profiler (e.g. localhost:6060)") fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, - "Enable block profiling, if profiler-address is set.") + "Enable block profiling") fs.IntVar(&clusterTopologyConcurrency, "clustertopology-concurrency", 10, "Number of clusters to process simultaneously") @@ -216,11 +213,16 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) flags.AddTLSOptions(fs, &tlsOptions) feature.MutableGates.AddFlag(fs) } +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + func main() { InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) @@ -266,6 +268,8 @@ func main() { os.Exit(1) } + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + var watchNamespaces map[string]cache.Config if watchNamespace != "" { watchNamespaces = map[string]cache.Config{ @@ -273,7 +277,7 @@ func main() { } } - if profilerAddress != "" && enableContentionProfiling { + if enableContentionProfiling { goruntime.SetBlockProfileRate(1) } @@ -290,9 +294,7 @@ func main() { LeaderElectionResourceLock: resourcelock.LeasesResourceLock, HealthProbeBindAddress: healthAddr, PprofBindAddress: profilerAddress, - Metrics: metricsserver.Options{ - BindAddress: metricsBindAddr, - }, + Metrics: diagnosticsOpts, Cache: cache.Options{ DefaultNamespaces: watchNamespaces, SyncPeriod: &syncPeriod, @@ -390,6 +392,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Error(err, "unable to create cluster cache tracker") os.Exit(1) } + if err := (&remote.ClusterCacheReconciler{ Client: mgr.GetClient(), Tracker: tracker, diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index cf1aafd762b2..35dba51cf929 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -65,8 +65,8 @@ providers: - name: v1.6.99 # next; use manifest from source files value: ../../../config/default replacements: - - old: --metrics-bind-addr=localhost:8080 - new: "--metrics-bind-addr=:8080\n - --logging-format=json" + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" files: - sourcePath: "../data/shared/main/metadata.yaml" @@ -103,8 +103,8 @@ providers: - name: v1.6.99 # next; use manifest from source files value: ../../../bootstrap/kubeadm/config/default replacements: - - old: --metrics-bind-addr=localhost:8080 - new: "--metrics-bind-addr=:8080\n - --logging-format=json" + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" files: - sourcePath: "../data/shared/main/metadata.yaml" @@ -141,8 +141,8 @@ providers: - name: v1.6.99 # next; use manifest from source files value: ../../../controlplane/kubeadm/config/default replacements: - - old: --metrics-bind-addr=localhost:8080 - new: "--metrics-bind-addr=:8080\n - --logging-format=json" + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" files: - sourcePath: "../data/shared/main/metadata.yaml" @@ -186,8 +186,8 @@ providers: - name: v1.6.99 # next; use manifest from source files value: ../../../test/infrastructure/docker/config/default replacements: - - old: --metrics-bind-addr=localhost:8080 - new: "--metrics-bind-addr=:8080\n - --logging-format=json" + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" files: # Add cluster templates - sourcePath: "../data/infrastructure-docker/main/cluster-template.yaml" @@ -213,16 +213,16 @@ providers: - name: in-memory type: InfrastructureProvider versions: - - name: v1.6.99 # next; use manifest from source files - value: ../../../test/infrastructure/inmemory/config/default - replacements: - - old: --metrics-bind-addr=localhost:8080 - new: "--metrics-bind-addr=:8080\n - --logging-format=json" - files: - # Add cluster templates - - sourcePath: "../data/infrastructure-inmemory/main/clusterclass-in-memory.yaml" - - sourcePath: "../data/infrastructure-inmemory/main/cluster-template.yaml" - - sourcePath: "../data/shared/main/metadata.yaml" + - name: v1.6.99 # next; use manifest from source files + value: ../../../test/infrastructure/inmemory/config/default + replacements: + - old: "--leader-elect" + new: "--leader-elect\n - --logging-format=json" + files: + # Add cluster templates + - sourcePath: "../data/infrastructure-inmemory/main/clusterclass-in-memory.yaml" + - sourcePath: "../data/infrastructure-inmemory/main/cluster-template.yaml" + - sourcePath: "../data/shared/main/metadata.yaml" - name: test-extension type: RuntimeExtensionProvider @@ -262,6 +262,8 @@ variables: CLUSTER_TOPOLOGY: "true" EXP_RUNTIME_SDK: "true" EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true" + CAPI_DIAGNOSTICS_ADDRESS: ":8080" + CAPI_INSECURE_DIAGNOSTICS: "true" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/go.mod b/test/go.mod index 72fde0e6e59a..8e2345c897bf 100644 --- a/test/go.mod +++ b/test/go.mod @@ -150,10 +150,12 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/cluster-bootstrap v0.28.1 // indirect + k8s.io/kms v0.28.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/test/go.sum b/test/go.sum index 7bb101f9a541..c35977944cb1 100644 --- a/test/go.sum +++ b/test/go.sum @@ -961,6 +961,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1000,6 +1001,8 @@ k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.28.1 h1:QLNTIc0k7Yebkt9yobj9Y9qBoRCMB4dq+pFCxVXVBnY= +k8s.io/kms v0.28.1/go.mod h1:I2TwA8oerDRInHWWBOqSUzv1EJDC1+55FQKYkxaPxh0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= diff --git a/test/infrastructure/docker/config/manager/manager.yaml b/test/infrastructure/docker/config/manager/manager.yaml index 0a6fcafe94b8..56fc71bb9239 100644 --- a/test/infrastructure/docker/config/manager/manager.yaml +++ b/test/infrastructure/docker/config/manager/manager.yaml @@ -18,7 +18,8 @@ spec: containers: - args: - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},ClusterTopology=${CLUSTER_TOPOLOGY:=false}" image: controller:latest name: manager @@ -41,6 +42,9 @@ spec: - containerPort: 9440 name: healthz protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz diff --git a/test/infrastructure/docker/config/rbac/role.yaml b/test/infrastructure/docker/config/rbac/role.yaml index f9ee53e84577..91fe8a21b068 100644 --- a/test/infrastructure/docker/config/rbac/role.yaml +++ b/test/infrastructure/docker/config/rbac/role.yaml @@ -12,6 +12,18 @@ rules: - get - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - cluster.x-k8s.io resources: diff --git a/test/infrastructure/docker/main.go b/test/infrastructure/docker/main.go index 703fac9507cf..526a39387d26 100644 --- a/test/infrastructure/docker/main.go +++ b/test/infrastructure/docker/main.go @@ -42,7 +42,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -66,25 +65,26 @@ var ( controllerName = "cluster-api-docker-controller-manager" // flags. - metricsBindAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - watchNamespace string - watchFilterValue string - profilerAddress string - enableContentionProfiling bool + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + watchFilterValue string + watchNamespace string + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + webhookPort int + webhookCertDir string + healthAddr string + tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() + // CAPD specific flags. concurrency int clusterCacheTrackerConcurrency int - syncPeriod time.Duration - restConfigQPS float32 - restConfigBurst int - webhookPort int - webhookCertDir string - healthAddr string - tlsOptions = flags.TLSOptions{} - logOptions = logs.NewOptions() ) func init() { @@ -98,12 +98,9 @@ func init() { } // InitFlags initializes the flags. -func initFlags(fs *pflag.FlagSet) { +func InitFlags(fs *pflag.FlagSet) { logsv1.AddFlags(logOptions, fs) - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -126,7 +123,7 @@ func initFlags(fs *pflag.FlagSet) { "Bind address to expose the pprof profiler (e.g. localhost:6060)") fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, - "Enable block profiling, if profiler-address is set.") + "Enable block profiling") fs.IntVar(&concurrency, "concurrency", 10, "The number of docker machines to process simultaneously") @@ -152,18 +149,23 @@ func initFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) flags.AddTLSOptions(fs, &tlsOptions) feature.MutableGates.AddFlag(fs) } +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + func main() { if _, err := os.ReadDir("/tmp/"); err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } - initFlags(pflag.CommandLine) + InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) pflag.CommandLine.AddGoFlagSet(flag.CommandLine) // Set log level 2 as default. @@ -192,6 +194,8 @@ func main() { os.Exit(1) } + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + var watchNamespaces map[string]cache.Config if watchNamespace != "" { watchNamespaces = map[string]cache.Config{ @@ -199,7 +203,7 @@ func main() { } } - if profilerAddress != "" && enableContentionProfiling { + if enableContentionProfiling { goruntime.SetBlockProfileRate(1) } @@ -216,9 +220,7 @@ func main() { LeaderElectionResourceLock: resourcelock.LeasesResourceLock, HealthProbeBindAddress: healthAddr, PprofBindAddress: profilerAddress, - Metrics: metricsserver.Options{ - BindAddress: metricsBindAddr, - }, + Metrics: diagnosticsOpts, Cache: cache.Options{ DefaultNamespaces: watchNamespaces, SyncPeriod: &syncPeriod, diff --git a/test/infrastructure/inmemory/config/manager/manager.yaml b/test/infrastructure/inmemory/config/manager/manager.yaml index 3c412cf85ccd..fe7bc67cca04 100644 --- a/test/infrastructure/inmemory/config/manager/manager.yaml +++ b/test/infrastructure/inmemory/config/manager/manager.yaml @@ -20,7 +20,8 @@ spec: - /manager args: - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" image: controller:latest name: manager env: @@ -32,6 +33,9 @@ spec: - containerPort: 9440 name: healthz protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP readinessProbe: httpGet: path: /readyz diff --git a/test/infrastructure/inmemory/config/rbac/role.yaml b/test/infrastructure/inmemory/config/rbac/role.yaml index abbf55eb0d13..91e3482a64e9 100644 --- a/test/infrastructure/inmemory/config/rbac/role.yaml +++ b/test/infrastructure/inmemory/config/rbac/role.yaml @@ -12,6 +12,18 @@ rules: - get - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - cluster.x-k8s.io resources: diff --git a/test/infrastructure/inmemory/main.go b/test/infrastructure/inmemory/main.go index 63bd1fd4ea34..dbcdbcb87e47 100644 --- a/test/infrastructure/inmemory/main.go +++ b/test/infrastructure/inmemory/main.go @@ -41,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -57,22 +56,20 @@ import ( ) var ( - scheme = runtime.NewScheme() - cloudScheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + cloudScheme = runtime.NewScheme() + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + controllerName = "cluster-api-inmemory-controller-manager" // flags. - metricsBindAddr string enableLeaderElection bool leaderElectionLeaseDuration time.Duration leaderElectionRenewDeadline time.Duration leaderElectionRetryPeriod time.Duration - watchNamespace string watchFilterValue string + watchNamespace string profilerAddress string enableContentionProfiling bool - clusterConcurrency int - machineConcurrency int syncPeriod time.Duration restConfigQPS float32 restConfigBurst int @@ -80,7 +77,11 @@ var ( webhookCertDir string healthAddr string tlsOptions = flags.TLSOptions{} + diagnosticsOptions = flags.DiagnosticsOptions{} logOptions = logs.NewOptions() + // CAPIM specific flags. + clusterConcurrency int + machineConcurrency int ) func init() { @@ -100,9 +101,6 @@ func init() { func InitFlags(fs *pflag.FlagSet) { logsv1.AddFlags(logOptions, fs) - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - fs.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -125,7 +123,7 @@ func InitFlags(fs *pflag.FlagSet) { "Bind address to expose the pprof profiler (e.g. localhost:6060)") fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, - "Enable block profiling, if profiler-address is set.") + "Enable block profiling") fs.IntVar(&clusterConcurrency, "cluster-concurrency", 10, "Number of clusters to process simultaneously") @@ -151,11 +149,16 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) flags.AddTLSOptions(fs, &tlsOptions) feature.MutableGates.AddFlag(fs) } +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + func main() { InitFlags(pflag.CommandLine) pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) @@ -178,7 +181,7 @@ func main() { restConfig := ctrl.GetConfigOrDie() restConfig.QPS = restConfigQPS restConfig.Burst = restConfigBurst - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-inmemory-controller-manager") + restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName) tlsOptionOverrides, err := flags.GetTLSOptionOverrideFuncs(tlsOptions) if err != nil { @@ -186,6 +189,8 @@ func main() { os.Exit(1) } + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + var watchNamespaces map[string]cache.Config if watchNamespace != "" { watchNamespaces = map[string]cache.Config{ @@ -193,7 +198,7 @@ func main() { } } - if profilerAddress != "" && enableContentionProfiling { + if enableContentionProfiling { goruntime.SetBlockProfileRate(1) } @@ -207,9 +212,7 @@ func main() { LeaderElectionResourceLock: resourcelock.LeasesResourceLock, HealthProbeBindAddress: healthAddr, PprofBindAddress: profilerAddress, - Metrics: metricsserver.Options{ - BindAddress: metricsBindAddr, - }, + Metrics: diagnosticsOpts, Cache: cache.Options{ DefaultNamespaces: watchNamespaces, SyncPeriod: &syncPeriod, diff --git a/util/flags/diagnostics.go b/util/flags/diagnostics.go new file mode 100644 index 000000000000..f88dfdc20b6f --- /dev/null +++ b/util/flags/diagnostics.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package flags implements the webhook server TLS options utilities. +package flags + +import ( + "net/http" + "net/http/pprof" + + "github.com/spf13/pflag" + "k8s.io/apiserver/pkg/server/routes" + "k8s.io/component-base/logs" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// DiagnosticsOptions has the options to configure diagnostics. +type DiagnosticsOptions struct { + // MetricsBindAddr + // + // Deprecated: This field will be removed in an upcoming release. + MetricsBindAddr string + DiagnosticsAddress string + InsecureDiagnostics bool +} + +// AddDiagnosticsOptions adds the diagnostics flags to the flag set. +func AddDiagnosticsOptions(fs *pflag.FlagSet, options *DiagnosticsOptions) { + fs.StringVar(&options.MetricsBindAddr, "metrics-bind-addr", "", + "The address the metrics endpoint binds to.") + _ = fs.MarkDeprecated("metrics-bind-addr", "Please use --diagnostics-address instead. To continue to serve"+ + "metrics via http and without authentication/authorization set --insecure-diagnostics as well.") + + fs.StringVar(&options.DiagnosticsAddress, "diagnostics-address", ":8443", + "The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+ + "authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+ + "If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints and an endpoint to change the log level.") + + fs.BoolVar(&options.InsecureDiagnostics, "insecure-diagnostics", false, + "Enable insecure diagnostics serving. For more details see the description of --diagnostics-address.") +} + +// GetDiagnosticsOptions returns metrics options which can be used to configure a Manager. +func GetDiagnosticsOptions(options DiagnosticsOptions) metricsserver.Options { + // If the deprecated "--metrics-bind-addr" flag is set, continue to serve metrics via http + // and without authentication/authorization. + if options.MetricsBindAddr != "" { + return metricsserver.Options{ + BindAddress: options.MetricsBindAddr, + } + } + + // If "--insecure-diagnostics" is set, serve metrics via http + // and without authentication/authorization. + if options.InsecureDiagnostics { + return metricsserver.Options{ + BindAddress: options.DiagnosticsAddress, + SecureServing: false, + } + } + + // If "--insecure-diagnostics" is not set, serve metrics via https + // and with authentication/authorization. As the endpoint is protected, + // we also serve pprof endpoints and an endpoint to change the log level. + return metricsserver.Options{ + BindAddress: options.DiagnosticsAddress, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + ExtraHandlers: map[string]http.Handler{ + // Add handler to dynamically change log level. + "/debug/flags/v": routes.StringFlagPutHandler(logs.GlogSetter), + // Add pprof handler. + "/debug/pprof/": http.HandlerFunc(pprof.Index), + "/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline), + "/debug/pprof/profile": http.HandlerFunc(pprof.Profile), + "/debug/pprof/symbol": http.HandlerFunc(pprof.Symbol), + "/debug/pprof/trace": http.HandlerFunc(pprof.Trace), + }, + } +}