From de1cc60900b896b2195e403a40c976a892df4921 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Fri, 5 Jul 2024 09:21:22 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20protection=20to=20metrics=20e?= =?UTF-8?q?ndpoint=20using=20authn/authz=20via=20controller-runtime=20feat?= =?UTF-8?q?ure=20(#4003)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add protection to metrics endpoint using authn/authz via controller-runtime feature This PR re-introduce authn/authz protection for the endpoint but without use the kube-rbac-proxy project and image. Signed-off-by: Joe Lanford Co-authored-by: Joe Lanford --- .github/workflows/test-sample-go.yml | 2 +- .../testdata/project/cmd/main.go | 26 ++- .../project/config/default/kustomization.yaml | 13 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../project/config/prometheus/monitor.yaml | 16 +- .../project/config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../cronjob-tutorial/testdata/project/go.mod | 23 +++ .../cronjob-tutorial/testdata/project/go.sum | 53 +++++ .../testdata/project/cmd/main.go | 26 ++- .../project/config/default/kustomization.yaml | 15 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../project/config/prometheus/monitor.yaml | 16 +- .../project/config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../getting-started/testdata/project/go.mod | 23 +++ .../getting-started/testdata/project/go.sum | 53 +++++ docs/book/src/reference/metrics.md | 181 ++++++++++++++---- pkg/plugin/util/util.go | 42 ++++ .../common/kustomize/v2/scaffolds/init.go | 3 + .../config/kdefault/kustomization.go | 15 +- .../config/kdefault/manager_metrics_patch.go | 4 +- .../config/kdefault/metrics_service.go | 6 +- .../templates/config/prometheus/monitor.go | 16 +- .../templates/config/rbac/kustomization.go | 9 + .../config/rbac/metrics_auth_role.go | 58 ++++++ .../config/rbac/metrics_auth_role_binding.go | 53 +++++ .../config/rbac/metrics_reader_role.go | 50 +++++ pkg/plugins/golang/v4/scaffolds/init.go | 4 +- .../v4/scaffolds/internal/templates/main.go | 29 ++- .../grafana/v1alpha/scaffolds/init.go | 1 + test/e2e/deployimage/plugin_cluster_test.go | 2 +- test/e2e/utils/test_context.go | 17 -- test/e2e/v4/generate_test.go | 28 +-- test/e2e/v4/plugin_cluster_test.go | 88 ++++++++- .../cmd/main.go | 26 ++- .../config/default/kustomization.yaml | 13 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../config/prometheus/monitor.yaml | 16 +- .../config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../dist/install.yaml | 60 ++++++ .../go.mod | 23 +++ testdata/project-v4-multigroup/cmd/main.go | 26 ++- .../config/default/kustomization.yaml | 13 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../config/prometheus/monitor.yaml | 16 +- .../config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../project-v4-multigroup/dist/install.yaml | 60 ++++++ testdata/project-v4-multigroup/go.mod | 23 +++ .../project-v4-with-deploy-image/cmd/main.go | 26 ++- .../config/default/kustomization.yaml | 13 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../config/prometheus/monitor.yaml | 16 +- .../config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../dist/install.yaml | 60 ++++++ testdata/project-v4-with-deploy-image/go.mod | 23 +++ testdata/project-v4-with-grafana/cmd/main.go | 26 ++- .../config/default/kustomization.yaml | 15 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../config/prometheus/monitor.yaml | 16 +- .../config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + .../project-v4-with-grafana/dist/install.yaml | 60 ++++++ testdata/project-v4-with-grafana/go.mod | 23 +++ testdata/project-v4/cmd/main.go | 26 ++- .../config/default/kustomization.yaml | 13 +- .../config/default/manager_metrics_patch.yaml | 4 +- .../config/default/metrics_service.yaml | 6 +- .../project-v4/config/prometheus/monitor.yaml | 16 +- .../project-v4/config/rbac/kustomization.yaml | 9 + .../config/rbac/metrics_auth_role.yaml | 17 ++ .../rbac/metrics_auth_role_binding.yaml | 12 ++ .../config/rbac/metrics_reader_role.yaml | 9 + testdata/project-v4/dist/install.yaml | 60 ++++++ testdata/project-v4/go.mod | 23 +++ 96 files changed, 1722 insertions(+), 241 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role_binding.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_reader_role.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role_binding.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/rbac/metrics_reader_role.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role.go create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role_binding.go create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_reader_role.go create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role.yaml create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_reader_role.yaml create mode 100644 testdata/project-v4-multigroup/config/rbac/metrics_auth_role.yaml create mode 100644 testdata/project-v4-multigroup/config/rbac/metrics_auth_role_binding.yaml create mode 100644 testdata/project-v4-multigroup/config/rbac/metrics_reader_role.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/rbac/metrics_reader_role.yaml create mode 100644 testdata/project-v4-with-grafana/config/rbac/metrics_auth_role.yaml create mode 100644 testdata/project-v4-with-grafana/config/rbac/metrics_auth_role_binding.yaml create mode 100644 testdata/project-v4-with-grafana/config/rbac/metrics_reader_role.yaml create mode 100644 testdata/project-v4/config/rbac/metrics_auth_role.yaml create mode 100644 testdata/project-v4/config/rbac/metrics_auth_role_binding.yaml create mode 100644 testdata/project-v4/config/rbac/metrics_reader_role.yaml diff --git a/.github/workflows/test-sample-go.yml b/.github/workflows/test-sample-go.yml index 71ef619e9c5..829f2e5642f 100644 --- a/.github/workflows/test-sample-go.yml +++ b/.github/workflows/test-sample-go.yml @@ -24,7 +24,7 @@ jobs: run: | KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '47s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '46s/^#//' $KUSTOMIZATION_FILE_PATH - name: Test run: | diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index 0408fffa13b..4ef7e818c74 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -32,6 +32,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -76,14 +77,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -116,10 +117,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index c70440ce666..8dc238281b8 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. - ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/metrics_service.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/metrics_service.yaml index 1cb008b3b59..4425b9b8977 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/metrics_service.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml index 91d41742932..1dea5d5fd7b 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml index 46cb71e7bf1..39fe987357a 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role_binding.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_reader_role.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/docs/book/src/cronjob-tutorial/testdata/project/go.mod b/docs/book/src/cronjob-tutorial/testdata/project/go.mod index 035b70ecb77..28820a0871c 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/go.mod +++ b/docs/book/src/cronjob-tutorial/testdata/project/go.mod @@ -15,13 +15,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -30,11 +35,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -49,11 +56,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -61,14 +78,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/docs/book/src/cronjob-tutorial/testdata/project/go.sum b/docs/book/src/cronjob-tutorial/testdata/project/go.sum index 7c7479a5a93..38f0c5b3887 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/go.sum +++ b/docs/book/src/cronjob-tutorial/testdata/project/go.sum @@ -1,5 +1,11 @@ +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -15,10 +21,15 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -31,12 +42,16 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -49,6 +64,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -98,10 +115,13 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -110,6 +130,22 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -136,6 +172,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -165,6 +203,14 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -172,6 +218,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -184,14 +231,20 @@ k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xa k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index 89c8531467b..a2568e3ce3f 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -57,14 +58,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -97,10 +98,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 7042e87ebde..0fe6fa9c410 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -#patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/docs/book/src/getting-started/testdata/project/config/default/metrics_service.yaml b/docs/book/src/getting-started/testdata/project/config/default/metrics_service.yaml index 1cb008b3b59..4425b9b8977 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/metrics_service.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml index 91d41742932..1dea5d5fd7b 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml index 75c7d0c2b48..603bdba9fb5 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role_binding.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/metrics_reader_role.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/docs/book/src/getting-started/testdata/project/go.mod b/docs/book/src/getting-started/testdata/project/go.mod index c40c0002c35..1c3baff1cd9 100644 --- a/docs/book/src/getting-started/testdata/project/go.mod +++ b/docs/book/src/getting-started/testdata/project/go.mod @@ -14,13 +14,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -29,11 +34,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,11 +55,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -60,14 +77,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/docs/book/src/getting-started/testdata/project/go.sum b/docs/book/src/getting-started/testdata/project/go.sum index e6c179a6112..9ce9c0dd5da 100644 --- a/docs/book/src/getting-started/testdata/project/go.sum +++ b/docs/book/src/getting-started/testdata/project/go.sum @@ -1,5 +1,11 @@ +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -15,10 +21,15 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -31,12 +42,16 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -49,6 +64,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -96,10 +113,13 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -108,6 +128,22 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -134,6 +170,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -163,6 +201,14 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -170,6 +216,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -182,14 +229,20 @@ k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xa k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index b9a68547ddf..bd94364530e 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -40,31 +40,37 @@ Further information can be found bellow in this document. -## Enabling the Metrics +## Metrics Configuration -First, you will need enable the Metrics by uncommenting the following line -in the file `config/default/kustomization.yaml`, see: +By looking at the file `config/default/kustomization.yaml` you can +check the metrics are exposed by default: ```yaml -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml ``` ```yaml -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. -# More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +patches: + # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. + # More info: https://book.kubebuilder.io/reference/metrics + - path: manager_metrics_patch.yaml + target: + kind: Deployment ``` -Note that projects are scaffolded by default passing the flag `--metrics-bind-address=0` -to the manager to ensure that metrics are disabled. See the [controller-runtime -implementation](https://github.com/kubernetes-sigs/controller-runtime/blob/834905b07c7b5a78e86d21d764f7c2fdaa9602e0/pkg/metrics/server/server.go#L119-L122) -where the server creation will be skipped in this case. +Then, you can check in the `cmd/main.go` where metrics server +is configured: -## Protecting the Metrics +```go +// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. +// For more info: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/metrics/server +Metrics: metricsserver.Options{ + ... +}, +``` + +## Metrics Protection Unprotected metrics endpoints can expose valuable data to unauthorized users, such as system performance, application behavior, and potentially confidential @@ -72,6 +78,125 @@ operational metrics. This exposure can lead to security vulnerabilities where an attacker could gain insights into the system's operation and exploit weaknesses. +### By using authn/authz (Enabled by default) + +To mitigate these risks, Kubebuilder projects utilize authentication (authn) and authorization (authz) to protect the +metrics endpoint. This approach ensures that only authorized users and service accounts can access sensitive metrics +data, enhancing the overall security of the system. + +In the past, the [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) was employed to provide this protection. +However, its usage has been discontinued in recent versions. Since the release of `v4.1.0`, projects have had the +metrics endpoint enabled and protected by default using the [WithAuthenticationAndAuthorization](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/metrics/server) +feature provided by controller-runtime. + +Therefore, you will find the following configuration: + +- In the `cmd/main.go`: + +```go +Metrics: metricsserver.Options{ + ... + FilterProvider: filters.WithAuthenticationAndAuthorization, + ... +} +``` + +This configuration leverages the FilterProvider to enforce authentication and authorization on the metrics endpoint. +By using this method, you ensure that the endpoint is accessible only to those with the appropriate permissions. + +- In the `config/rbac/kustomization.yaml`: + +```yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml +``` + +In this way, only Pods using the `ServiceAccount` token are authorized to read the metrics endpoint. For example: + +```ymal +apiVersion: v1 +kind: Pod +metadata: + name: metrics-consumer + namespace: system +spec: + # Use the scaffolded service account name to allow authn/authz + serviceAccountName: controller-manager + containers: + - name: metrics-consumer + image: curlimages/curl:7.78.0 + command: ["/bin/sh"] + args: + - "-c" + - > + while true; + do + # Note here that we are passing the token obtained from the ServiceAccount to curl the metrics endpoint + curl -s -k -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + https://controller-manager-metrics-service.system.svc.cluster.local:8443/metrics; + sleep 60; + done +``` + + + + + ### By using Network Policy NetworkPolicy acts as a basic firewall for pods within a Kubernetes cluster, controlling traffic @@ -88,22 +213,6 @@ the `Service` under `config/default/metrics_service.yaml` and the `ServiceMonitor` under `config/prometheus/monitor.yaml` to use a secure HTTPS port and ensure the necessary certificate is applied. -### By using Controller-Runtime new feature - -Also, you might want to check the new feature added in Controller-Runtime via -the [pr](https://github.com/kubernetes-sigs/controller-runtime/pull/2407) which can handle authentication (`authn`), -authorization (`authz`) similar to [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) has been doing. - - - ## Exporting Metrics for Prometheus Follow the steps below to export the metrics using the Prometheus Operator: @@ -152,14 +261,6 @@ for the metrics exported from the namespace where the project is running Screenshot 2019-10-02 at 13 07 13 -## Consuming the Metrics from other Pods. - -Then, see an example to create a Pod using Curl to reach out the metrics: - -```sh -kubectl run curl --restart=Never -n --image=curlimages/curl:7.78.0 -- /bin/sh -c "curl -v http://-controller-manager-metrics-service..svc.cluster.local:8080/metrics" -``` - ## Publishing Additional Metrics If you wish to publish additional metrics from your controllers, this diff --git a/pkg/plugin/util/util.go b/pkg/plugin/util/util.go index bae4e76f47f..407e3258407 100644 --- a/pkg/plugin/util/util.go +++ b/pkg/plugin/util/util.go @@ -176,6 +176,48 @@ func UncommentCode(filename, target, prefix string) error { return os.WriteFile(filename, out.Bytes(), 0644) } +// CommentCode searches for target in the file and adds the comment prefix +// to the target content. The target content may span multiple lines. +func CommentCode(filename, target, prefix string) error { + // Read the file content + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + // Find the target code to be commented + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be commented", target) + } + + // Create a buffer to hold the modified content + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + // Add the comment prefix to each line of the target code + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + for scanner.Scan() { + _, err := out.WriteString(prefix + scanner.Text() + "\n") + if err != nil { + return err + } + } + + // Write the rest of the file content + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + + // Write the modified content back to the file + return os.WriteFile(filename, out.Bytes(), 0644) +} + // ImplementWebhooks will mock an webhook data func ImplementWebhooks(filename string) error { // false positive diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/init.go b/pkg/plugins/common/kustomize/v2/scaffolds/init.go index cac7231885a..0d94427872e 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/init.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/init.go @@ -69,6 +69,9 @@ func (s *initScaffolder) Scaffold() error { // We need to create a Role because if the project // has not CRD define the controller-gen will not generate this file &rbac.Role{}, + &rbac.MetricsAuthRole{}, + &rbac.MetricsAuthRoleBinding{}, + &rbac.MetricsReaderRole{}, &rbac.LeaderElectionRole{}, &rbac.LeaderElectionRoleBinding{}, &rbac.ServiceAccount{}, diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 94ebda6d043..7c9f9ec8496 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -70,17 +70,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -#patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go index 2af98f59502..6cd3c4bd649 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_metrics_patch.go @@ -42,8 +42,8 @@ func (f *ManagerMetricsPatch) SetTemplateDefaults() error { return nil } -const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow exposing the metrics endpoint securely +const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/metrics_service.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/metrics_service.go index 7b6b40b564b..06a29990221 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/metrics_service.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/metrics_service.go @@ -52,10 +52,10 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go index d53b0b18a56..73ce389aa01 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go @@ -54,8 +54,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go index 099c099ea58..aad8b278d36 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go @@ -53,4 +53,13 @@ const kustomizeRBACTemplate = `resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role.go new file mode 100644 index 00000000000..8d353b12830 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role.go @@ -0,0 +1,58 @@ +/* + Copyright 2020 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rbac + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsAuthRole{} + +// MetricsAuthRole scaffolds a file that defines the role for the auth proxy +type MetricsAuthRole struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements file.Template +func (f *MetricsAuthRole) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "rbac", "metrics_auth_role.yaml") + } + + f.TemplateBody = metricsAuthRoleTemplate + + return nil +} + +const metricsAuthRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role_binding.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role_binding.go new file mode 100644 index 00000000000..dc7f1630d69 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_auth_role_binding.go @@ -0,0 +1,53 @@ +/* + Copyright 2020 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rbac + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsAuthRole{} + +// MetricsAuthRoleBinding scaffolds a file that defines the role binding for the auth proxy +type MetricsAuthRoleBinding struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements file.Template +func (f *MetricsAuthRoleBinding) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "rbac", "metrics_auth_role_binding.yaml") + } + + f.TemplateBody = metricsAuthRoleBindingTemplate + + return nil +} + +const metricsAuthRoleBindingTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_reader_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_reader_role.go new file mode 100644 index 00000000000..4e037536d34 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_reader_role.go @@ -0,0 +1,50 @@ +/* + Copyright 2020 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package rbac + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &MetricsReaderRole{} + +// MetricsReaderRole scaffolds a file that defines the role for the auth proxy +type MetricsReaderRole struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements file.Template +func (f *MetricsReaderRole) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "rbac", "metrics_reader_role.yaml") + } + + f.TemplateBody = metricsReaderRoleTemplate + + return nil +} + +const metricsReaderRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get +` diff --git a/pkg/plugins/golang/v4/scaffolds/init.go b/pkg/plugins/golang/v4/scaffolds/init.go index 56e13c29002..847e699e2bd 100644 --- a/pkg/plugins/golang/v4/scaffolds/init.go +++ b/pkg/plugins/golang/v4/scaffolds/init.go @@ -140,7 +140,9 @@ func (s *initScaffolder) Scaffold() error { } return scaffold.Execute( - &templates.Main{}, + &templates.Main{ + ControllerRuntimeVersion: ControllerRuntimeVersion, + }, &templates.GoMod{ ControllerRuntimeVersion: ControllerRuntimeVersion, }, diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/main.go index 78771892525..7f0fa141456 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/main.go @@ -33,6 +33,8 @@ type Main struct { machinery.BoilerplateMixin machinery.DomainMixin machinery.RepositoryMixin + + ControllerRuntimeVersion string } // SetTemplateDefaults implements file.Template @@ -181,6 +183,7 @@ func (f *MainUpdater) GetCodeFragments() machinery.CodeFragmentsMap { return fragments } +// nolint:lll var mainTemplate = `{{ .Boilerplate }} package main @@ -200,6 +203,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/webhook" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" %s @@ -222,14 +226,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. " + - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. " + + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. " + "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -262,10 +266,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@{{ .ControllerRuntimeVersion }}/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/pkg/plugins/optional/grafana/v1alpha/scaffolds/init.go b/pkg/plugins/optional/grafana/v1alpha/scaffolds/init.go index 7c220a578b4..09fa2feb03d 100644 --- a/pkg/plugins/optional/grafana/v1alpha/scaffolds/init.go +++ b/pkg/plugins/optional/grafana/v1alpha/scaffolds/init.go @@ -53,4 +53,5 @@ func (s *initScaffolder) Scaffold() error { &templates.ResourcesManifest{}, &templates.CustomMetricsConfigManifest{ConfigPath: string(configFilePath)}, ) + } diff --git a/test/e2e/deployimage/plugin_cluster_test.go b/test/e2e/deployimage/plugin_cluster_test.go index 3ce1bc6d82b..99900b6c4e3 100644 --- a/test/e2e/deployimage/plugin_cluster_test.go +++ b/test/e2e/deployimage/plugin_cluster_test.go @@ -45,7 +45,7 @@ var _ = Describe("kubebuilder", func() { AfterEach(func() { By("clean up API objects created during the test") - kbc.CleanupManifests(filepath.Join("config", "default")) + Expect(kbc.Make("undeploy")).To(Succeed()) By("removing controller image and working dir") kbc.Destroy() diff --git a/test/e2e/utils/test_context.go b/test/e2e/utils/test_context.go index 60ae5c4bbaa..6f3523d5013 100644 --- a/test/e2e/utils/test_context.go +++ b/test/e2e/utils/test_context.go @@ -166,23 +166,6 @@ func (t *TestContext) UninstallPrometheusOperManager() { } } -// CleanupManifests is a helper func to run kustomize build and pipe the output to kubectl delete -f - -func (t *TestContext) CleanupManifests(dir string) { - kustomizePath := filepath.Join(t.Dir, "bin", "kustomize") - if _, err := os.Stat(kustomizePath); err != nil { - // Just fail below with an error about kustomize not being installed globally. - kustomizePath = "kustomize" - } - cmd := exec.Command(kustomizePath, "build", dir) - output, err := t.Run(cmd) - if err != nil { - warnError(err) - } - if _, err := t.Kubectl.WithInput(string(output)).Command("delete", "-f", "-"); err != nil { - warnError(err) - } -} - // Init is for running `kubebuilder init` func (t *TestContext) Init(initOptions ...string) error { initOptions = append([]string{"init"}, initOptions...) diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index bd010acf036..83f9e49aa33 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -63,12 +63,6 @@ func GenerateV4(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - "#- metrics_service.yaml", "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - metricsTarget, "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) @@ -111,6 +105,13 @@ func GenerateV4WithoutMetrics(kbc *utils.TestContext) { "#- path: webhookcainjection_patch.yaml", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) + // Disable metrics + ExpectWithOffset(1, pluginutil.CommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + "- metrics_service.yaml", "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.CommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + metricsTarget, "#")).To(Succeed()) if kbc.IsRestricted { By("uncomment kustomize files to ensure that pods are restricted") @@ -123,18 +124,9 @@ func GenerateV4WithoutWebhooks(kbc *utils.TestContext) { initingTheProject(kbc) creatingAPI(kbc) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - "#patches:", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), "#- ../prometheus", "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - "#- metrics_service.yaml", "#")).To(Succeed()) - ExpectWithOffset(1, pluginutil.UncommentCode( - filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), - metricsTarget, "#")).To(Succeed()) if kbc.IsRestricted { By("uncomment kustomize files to ensure that pods are restricted") @@ -175,9 +167,9 @@ func initingTheProject(kbc *utils.TestContext) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } -const metricsTarget = `#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment` +const metricsTarget = `- path: manager_metrics_patch.yaml + target: + kind: Deployment` //nolint:lll const certManagerTarget = `#replacements: diff --git a/test/e2e/v4/plugin_cluster_test.go b/test/e2e/v4/plugin_cluster_test.go index 452289e3cbb..8c1f83fb6ec 100644 --- a/test/e2e/v4/plugin_cluster_test.go +++ b/test/e2e/v4/plugin_cluster_test.go @@ -17,6 +17,7 @@ limitations under the License. package v4 import ( + "encoding/json" "fmt" "os" "os/exec" @@ -32,6 +33,18 @@ import ( "sigs.k8s.io/kubebuilder/v4/test/e2e/utils" ) +const ( + tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` +) + +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + var _ = Describe("kubebuilder", func() { Context("plugin go/v4", func() { var kbc *utils.TestContext @@ -48,7 +61,7 @@ var _ = Describe("kubebuilder", func() { _ = kbc.RemoveNamespaceLabelToWarnAboutRestricted() By("clean up API objects created during the test") - kbc.CleanupManifests(filepath.Join("config", "default")) + Expect(kbc.Make("undeploy")).To(Succeed()) By("removing controller image and working dir") kbc.Destroy() @@ -284,9 +297,19 @@ func getControllerName(kbc *utils.TestContext) string { // getMetricsOutput return the metrics output from curl pod func getMetricsOutput(kbc *utils.TestContext) string { + _, err := kbc.Kubectl.Command( + "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), + fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", kbc.TestSuffix), + fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + token, err := serviceAccountToken(kbc) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + var metricsOutput string By("validating that the controller-manager service is available") - _, err := kbc.Kubectl.Get( + _, err = kbc.Kubectl.Get( true, "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), ) @@ -311,7 +334,7 @@ func getMetricsOutput(kbc *utils.TestContext) string { "Service endpoint should be ready") By("creating a curl pod to access the metrics endpoint") - cmdOpts := cmdOptsToCreateCurlPod(kbc) + cmdOpts := cmdOptsToCreateCurlPod(kbc, token) _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) ExpectWithOffset(2, err).NotTo(HaveOccurred()) @@ -340,9 +363,19 @@ func getMetricsOutput(kbc *utils.TestContext) string { } func metricsShouldBeUnavailable(kbc *utils.TestContext) { + _, err := kbc.Kubectl.Command( + "create", "clusterrolebinding", fmt.Sprintf("metrics-%s", kbc.TestSuffix), + fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", kbc.TestSuffix), + fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + token, err := serviceAccountToken(kbc) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + ExpectWithOffset(2, token).NotTo(BeEmpty()) + By("creating a curl pod to access the metrics endpoint") - cmdOpts := cmdOptsToCreateCurlPod(kbc) - _, err := kbc.Kubectl.CommandInNamespace(cmdOpts...) + cmdOpts := cmdOptsToCreateCurlPod(kbc, token) + _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) ExpectWithOffset(2, err).NotTo(HaveOccurred()) By("validating that the curl pod fail as expected") @@ -369,7 +402,7 @@ func metricsShouldBeUnavailable(kbc *utils.TestContext) { removeCurlPod(kbc) } -func cmdOptsToCreateCurlPod(kbc *utils.TestContext) []string { +func cmdOptsToCreateCurlPod(kbc *utils.TestContext, token string) []string { // nolint:lll cmdOpts := []string{ "run", "curl", @@ -377,8 +410,8 @@ func cmdOptsToCreateCurlPod(kbc *utils.TestContext) []string { "--namespace", kbc.Kubectl.Namespace, "--image=curlimages/curl:7.78.0", "--", - "/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics", - kbc.TestSuffix, kbc.Kubectl.Namespace), + "/bin/sh", "-c", fmt.Sprintf("curl -v -k -H 'Authorization: Bearer %s' https://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8443/metrics", + token, kbc.TestSuffix, kbc.Kubectl.Namespace), } return cmdOpts } @@ -388,3 +421,42 @@ func removeCurlPod(kbc *utils.TestContext) { _, err := kbc.Kubectl.Delete(true, "pods/curl") ExpectWithOffset(3, err).NotTo(HaveOccurred()) } + +// serviceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. This function leverages the k8s' +// TokenRequest API in raw format in order to make it generic for all version of the k8s that +// is currently being supported in kubebuilder test infra. +// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. +func serviceAccountToken(kbc *utils.TestContext) (out string, err error) { + secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) + tokenRequestFile := filepath.Join(kbc.Dir, secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) + if err != nil { + return out, err + } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + rawJson, err = kbc.Kubectl.Command( + "create", + "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + kbc.Kubectl.Namespace, + kbc.Kubectl.ServiceAccount, + ), + "-f", tokenRequestFile, + ) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) + + return out, err +} diff --git a/testdata/project-v4-multigroup-with-deploy-image/cmd/main.go b/testdata/project-v4-multigroup-with-deploy-image/cmd/main.go index ecd51d78d7e..af76f24471a 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/cmd/main.go +++ b/testdata/project-v4-multigroup-with-deploy-image/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -82,14 +83,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -122,10 +123,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml index b81da332f7b..303bbc1d6e8 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/metrics_service.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/metrics_service.yaml index a01a6872e4a..d2be1833a4c 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/metrics_service.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml index afc86491c6f..34e71995b62 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml index a2f9efd10a3..5bdd66e47ff 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_reader_role.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml index 9a4abb101e0..d12dacfaf85 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml @@ -1191,6 +1191,34 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: project-v4-multigroup-with-deploy-image-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-multigroup-with-deploy-image-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -1472,6 +1500,37 @@ subjects: name: project-v4-multigroup-with-deploy-image-controller-manager namespace: project-v4-multigroup-with-deploy-image-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-v4-multigroup-with-deploy-image-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-v4-multigroup-with-deploy-image-metrics-auth-role +subjects: +- kind: ServiceAccount + name: project-v4-multigroup-with-deploy-image-controller-manager + namespace: project-v4-multigroup-with-deploy-image-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-multigroup-with-deploy-image + control-plane: controller-manager + name: project-v4-multigroup-with-deploy-image-controller-manager-metrics-service + namespace: project-v4-multigroup-with-deploy-image-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: v1 kind: Service metadata: @@ -1511,6 +1570,7 @@ spec: spec: containers: - args: + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 command: diff --git a/testdata/project-v4-multigroup-with-deploy-image/go.mod b/testdata/project-v4-multigroup-with-deploy-image/go.mod index 29f6fe9ab3f..298af54172e 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/go.mod +++ b/testdata/project-v4-multigroup-with-deploy-image/go.mod @@ -14,13 +14,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -29,11 +34,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,11 +55,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -60,14 +77,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index f89a7544aae..0fc65deabea 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -82,14 +83,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -122,10 +123,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index fd22f14ceb3..88ea8e9f042 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/testdata/project-v4-multigroup/config/default/metrics_service.yaml b/testdata/project-v4-multigroup/config/default/metrics_service.yaml index efdd5394534..afd67bb43af 100644 --- a/testdata/project-v4-multigroup/config/default/metrics_service.yaml +++ b/testdata/project-v4-multigroup/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml index 7b9bc2077a7..89d2f351f5b 100644 --- a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml index a2f9efd10a3..5bdd66e47ff 100644 --- a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/testdata/project-v4-multigroup/config/rbac/metrics_auth_role.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/testdata/project-v4-multigroup/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/testdata/project-v4-multigroup/config/rbac/metrics_auth_role_binding.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/testdata/project-v4-multigroup/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/testdata/project-v4-multigroup/config/rbac/metrics_reader_role.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/testdata/project-v4-multigroup/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/testdata/project-v4-multigroup/dist/install.yaml b/testdata/project-v4-multigroup/dist/install.yaml index 1694d276d0f..62f26d13a4f 100644 --- a/testdata/project-v4-multigroup/dist/install.yaml +++ b/testdata/project-v4-multigroup/dist/install.yaml @@ -1191,6 +1191,34 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: project-v4-multigroup-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-multigroup-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: labels: app.kubernetes.io/managed-by: kustomize @@ -1472,6 +1500,37 @@ subjects: name: project-v4-multigroup-controller-manager namespace: project-v4-multigroup-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-v4-multigroup-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-v4-multigroup-metrics-auth-role +subjects: +- kind: ServiceAccount + name: project-v4-multigroup-controller-manager + namespace: project-v4-multigroup-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-multigroup + control-plane: controller-manager + name: project-v4-multigroup-controller-manager-metrics-service + namespace: project-v4-multigroup-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: v1 kind: Service metadata: @@ -1511,6 +1570,7 @@ spec: spec: containers: - args: + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 command: diff --git a/testdata/project-v4-multigroup/go.mod b/testdata/project-v4-multigroup/go.mod index 4ebad3298e0..720463df8ec 100644 --- a/testdata/project-v4-multigroup/go.mod +++ b/testdata/project-v4-multigroup/go.mod @@ -14,13 +14,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -29,11 +34,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,11 +55,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -60,14 +77,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/testdata/project-v4-with-deploy-image/cmd/main.go b/testdata/project-v4-with-deploy-image/cmd/main.go index da45f1c23f5..f3991736c25 100644 --- a/testdata/project-v4-with-deploy-image/cmd/main.go +++ b/testdata/project-v4-with-deploy-image/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -57,14 +58,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -97,10 +98,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml index a446d6d4a0a..93ec80cc093 100644 --- a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/testdata/project-v4-with-deploy-image/config/default/metrics_service.yaml b/testdata/project-v4-with-deploy-image/config/default/metrics_service.yaml index 905ebc18c94..d774226430d 100644 --- a/testdata/project-v4-with-deploy-image/config/default/metrics_service.yaml +++ b/testdata/project-v4-with-deploy-image/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml index 2d08ceebac3..c5a21a37e1d 100644 --- a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml index 1a988b476a3..60653a8cf59 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/testdata/project-v4-with-deploy-image/config/rbac/metrics_reader_role.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/testdata/project-v4-with-deploy-image/dist/install.yaml b/testdata/project-v4-with-deploy-image/dist/install.yaml index 6a99fdc2f2e..7b9849b6a18 100644 --- a/testdata/project-v4-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-with-deploy-image/dist/install.yaml @@ -516,6 +516,34 @@ rules: - get --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-with-deploy-image-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-with-deploy-image-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -548,6 +576,37 @@ subjects: name: project-v4-with-deploy-image-controller-manager namespace: project-v4-with-deploy-image-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-v4-with-deploy-image-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-v4-with-deploy-image-metrics-auth-role +subjects: +- kind: ServiceAccount + name: project-v4-with-deploy-image-controller-manager + namespace: project-v4-with-deploy-image-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-with-deploy-image + control-plane: controller-manager + name: project-v4-with-deploy-image-controller-manager-metrics-service + namespace: project-v4-with-deploy-image-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: v1 kind: Service metadata: @@ -587,6 +646,7 @@ spec: spec: containers: - args: + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 command: diff --git a/testdata/project-v4-with-deploy-image/go.mod b/testdata/project-v4-with-deploy-image/go.mod index a8b57c3114d..7df5b2ebf37 100644 --- a/testdata/project-v4-with-deploy-image/go.mod +++ b/testdata/project-v4-with-deploy-image/go.mod @@ -14,13 +14,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -29,11 +34,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,11 +55,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -60,14 +77,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/testdata/project-v4-with-grafana/cmd/main.go b/testdata/project-v4-with-grafana/cmd/main.go index de05a8dc7ac..e20a8db0b21 100644 --- a/testdata/project-v4-with-grafana/cmd/main.go +++ b/testdata/project-v4-with-grafana/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" // +kubebuilder:scaffold:imports @@ -53,14 +54,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -93,10 +94,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/testdata/project-v4-with-grafana/config/default/kustomization.yaml b/testdata/project-v4-with-grafana/config/default/kustomization.yaml index d17c01b9af8..1f9f8a6f785 100644 --- a/testdata/project-v4-with-grafana/config/default/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -#patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/testdata/project-v4-with-grafana/config/default/metrics_service.yaml b/testdata/project-v4-with-grafana/config/default/metrics_service.yaml index 4c40c3579cb..fded59c2325 100644 --- a/testdata/project-v4-with-grafana/config/default/metrics_service.yaml +++ b/testdata/project-v4-with-grafana/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml index 5f3e9887eae..4db5a1f1396 100644 --- a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml index 166fe79868f..5619aa00943 100644 --- a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml @@ -9,3 +9,12 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role_binding.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/testdata/project-v4-with-grafana/config/rbac/metrics_reader_role.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/testdata/project-v4-with-grafana/dist/install.yaml b/testdata/project-v4-with-grafana/dist/install.yaml index 6831125a6a1..ac9427cd5ef 100644 --- a/testdata/project-v4-with-grafana/dist/install.yaml +++ b/testdata/project-v4-with-grafana/dist/install.yaml @@ -75,6 +75,34 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-with-grafana-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-with-grafana-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -107,6 +135,37 @@ subjects: name: project-v4-with-grafana-controller-manager namespace: project-v4-with-grafana-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-v4-with-grafana-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-v4-with-grafana-metrics-auth-role +subjects: +- kind: ServiceAccount + name: project-v4-with-grafana-controller-manager + namespace: project-v4-with-grafana-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-with-grafana + control-plane: controller-manager + name: project-v4-with-grafana-controller-manager-metrics-service + namespace: project-v4-with-grafana-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -130,6 +189,7 @@ spec: spec: containers: - args: + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 command: diff --git a/testdata/project-v4-with-grafana/go.mod b/testdata/project-v4-with-grafana/go.mod index 88a7123174f..3b77a9ef619 100644 --- a/testdata/project-v4-with-grafana/go.mod +++ b/testdata/project-v4-with-grafana/go.mod @@ -13,13 +13,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -28,11 +33,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -47,11 +54,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -59,15 +76,21 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.30.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index 861436bc500..4d38cc131e8 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -57,14 +58,14 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ - "Use the port :8080. If not set, it will be 0 in order to disable the metrics server") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -97,10 +98,25 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html Metrics: metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - TLSOpts: tlsOpts, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + FilterProvider: filters.WithAuthenticationAndAuthorization, }, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index 9bb07eabee4..ef81f77b8fa 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -25,17 +25,16 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus -# [METRICS] To enable the controller manager metrics service, uncomment the following line. -#- metrics_service.yaml +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint. +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -# If you want to expose the metric endpoint of your controller-manager uncomment the following line. -#- path: manager_metrics_patch.yaml -# target: -# kind: Deployment +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4/config/default/manager_metrics_patch.yaml b/testdata/project-v4/config/default/manager_metrics_patch.yaml index 6c546ae4ca7..2aaef6536f4 100644 --- a/testdata/project-v4/config/default/manager_metrics_patch.yaml +++ b/testdata/project-v4/config/default/manager_metrics_patch.yaml @@ -1,4 +1,4 @@ -# This patch adds the args to allow exposing the metrics endpoint securely +# This patch adds the args to allow exposing the metrics endpoint using HTTPS - op: add path: /spec/template/spec/containers/0/args/0 - value: --metrics-bind-address=:8080 + value: --metrics-bind-address=:8443 diff --git a/testdata/project-v4/config/default/metrics_service.yaml b/testdata/project-v4/config/default/metrics_service.yaml index 3cd90980fa3..d29d5ff207a 100644 --- a/testdata/project-v4/config/default/metrics_service.yaml +++ b/testdata/project-v4/config/default/metrics_service.yaml @@ -9,9 +9,9 @@ metadata: namespace: system spec: ports: - - name: http - port: 8080 + - name: https + port: 8443 protocol: TCP - targetPort: 8080 + targetPort: 8443 selector: control-plane: controller-manager diff --git a/testdata/project-v4/config/prometheus/monitor.yaml b/testdata/project-v4/config/prometheus/monitor.yaml index d4a09ebf9fe..1e3f1aec14c 100644 --- a/testdata/project-v4/config/prometheus/monitor.yaml +++ b/testdata/project-v4/config/prometheus/monitor.yaml @@ -11,8 +11,20 @@ metadata: spec: endpoints: - path: /metrics - port: http # Ensure this is the name of the port that exposes HTTP metrics - scheme: http + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key + insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4/config/rbac/kustomization.yaml b/testdata/project-v4/config/rbac/kustomization.yaml index 2a7debcd3a1..b6ac31fc556 100644 --- a/testdata/project-v4/config/rbac/kustomization.yaml +++ b/testdata/project-v4/config/rbac/kustomization.yaml @@ -9,6 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/testdata/project-v4/config/rbac/metrics_auth_role.yaml b/testdata/project-v4/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..32d2e4ec6b0 --- /dev/null +++ b/testdata/project-v4/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/testdata/project-v4/config/rbac/metrics_auth_role_binding.yaml b/testdata/project-v4/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000000..e775d67ff08 --- /dev/null +++ b/testdata/project-v4/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/testdata/project-v4/config/rbac/metrics_reader_role.yaml b/testdata/project-v4/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000000..51a75db47a5 --- /dev/null +++ b/testdata/project-v4/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/testdata/project-v4/dist/install.yaml b/testdata/project-v4/dist/install.yaml index 4144d2f3017..7bd99099755 100644 --- a/testdata/project-v4/dist/install.yaml +++ b/testdata/project-v4/dist/install.yaml @@ -510,6 +510,34 @@ rules: - update --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: project-v4-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -542,6 +570,37 @@ subjects: name: project-v4-controller-manager namespace: project-v4-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: project-v4-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: project-v4-metrics-auth-role +subjects: +- kind: ServiceAccount + name: project-v4-controller-manager + namespace: project-v4-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4 + control-plane: controller-manager + name: project-v4-controller-manager-metrics-service + namespace: project-v4-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- apiVersion: v1 kind: Service metadata: @@ -581,6 +640,7 @@ spec: spec: containers: - args: + - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 command: diff --git a/testdata/project-v4/go.mod b/testdata/project-v4/go.mod index c6fbfd9a28c..ab479be096e 100644 --- a/testdata/project-v4/go.mod +++ b/testdata/project-v4/go.mod @@ -14,13 +14,18 @@ require ( ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -29,11 +34,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,11 +55,21 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -60,14 +77,20 @@ require ( golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect