From 234d7bb0a96ffb7021e2042374e1f32bd1dc12c5 Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Sun, 7 Apr 2024 05:55:44 +0100 Subject: [PATCH] :warning: replace the kube-rbac-proxy usage with NetworkPolicy --- .github/workflows/test-sample-go.yml | 4 +- .../project/config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 34 -------- .../config/default/manager_metrics_patch.yaml | 10 +++ .../project/config/policy/kustomization.yaml | 2 + .../project/config/policy/policy.yaml | 27 ++++++ .../project/config/prometheus/monitor.yaml | 7 +- .../project/config/rbac/kustomization.yaml | 11 +-- .../rbac/metrics_client_cluster_role.yaml} | 2 +- .../project/config/rbac/metrics_role.yaml} | 6 +- .../config/rbac/metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- .../project/config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 55 ------------ .../config/default/manager_metrics_patch.yaml | 15 ++++ .../project/config/policy/kustomization.yaml | 2 + .../project/config/policy/policy.yaml | 27 ++++++ .../project/config/prometheus/monitor.yaml | 7 +- .../project/config/rbac/kustomization.yaml | 11 +-- .../rbac/metrics_client_cluster_role.yaml} | 2 +- .../project/config/rbac/metrics_role.yaml} | 6 +- .../config/rbac/metrics_role_binding.yaml} | 8 +- .../project/config/rbac/metrics_service.yaml} | 8 +- .../project/config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../project/config/policy/kustomization.yaml | 2 + .../project/config/policy/policy.yaml | 27 ++++++ .../project/config/prometheus/monitor.yaml | 7 +- .../project/config/rbac/kustomization.yaml | 11 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- .../project/config/rbac/metrics_role.yaml} | 6 +- .../config/rbac/metrics_role_binding.yaml} | 8 +- .../project/config/rbac/metrics_service.yaml} | 8 +- docs/book/src/reference/metrics.md | 35 +++++++- .../cronjob-tutorial/generate_cronjob.go | 7 -- pkg/plugin/util/util.go | 30 +++++++ .../common/kustomize/v2/scaffolds/api.go | 27 ++++-- .../common/kustomize/v2/scaffolds/init.go | 13 +-- .../internal/templates/config/.DS_Store | Bin 0 -> 6148 bytes .../config/kdefault/kustomization.go | 11 ++- .../kdefault/manager_auth_proxy_patch.go | 40 ++------- .../templates/config/policy/kustomization.go | 45 ++++++++++ .../templates/config/policy/policy.go | 71 +++++++++++++++ .../templates/config/prometheus/monitor.go | 7 +- .../templates/config/rbac/kustomization.go | 10 +-- ..._client_role.go => metrics_client_role.go} | 16 ++-- .../{auth_proxy_role.go => metrics_role.go} | 20 ++--- ...ole_binding.go => metrics_role_binding.go} | 22 ++--- ...th_proxy_service.go => metrics_service.go} | 22 ++--- test/e2e/v4/plugin_cluster_test.go | 81 ++---------------- .../config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../config/policy/kustomization.yaml | 2 + .../config/policy/policy.yaml | 27 ++++++ .../config/prometheus/monitor.yaml | 7 +- .../config/rbac/kustomization.yaml | 11 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- ...auth_proxy_role.yaml => metrics_role.yaml} | 6 +- ...binding.yaml => metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- .../dist/install.yaml | 76 ++++++++-------- .../config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../config/policy/kustomization.yaml | 2 + .../config/policy/policy.yaml | 27 ++++++ .../config/prometheus/monitor.yaml | 7 +- .../config/rbac/kustomization.yaml | 11 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- ...auth_proxy_role.yaml => metrics_role.yaml} | 6 +- ...binding.yaml => metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- .../project-v4-multigroup/dist/install.yaml | 76 ++++++++-------- .../config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../config/policy/kustomization.yaml | 2 + .../config/policy/policy.yaml | 27 ++++++ .../config/prometheus/monitor.yaml | 7 +- .../config/rbac/kustomization.yaml | 11 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- ...auth_proxy_role.yaml => metrics_role.yaml} | 6 +- ...binding.yaml => metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- .../dist/install.yaml | 76 ++++++++-------- .../config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../config/policy/kustomization.yaml | 2 + .../config/policy/policy.yaml | 27 ++++++ .../config/prometheus/monitor.yaml | 7 +- .../config/rbac/kustomization.yaml | 10 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- ...auth_proxy_role.yaml => metrics_role.yaml} | 6 +- ...binding.yaml => metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- .../project-v4-with-grafana/dist/install.yaml | 76 ++++++++-------- .../config/default/kustomization.yaml | 11 ++- .../default/manager_auth_proxy_patch.yaml | 39 --------- .../config/default/manager_metrics_patch.yaml | 15 ++++ .../config/policy/kustomization.yaml | 2 + testdata/project-v4/config/policy/policy.yaml | 27 ++++++ .../project-v4/config/prometheus/monitor.yaml | 7 +- .../project-v4/config/rbac/kustomization.yaml | 11 +-- ....yaml => metrics_client_cluster_role.yaml} | 2 +- ...auth_proxy_role.yaml => metrics_role.yaml} | 6 +- ...binding.yaml => metrics_role_binding.yaml} | 8 +- ...roxy_service.yaml => metrics_service.yaml} | 8 +- testdata/project-v4/dist/install.yaml | 76 ++++++++-------- 111 files changed, 1035 insertions(+), 899 deletions(-) delete mode 100644 docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml create mode 100644 docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml create mode 100644 docs/book/src/component-config-tutorial/testdata/project/config/policy/kustomization.yaml create mode 100644 docs/book/src/component-config-tutorial/testdata/project/config/policy/policy.yaml rename docs/book/src/{cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml => component-config-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml} (88%) rename docs/book/src/{getting-started/testdata/project/config/rbac/auth_proxy_role.yaml => component-config-tutorial/testdata/project/config/rbac/metrics_role.yaml} (79%) rename docs/book/src/{cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml => component-config-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml} (73%) rename docs/book/src/component-config-tutorial/testdata/project/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (81%) delete mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/policy/kustomization.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/policy/policy.yaml rename docs/book/src/{component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml => cronjob-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml} (88%) rename docs/book/src/{component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml => cronjob-tutorial/testdata/project/config/rbac/metrics_role.yaml} (79%) rename docs/book/src/{getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml => cronjob-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml} (73%) rename docs/book/src/{getting-started/testdata/project/config/rbac/auth_proxy_service.yaml => cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml} (81%) delete mode 100644 docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/policy/kustomization.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/policy/policy.yaml rename docs/book/src/getting-started/testdata/project/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (88%) rename docs/book/src/{cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml => getting-started/testdata/project/config/rbac/metrics_role.yaml} (79%) rename docs/book/src/{component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml => getting-started/testdata/project/config/rbac/metrics_role_binding.yaml} (73%) rename docs/book/src/{cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml => getting-started/testdata/project/config/rbac/metrics_service.yaml} (81%) create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/.DS_Store create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/kustomization.go create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/policy.go rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/{auth_proxy_client_role.go => metrics_client_role.go} (70%) rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/{auth_proxy_role.go => metrics_role.go} (71%) rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/{auth_proxy_role_binding.go => metrics_role_binding.go} (67%) rename pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/{auth_proxy_service.go => metrics_service.go} (72%) delete mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/policy/kustomization.yaml create mode 100644 testdata/project-v4-multigroup-with-deploy-image/config/policy/policy.yaml rename testdata/project-v4-multigroup-with-deploy-image/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (89%) rename testdata/project-v4-multigroup-with-deploy-image/config/rbac/{auth_proxy_role.yaml => metrics_role.yaml} (81%) rename testdata/project-v4-multigroup-with-deploy-image/config/rbac/{auth_proxy_role_binding.yaml => metrics_role_binding.yaml} (76%) rename testdata/project-v4-multigroup-with-deploy-image/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (83%) delete mode 100644 testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml create mode 100644 testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml create mode 100644 testdata/project-v4-multigroup/config/policy/kustomization.yaml create mode 100644 testdata/project-v4-multigroup/config/policy/policy.yaml rename testdata/project-v4-multigroup/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (89%) rename testdata/project-v4-multigroup/config/rbac/{auth_proxy_role.yaml => metrics_role.yaml} (80%) rename testdata/project-v4-multigroup/config/rbac/{auth_proxy_role_binding.yaml => metrics_role_binding.yaml} (75%) rename testdata/project-v4-multigroup/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (82%) delete mode 100644 testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/policy/kustomization.yaml create mode 100644 testdata/project-v4-with-deploy-image/config/policy/policy.yaml rename testdata/project-v4-with-deploy-image/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (89%) rename testdata/project-v4-with-deploy-image/config/rbac/{auth_proxy_role.yaml => metrics_role.yaml} (81%) rename testdata/project-v4-with-deploy-image/config/rbac/{auth_proxy_role_binding.yaml => metrics_role_binding.yaml} (75%) rename testdata/project-v4-with-deploy-image/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (83%) delete mode 100644 testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml create mode 100644 testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml create mode 100644 testdata/project-v4-with-grafana/config/policy/kustomization.yaml create mode 100644 testdata/project-v4-with-grafana/config/policy/policy.yaml rename testdata/project-v4-with-grafana/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (89%) rename testdata/project-v4-with-grafana/config/rbac/{auth_proxy_role.yaml => metrics_role.yaml} (80%) rename testdata/project-v4-with-grafana/config/rbac/{auth_proxy_role_binding.yaml => metrics_role_binding.yaml} (75%) rename testdata/project-v4-with-grafana/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (82%) delete mode 100644 testdata/project-v4/config/default/manager_auth_proxy_patch.yaml create mode 100644 testdata/project-v4/config/default/manager_metrics_patch.yaml create mode 100644 testdata/project-v4/config/policy/kustomization.yaml create mode 100644 testdata/project-v4/config/policy/policy.yaml rename testdata/project-v4/config/rbac/{auth_proxy_client_clusterrole.yaml => metrics_client_cluster_role.yaml} (88%) rename testdata/project-v4/config/rbac/{auth_proxy_role.yaml => metrics_role.yaml} (79%) rename testdata/project-v4/config/rbac/{auth_proxy_role_binding.yaml => metrics_role_binding.yaml} (74%) rename testdata/project-v4/config/rbac/{auth_proxy_service.yaml => metrics_service.yaml} (82%) diff --git a/.github/workflows/test-sample-go.yml b/.github/workflows/test-sample-go.yml index 8cc44b3ded5..dfc31ba2067 100644 --- a/.github/workflows/test-sample-go.yml +++ b/.github/workflows/test-sample-go.yml @@ -39,8 +39,8 @@ jobs: KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml" sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '42s/^#//' $KUSTOMIZATION_FILE_PATH - sed -i '46,143s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '45s/^#//' $KUSTOMIZATION_FILE_PATH + sed -i '49,145s/^#//' $KUSTOMIZATION_FILE_PATH - name: Test run: | diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml index e0e588792cf..e0966459495 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 74c49152afb..00000000000 --- a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..d3fcd93ff93 --- /dev/null +++ b/docs/book/src/component-config-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,10 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/policy/kustomization.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/docs/book/src/component-config-tutorial/testdata/project/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/policy/policy.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/docs/book/src/component-config-tutorial/testdata/project/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml index d67c6106f87..fdaef9a1c30 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml index 9f6506d4c5b..2f863bdfdf8 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/kustomization.yaml @@ -10,15 +10,16 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - projectconfig_editor_role.yaml - projectconfig_viewer_role.yaml + diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml similarity index 88% rename from docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml rename to docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml index 500386b28f0..710646aa414 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role.yaml similarity index 79% rename from docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml rename to docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role.yaml index 85e39513cc1..fe6a2a6f2e0 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml similarity index 73% rename from docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml rename to docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml index 8b5ff114fa1..714fbbc1aa6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml similarity index 81% rename from docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml index f40b3d2c0bd..a172c8c0f2a 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/component-config-tutorial/testdata/project/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index e445fec445d..e782f79da7d 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. - ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 1064aa49c80..00000000000 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/policy/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/policy/policy.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml index d67c6106f87..fdaef9a1c30 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml index 8db606e9e72..6c042b4ff98 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml @@ -10,15 +10,16 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - cronjob_editor_role.yaml - cronjob_viewer_role.yaml + diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml similarity index 88% rename from docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml rename to docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml index 500386b28f0..710646aa414 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role.yaml similarity index 79% rename from docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml rename to docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role.yaml index 85e39513cc1..fe6a2a6f2e0 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml similarity index 73% rename from docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml rename to docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml index 8b5ff114fa1..714fbbc1aa6 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml similarity index 81% rename from docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml index f40b3d2c0bd..a172c8c0f2a 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index d851be9cae7..fdbc59615bb 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/docs/book/src/getting-started/testdata/project/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/docs/book/src/getting-started/testdata/project/config/policy/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/policy/policy.yaml b/docs/book/src/getting-started/testdata/project/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml index d67c6106f87..fdaef9a1c30 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml index 3dc289427b8..e5fbbeb20f0 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml @@ -10,15 +10,16 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines # if you do not want those helpers be installed with your Project. - memcached_editor_role.yaml - memcached_viewer_role.yaml + diff --git a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_client_cluster_role.yaml similarity index 88% rename from docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml rename to docs/book/src/getting-started/testdata/project/config/rbac/metrics_client_cluster_role.yaml index 500386b28f0..710646aa414 100644 --- a/docs/book/src/getting-started/testdata/project/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_role.yaml similarity index 79% rename from docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml rename to docs/book/src/getting-started/testdata/project/config/rbac/metrics_role.yaml index 85e39513cc1..fe6a2a6f2e0 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_role.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_role_binding.yaml similarity index 73% rename from docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml rename to docs/book/src/getting-started/testdata/project/config/rbac/metrics_role_binding.yaml index 8b5ff114fa1..714fbbc1aa6 100644 --- a/docs/book/src/component-config-tutorial/testdata/project/config/rbac/auth_proxy_role_binding.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml similarity index 81% rename from docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml rename to docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml index f40b3d2c0bd..a172c8c0f2a 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/rbac/auth_proxy_service.yaml +++ b/docs/book/src/getting-started/testdata/project/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project app.kubernetes.io/part-of: project app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index 33e7e3b0a13..57be1ec1c10 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -5,9 +5,38 @@ publishes [a collection of performance metrics](/reference/metrics-reference.md) ## Protecting the Metrics -These metrics are protected by [kube-rbac-proxy](https://github.com/brancz/kube-rbac-proxy) -by default if using kubebuilder. Kubebuilder v2.2.0+ scaffold a clusterrole which -can be found at `config/rbac/auth_proxy_client_clusterrole.yaml`. +These metrics are protected since the release `v3.15.0+` [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +by default if using Kubebuilder. Kubebuilder scaffold a clusterrole which +can be found at `config/rbac/metrics_client_cluster_role.yaml`. + + You will need to grant permissions to your Prometheus server so that it can scrape the protected metrics. To achieve that, you can create a diff --git a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go index 97258ee47aa..3de1859c668 100644 --- a/hack/docs/internal/cronjob-tutorial/generate_cronjob.go +++ b/hack/docs/internal/cronjob-tutorial/generate_cronjob.go @@ -583,13 +583,6 @@ func updateExample(sp *Sample) { filepath.Join(sp.ctx.Dir, "config/samples/batch_v1_cronjob.yaml"), `# TODO(user): Add fields here`, "") CheckError("fixing samples/batch_v1_cronjob.yaml", err) - - // update default/manager_auth_proxy_patch.yaml - err = pluginutil.InsertCode( - filepath.Join(sp.ctx.Dir, "config/default/manager_auth_proxy_patch.yaml"), - ` template: - spec:`, ManagerAuthProxySample) - CheckError("fixing default/manager_auth_proxy_patch.yaml", err) } func addControllerTest(sp *Sample) { diff --git a/pkg/plugin/util/util.go b/pkg/plugin/util/util.go index ebf5418adda..1495c850ed0 100644 --- a/pkg/plugin/util/util.go +++ b/pkg/plugin/util/util.go @@ -80,6 +80,36 @@ func InsertCode(filename, target, code string) error { return os.WriteFile(filename, []byte(out), 0644) } +// AppendCodeIfNotExist checks if the code does not already exist in the file, and if not, appends it to the end. +func AppendCodeIfNotExist(filename, code string) error { + contents, err := os.ReadFile(filename) + if err != nil { + return err + } + + if strings.Contains(string(contents), code) { + return nil // Code already exists, no need to append. + } + + return AppendCodeAtTheEnd(filename, code) +} + +// AppendCodeAtTheEnd appends the given code at the end of the file. +func AppendCodeAtTheEnd(filename, code string) error { + f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + return + } + }() + + _, err = f.WriteString(code) + return err +} + // InsertCodeIfNotExist insert code if it does not already exists func InsertCodeIfNotExist(filename, target, code string) error { // false positive diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/api.go b/pkg/plugins/common/kustomize/v2/scaffolds/api.go index dc875d01235..03e3ec2b8e6 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/api.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/api.go @@ -102,28 +102,37 @@ func (s *apiScaffolder) Scaffold() error { // Add scaffolded CRD Editor and Viewer roles in config/rbac/kustomization.yaml rbacKustomizeFilePath := "config/rbac/kustomization.yaml" - comment := ` -# For each CRD, "Editor" and "Viewer" roles are scaffolded by -# default, aiding admins in cluster management. Those roles are -# not used by the Project itself. You can comment the following lines -# if you do not want those helpers be installed with your Project.` - err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, - "- auth_proxy_client_clusterrole.yaml", comment) + err = pluginutil.AppendCodeIfNotExist(rbacKustomizeFilePath, + editViewRulesCommentFragment) if err != nil { - log.Errorf("Unable to add a comment in the file "+ + log.Errorf("Unable to append the edit/view roles editViewRulesCommentFragment in the file "+ "%s.", rbacKustomizeFilePath) } crdName := strings.ToLower(s.resource.Kind) if s.config.IsMultiGroup() && s.resource.Group != "" { crdName = strings.ToLower(s.resource.Group) + "_" + crdName } - err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, comment, + err = pluginutil.InsertCodeIfNotExist(rbacKustomizeFilePath, editViewRulesCommentFragment, fmt.Sprintf("\n- %[1]s_editor_role.yaml\n- %[1]s_viewer_role.yaml", crdName)) if err != nil { log.Errorf("Unable to add Editor and Viewer roles in the file "+ "%s.", rbacKustomizeFilePath) } + // Add an empty line at the end of the file + err = pluginutil.AppendCodeIfNotExist(rbacKustomizeFilePath, + ` + +`) + if err != nil { + log.Errorf("Unable to append empty line at the end of the file"+ + "%s.", rbacKustomizeFilePath) + } } return nil } + +const editViewRulesCommentFragment = `# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the Project itself. You can comment the following lines +# if you do not want those helpers be installed with your Project.` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/init.go b/pkg/plugins/common/kustomize/v2/scaffolds/init.go index baea4bb55c6..4ed341c8ba6 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/init.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/init.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/plugins" "sigs.k8s.io/kubebuilder/v3/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault" "sigs.k8s.io/kubebuilder/v3/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager" + "sigs.k8s.io/kubebuilder/v3/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy" "sigs.k8s.io/kubebuilder/v3/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus" "sigs.k8s.io/kubebuilder/v3/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac" ) @@ -64,10 +65,10 @@ func (s *initScaffolder) Scaffold() error { templates := []machinery.Builder{ &rbac.Kustomization{}, - &rbac.AuthProxyRole{}, - &rbac.AuthProxyRoleBinding{}, - &rbac.AuthProxyService{}, - &rbac.AuthProxyClientRole{}, + &rbac.MetricsRole{}, + &rbac.MonitoringBinding{}, + &rbac.MetricsService{}, + &rbac.MetricsClientRole{}, &rbac.RoleBinding{}, // We need to create a Role because if the project // has not CRD define the controller-gen will not generate this file @@ -78,7 +79,9 @@ func (s *initScaffolder) Scaffold() error { &manager.Kustomization{}, &manager.Config{Image: imageName}, &kdefault.Kustomization{}, - &kdefault.ManagerAuthProxyPatch{}, + &kdefault.ManagerMetricsPatch{}, + &policy.Kustomization{}, + &policy.NetworkPolicy{}, &kdefault.ManagerConfigPatch{}, &prometheus.Kustomization{}, &prometheus.Monitor{}, diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/.DS_Store b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4f2e901cbd366a750e60b3950c9dfb751d5b3416 GIT binary patch literal 6148 zcmeHKyG{c^3>-s>2%40X`wRTRDhgkaA3zXLil9S;^jGm+`839lg6N8c9h8O{eGp571 zj#+}(JVES*VLuSI;@5dt0!AcC>BrW{VmF2JyB5#NP$xY zu5-Ka{{KM#WBxxSX(t7wz(pxwv(LtnI`cs%>Sw@pkx7C7R^S^4C>6f| literal 0 HcmV?d00001 diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index 319bcf1a6b3..e12a49ca7dd 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -71,12 +71,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml {{ if .ComponentConfig -}} # Mount the controller config file for loading manager configurations diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go index d8d57261952..235a688b5a4 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/manager_auth_proxy_patch.go @@ -22,29 +22,28 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &ManagerAuthProxyPatch{} +var _ machinery.Template = &ManagerMetricsPatch{} -// ManagerAuthProxyPatch scaffolds a file that defines the patch that enables prometheus metrics for the manager -type ManagerAuthProxyPatch struct { +// ManagerMetricsPatch scaffolds a file that defines the patch that enables prometheus metrics for the manager +type ManagerMetricsPatch struct { machinery.TemplateMixin machinery.ComponentConfigMixin } // SetTemplateDefaults implements file.Template -func (f *ManagerAuthProxyPatch) SetTemplateDefaults() error { +func (f *ManagerMetricsPatch) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "default", "manager_auth_proxy_patch.yaml") + f.Path = filepath.Join("config", "default", "manager_metrics_patch.yaml") } - f.TemplateBody = kustomizeAuthProxyPatchTemplate + f.TemplateBody = kustomizeMetricsPatchTemplate f.IfExistsAction = machinery.Error return nil } -const kustomizeAuthProxyPatchTemplate = `# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +const kustomizeMetricsPatchTemplate = `# This patch adds the args to allow expose the metrics endpoint apiVersion: apps/v1 kind: Deployment metadata: @@ -54,34 +53,11 @@ spec: template: spec: containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi {{- if not .ComponentConfig }} - name: manager args: - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" + - "--metrics-bind-address=0.0.0.0:8080" - "--leader-elect" {{- end }} ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/kustomization.go new file mode 100644 index 00000000000..273d0c3d469 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/kustomization.go @@ -0,0 +1,45 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v3/pkg/machinery" +) + +var _ machinery.Template = &Kustomization{} + +// Kustomization scaffolds a file that defines the kustomization scheme for the prometheus folder +type Kustomization struct { + machinery.TemplateMixin +} + +// SetTemplateDefaults implements file.Template +func (f *Kustomization) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "policy", "kustomization.yaml") + } + + f.TemplateBody = kustomizationTemplate + + return nil +} + +const kustomizationTemplate = `resources: +- policy.yaml +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/policy.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/policy.go new file mode 100644 index 00000000000..c4a7f204762 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/policy/policy.go @@ -0,0 +1,71 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v3/pkg/machinery" +) + +var _ machinery.Template = &NetworkPolicy{} + +// NetworkPolicy scaffolds a file that defines the NetworkPolicy to protect the metrics endpoint +type NetworkPolicy struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements file.Template +func (f *NetworkPolicy) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "policy", "policy.yaml") + } + + f.TemplateBody = networkPolicyTemplate + + return nil +} + +const networkPolicyTemplate = `# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go index 5408fff0f59..abab6399c6d 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go @@ -58,11 +58,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go index d3ea9b22fd9..2f5e780bd88 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go @@ -54,10 +54,10 @@ const kustomizeRBACTemplate = `resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_client_role.go similarity index 70% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_client_role.go index 680eae6375b..07153af544c 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_client_role.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_client_role.go @@ -22,32 +22,32 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyClientRole{} +var _ machinery.Template = &MetricsClientRole{} -// AuthProxyClientRole scaffolds a file that defines the role for the metrics reader -type AuthProxyClientRole struct { +// MetricsClientRole scaffolds a file that defines the role for the metrics reader +type MetricsClientRole struct { machinery.TemplateMixin machinery.ProjectNameMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyClientRole) SetTemplateDefaults() error { +func (f *MetricsClientRole) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_client_clusterrole.yaml") + f.Path = filepath.Join("config", "rbac", "metrics_client_cluster_role.yaml") } - f.TemplateBody = clientClusterRoleTemplate + f.TemplateBody = metricsClientClusterRoleTemplate return nil } -const clientClusterRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +const metricsClientClusterRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: {{ .ProjectName }} app.kubernetes.io/part-of: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role.go similarity index 71% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role.go index c08756c3295..f494b2c9821 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role.go @@ -22,36 +22,36 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyRole{} +var _ machinery.Template = &MetricsRole{} -// AuthProxyRole scaffolds a file that defines the role for the auth proxy -type AuthProxyRole struct { +// MetricsRole scaffolds a file that defines the role for the auth proxy +type MetricsRole struct { machinery.TemplateMixin machinery.ProjectNameMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyRole) SetTemplateDefaults() error { +func (f *MetricsRole) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_role.yaml") + f.Path = filepath.Join("config", "rbac", "metrics_role.yaml") } - f.TemplateBody = proxyRoleTemplate + f.TemplateBody = metricsRoleTemplate return nil } -const proxyRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +const metricsRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: {{ .ProjectName }} app.kubernetes.io/part-of: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role_binding.go similarity index 67% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role_binding.go index c0df0d2d8c6..b11eb7d4e73 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_role_binding.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_role_binding.go @@ -22,40 +22,40 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyRoleBinding{} +var _ machinery.Template = &MonitoringBinding{} -// AuthProxyRoleBinding scaffolds a file that defines the role binding for the auth proxy -type AuthProxyRoleBinding struct { +// MonitoringBinding scaffolds a file that defines the role binding for the auth proxy +type MonitoringBinding struct { machinery.TemplateMixin machinery.ProjectNameMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyRoleBinding) SetTemplateDefaults() error { +func (f *MonitoringBinding) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_role_binding.yaml") + f.Path = filepath.Join("config", "rbac", "metrics_role_binding.yaml") } - f.TemplateBody = proxyRoleBindinggTemplate + f.TemplateBody = metricsRoleBindinggTemplate return nil } -const proxyRoleBindinggTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +const metricsRoleBindinggTemplate = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: {{ .ProjectName }} app.kubernetes.io/part-of: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go similarity index 72% rename from pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go rename to pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go index beda972e91e..e5a5a465eb3 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/auth_proxy_service.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go @@ -22,33 +22,33 @@ import ( "sigs.k8s.io/kubebuilder/v3/pkg/machinery" ) -var _ machinery.Template = &AuthProxyService{} +var _ machinery.Template = &MetricsService{} -// AuthProxyService scaffolds a file that defines the service for the auth proxy -type AuthProxyService struct { +// MetricsService scaffolds a file that defines the service for the auth proxy +type MetricsService struct { machinery.TemplateMixin machinery.ProjectNameMixin } // SetTemplateDefaults implements file.Template -func (f *AuthProxyService) SetTemplateDefaults() error { +func (f *MetricsService) SetTemplateDefaults() error { if f.Path == "" { - f.Path = filepath.Join("config", "rbac", "auth_proxy_service.yaml") + f.Path = filepath.Join("config", "rbac", "metrics_service.yaml") } - f.TemplateBody = authProxyServiceTemplate + f.TemplateBody = metricsServiceTemplate return nil } -const authProxyServiceTemplate = `apiVersion: v1 +const metricsServiceTemplate = `apiVersion: v1 kind: Service metadata: labels: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: {{ .ProjectName }} app.kubernetes.io/part-of: {{ .ProjectName }} app.kubernetes.io/managed-by: kustomize @@ -56,10 +56,10 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager ` diff --git a/test/e2e/v4/plugin_cluster_test.go b/test/e2e/v4/plugin_cluster_test.go index 266bf7dc210..d96646da115 100644 --- a/test/e2e/v4/plugin_cluster_test.go +++ b/test/e2e/v4/plugin_cluster_test.go @@ -17,7 +17,6 @@ limitations under the License. package v4 import ( - "encoding/json" "fmt" "os" "os/exec" @@ -37,18 +36,6 @@ import ( "sigs.k8s.io/kubebuilder/v3/test/e2e/utils" ) -const ( - tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` -) - -// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type -// that we want to use for extracting the token. -type tokenRequest struct { - Status struct { - Token string `json:"token"` - } `json:"status"` -} - var _ = Describe("kubebuilder", func() { Context("plugin go/v4", func() { var kbc *utils.TestContext @@ -121,13 +108,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { var output []byte if !isToUseInstaller { - // NOTE: If you want to run the test against a GKE cluster, you will need to grant yourself permission. - // Otherwise, you may see "... is forbidden: attempt to grant extra privileges" - // $ kubectl create clusterrolebinding myname-cluster-admin-binding \ - // --clusterrole=cluster-admin --user=myname@mycompany.com - // https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control By("deploying the controller-manager") - cmd := exec.Command("make", "deploy", "IMG="+kbc.ImageName) output, err = kbc.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -136,13 +117,7 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { err = kbc.Make("build-installer", "IMG="+kbc.ImageName) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // NOTE: If you want to run the test against a GKE cluster, you will need to grant yourself permission. - // Otherwise, you may see "... is forbidden: attempt to grant extra privileges" - // $ kubectl create clusterrolebinding myname-cluster-admin-binding \ - // --clusterrole=cluster-admin --user=myname@mycompany.com - // https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control By("deploying the controller-manager with the installer") - _, err = kbc.Kubectl.Apply(true, "-f", "dist/install.yaml") ExpectWithOffset(1, err).NotTo(HaveOccurred()) } @@ -291,20 +266,14 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller bool) { // curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. func curlMetrics(kbc *utils.TestContext) string { - By("reading the metrics token") - // Filter token query by service account in case more than one exists in a namespace. - token, err := ServiceAccountToken(kbc) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) - - By("creating a curl pod") + By("creating a curl pod to access the metrics endpoint") cmdOpts := []string{ - "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", - "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, strings.TrimSpace(token)), - fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics", + "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--labels=role=metrics", + "--namespace", kbc.Kubectl.Namespace, "--", + fmt.Sprintf("curl -v http://e2e-%s-controller-manager-metrics-service.%s.svc:8080/metrics", kbc.TestSuffix, kbc.Kubectl.Namespace), } - _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) + _, err := kbc.Kubectl.CommandInNamespace(cmdOpts...) ExpectWithOffset(2, err).NotTo(HaveOccurred()) By("validating that the curl pod is running as expected") @@ -336,43 +305,3 @@ func curlMetrics(kbc *utils.TestContext) string { return metricsOutput } - -// ServiceAccountToken provides a helper function that can provide you with a service account -// token that you can use to interact with the service. This function leverages the k8s' -// TokenRequest API in raw format in order to make it generic for all version of the k8s that -// is currently being supported in kubebuilder test infra. -// TokenRequest API returns the token in raw JWT format itself. There is no conversion required. -func ServiceAccountToken(kbc *utils.TestContext) (out string, err error) { - By("Creating the ServiceAccount token") - secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) - tokenRequestFile := filepath.Join(kbc.Dir, secretName) - err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755)) - if err != nil { - return out, err - } - var rawJson string - Eventually(func() error { - // Output of this is already a valid JWT token. No need to covert this from base64 to string format - rawJson, err = kbc.Kubectl.Command( - "create", - "--raw", fmt.Sprintf( - "/api/v1/namespaces/%s/serviceaccounts/%s/token", - kbc.Kubectl.Namespace, - kbc.Kubectl.ServiceAccount, - ), - "-f", tokenRequestFile, - ) - if err != nil { - return err - } - var token tokenRequest - err = json.Unmarshal([]byte(rawJson), &token) - if err != nil { - return err - } - out = token.Status.Token - return nil - }, time.Minute, time.Second).Should(Succeed()) - - return out, err -} diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml index 2f78dfb54aa..20e5ceb5d32 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/policy/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/policy/policy.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/testdata/project-v4-multigroup-with-deploy-image/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml index bf55d64ae3a..595c4ed87c3 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml index 08b359e46b5..f730eb57442 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml @@ -10,12 +10,12 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -40,3 +40,4 @@ resources: - ship_frigate_viewer_role.yaml - crew_captain_editor_role.yaml - crew_captain_viewer_role.yaml + diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml similarity index 89% rename from testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml rename to testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml index 3c9ad11fc52..63c38954077 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image app.kubernetes.io/managed-by: kustomize diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role.yaml similarity index 81% rename from testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml rename to testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role.yaml index 0050db22e36..db8b4f1eec8 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role_binding.yaml similarity index 76% rename from testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml rename to testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role_binding.yaml index 2865bf6007b..6dd925789d3 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml similarity index 83% rename from testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml index eaa3581887a..f6c31b3d478 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml index de7ab9a9859..8b3ddbe4419 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml @@ -1241,7 +1241,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/instance: metrics-reader app.kubernetes.io/managed-by: kustomize @@ -1258,13 +1258,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image - app.kubernetes.io/instance: proxy-role + app.kubernetes.io/instance: metrics-role app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrole app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image - name: project-v4-multigroup-with-deploy-image-proxy-role + name: project-v4-multigroup-with-deploy-image-metrics-role rules: - apiGroups: - authentication.k8s.io @@ -1614,17 +1614,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image - app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/instance: metrics-rolebinding app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/part-of: project-v4-multigroup-with-deploy-image - name: project-v4-multigroup-with-deploy-image-proxy-rolebinding + name: project-v4-multigroup-with-deploy-image-metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: project-v4-multigroup-with-deploy-image-proxy-role + name: project-v4-multigroup-with-deploy-image-metrics-role subjects: - kind: ServiceAccount name: project-v4-multigroup-with-deploy-image-controller-manager @@ -1634,7 +1634,7 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup-with-deploy-image app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize @@ -1645,10 +1645,10 @@ metadata: namespace: project-v4-multigroup-with-deploy-image-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -1696,7 +1696,7 @@ spec: containers: - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=0.0.0.0:8080 - --leader-elect command: - /manager @@ -1734,29 +1734,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-multigroup-with-deploy-image-controller-manager @@ -1767,6 +1744,33 @@ spec: defaultMode: 420 secretName: webhook-server-cert --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/part-of: project-v4 + control-plane: controller-manager + name: project-v4-multigroup-with-deploy-image-manager-metrics-policy + namespace: project-v4-multigroup-with-deploy-image-system +spec: + ingress: + - from: + - podSelector: + matchLabels: + role: metrics + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 9fe6e3630df..acb8bb44885 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-multigroup/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/testdata/project-v4-multigroup/config/policy/kustomization.yaml b/testdata/project-v4-multigroup/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/testdata/project-v4-multigroup/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/testdata/project-v4-multigroup/config/policy/policy.yaml b/testdata/project-v4-multigroup/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/testdata/project-v4-multigroup/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml index b4435aa9aaa..10c4becaedb 100644 --- a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml index 08b359e46b5..f730eb57442 100644 --- a/testdata/project-v4-multigroup/config/rbac/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/rbac/kustomization.yaml @@ -10,12 +10,12 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -40,3 +40,4 @@ resources: - ship_frigate_viewer_role.yaml - crew_captain_editor_role.yaml - crew_captain_viewer_role.yaml + diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_client_cluster_role.yaml similarity index 89% rename from testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml rename to testdata/project-v4-multigroup/config/rbac/metrics_client_cluster_role.yaml index bc61e75af6b..03fa98baae5 100644 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/testdata/project-v4-multigroup/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/part-of: project-v4-multigroup app.kubernetes.io/managed-by: kustomize diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_role.yaml similarity index 80% rename from testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml rename to testdata/project-v4-multigroup/config/rbac/metrics_role.yaml index fa5805cf8a5..4af8d2e2deb 100644 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role.yaml +++ b/testdata/project-v4-multigroup/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/part-of: project-v4-multigroup app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_role_binding.yaml similarity index 75% rename from testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml rename to testdata/project-v4-multigroup/config/rbac/metrics_role_binding.yaml index 0bb48978f41..415039c9a69 100644 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_role_binding.yaml +++ b/testdata/project-v4-multigroup/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/part-of: project-v4-multigroup app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-multigroup/config/rbac/metrics_service.yaml similarity index 82% rename from testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-multigroup/config/rbac/metrics_service.yaml index 88321dc14d5..274c9389286 100644 --- a/testdata/project-v4-multigroup/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-multigroup/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/part-of: project-v4-multigroup app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-multigroup/dist/install.yaml b/testdata/project-v4-multigroup/dist/install.yaml index 28c4aca2293..572e1e48ae4 100644 --- a/testdata/project-v4-multigroup/dist/install.yaml +++ b/testdata/project-v4-multigroup/dist/install.yaml @@ -1241,7 +1241,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/instance: metrics-reader app.kubernetes.io/managed-by: kustomize @@ -1258,13 +1258,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/instance: proxy-role + app.kubernetes.io/instance: metrics-role app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrole app.kubernetes.io/part-of: project-v4-multigroup - name: project-v4-multigroup-proxy-role + name: project-v4-multigroup-metrics-role rules: - apiGroups: - authentication.k8s.io @@ -1614,17 +1614,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup - app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/instance: metrics-rolebinding app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/part-of: project-v4-multigroup - name: project-v4-multigroup-proxy-rolebinding + name: project-v4-multigroup-metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: project-v4-multigroup-proxy-role + name: project-v4-multigroup-metrics-role subjects: - kind: ServiceAccount name: project-v4-multigroup-controller-manager @@ -1634,7 +1634,7 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-multigroup app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize @@ -1645,10 +1645,10 @@ metadata: namespace: project-v4-multigroup-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -1696,7 +1696,7 @@ spec: containers: - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=0.0.0.0:8080 - --leader-elect command: - /manager @@ -1734,29 +1734,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-multigroup-controller-manager @@ -1767,6 +1744,33 @@ spec: defaultMode: 420 secretName: webhook-server-cert --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/part-of: project-v4 + control-plane: controller-manager + name: project-v4-multigroup-manager-metrics-policy + namespace: project-v4-multigroup-system +spec: + ingress: + - from: + - podSelector: + matchLabels: + role: metrics + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml index 62e78ccdbbe..e3c36855d32 100644 --- a/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-with-deploy-image/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/testdata/project-v4-with-deploy-image/config/policy/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/testdata/project-v4-with-deploy-image/config/policy/policy.yaml b/testdata/project-v4-with-deploy-image/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/testdata/project-v4-with-deploy-image/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml index 7f52a66ad36..693106e3547 100644 --- a/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-deploy-image/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml index 67076dab990..d91779cd011 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml @@ -10,12 +10,12 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -24,3 +24,4 @@ resources: - busybox_viewer_role.yaml - memcached_editor_role.yaml - memcached_viewer_role.yaml + diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml similarity index 89% rename from testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml rename to testdata/project-v4-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml index 3b930da4bb1..5708a38d8ad 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/part-of: project-v4-with-deploy-image app.kubernetes.io/managed-by: kustomize diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_role.yaml similarity index 81% rename from testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml rename to testdata/project-v4-with-deploy-image/config/rbac/metrics_role.yaml index 149a84a43c5..297f68398e6 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/part-of: project-v4-with-deploy-image app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_role_binding.yaml similarity index 75% rename from testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml rename to testdata/project-v4-with-deploy-image/config/rbac/metrics_role_binding.yaml index 450754625d0..aaa8ebb2844 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_role_binding.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/part-of: project-v4-with-deploy-image app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml similarity index 83% rename from testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml index ac5f66d3182..d8721d9ea76 100644 --- a/testdata/project-v4-with-deploy-image/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-with-deploy-image/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/part-of: project-v4-with-deploy-image app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-deploy-image/dist/install.yaml b/testdata/project-v4-with-deploy-image/dist/install.yaml index a6b113e4fe1..a8334ed840a 100644 --- a/testdata/project-v4-with-deploy-image/dist/install.yaml +++ b/testdata/project-v4-with-deploy-image/dist/install.yaml @@ -543,7 +543,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/instance: metrics-reader app.kubernetes.io/managed-by: kustomize @@ -560,13 +560,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image - app.kubernetes.io/instance: proxy-role + app.kubernetes.io/instance: metrics-role app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrole app.kubernetes.io/part-of: project-v4-with-deploy-image - name: project-v4-with-deploy-image-proxy-role + name: project-v4-with-deploy-image-metrics-role rules: - apiGroups: - authentication.k8s.io @@ -626,17 +626,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image - app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/instance: metrics-rolebinding app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/part-of: project-v4-with-deploy-image - name: project-v4-with-deploy-image-proxy-rolebinding + name: project-v4-with-deploy-image-metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: project-v4-with-deploy-image-proxy-role + name: project-v4-with-deploy-image-metrics-role subjects: - kind: ServiceAccount name: project-v4-with-deploy-image-controller-manager @@ -646,7 +646,7 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-deploy-image app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize @@ -657,10 +657,10 @@ metadata: namespace: project-v4-with-deploy-image-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -708,7 +708,7 @@ spec: containers: - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=0.0.0.0:8080 - --leader-elect command: - /manager @@ -751,29 +751,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-with-deploy-image-controller-manager @@ -784,6 +761,33 @@ spec: defaultMode: 420 secretName: webhook-server-cert --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/part-of: project-v4 + control-plane: controller-manager + name: project-v4-with-deploy-image-manager-metrics-policy + namespace: project-v4-with-deploy-image-system +spec: + ingress: + - from: + - podSelector: + matchLabels: + role: metrics + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress +--- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/testdata/project-v4-with-grafana/config/default/kustomization.yaml b/testdata/project-v4-with-grafana/config/default/kustomization.yaml index 7fca0820b0c..a55a4c4a95c 100644 --- a/testdata/project-v4-with-grafana/config/default/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4-with-grafana/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/testdata/project-v4-with-grafana/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/testdata/project-v4-with-grafana/config/policy/kustomization.yaml b/testdata/project-v4-with-grafana/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/testdata/project-v4-with-grafana/config/policy/policy.yaml b/testdata/project-v4-with-grafana/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/testdata/project-v4-with-grafana/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml index 910e297d3bb..0eda59d0524 100644 --- a/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-grafana/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml index 731832a6ac3..a4438519005 100644 --- a/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/kustomization.yaml @@ -10,9 +10,9 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_client_cluster_role.yaml similarity index 89% rename from testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml rename to testdata/project-v4-with-grafana/config/rbac/metrics_client_cluster_role.yaml index 1a349448805..3b58c36c1b7 100644 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/part-of: project-v4-with-grafana app.kubernetes.io/managed-by: kustomize diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_role.yaml similarity index 80% rename from testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml rename to testdata/project-v4-with-grafana/config/rbac/metrics_role.yaml index 78751f31b46..9dedbbfab9f 100644 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/part-of: project-v4-with-grafana app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_role_binding.yaml similarity index 75% rename from testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml rename to testdata/project-v4-with-grafana/config/rbac/metrics_role_binding.yaml index 65e551ad032..16a79feb907 100644 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_role_binding.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/part-of: project-v4-with-grafana app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml b/testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml similarity index 82% rename from testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml index 55993935fab..daaaaf8b618 100644 --- a/testdata/project-v4-with-grafana/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4-with-grafana/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/part-of: project-v4-with-grafana app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4-with-grafana/dist/install.yaml b/testdata/project-v4-with-grafana/dist/install.yaml index 4432c399bc2..b8d67811fd2 100644 --- a/testdata/project-v4-with-grafana/dist/install.yaml +++ b/testdata/project-v4-with-grafana/dist/install.yaml @@ -90,7 +90,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/instance: metrics-reader app.kubernetes.io/managed-by: kustomize @@ -107,13 +107,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana - app.kubernetes.io/instance: proxy-role + app.kubernetes.io/instance: metrics-role app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrole app.kubernetes.io/part-of: project-v4-with-grafana - name: project-v4-with-grafana-proxy-role + name: project-v4-with-grafana-metrics-role rules: - apiGroups: - authentication.k8s.io @@ -173,17 +173,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana - app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/instance: metrics-rolebinding app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/part-of: project-v4-with-grafana - name: project-v4-with-grafana-proxy-rolebinding + name: project-v4-with-grafana-metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: project-v4-with-grafana-proxy-role + name: project-v4-with-grafana-metrics-role subjects: - kind: ServiceAccount name: project-v4-with-grafana-controller-manager @@ -193,7 +193,7 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4-with-grafana app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize @@ -204,10 +204,10 @@ metadata: namespace: project-v4-with-grafana-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -237,32 +237,9 @@ spec: control-plane: controller-manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=0.0.0.0:8080 - --leader-elect command: - /manager @@ -296,3 +273,30 @@ spec: runAsNonRoot: true serviceAccountName: project-v4-with-grafana-controller-manager terminationGracePeriodSeconds: 10 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/part-of: project-v4 + control-plane: controller-manager + name: project-v4-with-grafana-manager-metrics-policy + namespace: project-v4-with-grafana-system +spec: + ingress: + - from: + - podSelector: + matchLabels: + role: metrics + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index ae7fc170730..04e69227883 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -25,12 +25,15 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [NETWORK POLICY] Protect the /metrics endpoint. If you want your controller-manager to expose +# the /metrics w/o any authn/z, please comment the following line. +- ../policy patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +# The /metrics endpoint is protected by the NetworkPolicy +# If you want your controller-manager to not expose the /metrics +# endpoint please comment the following line. +- path: manager_metrics_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml b/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/testdata/project-v4/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/testdata/project-v4/config/default/manager_metrics_patch.yaml b/testdata/project-v4/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..c23d8d9268f --- /dev/null +++ b/testdata/project-v4/config/default/manager_metrics_patch.yaml @@ -0,0 +1,15 @@ +# This patch adds the args to allow expose the metrics endpoint +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" diff --git a/testdata/project-v4/config/policy/kustomization.yaml b/testdata/project-v4/config/policy/kustomization.yaml new file mode 100644 index 00000000000..dc4571ccb16 --- /dev/null +++ b/testdata/project-v4/config/policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- policy.yaml diff --git a/testdata/project-v4/config/policy/policy.yaml b/testdata/project-v4/config/policy/policy.yaml new file mode 100644 index 00000000000..33325d2a0e3 --- /dev/null +++ b/testdata/project-v4/config/policy/policy.yaml @@ -0,0 +1,27 @@ +# NetworkPolicy to protected metrics endpoint +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: manager-metrics-policy + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + role: metrics # Pod(s) which will collect the metrics must have this label + ports: + - protocol: TCP + port: 8080 # HTTP port for metrics diff --git a/testdata/project-v4/config/prometheus/monitor.yaml b/testdata/project-v4/config/prometheus/monitor.yaml index 905e15b285c..0d22b7258a9 100644 --- a/testdata/project-v4/config/prometheus/monitor.yaml +++ b/testdata/project-v4/config/prometheus/monitor.yaml @@ -15,11 +15,8 @@ metadata: spec: endpoints: - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true + port: http # Ensure this is the name of the port that exposes HTTP metrics + scheme: http selector: matchLabels: control-plane: controller-manager diff --git a/testdata/project-v4/config/rbac/kustomization.yaml b/testdata/project-v4/config/rbac/kustomization.yaml index 8518bf9e24d..4ac4e50ee4f 100644 --- a/testdata/project-v4/config/rbac/kustomization.yaml +++ b/testdata/project-v4/config/rbac/kustomization.yaml @@ -10,12 +10,12 @@ resources: - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# the metrics network policy # which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- metrics_service.yaml +- metrics_role.yaml +- metrics_role_binding.yaml +- metrics_client_cluster_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines @@ -26,3 +26,4 @@ resources: - firstmate_viewer_role.yaml - captain_editor_role.yaml - captain_viewer_role.yaml + diff --git a/testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml b/testdata/project-v4/config/rbac/metrics_client_cluster_role.yaml similarity index 88% rename from testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml rename to testdata/project-v4/config/rbac/metrics_client_cluster_role.yaml index 6eb655532e8..22ce8caa874 100644 --- a/testdata/project-v4/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/testdata/project-v4/config/rbac/metrics_client_cluster_role.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: clusterrole app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/part-of: project-v4 app.kubernetes.io/managed-by: kustomize diff --git a/testdata/project-v4/config/rbac/auth_proxy_role.yaml b/testdata/project-v4/config/rbac/metrics_role.yaml similarity index 79% rename from testdata/project-v4/config/rbac/auth_proxy_role.yaml rename to testdata/project-v4/config/rbac/metrics_role.yaml index 28de66c7882..17865e2a097 100644 --- a/testdata/project-v4/config/rbac/auth_proxy_role.yaml +++ b/testdata/project-v4/config/rbac/metrics_role.yaml @@ -3,12 +3,12 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-role + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/part-of: project-v4 app.kubernetes.io/managed-by: kustomize - name: proxy-role + name: metrics-role rules: - apiGroups: - authentication.k8s.io diff --git a/testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml b/testdata/project-v4/config/rbac/metrics_role_binding.yaml similarity index 74% rename from testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml rename to testdata/project-v4/config/rbac/metrics_role_binding.yaml index 609d1c5e0e0..0e7d5298bce 100644 --- a/testdata/project-v4/config/rbac/auth_proxy_role_binding.yaml +++ b/testdata/project-v4/config/rbac/metrics_role_binding.yaml @@ -3,16 +3,16 @@ kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/instance: metrics-rolebinding + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/part-of: project-v4 app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/testdata/project-v4/config/rbac/auth_proxy_service.yaml b/testdata/project-v4/config/rbac/metrics_service.yaml similarity index 82% rename from testdata/project-v4/config/rbac/auth_proxy_service.yaml rename to testdata/project-v4/config/rbac/metrics_service.yaml index 81fb97cbe94..1a6cf8a6562 100644 --- a/testdata/project-v4/config/rbac/auth_proxy_service.yaml +++ b/testdata/project-v4/config/rbac/metrics_service.yaml @@ -5,7 +5,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: service app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/part-of: project-v4 app.kubernetes.io/managed-by: kustomize @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager diff --git a/testdata/project-v4/dist/install.yaml b/testdata/project-v4/dist/install.yaml index adc1f4bb4e5..29488de01e3 100644 --- a/testdata/project-v4/dist/install.yaml +++ b/testdata/project-v4/dist/install.yaml @@ -545,7 +545,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/instance: metrics-reader app.kubernetes.io/managed-by: kustomize @@ -562,13 +562,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/instance: proxy-role + app.kubernetes.io/instance: metrics-role app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrole app.kubernetes.io/part-of: project-v4 - name: project-v4-proxy-role + name: project-v4-metrics-role rules: - apiGroups: - authentication.k8s.io @@ -628,17 +628,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 - app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/instance: metrics-rolebinding app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: clusterrolebinding app.kubernetes.io/part-of: project-v4 - name: project-v4-proxy-rolebinding + name: project-v4-metrics-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: project-v4-proxy-role + name: project-v4-metrics-role subjects: - kind: ServiceAccount name: project-v4-controller-manager @@ -648,7 +648,7 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: metrics app.kubernetes.io/created-by: project-v4 app.kubernetes.io/instance: controller-manager-metrics-service app.kubernetes.io/managed-by: kustomize @@ -659,10 +659,10 @@ metadata: namespace: project-v4-system spec: ports: - - name: https - port: 8443 + - name: http + port: 8080 protocol: TCP - targetPort: https + targetPort: 8080 selector: control-plane: controller-manager --- @@ -710,7 +710,7 @@ spec: containers: - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=0.0.0.0:8080 - --leader-elect command: - /manager @@ -748,29 +748,6 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL securityContext: runAsNonRoot: true serviceAccountName: project-v4-controller-manager @@ -781,6 +758,33 @@ spec: defaultMode: 420 secretName: webhook-server-cert --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: project-v4-network-policy + app.kubernetes.io/part-of: project-v4 + control-plane: controller-manager + name: project-v4-manager-metrics-policy + namespace: project-v4-system +spec: + ingress: + - from: + - podSelector: + matchLabels: + role: metrics + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: