From 63c3bc8497766564d7e76a3da2dabc53ba3b7d54 Mon Sep 17 00:00:00 2001 From: Tyler Gillson Date: Wed, 29 Nov 2023 11:48:26 -0700 Subject: [PATCH] feat: support proxy configuration (env vars & CA certificate) (#137) * fix: add yaml tags to ValidatorConfig types Signed-off-by: Tyler Gillson * feat: support proxy config Signed-off-by: Tyler Gillson * feat: support proxy config Signed-off-by: Tyler Gillson * chore: add Make target Signed-off-by: Tyler Gillson * chore: update chart README.md Signed-off-by: Tyler Gillson * fix: include root CAs and ca-certificates.conf in certs-init image Signed-off-by: Tyler Gillson * fix: configure root CAs properly Signed-off-by: Tyler Gillson * refactor: copy env & proxy config into each plugin's values Signed-off-by: Tyler Gillson * feat: support proxy CA cert secret creation Signed-off-by: Tyler Gillson * fix: use non-root user for cert-init image Signed-off-by: Tyler Gillson * chore: update network plugin version Signed-off-by: Tyler Gillson --------- Signed-off-by: Tyler Gillson --- Makefile | 5 ++ chart/validator/README.md | 21 +++++-- chart/validator/templates/deployment.yaml | 61 +++++++++++++++---- chart/validator/templates/proxy-secret.yaml | 9 +++ .../validator/templates/validator-config.yaml | 15 ++++- chart/validator/values.yaml | 39 +++++++----- hack/validator-certs-init.Dockerfile | 16 +++++ 7 files changed, 132 insertions(+), 34 deletions(-) create mode 100644 chart/validator/templates/proxy-secret.yaml create mode 100644 hack/validator-certs-init.Dockerfile diff --git a/Makefile b/Makefile index 3d0017f2..04163be2 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # Image URL to use all building/pushing image targets IMG ?= quay.io/spectrocloud-labs/validator:latest +CERTS_INIT_IMG ?= quay.io/spectrocloud-labs/validator-certs-init:latest GOARCH ?= $(shell go env GOARCH) @@ -81,6 +82,10 @@ run: manifests generate fmt vet ## Run a controller from your host. docker-build: test ## Build docker image with the manager. $(CONTAINER_TOOL) build -t ${IMG} . --platform linux/$(GOARCH) +.PHONY: docker-build-certs-init +docker-build-certs-init: ## Build validator-certs-init docker image. + $(CONTAINER_TOOL) build -f hack/validator-certs-init.Dockerfile -t ${CERTS_INIT_IMG} . --platform linux/$(GOARCH) + .PHONY: docker-push docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} diff --git a/chart/validator/README.md b/chart/validator/README.md index 0b82f7d1..04410fdd 100644 --- a/chart/validator/README.md +++ b/chart/validator/README.md @@ -2,7 +2,7 @@ Validator =========== -validator monitors ValidationResults created by one or more validator plugins and uploads them to a configurable sink +Monitor results created by validator plugins and upload them to a configurable sink ## Configuration @@ -15,26 +15,37 @@ The following table lists the configurable parameters of the Validator chart and | `controllerManager.kubeRbacProxy.containerSecurityContext.allowPrivilegeEscalation` | | `false` | | `controllerManager.kubeRbacProxy.containerSecurityContext.capabilities.drop` | | `["ALL"]` | | `controllerManager.kubeRbacProxy.image.repository` | | `"gcr.io/kubebuilder/kube-rbac-proxy"` | -| `controllerManager.kubeRbacProxy.image.tag` | | `"v0.14.1"` | +| `controllerManager.kubeRbacProxy.image.tag` | | `"v0.15.0"` | | `controllerManager.kubeRbacProxy.resources.limits.cpu` | | `"500m"` | | `controllerManager.kubeRbacProxy.resources.limits.memory` | | `"128Mi"` | | `controllerManager.kubeRbacProxy.resources.requests.cpu` | | `"5m"` | | `controllerManager.kubeRbacProxy.resources.requests.memory` | | `"64Mi"` | -| `controllerManager.manager.args` | | `["--health-probe-bind-address=:8081", "--metrics-bind-address=127.0.0.1:8080", "--leader-elect"]` | +| `controllerManager.manager.args` | | `["--health-probe-bind-address=:8081", "--leader-elect"]` | | `controllerManager.manager.containerSecurityContext.allowPrivilegeEscalation` | | `false` | | `controllerManager.manager.containerSecurityContext.capabilities.drop` | | `["ALL"]` | | `controllerManager.manager.image.repository` | | `"quay.io/spectrocloud-labs/validator"` | | `controllerManager.manager.image.tag` | x-release-please-version | `"v0.0.26"` | | `controllerManager.manager.resources.limits.cpu` | | `"500m"` | -| `controllerManager.manager.resources.limits.memory` | | `"128Mi"` | +| `controllerManager.manager.resources.limits.memory` | | `"512Mi"` | | `controllerManager.manager.resources.requests.cpu` | | `"10m"` | | `controllerManager.manager.resources.requests.memory` | | `"64Mi"` | +| `controllerManager.manager.sinkWebhookTimeout` | | `"30s"` | | `controllerManager.replicas` | | `1` | | `controllerManager.serviceAccount.annotations` | | `{}` | | `kubernetesClusterDomain` | | `"cluster.local"` | | `metricsService.ports` | | `[{"name": "https", "port": 8443, "protocol": "TCP", "targetPort": "https"}]` | | `metricsService.type` | | `"ClusterIP"` | -| `plugins` | | `[{"chart": {"name": "validator-plugin-aws", "repository": "https://spectrocloud-labs.github.io/validator-plugin-aws", "version": "v0.0.2"}, "values": "controllerManager:\n kubeRbacProxy:\n args:\n - --secure-listen-address=0.0.0.0:8443\n - --upstream=http://127.0.0.1:8080/\n - --logtostderr=true\n - --v=0\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: gcr.io/kubebuilder/kube-rbac-proxy\n tag: v0.14.1\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 5m\n memory: 64Mi\n manager:\n args:\n - --health-probe-bind-address=:8081\n - --metrics-bind-address=127.0.0.1:8080\n - --leader-elect\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: quay.io/spectrocloud-labs/validator-plugin-aws\n tag: v0.0.2\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 10m\n memory: 64Mi\n replicas: 1\n serviceAccount:\n annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n ports:\n - name: https\n port: 8443\n protocol: TCP\n targetPort: https\n type: ClusterIP\nauth:\n secretName: aws-creds\n accessKeyId: \"\"\n secretAccessKey: \"\""}]` | +| `env` | | `[]` | +| `proxy.enabled` | | `false` | +| `proxy.image` | | `"quay.io/spectrocloud-labs/validator-certs-init:latest"` | +| `proxy.secretName` | | `"proxy-cert"` | +| `proxy.createSecret` | | `false` | +| `proxy.caCert` | Raw CA certificate, required if createSecret is true | `""` | +| `sink` | | `{}` | +| `plugins` | | `[{"chart": {"name": "validator-plugin-aws", "repository": "https://spectrocloud-labs.github.io/validator-plugin-aws", "version": "v0.0.18"}, "values": "controllerManager:\n kubeRbacProxy:\n args:\n - --secure-listen-address=0.0.0.0:8443\n - --upstream=http://127.0.0.1:8080/\n - --logtostderr=true\n - --v=0\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: gcr.io/kubebuilder/kube-rbac-proxy\n tag: v0.15.0\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 5m\n memory: 64Mi\n manager:\n args:\n - --health-probe-bind-address=:8081\n - --leader-elect\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: quay.io/spectrocloud-labs/validator-plugin-aws\n tag: v0.0.18\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 10m\n memory: 64Mi\n replicas: 1\n serviceAccount:\n annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n ports:\n - name: https\n port: 8443\n protocol: TCP\n targetPort: https\n type: ClusterIP\nauth:\n # Option 1: Leave secret undefined for implicit auth (node instance IAM role, IMDSv2, etc.)\n # Option 2: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.aws.secretName must match.\n # Option 3: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.aws undefined.\n #\n secret: {} # Delete these curly braces if you're specifying secretName!\n # secretName: aws-creds\n\n # Override the service account used by AWS validator (optional, could be used for IMDSv2 on EKS)\n # WARNING: the chosen service account must include all RBAC privileges found in the AWS plugin template:\n # https://github.com/spectrocloud-labs/validator-plugin-aws/blob/main/chart/validator-plugin-aws/templates/manager-rbac.yaml\n serviceAccountName: \"\""}, {"chart": {"name": "validator-plugin-azure", "repository": "https://spectrocloud-labs.github.io/validator-plugin-azure", "version": "v0.0.2"}, "values": "controllerManager:\n kubeRbacProxy:\n args:\n - --secure-listen-address=0.0.0.0:8443\n - --upstream=http://127.0.0.1:8080/\n - --logtostderr=true\n - --v=0\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: gcr.io/kubebuilder/kube-rbac-proxy\n tag: v0.15.0\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 5m\n memory: 64Mi\n manager:\n args:\n - --health-probe-bind-address=:8081\n - --leader-elect\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: quay.io/spectrocloud-labs/validator-plugin-azure\n tag: v0.0.2\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 10m\n memory: 64Mi\n # Optionally specify a volumeMount to mount a volume containing a private key\n # to leverage Azure Service principal with certificate authentication.\n volumeMounts: []\n replicas: 1\n serviceAccount:\n annotations: {}\n # Optionally specify a volume containing a private key to leverage Azure Service\n # principal with certificate authentication.\n volumes: []\nkubernetesClusterDomain: cluster.local\nmetricsService:\n ports:\n - name: https\n port: 8443\n protocol: TCP\n targetPort: https\n type: ClusterIP\nauth:\n # Option 1: Leave secret undefined for WorkloadIdentityCredential authentication.\n # Option 2: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.azure.secretName must match.\n # Option 3: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.azure undefined.\n #\n secret: {} # Delete these curly braces if you're specifying secretName!\n # secretName: azure-creds\n\n # Override the service account used by Azure validator (optional, could be used for WorkloadIdentityCredentials on AKS)\n # WARNING: the chosen service account must include all RBAC privileges found in the Azure plugin template:\n # https://github.com/spectrocloud-labs/validator-plugin-aws/blob/main/chart/validator-plugin-azure/templates/manager-rbac.yaml\n serviceAccountName: \"\""}, {"chart": {"name": "validator-plugin-vsphere", "repository": "https://spectrocloud-labs.github.io/validator-plugin-vsphere", "version": "v0.0.13"}, "values": "controllerManager:\n kubeRbacProxy:\n args:\n - --secure-listen-address=0.0.0.0:8443\n - --upstream=http://127.0.0.1:8080/\n - --logtostderr=true\n - --v=0\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: gcr.io/kubebuilder/kube-rbac-proxy\n tag: v0.14.1\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 5m\n memory: 64Mi\n manager:\n args:\n - --health-probe-bind-address=:8081\n - --leader-elect\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: quay.io/spectrocloud-labs/validator-plugin-vsphere\n tag: v0.0.13\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 10m\n memory: 64Mi\n replicas: 1\n serviceAccount:\n annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n ports:\n - name: https\n port: 8443\n protocol: TCP\n targetPort: https\n type: ClusterIP\nauth:\n # Option 1: Create a secret via pluginSecrets (see below). Note: secretName and pluginSecrets.vSphere.secretName must match.\n # Option 2: Specify the name of a preexisting secret in your target cluster and leave pluginSecrets.vSphere undefined.\n secretName: vsphere-creds"}, {"chart": {"name": "validator-plugin-network", "repository": "https://spectrocloud-labs.github.io/validator-plugin-network", "version": "v0.0.9"}, "values": "controllerManager:\n kubeRbacProxy:\n args:\n - --secure-listen-address=0.0.0.0:8443\n - --upstream=http://127.0.0.1:8080/\n - --logtostderr=true\n - --v=0\n containerSecurityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n image:\n repository: gcr.io/kubebuilder/kube-rbac-proxy\n tag: v0.15.0\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 5m\n memory: 64Mi\n manager:\n args:\n - --health-probe-bind-address=:8081\n - --leader-elect\n containerSecurityContext:\n allowPrivilegeEscalation: true\n capabilities:\n add:\n - NET_RAW\n drop:\n - ALL\n image:\n repository: quay.io/spectrocloud-labs/validator-plugin-network\n tag: v0.0.9\n resources:\n limits:\n cpu: 500m\n memory: 128Mi\n requests:\n cpu: 10m\n memory: 64Mi\n replicas: 1\n serviceAccount:\n annotations: {}\nkubernetesClusterDomain: cluster.local\nmetricsService:\n ports:\n - name: https\n port: 8443\n protocol: TCP\n targetPort: https\n type: ClusterIP"}]` | +| `pluginSecrets.aws` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` | +| `pluginSecrets.azure` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` | +| `pluginSecrets.vSphere` | Don't forget to delete these curly braces if you're specifying credentials here! | `{}` | diff --git a/chart/validator/templates/deployment.yaml b/chart/validator/templates/deployment.yaml index 291c2434..9126fffd 100644 --- a/chart/validator/templates/deployment.yaml +++ b/chart/validator/templates/deployment.yaml @@ -22,22 +22,35 @@ spec: annotations: kubectl.kubernetes.io/default-container: manager spec: + {{- if .Values.proxy.enabled }} + initContainers: + - name: init-certs + image: {{ required ".Values.proxy.image is required!" .Values.proxy.image }} + command: ["/bin/bash", "-c"] + args: ["update-ca-certificates && cp -r /etc/ca-certificates.conf /usr/share/ca-certificates /etc/ssl/certs"] + volumeMounts: + - name: ca-pemstore + mountPath: /usr/local/share/ca-certificates + readOnly: false + - name: certs + mountPath: /etc/ssl/certs + readOnly: false + securityContext: + runAsNonRoot: false + {{- end }} containers: - args: {{- toYaml .Values.controllerManager.kubeRbacProxy.args | nindent 8 }} env: - name: KUBERNETES_CLUSTER_DOMAIN value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag - | default .Chart.AppVersion }} + image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag | default .Chart.AppVersion }} name: kube-rbac-proxy ports: - containerPort: 8443 name: https protocol: TCP - resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent - 10 }} - securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext - | nindent 10 }} + resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent 10 }} + securityContext: {{- toYaml .Values.controllerManager.kubeRbacProxy.containerSecurityContext | nindent 10 }} - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} command: - /manager @@ -48,10 +61,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 8 }} + {{- end }} - name: SINK_WEBHOOK_TIMEOUT_SECONDS value: {{ quote .Values.controllerManager.manager.sinkWebhookTimeout }} - image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag - | default .Chart.AppVersion }} + image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag | default .Chart.AppVersion }} livenessProbe: httpGet: path: /healthz @@ -65,13 +80,33 @@ spec: port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 - }} - securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext - | nindent 10 }} + resources: {{- toYaml .Values.controllerManager.manager.resources | nindent 10 }} + securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext | nindent 10 }} + {{- if .Values.proxy.enabled }} + volumeMounts: + - name: certs + mountPath: /etc/ca-certificates.conf + subPath: ca-certificates.conf + readOnly: true + - name: certs + mountPath: /usr/share/ca-certificates + subPath: ca-certificates + readOnly: true + - name: certs + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault serviceAccountName: {{ include "chart.fullname" . }}-controller-manager - terminationGracePeriodSeconds: 10 \ No newline at end of file + terminationGracePeriodSeconds: 10 + {{- if .Values.proxy.enabled }} + volumes: + - name: ca-pemstore + secret: + secretName: {{ required ".Values.proxy.secretName is required!" .Values.proxy.secretName }} + - name: certs + emptyDir: {} + {{- end}} \ No newline at end of file diff --git a/chart/validator/templates/proxy-secret.yaml b/chart/validator/templates/proxy-secret.yaml new file mode 100644 index 00000000..5b80f3c6 --- /dev/null +++ b/chart/validator/templates/proxy-secret.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.proxy.enabled .Values.proxy.createSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ required ".Values.proxy.secretName is required!" .Values.proxy.secretName }} +stringData: + ca.crt: | +{{ required ".Values.proxy.caCert is required!" .Values.proxy.caCert | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/chart/validator/templates/validator-config.yaml b/chart/validator/templates/validator-config.yaml index a2a09217..2f9071be 100644 --- a/chart/validator/templates/validator-config.yaml +++ b/chart/validator/templates/validator-config.yaml @@ -1,10 +1,23 @@ +{{ $env := .Values.env }} +{{ $proxy := .Values.proxy }} apiVersion: validation.spectrocloud.labs/v1alpha1 kind: ValidatorConfig metadata: name: validator-config spec: plugins: -{{ toYaml .Values.plugins | indent 2 }} +{{- range .Values.plugins }} + - +{{- toYaml . | nindent 4 }} + {{- if $env }} + env: + {{- toYaml $env | nindent 8 }} + {{- else }} + env: [] + {{- end }} + proxy: + {{- toYaml $proxy | nindent 8 }} +{{- end }} {{- if .Values.sink }} sink: type: {{ required ".Values.sink.type is required!" .Values.sink.type }} diff --git a/chart/validator/values.yaml b/chart/validator/values.yaml index 291ce47e..ab913afb 100644 --- a/chart/validator/values.yaml +++ b/chart/validator/values.yaml @@ -52,6 +52,27 @@ metricsService: targetPort: https type: ClusterIP +# Optional environment variable configuration +env: [] +# - name: HTTP_PROXY +# value: http://192.168.1.100:8080 +# - name: HTTPS_PROXY +# value: http://192.168.1.100:8080 +# - name: NO_PROXY +# value: foo.bar.com,127.0.0.1 + +# Optional proxy configuration. If enabled, the secret containing your proxy CA certificate +# will be mounted in the manager container and configured via an init container. +proxy: + enabled: false + # The image used by the init container. Must include the update-ca-certificates command. + image: quay.io/spectrocloud-labs/validator-certs-init:latest + # The name of a secret containing a proxy CA certificate. + secretName: proxy-cert + # If false (using an existing secret), the key whose value is the CA certificate must end with '.crt'. + createSecret: false + caCert: "" # Raw CA certificate, required if createSecret is true + # Optional sink configuration sink: {} # type: alertmanager @@ -282,7 +303,7 @@ plugins: - chart: name: validator-plugin-network repository: "https://spectrocloud-labs.github.io/validator-plugin-network" - version: v0.0.8 + version: v0.0.9 values: |- controllerManager: kubeRbacProxy: @@ -319,7 +340,7 @@ plugins: - ALL image: repository: quay.io/spectrocloud-labs/validator-plugin-network - tag: v0.0.8 + tag: v0.0.9 resources: limits: cpu: 500m @@ -338,19 +359,7 @@ plugins: protocol: TCP targetPort: https type: ClusterIP - env: {} - # Optional proxy configuration - # - name: HTTP_PROXY - # value: http://192.168.1.100:8080 - # - name: HTTPS_PROXY - # value: http://192.168.1.100:8080 - # - name: NO_PROXY - # value: foo.bar.com,127.0.0.1 - proxy: - enabled: false - image: ubuntu:latest - # The name of a Secret containing a proxy CA certificate - secretName: proxy-cert + pluginSecrets: # If installing the AWS plugin, the below config is required unless one of the following applies: # - The target cluster already contains a secret with the correct format and you've specified its name above. diff --git a/hack/validator-certs-init.Dockerfile b/hack/validator-certs-init.Dockerfile new file mode 100644 index 00000000..f99365f4 --- /dev/null +++ b/hack/validator-certs-init.Dockerfile @@ -0,0 +1,16 @@ +FROM --platform=$TARGETPLATFORM ubuntu:latest AS install + +RUN set -e; \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates + +# finalize, keeping only required artifacts +FROM --platform=$TARGETPLATFORM ubuntu:latest + +COPY --from=install /usr/bin/openssl /usr/bin/openssl +COPY --from=install /usr/sbin/update-ca-certificates /usr/sbin/update-ca-certificates +COPY --from=install /usr/share/ca-certificates /usr/share/ca-certificates +COPY --from=install /etc/ca-certificates.conf /etc/ca-certificates.conf + +USER 65532:65532 \ No newline at end of file