Skip to content

Commit

Permalink
add validating webhook for repository server CR (#2281)
Browse files Browse the repository at this point in the history
* add validating webhook for repository server CR

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* remove unncessary changes

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* add comment for in values.yaml for validating webhook flag

* Validating webhook can be enabled for versions < 1.25

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* add missing error messages

* 1. move common utilities validatingwebhook pkgs
2. rename the filenames
Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* change names of the port in service.yaml

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* move the validating webhook flag from repositoryservercontroller field to an independent field

* fix custom validator logic

* address cosmetic changes

---------

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>
  • Loading branch information
kale-amruta committed Sep 1, 2023
1 parent a52fa96 commit 19cc9a4
Show file tree
Hide file tree
Showing 11 changed files with 212 additions and 14 deletions.
44 changes: 43 additions & 1 deletion cmd/reposervercontroller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,27 @@ import (
"context"
"flag"
"os"
"strconv"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
"go.uber.org/zap/zapcore"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/discovery"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
crkanisteriov1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/kanisterio/kanister/pkg/controllers/repositoryserver"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kanisterio/kanister/pkg/resource"
"github.com/kanisterio/kanister/pkg/validatingwebhook"
//+kubebuilder:scaffold:imports
)

Expand All @@ -43,6 +48,12 @@ var (
defaultLogLevel = zapcore.InfoLevel
)

const (
whHandlePath = "/validate/v1alpha1/repositoryserver"
webhookServerPort = 8443
minorK8sVersion = 25
)

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))

Expand Down Expand Up @@ -70,7 +81,6 @@ func main() {
mgr, err := ctrl.NewManager(config, ctrl.Options{
Scheme: scheme,
MetricsBindAddress: metricsAddr,
Port: 9443,
HealthProbeBindAddress: probeAddr,
LeaderElection: false,
})
Expand Down Expand Up @@ -105,6 +115,38 @@ func main() {
}
}

discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
if err != nil {
setupLog.Error(err, "Failed to get discovery client")
os.Exit(1)
}

k8sserverVersion, err := discoveryClient.ServerVersion()
if err != nil {
setupLog.Error(err, "Failed to get server version using discovery client")
os.Exit(1)
}

minorVersion, err := strconv.Atoi(k8sserverVersion.Minor)
if err != nil {
setupLog.Error(err, "Failed to convert to integer")
os.Exit(1)
}

// We are using CEL validation rules for k8s server version > 1.25.
// More information about CEL can be found here - https://kubernetes.io/blog/2022/09/23/crd-validation-rules-beta/
// CEL is not supported for k8s server versions below 1.25. Hence for backward compatibility
// we can use validating webhook for k8s server versions < 1.25
if k8sserverVersion.Major == "1" && minorVersion < minorK8sVersion {
if validatingwebhook.IsCACertMounted() {
hookServer := mgr.GetWebhookServer()
webhook := admission.WithCustomValidator(&v1alpha1.RepositoryServer{}, &validatingwebhook.RepositoryServerValidator{})
hookServer.Register(whHandlePath, webhook)
hookServer.CertDir = validatingwebhook.WHCertsDir
hookServer.Port = webhookServerPort
}
}

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")
Expand Down
21 changes: 21 additions & 0 deletions helm/kanister-operator/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,27 @@ on the value of bpValidatingWebhook.enabled
{{- end -}}
{{- end -}}


{{/*
Figure out the target port of service, this depends
on the value of validatingWebhook.repositoryserver.enabled
*/}}
{{- define "reposerver-controller.targetPort" -}}
{{- if .Values.validatingWebhook.repositoryserver.enabled -}}
{{ 8443 }}
{{- end -}}
{{- end -}}

{{/*
Figure out the port of service, this depends
on the value of validatingWebhook.repositoryserver.enabled
*/}}
{{- define "reposerver-controller.servicePort" -}}
{{- if .Values.validatingWebhook.repositoryserver.enabled -}}
{{ .Values.repositoryServerController.service.port }}
{{- end -}}
{{- end -}}

{{/*
Define a custom kanister-tools image
*/}}
Expand Down
7 changes: 6 additions & 1 deletion helm/kanister-operator/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ spec:
{{ include "kanister-operator.helmLabels" . | indent 8}}
spec:
serviceAccountName: {{ template "kanister-operator.serviceAccountName" . }}
{{- if .Values.bpValidatingWebhook.enabled }}
{{- if or .Values.bpValidatingWebhook.enabled .Values.validatingWebhook.repositoryserver.enabled }}
volumes:
- name: webhook-certs
secret:
Expand Down Expand Up @@ -51,6 +51,11 @@ spec:
- name: {{ template "repository-server-controller.name" . }}
image: {{ .Values.repositoryServerControllerImage.registry }}/{{ .Values.repositoryServerControllerImage.name }}:{{ .Values.repositoryServerControllerImage.tag }}
imagePullPolicy: {{ .Values.repositoryServerControllerImage.pullPolicy }}
{{- if .Values.validatingWebhook.repositoryserver.enabled }}
volumeMounts:
- name: webhook-certs
mountPath: /var/run/webhook/serving-cert
{{- end }}
env:
- name: KOPIA_SERVER_START_TIMEOUT
value: {{ .Values.repositoryServerController.serverStartTimeout | quote }}
Expand Down
7 changes: 7 additions & 0 deletions helm/kanister-operator/templates/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ spec:
- port: {{ template "kanister-operator.servicePort" . }}
protocol: TCP
targetPort: {{ template "kanister-operator.targetPort" . }}
name: controller-port
{{- if .Values.validatingWebhook.repositoryserver.enabled }}
- port: {{ template "reposerver-controller.servicePort" . }}
protocol: TCP
targetPort: {{ template "reposerver-controller.targetPort" . }}
name: reposervercontroller-port
{{- end }}
selector:
app: {{ template "kanister-operator.name" . }}
status:
Expand Down
34 changes: 33 additions & 1 deletion helm/kanister-operator/templates/validating-webhook.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{{- if .Values.bpValidatingWebhook.enabled -}}
{{- if or .Values.bpValidatingWebhook.enabled .Values.validatingWebhook.repositoryserver.enabled }}
# generate ca cert with 365 days of validity
{{ $ca := genCA ( printf "%s-ca" ( include "kanister-operator.fullname" . ) ) 365 }}
{{- if eq (.Values.bpValidatingWebhook.tls.mode) "auto" }}
Expand All @@ -15,6 +15,7 @@ data:
tls.key: {{ $cert.Key | b64enc }}
---
{{- end }}
{{- if or .Values.bpValidatingWebhook.enabled }}
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
Expand All @@ -41,4 +42,35 @@ webhooks:
admissionReviewVersions: ["v1", "v1beta1"]
sideEffects: None
timeoutSeconds: 5
---
{{- end -}}
{{- if .Values.validatingWebhook.repositoryserver.enabled -}}
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
name: "repositoryservers.cr.kanister.io"
webhooks:
- name: repositoryserver.cr.kanister.io
admissionReviewVersions: ["v1"]
clientConfig:
service:
namespace: {{ .Release.Namespace }}
name: {{ template "kanister-operator.fullname" . }}
path: /validate/v1alpha1/repositoryserver
port: {{ .Values.repositoryServerController.service.port }}
# use same certificate used for blueprint validating webhook
{{- if eq (.Values.bpValidatingWebhook.tls.mode) "custom" }}
caBundle: {{ .Values.bpValidatingWebhook.tls.caBundle | required "Missing required caBundle, bpValidatingWebhook.tls.caBundle" }}
{{- else if eq (.Values.bpValidatingWebhook.tls.mode) "auto" }}
caBundle: {{ b64enc $ca.Cert }}
{{- end }}
failurePolicy: Fail
rules:
- apiGroups: ["cr.kanister.io"]
apiVersions: ["v1alpha1"]
operations: ["UPDATE"]
resources: ["repositoryservers"]
scope: "Namespaced"
sideEffects: None
{{- end -}}
{{- end -}}
14 changes: 14 additions & 0 deletions helm/kanister-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,20 @@ controller:
enabled: false
bpValidatingWebhook:
enabled: true
# `tls` field is used to specify TLS information for both blueprint and repositoryserver validating webhook server
tls:
mode: auto # If set to `custom` then secretName and caBundle should be provided
secretName: '' # An already created Secret in kanister controller namespace having tls cert details
caBundle: '' # A valid, CA bundle which is a PEM-encoded CA bundle for validating the webhook's server certificate
validatingWebhook:
# This flag is used to enable validating webhook for repository server CR
# The TLS certificates for blueprint validating webhook server and
# repositoryserver validing webhook server are same and can be provided
# under field `bpValidatingWebhook.tls`
# Webhook can only be enabled for k8s server versions < 1.25
# For versions > 1.25 we will be using k8s CEL validation rules -https://kubernetes.io/blog/2022/09/23/crd-validation-rules-beta/
repositoryserver:
enabled: false
repositoryServerController:
enabled: false
# startTimeout is used to specify the time in seconds to wait for starting the kopia repository server
Expand All @@ -51,6 +61,10 @@ repositoryServerController:
logLevel: 'info'
container:
name: 'repository-server-controller'
service:
# port is used as the secured service port if the validating
# webhook is enabled.
port: 444
resources:
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
Expand Down
3 changes: 1 addition & 2 deletions pkg/handler/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ const (
healthCheckPath = "/v0/healthz"
metricsPath = "/metrics"
healthCheckAddr = ":8000"
WHCertsDir = "/var/run/webhook/serving-cert"
whHandlePath = "/validate/v1alpha1/blueprint"
)

Expand Down Expand Up @@ -73,7 +72,7 @@ func RunWebhookServer(c *rest.Config) error {
hookServer.Register(healthCheckPath, &healthCheckHandler{})
hookServer.Register(metricsPath, promhttp.Handler())

hookServer.CertDir = WHCertsDir
hookServer.CertDir = validatingwebhook.WHCertsDir

if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
return err
Expand Down
11 changes: 2 additions & 9 deletions pkg/kancontroller/kancontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kanisterio/kanister/pkg/resource"
"github.com/kanisterio/kanister/pkg/validatingwebhook"
)

const (
Expand Down Expand Up @@ -78,7 +79,7 @@ func Execute() {

// Run HTTPS webhook server if webhook certificates are mounted in the pod
// otherwise normal HTTP server for health and prom endpoints
if isCACertMounted() {
if validatingwebhook.IsCACertMounted() {
go func(config *rest.Config) {
err := handler.RunWebhookServer(config)
if err != nil {
Expand Down Expand Up @@ -142,11 +143,3 @@ func Execute() {
log.Print("shutdown signal received, exiting...")
cancel()
}

func isCACertMounted() bool {
if _, err := os.Stat(fmt.Sprintf("%s/%s", handler.WHCertsDir, "tls.crt")); err != nil {
return false
}

return true
}
File renamed without changes.
55 changes: 55 additions & 0 deletions pkg/validatingwebhook/repositoryserver_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Copyright 2023 The Kanister Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validatingwebhook

import (
"context"
"fmt"

"github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

type RepositoryServerValidator struct{}

var _ webhook.CustomValidator = &RepositoryServerValidator{}

//+kubebuilder:webhook:path=/validate/v1alpha1/repositoryserver,mutating=false,failurePolicy=fail,sideEffects=None,groups=cr.kanister.io,resources=repositoryservers,verbs=update,versions=v1alpha1,name=repositoryserver.cr.kanister.io,admissionReviewVersions=v1

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *RepositoryServerValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error {
return nil
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (r *RepositoryServerValidator) ValidateUpdate(ctx context.Context, old runtime.Object, new runtime.Object) error {
oldrs, ook := old.(*v1alpha1.RepositoryServer)
newrs, nok := new.(*v1alpha1.RepositoryServer)
if !ook || !nok {
return errors.New("Either updated object or the old object is not of type RepositoryServer.cr.kanister.io")
}
errMsg := fmt.Sprintf("RepositoryServer.cr.kanister.io \"%s\" is invalid: spec.repository.rootPath: Invalid value, Value is immutable", newrs.Name)
if oldrs.Spec.Repository.RootPath != newrs.Spec.Repository.RootPath {
return errors.New(errMsg)
}
return nil
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
func (r *RepositoryServerValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error {
return nil
}
30 changes: 30 additions & 0 deletions pkg/validatingwebhook/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright 2023 The Kanister Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validatingwebhook

import (
"fmt"
"os"
)

const WHCertsDir = "/var/run/webhook/serving-cert"

func IsCACertMounted() bool {
if _, err := os.Stat(fmt.Sprintf("%s/%s", WHCertsDir, "tls.crt")); err != nil {
return false
}

return true
}

0 comments on commit 19cc9a4

Please sign in to comment.