From 9095af754ebf98f2d581bfd7930b4dfc98d51744 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Fri, 11 Aug 2023 16:02:23 +0300 Subject: [PATCH 1/3] Move IPPools CRD manifest to deploy/crds Signed-off-by: Yury Kulazhenkov --- deploy/crds/kustomization.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 deploy/crds/kustomization.yaml diff --git a/deploy/crds/kustomization.yaml b/deploy/crds/kustomization.yaml new file mode 100644 index 0000000..764ffa4 --- /dev/null +++ b/deploy/crds/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - nv-ipam.nvidia.com_ippools.yaml \ No newline at end of file From bf17352d953fe430d64daeeb927c29678d3ddc67 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Wed, 16 Aug 2023 14:11:02 +0300 Subject: [PATCH 2/3] Add Validating webhook Signed-off-by: Yury Kulazhenkov --- api/v1alpha1/ippool_test.go | 128 ++++++++++++++++++ api/v1alpha1/ippool_validate.go | 72 ++++++++++ api/v1alpha1/ippool_webhook.go | 60 ++++++++ api/v1alpha1/v1alpha1_suite_test.go | 13 ++ api/v1alpha1/validate_nodeselector.go | 123 +++++++++++++++++ cmd/ipam-controller/app/app.go | 7 + cmd/ipam-controller/app/options/options.go | 4 + pkg/ipam-controller/config/config.go | 43 ++---- .../controllers/ippool/ippool.go | 7 +- 9 files changed, 422 insertions(+), 35 deletions(-) create mode 100644 api/v1alpha1/ippool_test.go create mode 100644 api/v1alpha1/ippool_validate.go create mode 100644 api/v1alpha1/ippool_webhook.go create mode 100644 api/v1alpha1/v1alpha1_suite_test.go create mode 100644 api/v1alpha1/validate_nodeselector.go diff --git a/api/v1alpha1/ippool_test.go b/api/v1alpha1/ippool_test.go new file mode 100644 index 0000000..1523a70 --- /dev/null +++ b/api/v1alpha1/ippool_test.go @@ -0,0 +1,128 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" +) + +var _ = Describe("Validate", func() { + It("Valid", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Gateway: "192.168.0.1", + NodeSelector: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "foo.bar", + Operator: corev1.NodeSelectorOpExists, + }}, + }}, + }, + }, + } + Expect(ipPool.Validate()).To(BeEmpty()) + }) + It("Valid - ipv6", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "2001:db8:3333:4444::0/64", + PerNodeBlockSize: 1000, + Gateway: "2001:db8:3333:4444::1", + }, + } + Expect(ipPool.Validate()).To(BeEmpty()) + }) + It("Valid - no NodeSelector", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Gateway: "192.168.0.1", + }, + } + Expect(ipPool.Validate()).To(BeEmpty()) + }) + It("Empty object", func() { + ipPool := v1alpha1.IPPool{} + Expect(ipPool.Validate().ToAggregate().Error()). + To(And( + ContainSubstring("metadata.name"), + ContainSubstring("spec.subnet"), + ContainSubstring("spec.perNodeBlockSize"), + ContainSubstring("gateway"), + )) + }) + It("Invalid - perNodeBlockSize is too large", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/24", + PerNodeBlockSize: 300, + Gateway: "192.168.0.1", + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.perNodeBlockSize"), + ) + }) + It("Invalid - gateway outside of the subnet", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Gateway: "10.0.0.1", + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.gateway"), + ) + }) + It("Invalid - invalid NodeSelector", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Gateway: "192.168.0.1", + NodeSelector: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "foo.bar", + Operator: "unknown", + }}, + }}, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.nodeSelector"), + ) + }) +}) diff --git a/api/v1alpha1/ippool_validate.go b/api/v1alpha1/ippool_validate.go new file mode 100644 index 0000000..f4d7daa --- /dev/null +++ b/api/v1alpha1/ippool_validate.go @@ -0,0 +1,72 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1 + +import ( + "math" + "net" + + cniUtils "github.com/containernetworking/cni/pkg/utils" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Validate contains validation for the object fields +func (r *IPPool) Validate() field.ErrorList { + errList := field.ErrorList{} + if err := cniUtils.ValidateNetworkName(r.Name); err != nil { + errList = append(errList, field.Invalid( + field.NewPath("metadata", "name"), r.Name, + "invalid IP pool name, should be compatible with CNI network name")) + } + _, network, err := net.ParseCIDR(r.Spec.Subnet) + if err != nil { + errList = append(errList, field.Invalid( + field.NewPath("spec", "subnet"), r.Spec.Subnet, "is invalid subnet")) + } + + if r.Spec.PerNodeBlockSize < 2 { + errList = append(errList, field.Invalid( + field.NewPath("spec", "perNodeBlockSize"), + r.Spec.PerNodeBlockSize, "must be at least 2")) + } + + if network != nil && r.Spec.PerNodeBlockSize >= 2 { + setBits, bitsTotal := network.Mask.Size() + // possibleIPs = net size - network address - broadcast + possibleIPs := int(math.Pow(2, float64(bitsTotal-setBits))) - 2 + if possibleIPs < r.Spec.PerNodeBlockSize { + // config is not valid even if only one node exist in the cluster + errList = append(errList, field.Invalid( + field.NewPath("spec", "perNodeBlockSize"), r.Spec.PerNodeBlockSize, + "is larger then amount of IPs available in the subnet")) + } + } + parsedGW := net.ParseIP(r.Spec.Gateway) + if len(parsedGW) == 0 { + errList = append(errList, field.Invalid( + field.NewPath("spec", "gateway"), r.Spec.Gateway, + "is invalid IP address")) + } + + if network != nil && len(parsedGW) != 0 && !network.Contains(parsedGW) { + errList = append(errList, field.Invalid( + field.NewPath("spec", "gateway"), r.Spec.Gateway, + "is not part of the subnet")) + } + + if r.Spec.NodeSelector != nil { + errList = append(errList, validateNodeSelector(r.Spec.NodeSelector, field.NewPath("spec"))...) + } + return errList +} diff --git a/api/v1alpha1/ippool_webhook.go b/api/v1alpha1/ippool_webhook.go new file mode 100644 index 0000000..9910f08 --- /dev/null +++ b/api/v1alpha1/ippool_webhook.go @@ -0,0 +1,60 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logPkg "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var logger = logPkg.Log.WithName("IPPool-validator") + +// SetupWebhookWithManager registers webhook handler in the manager +func (r *IPPool) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +var _ webhook.Validator = &IPPool{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *IPPool) ValidateCreate() error { + logger.V(1).Info("validate create", "name", r.Name) + return r.validate() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *IPPool) ValidateUpdate(_ runtime.Object) error { + logger.V(1).Info("validate update", "name", r.Name) + return r.validate() +} + +func (r *IPPool) validate() error { + errList := r.Validate() + if len(errList) == 0 { + logger.V(1).Info("validation succeed") + return nil + } + err := errList.ToAggregate() + logger.V(1).Info("validation failed", "reason", err.Error()) + return err +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *IPPool) ValidateDelete() error { + return nil +} diff --git a/api/v1alpha1/v1alpha1_suite_test.go b/api/v1alpha1/v1alpha1_suite_test.go new file mode 100644 index 0000000..761ff02 --- /dev/null +++ b/api/v1alpha1/v1alpha1_suite_test.go @@ -0,0 +1,13 @@ +package v1alpha1_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestV1alpha1(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "V1alpha1 Suite") +} diff --git a/api/v1alpha1/validate_nodeselector.go b/api/v1alpha1/validate_nodeselector.go new file mode 100644 index 0000000..2341e9e --- /dev/null +++ b/api/v1alpha1/validate_nodeselector.go @@ -0,0 +1,123 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Copyright 2014 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// this package contains logic to validate NodeSelector field +// functions are copied from kubernetes/pkg/apis/core/validation to avoid import of the +// main k8s project + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + apimachineryValidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metaValidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// validateNodeSelector tests that the specified nodeSelector fields has valid data +func validateNodeSelector(nodeSelector *corev1.NodeSelector, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + termFldPath := fldPath.Child("nodeSelectorTerms") + if len(nodeSelector.NodeSelectorTerms) == 0 { + return append(allErrs, field.Required(termFldPath, "must have at least one node selector term")) + } + + for i, term := range nodeSelector.NodeSelectorTerms { + allErrs = append(allErrs, validateNodeSelectorTerm(term, termFldPath.Index(i))...) + } + + return allErrs +} + +// validateNodeSelectorTerm tests that the specified node selector term has valid data +func validateNodeSelectorTerm(term corev1.NodeSelectorTerm, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + for j, req := range term.MatchExpressions { + allErrs = append(allErrs, validateNodeSelectorRequirement(req, + fldPath.Child("matchExpressions").Index(j))...) + } + + for j, req := range term.MatchFields { + allErrs = append(allErrs, validateNodeFieldSelectorRequirement(req, + fldPath.Child("matchFields").Index(j))...) + } + + return allErrs +} + +// validateNodeSelectorRequirement tests that the specified NodeSelectorRequirement fields has valid data +func validateNodeSelectorRequirement(rq corev1.NodeSelectorRequirement, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + switch rq.Operator { + case corev1.NodeSelectorOpIn, corev1.NodeSelectorOpNotIn: + if len(rq.Values) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("values"), + "must be specified when `operator` is 'In' or 'NotIn'")) + } + case corev1.NodeSelectorOpExists, corev1.NodeSelectorOpDoesNotExist: + if len(rq.Values) > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("values"), + "may not be specified when `operator` is 'Exists' or 'DoesNotExist'")) + } + + case corev1.NodeSelectorOpGt, corev1.NodeSelectorOpLt: + if len(rq.Values) != 1 { + allErrs = append(allErrs, field.Required(fldPath.Child("values"), + "must be specified single value when `operator` is 'Lt' or 'Gt'")) + } + default: + allErrs = append(allErrs, field.Invalid(fldPath.Child("operator"), rq.Operator, + "not a valid selector operator")) + } + + allErrs = append(allErrs, metaValidation.ValidateLabelName(rq.Key, fldPath.Child("key"))...) + + return allErrs +} + +var nodeFieldSelectorValidators = map[string]func(string, bool) []string{ + metav1.ObjectNameField: apimachineryValidation.NameIsDNSSubdomain, +} + +// validateNodeFieldSelectorRequirement tests that the specified NodeSelectorRequirement fields has valid data +func validateNodeFieldSelectorRequirement(req corev1.NodeSelectorRequirement, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + switch req.Operator { + case corev1.NodeSelectorOpIn, corev1.NodeSelectorOpNotIn: + if len(req.Values) != 1 { + allErrs = append(allErrs, field.Required(fldPath.Child("values"), + "must be only one value when `operator` is 'In' or 'NotIn' for node field selector")) + } + default: + allErrs = append(allErrs, field.Invalid(fldPath.Child("operator"), req.Operator, + "not a valid selector operator")) + } + + if vf, found := nodeFieldSelectorValidators[req.Key]; !found { + allErrs = append(allErrs, field.Invalid(fldPath.Child("key"), req.Key, + "not a valid field selector key")) + } else { + for i, v := range req.Values { + for _, msg := range vf(v, false) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("values").Index(i), v, msg)) + } + } + } + + return allErrs +} diff --git a/cmd/ipam-controller/app/app.go b/cmd/ipam-controller/app/app.go index 70fe4db..f3a16b6 100644 --- a/cmd/ipam-controller/app/app.go +++ b/cmd/ipam-controller/app/app.go @@ -162,6 +162,13 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio return err } + if opts.EnableWebHook { + if err = (&ipamv1alpha1.IPPool{}).SetupWebhookWithManager(mgr); err != nil { + logger.Error(err, "unable to create webhook", "webhook", "IPPool") + os.Exit(1) + } + } + if err = (&poolctrl.IPPoolReconciler{ NodeEventCh: nodeEventCH, PoolsNamespace: opts.IPPoolsNamespace, diff --git a/cmd/ipam-controller/app/options/options.go b/cmd/ipam-controller/app/options/options.go index d209b50..1a29c9f 100644 --- a/cmd/ipam-controller/app/options/options.go +++ b/cmd/ipam-controller/app/options/options.go @@ -29,6 +29,7 @@ func New() *Options { MetricsAddr: ":8080", ProbeAddr: ":8081", EnableLeaderElection: false, + EnableWebHook: false, LeaderElectionNamespace: "kube-system", IPPoolsNamespace: "kube-system", } @@ -40,6 +41,7 @@ type Options struct { MetricsAddr string ProbeAddr string EnableLeaderElection bool + EnableWebHook bool LeaderElectionNamespace string IPPoolsNamespace string } @@ -58,6 +60,8 @@ func (o *Options) AddNamedFlagSets(sharedFS *cliflag.NamedFlagSets) { "The address the metric endpoint binds to.") controllerFS.StringVar(&o.ProbeAddr, "health-probe-bind-address", o.ProbeAddr, "The address the probe endpoint binds to.") + controllerFS.BoolVar(&o.EnableWebHook, "webhook", o.EnableWebHook, + "Enable validating webhook server as a part of the controller") controllerFS.BoolVar(&o.EnableLeaderElection, "leader-elect", o.EnableLeaderElection, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") diff --git a/pkg/ipam-controller/config/config.go b/pkg/ipam-controller/config/config.go index e4c78cc..3ed6356 100644 --- a/pkg/ipam-controller/config/config.go +++ b/pkg/ipam-controller/config/config.go @@ -15,12 +15,12 @@ package config import ( "fmt" - "math" - "net" - cniUtils "github.com/containernetworking/cni/pkg/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metaValidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" validationField "k8s.io/apimachinery/pkg/util/validation/field" + + "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" ) const ( @@ -65,33 +65,14 @@ func (c *Config) Validate() error { // ValidatePool validates the IPPool parameters func ValidatePool(name string, subnet string, gateway string, blockSize int) error { - if err := cniUtils.ValidateNetworkName(name); err != nil { - return fmt.Errorf("invalid IP pool name %s, should be compatible with CNI network name", name) - } - _, network, err := net.ParseCIDR(subnet) - if err != nil { - return fmt.Errorf("IP pool %s contains invalid subnet: %v", name, err) + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: subnet, + PerNodeBlockSize: blockSize, + Gateway: gateway, + NodeSelector: nil, + }, } - - if blockSize < 2 { - return fmt.Errorf("perNodeBlockSize should be at least 2") - } - - setBits, bitsTotal := network.Mask.Size() - // possibleIPs = net size - network address - broadcast - possibleIPs := int(math.Pow(2, float64(bitsTotal-setBits))) - 2 - if possibleIPs < blockSize { - // config is not valid even if only one node exist in the cluster - return fmt.Errorf("IP pool subnet contains less available IPs then " + - "requested by perNodeBlockSize parameter") - } - parsedGW := net.ParseIP(gateway) - if len(parsedGW) == 0 { - return fmt.Errorf("IP pool contains invalid gateway configuration: invalid IP") - } - if !network.Contains(parsedGW) { - return fmt.Errorf("IP pool contains invalid gateway configuration: " + - "gateway is outside of the subnet") - } - return nil + return ipPool.Validate().ToAggregate() } diff --git a/pkg/ipam-controller/controllers/ippool/ippool.go b/pkg/ipam-controller/controllers/ippool/ippool.go index 9221ba0..f540ea9 100644 --- a/pkg/ipam-controller/controllers/ippool/ippool.go +++ b/pkg/ipam-controller/controllers/ippool/ippool.go @@ -39,7 +39,6 @@ import ( ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" ) // IPPoolReconciler reconciles Pool objects @@ -78,9 +77,9 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } reqLog.Info("Notification on IPPool", "name", pool.Name) - err = config.ValidatePool(pool.Name, pool.Spec.Subnet, pool.Spec.Gateway, pool.Spec.PerNodeBlockSize) - if err != nil { - return r.handleInvalidSpec(ctx, err, pool) + errList := pool.Validate() + if len(errList) != 0 { + return r.handleInvalidSpec(ctx, errList.ToAggregate(), pool) } nodeList := &corev1.NodeList{} From 3a0c4ce30f4ad7c402a17e79d09d890da0d8a57d Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Wed, 16 Aug 2023 14:05:49 +0300 Subject: [PATCH 3/3] Use kustomize for deployment supported overlays: - no-webhook - deploy without webhook - certmanager - deploy with webhook to k8s cluster where certmanager is available - openshift - deploy with webhook to the Openshift cluster Signed-off-by: Yury Kulazhenkov --- README.md | 75 +++-- deploy/crds/kustomization.yaml | 2 +- deploy/manifests/certmanager/certificate.yaml | 25 ++ .../manifests/certmanager/kustomization.yaml | 5 + .../certmanager/kustomizeconfig.yaml | 16 + deploy/manifests/controller/deployment.yaml | 99 ++++++ .../manifests/controller/kustomization.yaml | 5 + deploy/manifests/controller/role.yaml | 60 ++++ deploy/manifests/controller/role_binding.yaml | 12 + .../manifests/controller/service_account.yaml | 5 + deploy/manifests/node/daemonset.yaml | 90 ++++++ deploy/manifests/node/kustomization.yaml | 5 + deploy/manifests/node/role.yaml | 21 ++ deploy/manifests/node/role_binding.yaml | 12 + deploy/manifests/node/service_account.yaml | 5 + .../openshift-privileged/kustomization.yaml | 3 + .../manifests/openshift-privileged/role.yaml | 14 + .../openshift-privileged/role_binding.yaml | 13 + deploy/manifests/webhook/kustomization.yaml | 6 + deploy/manifests/webhook/kustomizeconfig.yaml | 25 ++ deploy/manifests/webhook/manifests.yaml | 25 ++ deploy/manifests/webhook/service.yaml | 11 + deploy/nv-ipam.yaml | 301 ------------------ .../certmanager/controller_webhook_patch.yaml | 23 ++ .../overlays/certmanager/kustomization.yaml | 59 ++++ .../certmanager/webhook_config_patch.yaml | 6 + deploy/overlays/no-webhook/kustomization.yaml | 19 ++ .../node_openshift_cni_path_patch.yaml | 13 + .../openshilft/controller_webhook_patch.yaml | 23 ++ deploy/overlays/openshilft/kustomization.yaml | 33 ++ .../node_openshift_cni_path_patch.yaml | 13 + .../openshilft/webhook_config_patch.yaml | 6 + .../openshilft/webhook_service_patch.yaml | 7 + 33 files changed, 712 insertions(+), 325 deletions(-) create mode 100644 deploy/manifests/certmanager/certificate.yaml create mode 100644 deploy/manifests/certmanager/kustomization.yaml create mode 100644 deploy/manifests/certmanager/kustomizeconfig.yaml create mode 100644 deploy/manifests/controller/deployment.yaml create mode 100644 deploy/manifests/controller/kustomization.yaml create mode 100644 deploy/manifests/controller/role.yaml create mode 100644 deploy/manifests/controller/role_binding.yaml create mode 100644 deploy/manifests/controller/service_account.yaml create mode 100644 deploy/manifests/node/daemonset.yaml create mode 100644 deploy/manifests/node/kustomization.yaml create mode 100644 deploy/manifests/node/role.yaml create mode 100644 deploy/manifests/node/role_binding.yaml create mode 100644 deploy/manifests/node/service_account.yaml create mode 100644 deploy/manifests/openshift-privileged/kustomization.yaml create mode 100644 deploy/manifests/openshift-privileged/role.yaml create mode 100644 deploy/manifests/openshift-privileged/role_binding.yaml create mode 100644 deploy/manifests/webhook/kustomization.yaml create mode 100644 deploy/manifests/webhook/kustomizeconfig.yaml create mode 100644 deploy/manifests/webhook/manifests.yaml create mode 100644 deploy/manifests/webhook/service.yaml delete mode 100644 deploy/nv-ipam.yaml create mode 100644 deploy/overlays/certmanager/controller_webhook_patch.yaml create mode 100644 deploy/overlays/certmanager/kustomization.yaml create mode 100644 deploy/overlays/certmanager/webhook_config_patch.yaml create mode 100644 deploy/overlays/no-webhook/kustomization.yaml create mode 100644 deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml create mode 100644 deploy/overlays/openshilft/controller_webhook_patch.yaml create mode 100644 deploy/overlays/openshilft/kustomization.yaml create mode 100644 deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml create mode 100644 deploy/overlays/openshilft/webhook_config_patch.yaml create mode 100644 deploy/overlays/openshilft/webhook_service_patch.yaml diff --git a/README.md b/README.md index 0b0da21..f047f67 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,16 @@ NVIDIA IPAM plugin consists of 3 main components: A Kubernetes(K8s) controller that Watches on IPPools CRs in a predefined Namespace. It then proceeds by assiging each node via IPPools Status a cluster unique range of IPs of the defined IP Pools. +#### Validation webhook + +ipam-controller implements validation webhook for IPPool resource. +The webhook can prevent the creation of IPPool resources with invalid configurations. +Supported X.509 certificate management system should be available in the cluster to enable the webhook. +Currently supported systems are [certmanager](https://cert-manager.io/) and +[Openshift certificate management](https://docs.openshift.com/container-platform/4.13/security/certificates/service-serving-certificate.html) + +Activation of the validation webhook is optional. Check the [Deployment](#deployment) section for details. + ### ipam-node The daemon is responsible for: @@ -144,48 +154,50 @@ ipam-controller accepts configuration using command line flags and IPPools CRs. ```text Logging flags: - --log-flush-frequency duration + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) - --log-json-info-buffer-size quantity - [Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The - size can be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature - gate to use this. - --log-json-split-stream - [Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate - to use this. - --logging-format string + --log-json-info-buffer-size quantity + [Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The size can + be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature gate to use this. + --log-json-split-stream + [Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate to use + this. + --logging-format string Sets the log format. Permitted formats: "json" (gated by LoggingBetaOptions), "text". (default "text") - -v, --v Level + -v, --v Level number for the log level verbosity - --vmodule pattern=N,... + --vmodule pattern=N,... comma-separated list of pattern=N settings for file-filtered logging (only works for text log format) Common flags: - --feature-gates mapStringBool + --feature-gates mapStringBool A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: AllAlpha=true|false (ALPHA - default=false) AllBeta=true|false (BETA - default=false) ContextualLogging=true|false (ALPHA - default=false) LoggingAlphaOptions=true|false (ALPHA - default=false) LoggingBetaOptions=true|false (BETA - default=true) - --version + --version print binary version and exit Controller flags: - --health-probe-bind-address string + --health-probe-bind-address string The address the probe endpoint binds to. (default ":8081") - --kubeconfig string + --ippools-namespace string + The name of the namespace to watch for IPPools CRs (default "kube-system") + --kubeconfig string Paths to a kubeconfig. Only required if out-of-cluster. - --leader-elect + --leader-elect Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. - --leader-elect-namespace string + --leader-elect-namespace string Determines the namespace in which the leader election resource will be created. (default "kube-system") - --metrics-bind-address string + --metrics-bind-address string The address the metric endpoint binds to. (default ":8080") - --ippools-namespace string - The name of the namespace to watch for IPPools CRs. (default "kube-system") + --webhook + Enable validating webhook server as a part of the controller + ``` #### IPPool CR @@ -331,11 +343,28 @@ interface should have two IP addresses: one IPv4 and one IPv6. (default: network ### Deploy IPAM plugin -> _NOTE:_ This command will deploy latest dev build with default configuration +> _NOTE:_ These commands will deploy latest dev build with default configuration + +The plugin can be deployed with kustomize. + +Supported overlays are: + +`no-webhook` - deploy without webhook + +```shell +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/no-webhook?ref=main | kubectl apply -f - +``` + +`certmanager` - deploy with webhook to the Kubernetes cluster where certmanager is available + +```shell +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/certmanager?ref=main | kubectl apply -f - +``` + +`openshift` - deploy with webhook to the Openshift cluster ```shell -kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/crds/nv-ipam.nvidia.com_ippools.yaml -kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/nv-ipam.yaml +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/openshift?ref=main | kubectl apply -f - ``` ### Create IPPool CR diff --git a/deploy/crds/kustomization.yaml b/deploy/crds/kustomization.yaml index 764ffa4..a9c6a2f 100644 --- a/deploy/crds/kustomization.yaml +++ b/deploy/crds/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - nv-ipam.nvidia.com_ippools.yaml \ No newline at end of file + - nv-ipam.nvidia.com_ippools.yaml diff --git a/deploy/manifests/certmanager/certificate.yaml b/deploy/manifests/certmanager/certificate.yaml new file mode 100644 index 0000000..91713a2 --- /dev/null +++ b/deploy/manifests/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: nv-ipam-webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/deploy/manifests/certmanager/kustomization.yaml b/deploy/manifests/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/deploy/manifests/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/manifests/certmanager/kustomizeconfig.yaml b/deploy/manifests/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..e631f77 --- /dev/null +++ b/deploy/manifests/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/deploy/manifests/controller/deployment.yaml b/deploy/manifests/controller/deployment.yaml new file mode 100644 index 0000000..bc86924 --- /dev/null +++ b/deploy/manifests/controller/deployment.yaml @@ -0,0 +1,99 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system + annotations: + kubernetes.io/description: | + This deployment launches the nv-ipam controller for nv-ipam. +spec: + strategy: + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + name: controller + template: + metadata: + labels: + name: controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: controller + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: name + operator: In + values: + - controller + topologyKey: "kubernetes.io/hostname" + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: controller + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + command: [ "/ipam-controller" ] + args: + - --leader-elect=true + - --leader-elect-namespace=$(POD_NAMESPACE) + - --ippools-namespace=$(POD_NAMESPACE) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 300Mi + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP diff --git a/deploy/manifests/controller/kustomization.yaml b/deploy/manifests/controller/kustomization.yaml new file mode 100644 index 0000000..3246145 --- /dev/null +++ b/deploy/manifests/controller/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - deployment.yaml + - role.yaml + - role_binding.yaml + - service_account.yaml diff --git a/deploy/manifests/controller/role.yaml b/deploy/manifests/controller/role.yaml new file mode 100644 index 0000000..13a9b6b --- /dev/null +++ b/deploy/manifests/controller/role.yaml @@ -0,0 +1,60 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - delete + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch + - create + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools/status + verbs: + - get + - update + - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/deploy/manifests/controller/role_binding.yaml b/deploy/manifests/controller/role_binding.yaml new file mode 100644 index 0000000..fcd3a36 --- /dev/null +++ b/deploy/manifests/controller/role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: controller +subjects: + - kind: ServiceAccount + name: controller + namespace: system diff --git a/deploy/manifests/controller/service_account.yaml b/deploy/manifests/controller/service_account.yaml new file mode 100644 index 0000000..130eeec --- /dev/null +++ b/deploy/manifests/controller/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller + namespace: system diff --git a/deploy/manifests/node/daemonset.yaml b/deploy/manifests/node/daemonset.yaml new file mode 100644 index 0000000..7d4cd07 --- /dev/null +++ b/deploy/manifests/node/daemonset.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node +spec: + selector: + matchLabels: + name: nv-ipam-node + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node + spec: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + serviceAccountName: node + containers: + - name: node + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + command: [ "/ipam-node" ] + args: + - --node-name=$(NODE_NAME) + - --v=1 # log level for ipam-node + - --logging-format=json + - --bind-address=unix:///var/lib/cni/nv-ipam/daemon.sock + - --store-file=/var/lib/cni/nv-ipam/store + - --cni-daemon-socket=unix:///var/lib/cni/nv-ipam/daemon.sock + - --cni-daemon-call-timeout=5 # 5 seconds + - --cni-bin-dir=/opt/cni/bin + - --cni-conf-dir=/etc/cni/net.d/nv-ipam.d + - --cni-log-file=/var/log/nv-ipam-cni.log + - --cni-log-level=info # log level for shim CNI + - --ippools-namespace=$(POD_NAMESPACE) + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "300m" + memory: "300Mi" + volumeMounts: + - name: cnibin + mountPath: /opt/cni/bin + - name: cniconf + mountPath: /etc/cni/net.d/nv-ipam.d + - name: daemonstate + mountPath: /var/lib/cni/nv-ipam/ + terminationGracePeriodSeconds: 10 + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cniconf + hostPath: + path: /etc/cni/net.d/nv-ipam.d + type: DirectoryOrCreate + - name: daemonstate + hostPath: + path: /var/lib/cni/nv-ipam/ + type: DirectoryOrCreate diff --git a/deploy/manifests/node/kustomization.yaml b/deploy/manifests/node/kustomization.yaml new file mode 100644 index 0000000..799ffbc --- /dev/null +++ b/deploy/manifests/node/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - daemonset.yaml + - role.yaml + - role_binding.yaml + - service_account.yaml diff --git a/deploy/manifests/node/role.yaml b/deploy/manifests/node/role.yaml new file mode 100644 index 0000000..58519d2 --- /dev/null +++ b/deploy/manifests/node/role.yaml @@ -0,0 +1,21 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch diff --git a/deploy/manifests/node/role_binding.yaml b/deploy/manifests/node/role_binding.yaml new file mode 100644 index 0000000..c3db13c --- /dev/null +++ b/deploy/manifests/node/role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node +subjects: + - kind: ServiceAccount + name: node + namespace: system diff --git a/deploy/manifests/node/service_account.yaml b/deploy/manifests/node/service_account.yaml new file mode 100644 index 0000000..d55c1d4 --- /dev/null +++ b/deploy/manifests/node/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node + namespace: system diff --git a/deploy/manifests/openshift-privileged/kustomization.yaml b/deploy/manifests/openshift-privileged/kustomization.yaml new file mode 100644 index 0000000..1af844a --- /dev/null +++ b/deploy/manifests/openshift-privileged/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - role_binding.yaml + - role.yaml diff --git a/deploy/manifests/openshift-privileged/role.yaml b/deploy/manifests/openshift-privileged/role.yaml new file mode 100644 index 0000000..b9cc539 --- /dev/null +++ b/deploy/manifests/openshift-privileged/role.yaml @@ -0,0 +1,14 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node + namespace: system +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged diff --git a/deploy/manifests/openshift-privileged/role_binding.yaml b/deploy/manifests/openshift-privileged/role_binding.yaml new file mode 100644 index 0000000..cf60f70 --- /dev/null +++ b/deploy/manifests/openshift-privileged/role_binding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: node + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: node +subjects: + - kind: ServiceAccount + name: node + namespace: system diff --git a/deploy/manifests/webhook/kustomization.yaml b/deploy/manifests/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/deploy/manifests/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/manifests/webhook/kustomizeconfig.yaml b/deploy/manifests/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..25e21e3 --- /dev/null +++ b/deploy/manifests/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/deploy/manifests/webhook/manifests.yaml b/deploy/manifests/webhook/manifests.yaml new file mode 100644 index 0000000..d06f2b7 --- /dev/null +++ b/deploy/manifests/webhook/manifests.yaml @@ -0,0 +1,25 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-nv-ipam-nvidia-com-v1alpha1-ippool + failurePolicy: Fail + name: validate-ippool.nv-ipam.nvidia.com + rules: + - apiGroups: + - nv-ipam.nvidia.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - ippools + sideEffects: None diff --git a/deploy/manifests/webhook/service.yaml b/deploy/manifests/webhook/service.yaml new file mode 100644 index 0000000..50a79d1 --- /dev/null +++ b/deploy/manifests/webhook/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + name: controller diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml deleted file mode 100644 index 95c0f34..0000000 --- a/deploy/nv-ipam.yaml +++ /dev/null @@ -1,301 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools - verbs: - - get - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nv-ipam-node -subjects: - - kind: ServiceAccount - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nv-ipam-node-ds - namespace: kube-system - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node -spec: - selector: - matchLabels: - name: nv-ipam-node - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node - spec: - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - serviceAccountName: nv-ipam-node - containers: - - name: nv-ipam-node - image: ghcr.io/mellanox/nvidia-k8s-ipam:latest - imagePullPolicy: IfNotPresent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: IPPOOLS_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: [ "/ipam-node" ] - args: - - --node-name=$(NODE_NAME) - - --v=1 # log level for ipam-node - - --logging-format=json - - --bind-address=unix:///var/lib/cni/nv-ipam/daemon.sock - - --store-file=/var/lib/cni/nv-ipam/store - - --cni-daemon-socket=unix:///var/lib/cni/nv-ipam/daemon.sock - - --cni-daemon-call-timeout=5 # 5 seconds - - --cni-bin-dir=/opt/cni/bin - - --cni-conf-dir=/etc/cni/net.d/nv-ipam.d - - --cni-log-file=/var/log/nv-ipam-cni.log - - --cni-log-level=info # log level for shim CNI - - --ippools-namespace=$(IPPOOLS_NAMESPACE) - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "300m" - memory: "300Mi" - volumeMounts: - - name: cnibin - mountPath: /opt/cni/bin - - name: cniconf - mountPath: /etc/cni/net.d/nv-ipam.d - - name: daemonstate - mountPath: /var/lib/cni/nv-ipam/ - terminationGracePeriodSeconds: 10 - volumes: - - name: cnibin - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - - name: cniconf - hostPath: - path: /etc/cni/net.d/nv-ipam.d - type: DirectoryOrCreate - - name: daemonstate - hostPath: - path: /var/lib/cni/nv-ipam/ - type: DirectoryOrCreate ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-controller -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - delete - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools - verbs: - - get - - list - - watch - - create - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools/status - verbs: - - get - - update - - patch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nv-ipam-controller -subjects: - - kind: ServiceAccount - name: nv-ipam-controller - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nv-ipam-controller - namespace: kube-system ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: nv-ipam-controller - namespace: kube-system - annotations: - kubernetes.io/description: | - This deployment launches the nv-ipam controller for nv-ipam. -spec: - strategy: - type: RollingUpdate - replicas: 1 - selector: - matchLabels: - name: nv-ipam-controller - template: - metadata: - labels: - name: nv-ipam-controller - spec: - priorityClassName: system-cluster-critical - serviceAccountName: nv-ipam-controller - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: name - operator: In - values: - - nv-ipam-controller - topologyKey: "kubernetes.io/hostname" - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - containers: - - name: nv-ipam-controller - image: ghcr.io/mellanox/nvidia-k8s-ipam:latest - imagePullPolicy: IfNotPresent - command: [ "/ipam-controller" ] - args: - - --leader-elect=true - - --leader-elect-namespace=$(POD_NAMESPACE) - - --ippools-namespace=$(POD_NAMESPACE) - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 300Mi diff --git a/deploy/overlays/certmanager/controller_webhook_patch.yaml b/deploy/overlays/certmanager/controller_webhook_patch.yaml new file mode 100644 index 0000000..ca7011d --- /dev/null +++ b/deploy/overlays/certmanager/controller_webhook_patch.yaml @@ -0,0 +1,23 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: nv-ipam-webhook-server-cert diff --git a/deploy/overlays/certmanager/kustomization.yaml b/deploy/overlays/certmanager/kustomization.yaml new file mode 100644 index 0000000..b23adea --- /dev/null +++ b/deploy/overlays/certmanager/kustomization.yaml @@ -0,0 +1,59 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + - ../../manifests/webhook + - ../../manifests/certmanager + +patches: + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook=true + target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + - path: controller_webhook_patch.yaml + - path: webhook_config_patch.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/deploy/overlays/certmanager/webhook_config_patch.yaml b/deploy/overlays/certmanager/webhook_config_patch.yaml new file mode 100644 index 0000000..6b2c0f3 --- /dev/null +++ b/deploy/overlays/certmanager/webhook_config_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/deploy/overlays/no-webhook/kustomization.yaml b/deploy/overlays/no-webhook/kustomization.yaml new file mode 100644 index 0000000..1454e23 --- /dev/null +++ b/deploy/overlays/no-webhook/kustomization.yaml @@ -0,0 +1,19 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + +# Uncomment this to deploy to Openshift cluster +#patches: +# - path: node_openshift_cni_path_patch.yaml diff --git a/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml b/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml new file mode 100644 index 0000000..8c5438c --- /dev/null +++ b/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system +spec: + template: + spec: + volumes: + - name: cnibin + hostPath: + path: /var/lib/cni/bin + type: DirectoryOrCreate diff --git a/deploy/overlays/openshilft/controller_webhook_patch.yaml b/deploy/overlays/openshilft/controller_webhook_patch.yaml new file mode 100644 index 0000000..ca7011d --- /dev/null +++ b/deploy/overlays/openshilft/controller_webhook_patch.yaml @@ -0,0 +1,23 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: nv-ipam-webhook-server-cert diff --git a/deploy/overlays/openshilft/kustomization.yaml b/deploy/overlays/openshilft/kustomization.yaml new file mode 100644 index 0000000..abce16d --- /dev/null +++ b/deploy/overlays/openshilft/kustomization.yaml @@ -0,0 +1,33 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + - ../../manifests/openshift-privileged + - ../../manifests/webhook + +patches: + - path: node_openshift_cni_path_patch.yaml + - path: controller_webhook_patch.yaml + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook=true + target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + - path: webhook_service_patch.yaml + - path: webhook_config_patch.yaml diff --git a/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml b/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml new file mode 100644 index 0000000..8c5438c --- /dev/null +++ b/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system +spec: + template: + spec: + volumes: + - name: cnibin + hostPath: + path: /var/lib/cni/bin + type: DirectoryOrCreate diff --git a/deploy/overlays/openshilft/webhook_config_patch.yaml b/deploy/overlays/openshilft/webhook_config_patch.yaml new file mode 100644 index 0000000..e5a1054 --- /dev/null +++ b/deploy/overlays/openshilft/webhook_config_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/deploy/overlays/openshilft/webhook_service_patch.yaml b/deploy/overlays/openshilft/webhook_service_patch.yaml new file mode 100644 index 0000000..50084b6 --- /dev/null +++ b/deploy/overlays/openshilft/webhook_service_patch.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system + annotations: + service.alpha.openshift.io/serving-cert-secret-name: nv-ipam-webhook-server-cert