From 9a0db98c89019eb5d43746ad3bc0460f487ce99d Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Mon, 24 Jul 2023 14:13:16 +0300 Subject: [PATCH 1/5] NVIPAMPool CRD - Add CRD definition - Add Make targets - Add CR examples based on ConfigMap example Signed-off-by: Fred Rolland --- Makefile | 15 +- api/v1alpha1/groupversion_info.go | 33 ++++ api/v1alpha1/ippool_type.go | 72 ++++++++ api/v1alpha1/zz_generated.deepcopy.go | 137 +++++++++++++++ boilerplate.go.txt | 12 ++ deploy/crds/nv-ipam.nvidia.com_ippools.yaml | 175 ++++++++++++++++++++ examples/ippool-1.yaml | 16 ++ examples/ippool-2.yaml | 16 ++ 8 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 api/v1alpha1/groupversion_info.go create mode 100644 api/v1alpha1/ippool_type.go create mode 100644 api/v1alpha1/zz_generated.deepcopy.go create mode 100644 boilerplate.go.txt create mode 100644 deploy/crds/nv-ipam.nvidia.com_ippools.yaml create mode 100644 examples/ippool-1.yaml create mode 100644 examples/ippool-2.yaml diff --git a/Makefile b/Makefile index e2b9e1d..f755d6b 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,7 @@ PROTOC ?= $(LOCALBIN)/protoc/bin/protoc PROTOC_GEN_GO ?= $(LOCALBIN)/protoc-gen-go PROTOC_GEN_GO_GRPC ?= $(LOCALBIN)/protoc-gen-go-grpc BUF ?= $(LOCALBIN)/buf +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ## Tool Versions GOLANGCILINT_VERSION ?= v1.52.2 @@ -152,7 +153,7 @@ PROTOC_VER ?= 23.4 PROTOC_GEN_GO_VER ?= 1.31.0 PROTOC_GEN_GO_GRPC_VER ?= 1.3.0 BUF_VERSION ?= 1.23.1 - +CONTROLLER_GEN_VERSION ?= v0.13.0 .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. @@ -251,3 +252,15 @@ grpc-format: buf ## Format GRPC files @echo "format protobuf files"; cd $(PROTO_DIR) && \ $(BUF) format -w --exit-code +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary +$(CONTROLLER_GEN): | $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION) + +.PHONY: generate +generate: controller-gen + $(CONTROLLER_GEN) object:headerFile="./boilerplate.go.txt" paths="./api/..." + +.PHONY: manifests +manifests: controller-gen + $(CONTROLLER_GEN) crd paths="./api/..." output:dir="./deploy/crds" diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..2852a03 --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,33 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=nv-ipam.nvidia.com + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "nv-ipam.nvidia.com", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/ippool_type.go b/api/v1alpha1/ippool_type.go new file mode 100644 index 0000000..834921a --- /dev/null +++ b/api/v1alpha1/ippool_type.go @@ -0,0 +1,72 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Subnet",type="string",JSONPath=`.spec.subnet` +// +kubebuilder:printcolumn:name="Gateway",type="string",JSONPath=`.spec.gateway` +// +kubebuilder:printcolumn:name="Block Size",type="integer",JSONPath=`.spec.perNodeBlockSize` + +// IPPool contains configuration for IPAM controller +type IPPool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec IPPoolSpec `json:"spec"` + Status IPPoolStatus `json:"status,omitempty"` +} + +// IPPoolSpec contains configuration for IP pool +type IPPoolSpec struct { + // subnet of the pool + Subnet string `json:"subnet"` + // amount of IPs to allocate for each node, + // must be less than amount of available IPs in the subnet + PerNodeBlockSize int `json:"perNodeBlockSize"` + // gateway for the pool + Gateway string `json:"gateway"` + // selector for nodes, if empty match all nodes + NodeSelector *corev1.NodeSelector `json:"nodeSelector,omitempty"` +} + +// IPPoolStatus contains the IP ranges allocated to nodes +type IPPoolStatus struct { + // IP allocations for Nodes + Allocations []Allocation `json:"allocations"` +} + +// Allocation contains IP Allocation for a specific Node +type Allocation struct { + NodeName string `json:"nodeName"` + StartIP string `json:"startIP"` + EndIP string `json:"endIP"` +} + +// +kubebuilder:object:root=true + +// IPPoolList contains a list of IPPool +type IPPoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []IPPool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&IPPool{}, &IPPoolList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..f74af3e --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,137 @@ +//go:build !ignore_autogenerated + +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Allocation) DeepCopyInto(out *Allocation) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Allocation. +func (in *Allocation) DeepCopy() *Allocation { + if in == nil { + return nil + } + out := new(Allocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPool) DeepCopyInto(out *IPPool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPool. +func (in *IPPool) DeepCopy() *IPPool { + if in == nil { + return nil + } + out := new(IPPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPPool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolList) DeepCopyInto(out *IPPoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]IPPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolList. +func (in *IPPoolList) DeepCopy() *IPPoolList { + if in == nil { + return nil + } + out := new(IPPoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *IPPoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(v1.NodeSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolSpec. +func (in *IPPoolSpec) DeepCopy() *IPPoolSpec { + if in == nil { + return nil + } + out := new(IPPoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPPoolStatus) DeepCopyInto(out *IPPoolStatus) { + *out = *in + if in.Allocations != nil { + in, out := &in.Allocations, &out.Allocations + *out = make([]Allocation, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolStatus. +func (in *IPPoolStatus) DeepCopy() *IPPoolStatus { + if in == nil { + return nil + } + out := new(IPPoolStatus) + in.DeepCopyInto(out) + return out +} diff --git a/boilerplate.go.txt b/boilerplate.go.txt new file mode 100644 index 0000000..44d6a40 --- /dev/null +++ b/boilerplate.go.txt @@ -0,0 +1,12 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ diff --git a/deploy/crds/nv-ipam.nvidia.com_ippools.yaml b/deploy/crds/nv-ipam.nvidia.com_ippools.yaml new file mode 100644 index 0000000..0190190 --- /dev/null +++ b/deploy/crds/nv-ipam.nvidia.com_ippools.yaml @@ -0,0 +1,175 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: ippools.nv-ipam.nvidia.com +spec: + group: nv-ipam.nvidia.com + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.subnet + name: Subnet + type: string + - jsonPath: .spec.gateway + name: Gateway + type: string + - jsonPath: .spec.perNodeBlockSize + name: Block Size + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: IPPool contains configuration for IPAM controller + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains configuration for IP pool + properties: + gateway: + description: gateway for the pool + type: string + nodeSelector: + description: selector for nodes, if empty match all nodes + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + perNodeBlockSize: + description: amount of IPs to allocate for each node, must be less + than amount of available IPs in the subnet + type: integer + subnet: + description: subnet of the pool + type: string + required: + - gateway + - perNodeBlockSize + - subnet + type: object + status: + description: IPPoolStatus contains the IP ranges allocated to nodes + properties: + allocations: + description: IP allocations for Nodes + items: + description: Allocation contains IP Allocation for a specific Node + properties: + endIP: + type: string + nodeName: + type: string + startIP: + type: string + required: + - endIP + - nodeName + - startIP + type: object + type: array + required: + - allocations + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/examples/ippool-1.yaml b/examples/ippool-1.yaml new file mode 100644 index 0000000..8fd733b --- /dev/null +++ b/examples/ippool-1.yaml @@ -0,0 +1,16 @@ +apiVersion: nv-ipam.nvidia.com/v1alpha1 +kind: IPPool +metadata: + name: pool1 + namespace: kube-system +spec: + subnet: 192.168.0.0/16 + perNodeBlockSize: 128 + gateway: 192.168.0.1 + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: In + values: + - worker diff --git a/examples/ippool-2.yaml b/examples/ippool-2.yaml new file mode 100644 index 0000000..cb17207 --- /dev/null +++ b/examples/ippool-2.yaml @@ -0,0 +1,16 @@ +apiVersion: nv-ipam.nvidia.com/v1alpha1 +kind: IPPool +metadata: + name: pool2 + namespace: kube-system +spec: + subnet: 172.16.0.0/16 + perNodeBlockSize: 128 + gateway: 172.16.0.1 + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: In + values: + - worker From 2847204edac7388459525d0d9a8f17914f96eca0 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Sun, 6 Aug 2023 16:30:56 +0300 Subject: [PATCH 2/5] IPPool controller Add IPPool controller, watching Pools. The IP range allocations are available on the CR status. Node controller triggers event on all Pools when Node is added/updated/deleted. Signed-off-by: Fred Rolland --- cmd/ipam-controller/app/app.go | 54 +-- cmd/ipam-controller/app/app_suite_test.go | 15 +- cmd/ipam-controller/app/app_test.go | 292 +++++++------ cmd/ipam-controller/app/options/options.go | 12 +- deploy/nv-ipam.yaml | 19 +- examples/configmap.yaml | 14 - go.mod | 1 + go.sum | 2 + pkg/ipam-controller/allocator/allocator.go | 327 ++++----------- .../allocator/allocator_test.go | 394 +++++++++--------- pkg/ipam-controller/config/config.go | 58 +-- .../controllers/configmap/configmap.go | 132 ------ .../controllers/ippool/ippool.go | 165 ++++++++ pkg/ipam-controller/controllers/node/node.go | 102 +---- pkg/ipam-controller/selector/selector.go | 49 --- .../selector/selector_suite_test.go | 26 -- pkg/ipam-controller/selector/selector_test.go | 41 -- 17 files changed, 730 insertions(+), 973 deletions(-) delete mode 100644 examples/configmap.yaml delete mode 100644 pkg/ipam-controller/controllers/configmap/configmap.go create mode 100644 pkg/ipam-controller/controllers/ippool/ippool.go delete mode 100644 pkg/ipam-controller/selector/selector.go delete mode 100644 pkg/ipam-controller/selector/selector_suite_test.go delete mode 100644 pkg/ipam-controller/selector/selector_test.go diff --git a/cmd/ipam-controller/app/app.go b/cmd/ipam-controller/app/app.go index b659627..f668679 100644 --- a/cmd/ipam-controller/app/app.go +++ b/cmd/ipam-controller/app/app.go @@ -21,8 +21,6 @@ import ( "github.com/go-logr/logr" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -30,7 +28,6 @@ import ( "k8s.io/component-base/term" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -40,12 +37,11 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-controller/app/options" "github.com/Mellanox/nvidia-k8s-ipam/pkg/common" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" - configmapctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/configmap" + poolctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/ippool" nodectrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/node" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/selector" "github.com/Mellanox/nvidia-k8s-ipam/pkg/version" ) @@ -98,8 +94,7 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio ctrl.SetLogger(logger) logger.Info("start IPAM controller", - "version", version.GetVersionString(), "config", opts.ConfigMapName, - "configNamespace", opts.ConfigMapNamespace) + "version", version.GetVersionString(), "poolsNamespace", opts.IPPoolsNamespace) scheme := runtime.NewScheme() @@ -108,15 +103,14 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio return err } + if err := ipamv1alpha1.AddToScheme(scheme); err != nil { + logger.Error(err, "failed to register ipamv1alpha1 scheme") + return err + } + mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, - NewCache: cache.BuilderWithOptions(cache.Options{ - SelectorsByObject: cache.SelectorsByObject{&corev1.ConfigMap{}: cache.ObjectSelector{ - Field: fields.AndSelectors( - fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", opts.ConfigMapName)), - fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace=%s", opts.ConfigMapNamespace))), - }}, - }), + Scheme: scheme, + Namespace: opts.IPPoolsNamespace, MetricsBindAddress: opts.MetricsAddr, Port: 9443, HealthProbeBindAddress: opts.ProbeAddr, @@ -131,31 +125,25 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio return err } - netAllocator := allocator.New() - nodeSelector := selector.New() - configEventCH := make(chan event.GenericEvent, 1) + nodeEventCH := make(chan event.GenericEvent, 1) if err = (&nodectrl.NodeReconciler{ - Allocator: netAllocator, - Selector: nodeSelector, - ConfigEventCh: configEventCH, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + NodeEventCh: nodeEventCH, + PoolsNamespace: opts.IPPoolsNamespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "Node") return err } - if err = (&configmapctrl.ConfigMapReconciler{ - Allocator: netAllocator, - Selector: nodeSelector, - ConfigEventCh: configEventCH, - ConfigMapName: opts.ConfigMapName, - ConfigMapNamespace: opts.ConfigMapNamespace, - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + if err = (&poolctrl.IPPoolReconciler{ + NodeEventCh: nodeEventCH, + PoolsNamespace: opts.IPPoolsNamespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "ConfigMap") + logger.Error(err, "unable to create controller", "controller", "IPPool") return err } diff --git a/cmd/ipam-controller/app/app_suite_test.go b/cmd/ipam-controller/app/app_suite_test.go index c65b1bb..1cb15f5 100644 --- a/cmd/ipam-controller/app/app_suite_test.go +++ b/cmd/ipam-controller/app/app_suite_test.go @@ -21,9 +21,12 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" ) const ( @@ -46,17 +49,25 @@ func TestApp(t *testing.T) { var _ = BeforeSuite(func() { By("bootstrapping test environment") - testEnv = &envtest.Environment{} + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{"../../../deploy/crds"}, + CRDInstallOptions: envtest.CRDInstallOptions{ + ErrorIfPathMissing: true, + }, + } ctx, cFunc = context.WithCancel(context.Background()) var err error + err = ipamv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + // cfg is defined in this file globally. cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - k8sClient, err = client.New(cfg, client.Options{}) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) diff --git a/cmd/ipam-controller/app/app_test.go b/cmd/ipam-controller/app/app_test.go index ab9b56a..f606ea8 100644 --- a/cmd/ipam-controller/app/app_test.go +++ b/cmd/ipam-controller/app/app_test.go @@ -15,7 +15,7 @@ package app_test import ( "context" - "fmt" + "reflect" "time" . "github.com/onsi/ginkgo/v2" @@ -23,15 +23,13 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-controller/app" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-controller/app/options" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) const ( @@ -40,48 +38,6 @@ const ( pool3Name = "pool3" ) -func updateConfigMap(data string) { - d := map[string]string{config.ConfigMapKey: data} - err := k8sClient.Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: TestConfigMapName, Namespace: TestNamespace}, - Data: d, - }) - if err == nil { - return - } - if apiErrors.IsAlreadyExists(err) { - configMap := &corev1.ConfigMap{} - Expect(k8sClient.Get( - ctx, types.NamespacedName{Name: TestConfigMapName, Namespace: TestNamespace}, configMap)).NotTo(HaveOccurred()) - configMap.Data = d - Expect(k8sClient.Update( - ctx, configMap)).NotTo(HaveOccurred()) - } else { - Expect(err).NotTo(HaveOccurred()) - } -} - -var validConfig = fmt.Sprintf(` - { - "pools": { - "%s": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 10 , "gateway": "192.168.0.1"}, - "%s": { "subnet": "172.16.0.0/16", "perNodeBlockSize": 50 , "gateway": "172.16.0.1"} - } - } -`, pool1Name, pool2Name) - -// ranges for two nodes only can be allocated -var validConfig2 = fmt.Sprintf(` - { - "pools": { - "%s": { "subnet": "172.17.0.0/24", "perNodeBlockSize": 100 , "gateway": "172.17.0.1"} - }, - "nodeSelector": {"foo": "bar"} - } -`, pool3Name) - -var invalidConfig = `{{{` - func createNode(name string) *corev1.Node { node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}} Expect(k8sClient.Create(ctx, node)).NotTo(HaveOccurred()) @@ -99,13 +55,39 @@ func updateNode(node *corev1.Node) *corev1.Node { return node } -func getRangeFromNode(nodeName string) map[string]*pool.IPPool { - node := getNode(nodeName) - poolCfg, err := pool.NewConfigReader(node) - if err != nil { - return nil +func getRangeForNode(nodeName string, poolName string) *ipamv1alpha1.Allocation { + allocations := getAllocationsFromIPPools(poolName) + + for _, a := range allocations { + alloc := a + if a.NodeName == nodeName { + return &alloc + } + } + return nil +} + +func getAllocationsFromIPPools(poolName string) []ipamv1alpha1.Allocation { + pool := &ipamv1alpha1.IPPool{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: TestNamespace, Name: poolName}, pool)).NotTo(HaveOccurred()) + return pool.Status.Allocations +} + +func checkAllocationExists(allocations []ipamv1alpha1.Allocation, allocation ipamv1alpha1.Allocation) bool { + for _, a := range allocations { + if reflect.DeepEqual(a, allocation) { + return true + } } - return poolCfg.GetPools() + return false +} + +func updatePoolSpec(poolName string, spec ipamv1alpha1.IPPoolSpec) { + pool := &ipamv1alpha1.IPPool{} + Expect(k8sClient.Get( + ctx, types.NamespacedName{Name: poolName, Namespace: TestNamespace}, pool)).NotTo(HaveOccurred()) + pool.Spec = spec + Expect(k8sClient.Update(ctx, pool)).NotTo(HaveOccurred()) } // WaitAndCheckForStability wait for condition and then check it is stable for 1 second @@ -124,39 +106,62 @@ var _ = Describe("App", func() { cfg1pools := []string{pool1Name, pool2Name} - By("Create valid cfg1") - updateConfigMap(validConfig) - - By("Set annotation with valid ranges for node1") - node1 := createNode(testNode1) - node1InitialRanges := map[string]*pool.IPPool{pool1Name: { - Name: pool1Name, - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.11", - EndIP: "192.168.0.20", - Gateway: "192.168.0.1", + By("Create valid pools") + pool1 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: pool1Name, + Namespace: TestNamespace, + }, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + Gateway: "192.168.0.1", + PerNodeBlockSize: 10, + }, + } + Expect(k8sClient.Create(ctx, pool1)).NotTo(HaveOccurred()) + + pool2 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: pool2Name, + Namespace: TestNamespace, + }, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "172.16.0.0/16", + Gateway: "172.16.0.1", + PerNodeBlockSize: 50, + }, + } + Expect(k8sClient.Create(ctx, pool2)).NotTo(HaveOccurred()) + + By("Update Pools Status with valid ranges for node1 and invalid for node2 (wrong IP count)") + createNode(testNode1) + createNode(testNode2) + node1InitialRanges := map[string]ipamv1alpha1.Allocation{pool1Name: { + NodeName: testNode1, + StartIP: "192.168.0.11", + EndIP: "192.168.0.20", }, pool2Name: { - Name: pool2Name, - Subnet: "172.16.0.0/16", - StartIP: "172.16.0.1", - EndIP: "172.16.0.50", - Gateway: "172.16.0.1", - }} - Expect(pool.SetIPBlockAnnotation(node1, node1InitialRanges)).NotTo(HaveOccurred()) - Expect(updateNode(node1)) - - By("Set annotation with invalid ranges for node2") - // create annotation for node2 with invalid config (wrong IP count) - node2 := createNode(testNode2) - node2InitialRanges := map[string]*pool.IPPool{pool1Name: { - Name: pool1Name, - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.11", - EndIP: "192.168.0.14", - Gateway: "192.168.0.1", + NodeName: testNode1, + StartIP: "172.16.0.1", + EndIP: "172.16.0.50", }} - Expect(pool.SetIPBlockAnnotation(node2, node2InitialRanges)).NotTo(HaveOccurred()) - Expect(updateNode(node2)) + pool1.Status = ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + node1InitialRanges[pool1Name], + { + NodeName: testNode2, + StartIP: "192.168.0.11", + EndIP: "192.168.0.14", + }, + }, + } + pool2.Status = ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + node1InitialRanges[pool2Name], + }, + } + Expect(k8sClient.Status().Update(ctx, pool1)).NotTo(HaveOccurred()) + Expect(k8sClient.Status().Update(ctx, pool2)).NotTo(HaveOccurred()) By("Start controller") @@ -166,75 +171,103 @@ var _ = Describe("App", func() { go func() { Expect(app.RunController(logr.NewContext(ctrlCtx, klog.NewKlogr()), cfg, &options.Options{ - ConfigMapName: TestConfigMapName, - ConfigMapNamespace: TestNamespace, - MetricsAddr: "0", // disable - ProbeAddr: "0", // disable + MetricsAddr: "0", // disable + ProbeAddr: "0", // disable + IPPoolsNamespace: TestNamespace, })).NotTo(HaveOccurred()) close(controllerStopped) }() - By("Create node3 without annotation") - + By("Create node3") createNode(testNode3) By("Check how controller handled the state") WaitAndCheckForStability(func(g Gomega) { uniqStartEndIPs := map[string]struct{}{} - node1Ranges := getRangeFromNode(testNode1) - node2Ranges := getRangeFromNode(testNode2) - node3Ranges := getRangeFromNode(testNode3) - for _, r := range []map[string]*pool.IPPool{node1Ranges, node2Ranges, node3Ranges} { - g.Expect(r).To(HaveLen(len(cfg1pools))) - for _, p := range cfg1pools { - if r[p] != nil { - uniqStartEndIPs[r[p].StartIP] = struct{}{} - uniqStartEndIPs[r[p].EndIP] = struct{}{} - g.Expect(r[p].Gateway).NotTo(BeEmpty()) - g.Expect(r[p].Subnet).NotTo(BeEmpty()) - } + pool1Allocations := getAllocationsFromIPPools(pool1Name) + pool2Allocations := getAllocationsFromIPPools(pool2Name) + for _, r := range [][]ipamv1alpha1.Allocation{pool1Allocations, pool2Allocations} { + g.Expect(r).To(HaveLen(3)) // 3 allocations, 1 for each node + for _, a := range r { + uniqStartEndIPs[a.StartIP] = struct{}{} + uniqStartEndIPs[a.EndIP] = struct{}{} } } // we should have unique start/end IPs for each node for each pool g.Expect(uniqStartEndIPs).To(HaveLen(len(cfg1pools) * 3 * 2)) // node1 should have restored ranges - g.Expect(node1Ranges).To(Equal(node1InitialRanges)) + g.Expect(checkAllocationExists(pool1Allocations, node1InitialRanges[pool1Name])).To(BeTrue()) + g.Expect(checkAllocationExists(pool2Allocations, node1InitialRanges[pool2Name])).To(BeTrue()) }, 15, 2) - By("Set invalid config for controller") - updateConfigMap(invalidConfig) - time.Sleep(time.Second) - - By("Set valid cfg2, which ignores all nodes") - - updateConfigMap(validConfig2) - - By("Wait for controller to remove annotations") - WaitAndCheckForStability(func(g Gomega) { - g.Expect(getRangeFromNode(testNode1)).To(BeNil()) - g.Expect(getRangeFromNode(testNode2)).To(BeNil()) - g.Expect(getRangeFromNode(testNode3)).To(BeNil()) - }, 15, 2) - - By("Update nodes to match selector in cfg2") + By("Set invalid config for pool1") + invalidPool1Spec := ipamv1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + Gateway: "10.10.0.1", + PerNodeBlockSize: 10, + } + updatePoolSpec(pool1Name, invalidPool1Spec) + Eventually(func(g Gomega) bool { + pool := &ipamv1alpha1.IPPool{} + g.Expect(k8sClient.Get( + ctx, types.NamespacedName{Name: pool1Name, Namespace: TestNamespace}, pool)).NotTo(HaveOccurred()) + return len(pool.Status.Allocations) == 0 + }, 30, 5).Should(BeTrue()) + + By("Create Pool3, with selector which ignores all nodes") + // ranges for two nodes only can be allocated + pool3Spec := ipamv1alpha1.IPPoolSpec{ + Subnet: "172.17.0.0/24", + Gateway: "172.17.0.1", + PerNodeBlockSize: 100, + NodeSelector: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + MatchFields: nil, + }, + }, + }, + } + pool3 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: pool3Name, + Namespace: TestNamespace, + }, + Spec: pool3Spec, + } + Expect(k8sClient.Create(ctx, pool3)).NotTo(HaveOccurred()) + + Consistently(func(g Gomega) bool { + pool := &ipamv1alpha1.IPPool{} + g.Expect(k8sClient.Get( + ctx, types.NamespacedName{Name: pool3Name, Namespace: TestNamespace}, pool)).NotTo(HaveOccurred()) + return len(pool.Status.Allocations) == 0 + }, 30, 5).Should(BeTrue()) + + By("Update nodes to match selector in pool3") // node1 should have range - node1 = getNode(testNode1) + node1 := getNode(testNode1) node1.Labels = map[string]string{"foo": "bar"} updateNode(node1) WaitAndCheckForStability(func(g Gomega) { - g.Expect(getRangeFromNode(testNode1)).NotTo(BeNil()) + g.Expect(getRangeForNode(testNode1, pool3Name)).NotTo(BeNil()) }, 15, 2) // node2 should have range - node2 = getNode(testNode2) + node2 := getNode(testNode2) node2.Labels = map[string]string{"foo": "bar"} - var node2Ranges map[string]*pool.IPPool updateNode(node2) WaitAndCheckForStability(func(g Gomega) { - node2Ranges = getRangeFromNode(testNode2) - g.Expect(node2Ranges).NotTo(BeNil()) + g.Expect(getRangeForNode(testNode2, pool3Name)).NotTo(BeNil()) }, 15, 2) // node3 should have no range, because no free ranges available @@ -242,18 +275,21 @@ var _ = Describe("App", func() { node3.Labels = map[string]string{"foo": "bar"} updateNode(node3) WaitAndCheckForStability(func(g Gomega) { - g.Expect(getRangeFromNode(testNode3)).To(BeNil()) + g.Expect(getRangeForNode(testNode3, pool3Name)).To(BeNil()) }, 15, 5) + node2Range := getRangeForNode(testNode2, pool3Name) // remove label from node2, node3 should have a range now node2 = getNode(testNode2) node2.Labels = nil updateNode(node2) WaitAndCheckForStability(func(g Gomega) { - node3Ranges := getRangeFromNode(testNode3) - g.Expect(node3Ranges).NotTo(BeNil()) + node3Range := getRangeForNode(testNode3, pool3Name) + g.Expect(node3Range).NotTo(BeNil()) // should reuse ranges from node2 - g.Expect(node3Ranges).To(Equal(node2Ranges)) + matchRange := node3Range.StartIP == node2Range.StartIP && + node3Range.EndIP == node2Range.EndIP + g.Expect(matchRange).To(BeTrue()) }, 15, 2) By("Stop controller") diff --git a/cmd/ipam-controller/app/options/options.go b/cmd/ipam-controller/app/options/options.go index 1e323a0..d209b50 100644 --- a/cmd/ipam-controller/app/options/options.go +++ b/cmd/ipam-controller/app/options/options.go @@ -30,8 +30,7 @@ func New() *Options { ProbeAddr: ":8081", EnableLeaderElection: false, LeaderElectionNamespace: "kube-system", - ConfigMapName: "nvidia-k8s-ipam-config", - ConfigMapNamespace: "kube-system", + IPPoolsNamespace: "kube-system", } } @@ -42,8 +41,7 @@ type Options struct { ProbeAddr string EnableLeaderElection bool LeaderElectionNamespace string - ConfigMapName string - ConfigMapNamespace string + IPPoolsNamespace string } // AddNamedFlagSets register flags for common options in NamedFlagSets @@ -66,10 +64,8 @@ func (o *Options) AddNamedFlagSets(sharedFS *cliflag.NamedFlagSets) { controllerFS.StringVar(&o.LeaderElectionNamespace, "leader-elect-namespace", o.LeaderElectionNamespace, "Determines the namespace in which the leader "+ "election resource will be created.") - controllerFS.StringVar(&o.ConfigMapName, "config-name", - o.ConfigMapName, "The name of the ConfigMap which holds controller configuration") - controllerFS.StringVar(&o.ConfigMapNamespace, "config-namespace", - o.ConfigMapNamespace, "The name of the namespace where ConfigMap with controller configuration exist") + controllerFS.StringVar(&o.IPPoolsNamespace, "ippools-namespace", + o.IPPoolsNamespace, "The name of the namespace to watch for IPPools CRs") } // Validate registered options diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml index ae029d4..21380a7 100644 --- a/deploy/nv-ipam.yaml +++ b/deploy/nv-ipam.yaml @@ -135,6 +135,22 @@ rules: - get - list - watch + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools/status + verbs: + - get + - update + - patch - apiGroups: - coordination.k8s.io resources: @@ -240,10 +256,9 @@ spec: imagePullPolicy: IfNotPresent command: [ "/ipam-controller" ] args: - - --config-name=nvidia-k8s-ipam-config - - --config-namespace=$(POD_NAMESPACE) - --leader-elect=true - --leader-elect-namespace=$(POD_NAMESPACE) + - --ippools-namespace=$(POD_NAMESPACE) env: - name: POD_NAMESPACE valueFrom: diff --git a/examples/configmap.yaml b/examples/configmap.yaml deleted file mode 100644 index bb96fcd..0000000 --- a/examples/configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nvidia-k8s-ipam-config - namespace: kube-system -data: - config: | - { - "pools": { - "pool1": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 100 , "gateway": "192.168.0.1"}, - "pool2": { "subnet": "172.16.0.0/16", "perNodeBlockSize": 50 , "gateway": "172.16.0.1"} - }, - "nodeSelector": {"kubernetes.io/os": "linux"} - } diff --git a/go.mod b/go.mod index 7767783..5aafe3d 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( k8s.io/apimachinery v0.26.4 k8s.io/client-go v0.26.4 k8s.io/component-base v0.26.4 + k8s.io/component-helpers v0.26.4 k8s.io/klog/v2 v2.90.1 sigs.k8s.io/controller-runtime v0.14.6 ) diff --git a/go.sum b/go.sum index 0406159..569787f 100644 --- a/go.sum +++ b/go.sum @@ -663,6 +663,8 @@ k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= k8s.io/component-base v0.26.4 h1:Bg2xzyXNKL3eAuiTEu3XE198d6z22ENgFgGQv2GGOUk= k8s.io/component-base v0.26.4/go.mod h1:lTuWL1Xz/a4e80gmIC3YZG2JCO4xNwtKWHJWeJmsq20= +k8s.io/component-helpers v0.26.4 h1:qbZrh8QmfL+Yn7lWEI/BPrvITGgkBy33djP5Tzsu2hA= +k8s.io/component-helpers v0.26.4/go.mod h1:2Siz5eWmaKu0khASXMTCfJuASZAbCPX9mtjlCe5IWRs= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= diff --git a/pkg/ipam-controller/allocator/allocator.go b/pkg/ipam-controller/allocator/allocator.go index 97ea52a..1a8c66a 100644 --- a/pkg/ipam-controller/allocator/allocator.go +++ b/pkg/ipam-controller/allocator/allocator.go @@ -15,78 +15,60 @@ package allocator import ( "context" - "encoding/json" "errors" "fmt" "math" "net" "reflect" "sort" - "sync" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ip" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" ) var ErrNoFreeRanges = errors.New("no free IP ranges available") -// contains allocation information for the node -type nodeAllocationInfo struct { - Node string - Subnet *net.IPNet - Gateway net.IP - allocatedRange -} - -// allocatedRange contains range of IPs allocated for the node -type allocatedRange struct { +// AllocatedRange contains range of IPs allocated for the node +type AllocatedRange struct { StartIP net.IP EndIP net.IP } -func newPoolAllocator(cfg AllocationConfig) *poolAllocator { - return &poolAllocator{ +func newPoolAllocator(cfg AllocationConfig) *PoolAllocator { + return &PoolAllocator{ cfg: cfg, - allocations: map[string]allocatedRange{}} + allocations: map[string]AllocatedRange{}, + startIps: sets.New[string]()} } -// poolAllocator contains pool settings and related allocations -type poolAllocator struct { +// PoolAllocator contains pool settings and related allocations +type PoolAllocator struct { cfg AllocationConfig // allocations for nodes, key is the node name, value is allocated range - allocations map[string]allocatedRange + allocations map[string]AllocatedRange + // startIps map, key is StartIp as string. It is used to find overlaps + startIps sets.Set[string] } -func (pa *poolAllocator) getLog(ctx context.Context, cfg AllocationConfig) logr.Logger { +func (pa *PoolAllocator) getLog(ctx context.Context, cfg AllocationConfig) logr.Logger { return logr.FromContextOrDiscard(ctx).WithName(fmt.Sprintf("allocator/pool=%s", cfg.PoolName)) } -// Configure update configuration for pool allocator, resets allocations if required -func (pa *poolAllocator) Configure(ctx context.Context, cfg AllocationConfig) { - log := pa.getLog(ctx, cfg) - log.V(1).Info("pool configuration update") - if pa.cfg.Equal(&cfg) { - log.V(1).Info("pool configuration is the same, keep allocations") - return - } - pa.cfg = cfg - pa.allocations = map[string]allocatedRange{} - log.Info("pool configuration updated, reset allocations") -} - -// Allocate allocates a new range in the poolAllocator or +// AllocateFromPool allocates a new range in the poolAllocator or // return existing one, // returns ErrNoFreeRanges if no free ranges available -func (pa *poolAllocator) Allocate(ctx context.Context, node string) (nodeAllocationInfo, error) { +func (pa *PoolAllocator) AllocateFromPool(ctx context.Context, node string) (*AllocatedRange, error) { log := pa.getLog(ctx, pa.cfg).WithValues("node", node) existingAlloc, exist := pa.allocations[node] if exist { log.V(1).Info("allocation for the node already exist", "start", existingAlloc.StartIP, "end", existingAlloc.EndIP) - return pa.getNodeAllocationInfo(node, existingAlloc), nil + return &existingAlloc, nil } allocations := pa.getAllocationsAsSlice() var startIP net.IP @@ -115,76 +97,79 @@ func (pa *poolAllocator) Allocate(ctx context.Context, node string) (nodeAllocat ip.IsBroadcast(endIP, pa.cfg.Subnet) { // out of range log.Info("can't allocate: pool has no free ranges") - return nodeAllocationInfo{}, ErrNoFreeRanges + return &AllocatedRange{}, ErrNoFreeRanges } log.Info("range allocated", "start", startIP, "end", endIP) - pa.allocations[node] = allocatedRange{ + a := AllocatedRange{ StartIP: startIP, EndIP: endIP, } - return pa.getNodeAllocationInfo(node, pa.allocations[node]), nil + pa.allocations[node] = a + pa.startIps.Insert(startIP.String()) + return &a, nil } // Deallocate remove info about allocation for the node from the poolAllocator -func (pa *poolAllocator) Deallocate(ctx context.Context, node string) { +func (pa *PoolAllocator) Deallocate(ctx context.Context, node string) { log := pa.getLog(ctx, pa.cfg) log.Info("deallocate range for node", "node", node) - delete(pa.allocations, node) + a, ok := pa.allocations[node] + if ok { + pa.startIps.Delete(a.StartIP.String()) + delete(pa.allocations, node) + } } -// Load loads range to the pool allocator with validation for conflicts -func (pa *poolAllocator) Load(ctx context.Context, allocData nodeAllocationInfo) error { - log := pa.getLog(ctx, pa.cfg).WithValues("node", allocData.Node) - if err := pa.checkAllocation(allocData); err != nil { +// load loads range to the pool allocator with validation for conflicts +func (pa *PoolAllocator) load(ctx context.Context, nodeName string, allocRange AllocatedRange) error { + log := pa.getLog(ctx, pa.cfg).WithValues("node", nodeName) + if err := pa.checkAllocation(allocRange); err != nil { log.Info("range check failed", "reason", err.Error()) return err } - allocations := pa.getAllocationsAsSlice() - for _, a := range allocations { - // range size is always the same, then an overlap means the blocks are necessarily equal. - // it's enough to just compare StartIP which can technically act as an absolute "block index" in the subnet - if allocData.allocatedRange.StartIP.Equal(a.StartIP) { - err := fmt.Errorf("range overlaps with: %v", a) - log.Info("skip loading range", "reason", err.Error()) - return err - } + // range size is always the same, then an overlap means the blocks are necessarily equal. + // it's enough to just check StartIP which can technically act as an absolute "block index" in the subnet + if pa.startIps.Has(allocRange.StartIP.String()) { + err := fmt.Errorf("range overlaps") + log.Info("skip loading range", "reason", err.Error()) + return err } - log.Info("data loaded", "startIP", allocData.StartIP, "endIP", allocData.EndIP) - pa.allocations[allocData.Node] = allocData.allocatedRange + + log.Info("data loaded", "startIP", allocRange.StartIP, "endIP", allocRange.EndIP) + pa.allocations[nodeName] = allocRange + pa.startIps.Insert(allocRange.StartIP.String()) return nil } -func (pa *poolAllocator) checkAllocation(allocData nodeAllocationInfo) error { - if allocData.Subnet.String() != pa.cfg.Subnet.String() { - return fmt.Errorf("subnet mismatch") +func (pa *PoolAllocator) checkAllocation(allocRange AllocatedRange) error { + if !pa.cfg.Subnet.Contains(allocRange.StartIP) || !pa.cfg.Subnet.Contains(allocRange.EndIP) { + return fmt.Errorf("invalid allocation allocators: start or end IP is out of the subnet") } - if !allocData.Gateway.Equal(pa.cfg.Gateway) { - return fmt.Errorf("gateway mismatch") + + if ip.Cmp(allocRange.EndIP, allocRange.StartIP) <= 0 { + return fmt.Errorf("invalid allocation allocators: start IP must be less then end IP") } + // check that StartIP of the range has valid offset. // all ranges have same size, so we can simply check that (StartIP offset - 1) % pa.cfg.PerNodeBlockSize == 0 // -1 required because we skip network addressee (e.g. in 192.168.0.0/24, first allocation will be 192.168.0.1) - distanceFromNetworkStart := ip.Distance(pa.cfg.Subnet.IP, allocData.StartIP) + distanceFromNetworkStart := ip.Distance(pa.cfg.Subnet.IP, allocRange.StartIP) if distanceFromNetworkStart < 1 || math.Mod(float64(distanceFromNetworkStart)-1, float64(pa.cfg.PerNodeBlockSize)) != 0 { return fmt.Errorf("invalid start IP offset") } - if ip.Distance(allocData.StartIP, allocData.EndIP) != int64(pa.cfg.PerNodeBlockSize)-1 { + if ip.Distance(allocRange.StartIP, allocRange.EndIP) != int64(pa.cfg.PerNodeBlockSize)-1 { return fmt.Errorf("ip count mismatch") } return nil } -func (pa *poolAllocator) getNodeAllocationInfo(node string, allocRange allocatedRange) nodeAllocationInfo { - return nodeAllocationInfo{allocatedRange: allocRange, Subnet: pa.cfg.Subnet, Gateway: pa.cfg.Gateway, Node: node} -} - // return slice with allocated ranges. // ranges are not overlap and are sorted, but there can be "holes" between ranges -func (pa *poolAllocator) getAllocationsAsSlice() []allocatedRange { - allocatedRanges := make([]allocatedRange, 0, len(pa.allocations)) +func (pa *PoolAllocator) getAllocationsAsSlice() []AllocatedRange { + allocatedRanges := make([]AllocatedRange, 0, len(pa.allocations)) for _, a := range pa.allocations { allocatedRanges = append(allocatedRanges, a) } @@ -200,186 +185,56 @@ type AllocationConfig struct { Subnet *net.IPNet Gateway net.IP PerNodeBlockSize int + NodeSelector *corev1.NodeSelector } func (pc *AllocationConfig) Equal(other *AllocationConfig) bool { return reflect.DeepEqual(pc, other) } -// New create and initialize new allocator -func New() *Allocator { - return &Allocator{allocators: map[string]*poolAllocator{}} -} - -type Allocator struct { - lock sync.Mutex - allocators map[string]*poolAllocator - configured bool -} - -// IsConfigured returns true if allocator is configured -func (a *Allocator) IsConfigured() bool { - a.lock.Lock() - defer a.lock.Unlock() - return a.configured -} - -// Configure update allocator configuration -func (a *Allocator) Configure(ctx context.Context, configs []AllocationConfig) { - a.lock.Lock() - defer a.lock.Unlock() - a.configure(ctx, configs) -} - -// ConfigureAndLoadAllocations configures allocator and load data from the node objects -func (a *Allocator) ConfigureAndLoadAllocations(ctx context.Context, configs []AllocationConfig, nodes []corev1.Node) { +// CreatePoolAllocatorFromIPPool creates a PoolAllocator and load data from the IPPool CR +// the nodes Set contains the nodes that match the current NodeSelector of the IPPool +// it is used to filter out Allocations that are not relevant anymore +func CreatePoolAllocatorFromIPPool(ctx context.Context, + p *ipamv1alpha1.IPPool, nodes sets.Set[string]) *PoolAllocator { log := logr.FromContextOrDiscard(ctx) - a.lock.Lock() - defer a.lock.Unlock() - a.configure(ctx, configs) - for i := range nodes { - node := nodes[i] - nodeLog := log.WithValues("node", node.Name) - poolCfg, err := pool.NewConfigReader(&node) - if err != nil { - nodeLog.Info("skip loading data from the node", "reason", err.Error()) + _, subnet, _ := net.ParseCIDR(p.Spec.Subnet) + gateway := net.ParseIP(p.Spec.Gateway) + allocatorConfig := AllocationConfig{ + PoolName: p.Name, + Subnet: subnet, + Gateway: gateway, + PerNodeBlockSize: p.Spec.PerNodeBlockSize, + NodeSelector: p.Spec.NodeSelector, + } + pa := newPoolAllocator(allocatorConfig) + for i := range p.Status.Allocations { + alloc := p.Status.Allocations[i] + if !nodes.Has(alloc.NodeName) { continue } - // load allocators only for know pools (pools which are defined in the config) - for poolName, poolData := range a.allocators { - nodeIPPoolConfig := poolCfg.GetPoolByName(poolName) - allocInfo, err := ipPoolConfigToNodeAllocationInfo(node.Name, nodeIPPoolConfig) - logErr := func(err error) { - nodeLog.Info("ignore allocation info from node", - "pool", poolName, "reason", err.Error()) - } - if err != nil { - logErr(err) - continue - } - - if err := poolData.Load(ctx, allocInfo); err != nil { - logErr(err) - continue - } - a.allocators[poolName] = poolData + var err error + nodeAllocStart := net.ParseIP(alloc.StartIP) + if nodeAllocStart == nil { + err = fmt.Errorf("startIP is incorrect ip") } - } - a.configured = true -} - -// Allocate allocates ranges for node from all pools -func (a *Allocator) Allocate(ctx context.Context, nodeName string) (map[string]*pool.IPPool, error) { - log := logr.FromContextOrDiscard(ctx).WithValues("node", nodeName) - a.lock.Lock() - defer a.lock.Unlock() - - nodeAllocations := make(map[string]*pool.IPPool, len(a.allocators)) - for poolName, allocator := range a.allocators { - allocation, err := allocator.Allocate(ctx, nodeName) - if err != nil { - a.deallocate(ctx, nodeName) - return nil, err + nodeAllocEnd := net.ParseIP(alloc.EndIP) + if nodeAllocEnd == nil { + err = fmt.Errorf("endIP is incorrect ip") } - nodeAllocations[poolName] = nodeAllocationInfoToIPPoolConfig(poolName, allocation) - } - - if log.V(1).Enabled() { - //nolint:errchkjson - dump, _ := json.Marshal(nodeAllocations) - log.V(1).Info("allocated ranges", "ranges", dump) - } - return nodeAllocations, nil -} - -func (a *Allocator) deallocate(ctx context.Context, nodeName string) { - for _, allocator := range a.allocators { - allocator.Deallocate(ctx, nodeName) - } -} - -// Deallocate release all ranges allocated for node -func (a *Allocator) Deallocate(ctx context.Context, nodeName string) { - a.lock.Lock() - defer a.lock.Unlock() - a.deallocate(ctx, nodeName) -} - -func (a *Allocator) configure(ctx context.Context, configs []AllocationConfig) { - log := logr.FromContextOrDiscard(ctx) - for _, cfg := range configs { - poolLog := log.WithValues("pool", cfg.PoolName, - "gateway", cfg.Gateway.String(), "subnet", cfg.Subnet.String(), "perNodeBlockSize", cfg.PerNodeBlockSize) - pAlloc, exist := a.allocators[cfg.PoolName] - if exist { - poolLog.Info("update IP pool allocator config") - pAlloc.Configure(ctx, cfg) - } else { - poolLog.Info("initialize IP pool allocator") - a.allocators[cfg.PoolName] = newPoolAllocator(cfg) + logInfo := func(err error) { + log.Info("ignore allocation info from node", "node", alloc.NodeName, + "pool", p.Name, "reason", err.Error()) } - } - // remove outdated pools from controller state - for poolName := range a.allocators { - found := false - for _, cfg := range configs { - if poolName == cfg.PoolName { - found = true - break - } + if err != nil { + logInfo(err) + continue } - if !found { - delete(a.allocators, poolName) + allocRange := AllocatedRange{StartIP: nodeAllocStart, EndIP: nodeAllocEnd} + if err := pa.load(ctx, alloc.NodeName, allocRange); err != nil { + logInfo(err) + continue } } -} - -func nodeAllocationInfoToIPPoolConfig(poolName string, alloc nodeAllocationInfo) *pool.IPPool { - return &pool.IPPool{ - Name: poolName, - Subnet: alloc.Subnet.String(), - StartIP: alloc.StartIP.String(), - EndIP: alloc.EndIP.String(), - Gateway: alloc.Gateway.String(), - } -} - -func ipPoolConfigToNodeAllocationInfo(node string, alloc *pool.IPPool) (nodeAllocationInfo, error) { - if alloc == nil { - return nodeAllocationInfo{}, fmt.Errorf("node allocation is nil") - } - _, subnet, err := net.ParseCIDR(alloc.Subnet) - if subnet == nil || err != nil { - return nodeAllocationInfo{}, fmt.Errorf("subnet is incorrect network") - } - gateway := net.ParseIP(alloc.Gateway) - if gateway == nil { - return nodeAllocationInfo{}, fmt.Errorf("gateway is incorrect ip") - } - nodeAllocStart := net.ParseIP(alloc.StartIP) - if nodeAllocStart == nil { - return nodeAllocationInfo{}, fmt.Errorf("startIP is incorrect ip") - } - nodeAllocEnd := net.ParseIP(alloc.EndIP) - if nodeAllocEnd == nil { - return nodeAllocationInfo{}, fmt.Errorf("endIP is incorrect ip") - } - - if !subnet.Contains(gateway) { - return nodeAllocationInfo{}, fmt.Errorf("gateway is outside of the subnet") - } - - if !subnet.Contains(nodeAllocStart) || !subnet.Contains(nodeAllocEnd) { - return nodeAllocationInfo{}, fmt.Errorf("invalid allocation allocators: start or end IP is out of the subnet") - } - - if ip.Cmp(nodeAllocEnd, nodeAllocStart) <= 0 { - return nodeAllocationInfo{}, fmt.Errorf("invalid allocation allocators: start IP must be less then end IP") - } - return nodeAllocationInfo{ - Node: node, - Subnet: subnet, - Gateway: gateway, - allocatedRange: allocatedRange{StartIP: nodeAllocStart, EndIP: nodeAllocEnd}, - }, nil + return pa } diff --git a/pkg/ipam-controller/allocator/allocator_test.go b/pkg/ipam-controller/allocator/allocator_test.go index eae1dc1..4a6264d 100644 --- a/pkg/ipam-controller/allocator/allocator_test.go +++ b/pkg/ipam-controller/allocator/allocator_test.go @@ -20,10 +20,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) const ( @@ -37,23 +38,23 @@ const ( testPerNodeBlockCount2 = 10 ) -func getPool1Config() allocator.AllocationConfig { - _, network, _ := net.ParseCIDR("192.168.0.0/24") - return allocator.AllocationConfig{ - PoolName: testPoolName1, - Subnet: network, - Gateway: net.ParseIP("192.168.0.1"), - PerNodeBlockSize: testPerNodeBlockCount1, +func getPool1() *ipamv1alpha1.IPPool { + return &ipamv1alpha1.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: testPoolName1}, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/24", + PerNodeBlockSize: testPerNodeBlockCount1, + Gateway: "192.168.0.1"}, } } -func getPool2Config() allocator.AllocationConfig { - _, network, _ := net.ParseCIDR("172.16.0.0/16") - return allocator.AllocationConfig{ - PoolName: testPoolName2, - Subnet: network, - Gateway: net.ParseIP("172.16.0.1"), - PerNodeBlockSize: testPerNodeBlockCount2, +func getPool2() *ipamv1alpha1.IPPool { + return &ipamv1alpha1.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: testPoolName2}, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "172.16.0.0/16", + PerNodeBlockSize: testPerNodeBlockCount2, + Gateway: "172.16.0.1"}, } } @@ -66,131 +67,185 @@ var _ = Describe("Allocator", func() { ctx = context.Background() }) - It("Allocated/Deallocate without config", func() { - a := allocator.New() - alloc, err := a.Allocate(ctx, testNodeName1) - Expect(alloc).To(BeEmpty()) - Expect(err).To(BeNil()) - a.Deallocate(ctx, testNodeName1) - }) - It("Allocate/Deallocate", func() { - pool1 := getPool1Config() - pool2 := getPool2Config() - a := allocator.New() - a.Configure(ctx, []allocator.AllocationConfig{pool1, pool2}) - node1Alloc, err := a.Allocate(ctx, testNodeName1) + pool1 := getPool1() + pool2 := getPool2() + pa1 := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, sets.New[string]()) + pa2 := allocator.CreatePoolAllocatorFromIPPool(ctx, pool2, sets.New[string]()) + node1AllocPool1, err := pa1.AllocateFromPool(ctx, testNodeName1) + Expect(err).ToNot(HaveOccurred()) + node1AllocPool2, err := pa2.AllocateFromPool(ctx, testNodeName1) + Expect(err).ToNot(HaveOccurred()) + Expect(node1AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.1")) + Expect(node1AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.15")) + Expect(node1AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.1")) + Expect(node1AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.10")) + + node1AllocSecondCall, err := pa1.AllocateFromPool(ctx, testNodeName1) Expect(err).NotTo(HaveOccurred()) - Expect(node1Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.1")) - Expect(node1Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.15")) - Expect(node1Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.1")) - Expect(node1Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.10")) + Expect(node1AllocSecondCall).To(Equal(node1AllocPool1)) - node1AllocSecondCall, err := a.Allocate(ctx, testNodeName1) + node1AllocSecondCall, err = pa2.AllocateFromPool(ctx, testNodeName1) Expect(err).NotTo(HaveOccurred()) - Expect(node1AllocSecondCall).To(Equal(node1Alloc)) + Expect(node1AllocSecondCall).To(Equal(node1AllocPool2)) - node2Alloc, err := a.Allocate(ctx, testNodeName2) + node2AllocPool1, err := pa1.AllocateFromPool(ctx, testNodeName2) Expect(err).NotTo(HaveOccurred()) - Expect(node2Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.16")) - Expect(node2Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.30")) - Expect(node2Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.11")) - Expect(node2Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.20")) + node2AllocPool2, err := pa2.AllocateFromPool(ctx, testNodeName2) + Expect(err).NotTo(HaveOccurred()) + Expect(node2AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.16")) + Expect(node2AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.30")) + Expect(node2AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.11")) + Expect(node2AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.20")) - node3Alloc, err := a.Allocate(ctx, testNodeName3) - Expect(node3Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.31")) - Expect(node3Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.45")) - Expect(node3Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.21")) - Expect(node3Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.30")) + node3AllocPool1, err := pa1.AllocateFromPool(ctx, testNodeName3) + Expect(err).NotTo(HaveOccurred()) + node3AllocPool2, err := pa2.AllocateFromPool(ctx, testNodeName3) + Expect(err).NotTo(HaveOccurred()) + Expect(node3AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.31")) + Expect(node3AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.45")) + Expect(node3AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.21")) + Expect(node3AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.30")) - node4Alloc, err := a.Allocate(ctx, testNodeName4) - Expect(node4Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.46")) - Expect(node4Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.60")) - Expect(node4Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.31")) - Expect(node4Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.40")) + node4AllocPool1, err := pa1.AllocateFromPool(ctx, testNodeName4) + Expect(err).NotTo(HaveOccurred()) + node4AllocPool2, err := pa2.AllocateFromPool(ctx, testNodeName4) + Expect(err).NotTo(HaveOccurred()) + Expect(node4AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.46")) + Expect(node4AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.60")) + Expect(node4AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.31")) + Expect(node4AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.40")) // deallocate for node3 and node1 - a.Deallocate(ctx, testNodeName1) - a.Deallocate(ctx, testNodeName3) + pa1.Deallocate(ctx, testNodeName1) + pa1.Deallocate(ctx, testNodeName3) + pa2.Deallocate(ctx, testNodeName1) + pa2.Deallocate(ctx, testNodeName3) // allocate again, testNodeName3 should have IPs from index 0, testNodeName3 IPs from index 2 - node3Alloc, err = a.Allocate(ctx, testNodeName3) - Expect(node3Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.1")) - Expect(node3Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.15")) - Expect(node3Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.1")) - Expect(node3Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.10")) + node3AllocPool1, err = pa1.AllocateFromPool(ctx, testNodeName3) + Expect(err).NotTo(HaveOccurred()) + node3AllocPool2, err = pa2.AllocateFromPool(ctx, testNodeName3) + Expect(err).NotTo(HaveOccurred()) + Expect(node3AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.1")) + Expect(node3AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.15")) + Expect(node3AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.1")) + Expect(node3AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.10")) - node1Alloc, err = a.Allocate(ctx, testNodeName1) - Expect(node1Alloc[testPoolName1].StartIP).To(BeEquivalentTo("192.168.0.31")) - Expect(node1Alloc[testPoolName1].EndIP).To(BeEquivalentTo("192.168.0.45")) - Expect(node1Alloc[testPoolName2].StartIP).To(BeEquivalentTo("172.16.0.21")) - Expect(node1Alloc[testPoolName2].EndIP).To(BeEquivalentTo("172.16.0.30")) + node1AllocPool1, err = pa1.AllocateFromPool(ctx, testNodeName1) + Expect(err).ToNot(HaveOccurred()) + node1AllocPool2, err = pa2.AllocateFromPool(ctx, testNodeName1) + Expect(err).ToNot(HaveOccurred()) + Expect(node1AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.31")) + Expect(node1AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.45")) + Expect(node1AllocPool2.StartIP.String()).To(BeEquivalentTo("172.16.0.21")) + Expect(node1AllocPool2.EndIP.String()).To(BeEquivalentTo("172.16.0.30")) + }) + + It("Deallocate from pool", func() { + pool1 := getPool1() + a := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, sets.New[string]()) + node1AllocPool1, err := a.AllocateFromPool(ctx, testNodeName1) + Expect(err).ToNot(HaveOccurred()) + Expect(node1AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.1")) + Expect(node1AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.15")) + + a.Deallocate(ctx, testNodeName1) + + //Allocate to Node2, should get first range + node2AllocPool1, err := a.AllocateFromPool(ctx, testNodeName2) + Expect(err).NotTo(HaveOccurred()) + Expect(node2AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.1")) + Expect(node2AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.15")) }) It("No free ranges", func() { - pool1 := getPool1Config() + pool1 := getPool1() // pool is /24, must fail on the second allocation - pool1.PerNodeBlockSize = 200 - a := allocator.New() - a.Configure(ctx, []allocator.AllocationConfig{pool1}) - node1Alloc, err := a.Allocate(ctx, testNodeName1) + pool1.Spec.PerNodeBlockSize = 200 + a := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, sets.New[string]()) + node1Alloc, err := a.AllocateFromPool(ctx, testNodeName1) Expect(err).NotTo(HaveOccurred()) - Expect(node1Alloc).To(HaveLen(1)) + Expect(node1Alloc).NotTo(BeNil()) - _, err = a.Allocate(ctx, testNodeName2) + _, err = a.AllocateFromPool(ctx, testNodeName2) Expect(errors.Is(err, allocator.ErrNoFreeRanges)).To(BeTrue()) }) + It("return NoFreeRanges in case if IP is too large", func() { - _, subnet, _ := net.ParseCIDR("255.255.255.0/24") - a := allocator.New() - a.Configure(ctx, []allocator.AllocationConfig{{ - PoolName: "pool", - Subnet: subnet, - Gateway: net.ParseIP("255.255.255.1"), - PerNodeBlockSize: 200}}) - _, err := a.Allocate(ctx, testNodeName1) + testPool := &ipamv1alpha1.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool"}, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "255.255.255.0/24", + PerNodeBlockSize: 200, + Gateway: "255.255.255.1"}, + } + + a := allocator.CreatePoolAllocatorFromIPPool(ctx, testPool, sets.New[string]()) + _, err := a.AllocateFromPool(ctx, testNodeName1) Expect(err).NotTo(HaveOccurred()) - _, err = a.Allocate(ctx, testNodeName2) + _, err = a.AllocateFromPool(ctx, testNodeName2) Expect(errors.Is(err, allocator.ErrNoFreeRanges)).To(BeTrue()) }) - It("Configure - reset allocations", func() { - a := allocator.New() - origConfig := getPool1Config() - a.Configure(ctx, []allocator.AllocationConfig{origConfig}) - _, err := a.Allocate(ctx, testNodeName1) - Expect(err).NotTo(HaveOccurred()) - node2Alloc, err := a.Allocate(ctx, testNodeName2) - Expect(err).NotTo(HaveOccurred()) - - // update config with same configuration, should not reset allocations - a.Configure(ctx, []allocator.AllocationConfig{origConfig}) - node2AllocSecondCall, err := a.Allocate(ctx, testNodeName2) - Expect(err).NotTo(HaveOccurred()) - Expect(node2AllocSecondCall[testPoolName1].StartIP).To(Equal(node2Alloc[testPoolName1].StartIP)) + It("Allocate with Pool Status containing not selected Node", func() { + pool1 := getPool1() + pool1.Status = ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + { + NodeName: "not-in-selector", + StartIP: "192.168.0.1", + EndIP: "192.168.0.15", + }, + { + NodeName: testNodeName1, + StartIP: "192.168.0.16", + EndIP: "192.168.0.30", + }, + }, + } + selectedNodes := sets.New[string](testNodeName1) + a := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, selectedNodes) + node2AllocPool1, err := a.AllocateFromPool(ctx, testNodeName2) + Expect(err).ToNot(HaveOccurred()) + Expect(node2AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.1")) + Expect(node2AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.15")) + }) - // reset config - newCfg := origConfig - newCfg.Gateway = net.ParseIP("192.168.0.2") - a.Configure(ctx, []allocator.AllocationConfig{newCfg}) - node2AllocThirdCall, err := a.Allocate(ctx, testNodeName2) - Expect(err).NotTo(HaveOccurred()) - // allocation begins from the start of the subnet - Expect(node2AllocThirdCall[testPoolName1].StartIP).NotTo(Equal(node2Alloc[testPoolName1].StartIP)) + It("Allocate with Pool Status containing duplicate StartIP", func() { + pool1 := getPool1() + pool1.Status = ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + { + NodeName: testNodeName1, + StartIP: "192.168.0.1", + EndIP: "192.168.0.15", + }, + { + NodeName: testNodeName2, + StartIP: "192.168.0.1", + EndIP: "192.168.0.15", + }, + }, + } + selectedNodes := sets.New[string](testNodeName1, testNodeName2) + a := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, selectedNodes) + node2AllocPool1, err := a.AllocateFromPool(ctx, testNodeName2) + Expect(err).ToNot(HaveOccurred()) + Expect(node2AllocPool1.StartIP.String()).To(BeEquivalentTo("192.168.0.16")) + Expect(node2AllocPool1.EndIP.String()).To(BeEquivalentTo("192.168.0.30")) }) It("ConfigureAndLoadAllocations - Data load test", func() { - getValidData := func() *pool.IPPool { - return &pool.IPPool{Name: testPoolName1, - Subnet: "192.168.0.0/24", - StartIP: "192.168.0.16", - EndIP: "192.168.0.30", - Gateway: "192.168.0.1", + getValidData := func() *allocator.AllocatedRange { + return &allocator.AllocatedRange{ + StartIP: net.ParseIP("192.168.0.16"), + EndIP: net.ParseIP("192.168.0.30"), } } testCases := []struct { - in *pool.IPPool + in *allocator.AllocatedRange loaded bool }{ { // valid data @@ -198,141 +253,98 @@ var _ = Describe("Allocator", func() { loaded: true, }, { // different subnet, should ignore - in: &pool.IPPool{Name: testPoolName1, - Subnet: "1.1.1.0/24", - StartIP: "1.1.1.1", - EndIP: "1.1.1.2", - Gateway: "1.1.1.1", + in: &allocator.AllocatedRange{ + StartIP: net.ParseIP("1.1.1.1"), + EndIP: net.ParseIP("1.1.1.2"), }, loaded: false, }, - { // no subnet, should ignore - in: func() *pool.IPPool { - d := getValidData() - d.Subnet = "" - return d - }(), - loaded: false, - }, - { // no gw, should ignore - in: func() *pool.IPPool { - d := getValidData() - d.Gateway = "" - return d - }(), - loaded: false, - }, { // no startIP, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.StartIP = "" + d.StartIP = nil return d }(), loaded: false, }, { // no endIP, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.EndIP = "" + d.EndIP = net.IPv4allrouter return d }(), loaded: false, }, { // start and end IPs are the same, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.StartIP = "192.168.0.1" - d.EndIP = "192.168.0.1" + d.StartIP = net.ParseIP("192.168.0.1") + d.EndIP = net.ParseIP("192.168.0.1") return d }(), loaded: false, }, { // IPs out of subnet, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.StartIP = "192.168.1.1" - d.EndIP = "192.168.1.15" + d.StartIP = net.ParseIP("192.168.1.1") + d.EndIP = net.ParseIP("192.168.1.15") return d }(), loaded: false, }, { // duplicate range, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.StartIP = "192.168.0.1" - d.EndIP = "192.168.0.15" + d.StartIP = net.ParseIP("192.168.0.1") + d.EndIP = net.ParseIP("192.168.0.15") return d }(), loaded: false, }, { // ip invalid offset, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.StartIP = "192.168.0.17" - d.EndIP = "192.168.0.31" + d.StartIP = net.ParseIP("192.168.0.17") + d.EndIP = net.ParseIP("192.168.0.31") return d }(), loaded: false, }, { // bad IP count, should ignore - in: func() *pool.IPPool { - d := getValidData() - d.StartIP = "192.168.0.16" - d.EndIP = "192.168.0.25" - return d - }(), - loaded: false, - }, - { // different GW, should ignore - in: func() *pool.IPPool { + in: func() *allocator.AllocatedRange { d := getValidData() - d.Gateway = "192.168.0.2" - return d - }(), - loaded: false, - }, - { // wrong GW, should ignore - in: func() *pool.IPPool { - d := getValidData() - d.Gateway = "192.168.100.1" + d.StartIP = net.ParseIP("192.168.0.16") + d.EndIP = net.ParseIP("192.168.0.25") return d }(), loaded: false, }, } for _, test := range testCases { - a := allocator.New() - Expect(a.IsConfigured()).To(BeFalse()) - - pool1 := getPool1Config() - - defNode := corev1.Node{} - defNode.SetName(testPoolName1) - defNodeAlloc := map[string]*pool.IPPool{ - testPoolName1: {Name: testPoolName1, - Subnet: "192.168.0.0/24", - StartIP: "192.168.0.1", - EndIP: "192.168.0.15", - Gateway: "192.168.0.1", - }, - } - Expect(pool.SetIPBlockAnnotation(&defNode, defNodeAlloc)).NotTo(HaveOccurred()) - - testNodeAlloc := map[string]*pool.IPPool{ - testPoolName1: test.in, - } - testNode := corev1.Node{} - testNode.SetName(testNodeName2) - Expect(pool.SetIPBlockAnnotation(&testNode, testNodeAlloc)).NotTo(HaveOccurred()) + pool1 := getPool1() + pool1.Status = ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + { + NodeName: testNodeName1, + StartIP: "192.168.0.1", + EndIP: "192.168.0.15", + }, + { + NodeName: testNodeName2, + StartIP: test.in.StartIP.String(), + EndIP: test.in.EndIP.String(), + }, + }} + nodes := sets.New[string](testNodeName1, testNodeName2) + pa1 := allocator.CreatePoolAllocatorFromIPPool(ctx, pool1, nodes) - a.ConfigureAndLoadAllocations(ctx, []allocator.AllocationConfig{pool1}, []corev1.Node{defNode, {}, testNode}) - Expect(a.IsConfigured()).To(BeTrue()) - node1AllocFromAllocator, err := a.Allocate(ctx, testNodeName2) + node1AllocFromAllocator, err := pa1.AllocateFromPool(ctx, testNodeName2) Expect(err).NotTo(HaveOccurred()) if test.loaded { - Expect(node1AllocFromAllocator).To(Equal(testNodeAlloc)) + Expect(node1AllocFromAllocator).To(BeEquivalentTo(test.in)) } else { - Expect(node1AllocFromAllocator).NotTo(Equal(testNodeAlloc)) + Expect(node1AllocFromAllocator).NotTo(BeEquivalentTo(test.in)) } } }) diff --git a/pkg/ipam-controller/config/config.go b/pkg/ipam-controller/config/config.go index 1f7f9f0..e4c78cc 100644 --- a/pkg/ipam-controller/config/config.go +++ b/pkg/ipam-controller/config/config.go @@ -58,34 +58,40 @@ func (c *Config) Validate() error { return errList.ToAggregate() } for poolName, pool := range c.Pools { - if err := cniUtils.ValidateNetworkName(poolName); err != nil { - return fmt.Errorf("invalid IP pool name %s, should be compatible with CNI network name", poolName) - } - _, network, err := net.ParseCIDR(pool.Subnet) - if err != nil { - return fmt.Errorf("IP pool %s contains invalid subnet: %v", poolName, err) - } + return ValidatePool(poolName, pool.Subnet, pool.Gateway, pool.PerNodeBlockSize) + } + return nil +} - if pool.PerNodeBlockSize < 2 { - return fmt.Errorf("perNodeBlockSize should be at least 2") - } +// ValidatePool validates the IPPool parameters +func ValidatePool(name string, subnet string, gateway string, blockSize int) error { + if err := cniUtils.ValidateNetworkName(name); err != nil { + return fmt.Errorf("invalid IP pool name %s, should be compatible with CNI network name", name) + } + _, network, err := net.ParseCIDR(subnet) + if err != nil { + return fmt.Errorf("IP pool %s contains invalid subnet: %v", name, err) + } - setBits, bitsTotal := network.Mask.Size() - // possibleIPs = net size - network address - broadcast - possibleIPs := int(math.Pow(2, float64(bitsTotal-setBits))) - 2 - if possibleIPs < pool.PerNodeBlockSize { - // config is not valid even if only one node exist in the cluster - return fmt.Errorf("IP pool subnet contains less available IPs then " + - "requested by perNodeBlockSize parameter") - } - parsedGW := net.ParseIP(pool.Gateway) - if len(parsedGW) == 0 { - return fmt.Errorf("IP pool contains invalid gateway configuration: invalid IP") - } - if !network.Contains(parsedGW) { - return fmt.Errorf("IP pool contains invalid gateway configuration: " + - "gateway is outside of the subnet") - } + if blockSize < 2 { + return fmt.Errorf("perNodeBlockSize should be at least 2") + } + + setBits, bitsTotal := network.Mask.Size() + // possibleIPs = net size - network address - broadcast + possibleIPs := int(math.Pow(2, float64(bitsTotal-setBits))) - 2 + if possibleIPs < blockSize { + // config is not valid even if only one node exist in the cluster + return fmt.Errorf("IP pool subnet contains less available IPs then " + + "requested by perNodeBlockSize parameter") + } + parsedGW := net.ParseIP(gateway) + if len(parsedGW) == 0 { + return fmt.Errorf("IP pool contains invalid gateway configuration: invalid IP") + } + if !network.Contains(parsedGW) { + return fmt.Errorf("IP pool contains invalid gateway configuration: " + + "gateway is outside of the subnet") } return nil } diff --git a/pkg/ipam-controller/controllers/configmap/configmap.go b/pkg/ipam-controller/controllers/configmap/configmap.go deleted file mode 100644 index 8e150cf..0000000 --- a/pkg/ipam-controller/controllers/configmap/configmap.go +++ /dev/null @@ -1,132 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - "encoding/json" - "fmt" - "net" - - corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/selector" -) - -// ConfigMapReconciler reconciles ConfigMap objects -type ConfigMapReconciler struct { - Allocator *allocator.Allocator - Selector *selector.Selector - ConfigMapName string - ConfigMapNamespace string - - ConfigEventCh chan event.GenericEvent - client.Client - Scheme *runtime.Scheme -} - -// Reconcile contains logic to sync Node objects -func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - reqLog := log.FromContext(ctx) - if req.Name != r.ConfigMapName || req.Namespace != r.ConfigMapNamespace { - // this should never happen because of the watcher configuration of the manager from controller-runtime pkg - return ctrl.Result{}, nil - } - - cfg := &corev1.ConfigMap{} - err := r.Client.Get(ctx, req.NamespacedName, cfg) - if err != nil { - if apiErrors.IsNotFound(err) { - reqLog.Info("ConfigMap not found, wait for creation") - return ctrl.Result{}, nil - } - reqLog.Error(err, "failed to get ConfigMap object from the cache") - return ctrl.Result{}, err - } - - confData, exist := cfg.Data[config.ConfigMapKey] - if !exist { - reqLog.Error(nil, fmt.Sprintf("invalid configuration: ConfigMap %s doesn't contain %s key", - r.ConfigMapNamespace, config.ConfigMapKey)) - return ctrl.Result{}, nil - } - - controllerConfig := &config.Config{} - if err := json.Unmarshal([]byte(confData), controllerConfig); err != nil { - reqLog.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid JSON", - config.ConfigMapKey)) - return ctrl.Result{}, nil - } - - if err := controllerConfig.Validate(); err != nil { - reqLog.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid config", - config.ConfigMapKey)) - return ctrl.Result{}, nil - } - - r.Selector.Update(controllerConfig.NodeSelector) - - allocatorPools := make([]allocator.AllocationConfig, 0, len(controllerConfig.Pools)) - for pName, p := range controllerConfig.Pools { - // already validated by Validate function - _, subnet, _ := net.ParseCIDR(p.Subnet) - gateway := net.ParseIP(p.Gateway) - allocatorPools = append(allocatorPools, allocator.AllocationConfig{ - PoolName: pName, - Subnet: subnet, - Gateway: gateway, - PerNodeBlockSize: p.PerNodeBlockSize, - }) - } - - if r.Allocator.IsConfigured() { - r.Allocator.Configure(ctx, allocatorPools) - } else { - nodeList := &corev1.NodeList{} - if err := r.Client.List(ctx, nodeList); err != nil { - return ctrl.Result{}, err - } - r.Allocator.ConfigureAndLoadAllocations(ctx, allocatorPools, nodeList.Items) - } - - // config updated, trigger sync for all nodes - nodeList := &corev1.NodeList{} - if err := r.Client.List(ctx, nodeList); err != nil { - return ctrl.Result{}, err - } - for _, n := range nodeList.Items { - r.ConfigEventCh <- event.GenericEvent{ - Object: &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: n.Name}, - }} - } - - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.ConfigMap{}). - Complete(r) -} diff --git a/pkg/ipam-controller/controllers/ippool/ippool.go b/pkg/ipam-controller/controllers/ippool/ippool.go new file mode 100644 index 0000000..b889fef --- /dev/null +++ b/pkg/ipam-controller/controllers/ippool/ippool.go @@ -0,0 +1,165 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package controllers + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + v1helper "k8s.io/component-helpers/scheduling/corev1" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" +) + +// IPPoolReconciler reconciles Pool objects +type IPPoolReconciler struct { + PoolsNamespace string + NodeEventCh chan event.GenericEvent + client.Client + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +// Reconcile contains logic to sync IPPool objects +func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reqLog := log.FromContext(ctx) + if req.Namespace != r.PoolsNamespace { + // this should never happen because of the watcher configuration of the manager from controller-runtime pkg + reqLog.Info("Ignoring notification on IPPool from different namespace", "ns", req.Namespace) + return ctrl.Result{}, nil + } + + pool := &ipamv1alpha1.IPPool{} + err := r.Client.Get(ctx, req.NamespacedName, pool) + if err != nil { + if apiErrors.IsNotFound(err) { + reqLog.Info("IPPool not found") + return ctrl.Result{}, nil + } + reqLog.Error(err, "failed to get IPPool object from the cache") + return ctrl.Result{}, err + } + reqLog.Info("Notification on IPPool", "name", pool.Name) + + err = config.ValidatePool(pool.Name, pool.Spec.Subnet, pool.Spec.Gateway, pool.Spec.PerNodeBlockSize) + if err != nil { + return r.handleInvalidSpec(ctx, err, pool) + } + + nodeList := &corev1.NodeList{} + if err := r.Client.List(ctx, nodeList); err != nil { + reqLog.Error(err, "failed to list Nodes") + return ctrl.Result{}, err + } + nodeNames := make([]string, 0) + for i := range nodeList.Items { + node := nodeList.Items[i] + if pool.Spec.NodeSelector != nil { + match, err := v1helper.MatchNodeSelectorTerms(&node, pool.Spec.NodeSelector) + if err != nil { + reqLog.Error(err, "failed to match Node", "node", node.Name) + continue + } + if match { + nodeNames = append(nodeNames, node.Name) + } + } else { + nodeNames = append(nodeNames, node.Name) + } + } + + sort.Slice(nodeNames, func(i, j int) bool { + return nodeNames[i] < nodeNames[j] + }) + + pa := allocator.CreatePoolAllocatorFromIPPool(ctx, pool, sets.New[string]().Insert(nodeNames...)) + + allocations := make([]ipamv1alpha1.Allocation, 0) + for _, name := range nodeNames { + // AllocateFromPool will return same allocation if it was already allocated + a, err := pa.AllocateFromPool(ctx, name) + if err != nil { + if errors.Is(allocator.ErrNoFreeRanges, err) { + msg := fmt.Sprintf("failed to allocate IPs on Node: %s", name) + reqLog.Error(err, msg) + r.recorder.Event(pool, "Warning", "NoFreeRanges", msg) + continue + } + return ctrl.Result{}, err + } + alloc := ipamv1alpha1.Allocation{ + NodeName: name, + StartIP: a.StartIP.String(), + EndIP: a.EndIP.String(), + } + allocations = append(allocations, alloc) + } + + if !reflect.DeepEqual(pool.Status.Allocations, allocations) { + pool.Status.Allocations = allocations + if err := r.Status().Update(ctx, pool); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *IPPoolReconciler) handleInvalidSpec(ctx context.Context, + err error, pool *ipamv1alpha1.IPPool) (reconcile.Result, error) { + reqLog := log.FromContext(ctx) + reqLog.Error(err, "invalid IPPool Spec, clearing Status") + pool.Status.Allocations = make([]ipamv1alpha1.Allocation, 0) + if err2 := r.Status().Update(ctx, pool); err2 != nil { + return ctrl.Result{}, err2 + } + r.recorder.Event(pool, "Warning", "InvalidSpec", err.Error()) + return ctrl.Result{Requeue: false}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *IPPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.recorder = mgr.GetEventRecorderFor("IPPoolController") + return ctrl.NewControllerManagedBy(mgr). + For(&ipamv1alpha1.IPPool{}). + // catch notifications received through chan from Node controller + Watches(&source.Channel{Source: r.NodeEventCh}, handler.Funcs{ + GenericFunc: func(e event.GenericEvent, q workqueue.RateLimitingInterface) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: e.Object.GetNamespace(), + Name: e.Object.GetName(), + }}) + }}). + Complete(r) +} diff --git a/pkg/ipam-controller/controllers/node/node.go b/pkg/ipam-controller/controllers/node/node.go index 26ff442..495f092 100644 --- a/pkg/ipam-controller/controllers/node/node.go +++ b/pkg/ipam-controller/controllers/node/node.go @@ -15,35 +15,24 @@ package controllers import ( "context" - "errors" - "reflect" - "time" apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - corev1 "k8s.io/api/core/v1" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/allocator" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/selector" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" + corev1 "k8s.io/api/core/v1" ) // NodeReconciler reconciles Node objects type NodeReconciler struct { - Allocator *allocator.Allocator - Selector *selector.Selector - - ConfigEventCh chan event.GenericEvent + PoolsNamespace string + NodeEventCh chan event.GenericEvent client.Client Scheme *runtime.Scheme } @@ -51,74 +40,25 @@ type NodeReconciler struct { // Reconcile contains logic to sync Node objects func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { reqLog := log.FromContext(ctx) - if !r.Allocator.IsConfigured() { - reqLog.V(1).Info("allocator is not yet configured, requeue") - return ctrl.Result{RequeueAfter: time.Second, Requeue: true}, nil - } - + reqLog.Info("Notification on Node", "name", req.Name) node := &corev1.Node{} err := r.Client.Get(ctx, req.NamespacedName, node) if err != nil { - if apiErrors.IsNotFound(err) { - reqLog.Info("node object removed, deallocate ranges") - r.Allocator.Deallocate(ctx, req.Name) - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - if !r.Selector.Match(node) { - reqLog.Info("node doesn't match selector, ensure range is not allocated") - r.Allocator.Deallocate(ctx, node.Name) - return r.cleanAnnotation(ctx, node) - } - - var existingNodeAlloc map[string]*pool.IPPool - poolCfg, err := pool.NewConfigReader(node) - if err == nil { - existingNodeAlloc = poolCfg.GetPools() - } - - expectedAlloc, err := r.Allocator.Allocate(ctx, node.Name) - if err != nil { - if errors.Is(allocator.ErrNoFreeRanges, err) { - _, err := r.cleanAnnotation(ctx, node) - if err != nil { - return ctrl.Result{}, err - } - // keep retrying to allocated IP - return ctrl.Result{RequeueAfter: time.Second * 5}, nil + if !apiErrors.IsNotFound(err) { + return ctrl.Result{}, err } - return ctrl.Result{}, err + reqLog.Info("node object removed") } - if reflect.DeepEqual(existingNodeAlloc, expectedAlloc) { - reqLog.Info("node ranges are up-to-date") - return ctrl.Result{}, nil - } - - if err := pool.SetIPBlockAnnotation(node, expectedAlloc); err != nil { - return ctrl.Result{}, err - } - - if err := r.Client.Update(ctx, node); err != nil { - reqLog.Info("failed to set annotation on the node object, deallocate ranges and retry", - "reason", err.Error()) - r.Allocator.Deallocate(ctx, node.Name) + // node updated, trigger sync for all pools + poolList := &ipamv1alpha1.IPPoolList{} + if err := r.Client.List(ctx, poolList, client.InNamespace(r.PoolsNamespace)); err != nil { return ctrl.Result{}, err } - reqLog.Info("node object updated") - - return ctrl.Result{}, nil -} - -// remove annotation from the node object in the API -func (r *NodeReconciler) cleanAnnotation(ctx context.Context, node *corev1.Node) (ctrl.Result, error) { - if !pool.IPBlockAnnotationExists(node) { - return ctrl.Result{}, nil - } - pool.RemoveIPBlockAnnotation(node) - if err := r.Client.Update(ctx, node); err != nil { - return ctrl.Result{}, err + for _, p := range poolList.Items { + r.NodeEventCh <- event.GenericEvent{ + Object: &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Namespace: r.PoolsNamespace, Name: p.Name}, + }} } return ctrl.Result{}, nil } @@ -127,13 +67,5 @@ func (r *NodeReconciler) cleanAnnotation(ctx context.Context, node *corev1.Node) func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1.Node{}). - // catch notifications received through chan from ConfigMap controller - Watches(&source.Channel{Source: r.ConfigEventCh}, handler.Funcs{ - GenericFunc: func(e event.GenericEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: e.Object.GetNamespace(), - Name: e.Object.GetName(), - }}) - }}). Complete(r) } diff --git a/pkg/ipam-controller/selector/selector.go b/pkg/ipam-controller/selector/selector.go deleted file mode 100644 index 589496e..0000000 --- a/pkg/ipam-controller/selector/selector.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package selector - -import ( - "sync" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" -) - -// New creates a new selector instance -func New() *Selector { - return &Selector{} -} - -// Selector holds labels selector -type Selector struct { - lock sync.RWMutex - selector map[string]string -} - -// Match check if selector match the node -func (s *Selector) Match(node *corev1.Node) bool { - s.lock.RLock() - defer s.lock.RUnlock() - if len(s.selector) == 0 { - return true - } - return labels.Set(s.selector).AsSelector().Matches(labels.Set(node.GetLabels())) -} - -// Update label selector -func (s *Selector) Update(newSelector map[string]string) { - s.lock.Lock() - defer s.lock.Unlock() - s.selector = newSelector -} diff --git a/pkg/ipam-controller/selector/selector_suite_test.go b/pkg/ipam-controller/selector/selector_suite_test.go deleted file mode 100644 index f81f3a2..0000000 --- a/pkg/ipam-controller/selector/selector_suite_test.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package selector_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestSelector(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Selector Suite") -} diff --git a/pkg/ipam-controller/selector/selector_test.go b/pkg/ipam-controller/selector/selector_test.go deleted file mode 100644 index 7b21bbc..0000000 --- a/pkg/ipam-controller/selector/selector_test.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package selector_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/selector" -) - -var _ = Describe("Selector", func() { - It("Empty selector - match all", func() { - s := selector.New() - Expect(s.Match(&corev1.Node{})).To(BeTrue()) - }) - It("Match", func() { - s := selector.New() - labels := map[string]string{"foo": "bar"} - s.Update(labels) - node := &corev1.Node{} - node.SetLabels(labels) - Expect(s.Match(node)).To(BeTrue()) - Expect(s.Match(&corev1.Node{})).To(BeFalse()) - s.Update(map[string]string{"foobar": "foobar"}) - Expect(s.Match(node)).To(BeFalse()) - }) -}) From a95a67b26a31133caccfb2a8d81b6234198ea697 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Mon, 14 Aug 2023 11:12:37 +0300 Subject: [PATCH 3/5] Node: watch IPPool CR Move from watching Nodes object and read IP range from annotation, to watch IPPools objects and get Allocations from their Status. Signed-off-by: Fred Rolland --- cmd/ipam-node/app/app.go | 28 +++---- cmd/ipam-node/app/app_suite_test.go | 17 ++++- cmd/ipam-node/app/app_test.go | 84 +++++++++++++++------ cmd/ipam-node/app/options/options.go | 29 +++++--- deploy/nv-ipam.yaml | 14 +++- pkg/ipam-node/controllers/ippool/ippool.go | 86 ++++++++++++++++++++++ pkg/ipam-node/controllers/node/node.go | 73 ------------------ pkg/pool/manager.go | 57 +++++--------- pkg/pool/manager_test.go | 14 +--- 9 files changed, 233 insertions(+), 169 deletions(-) create mode 100644 pkg/ipam-node/controllers/ippool/ippool.go delete mode 100644 pkg/ipam-node/controllers/node/node.go diff --git a/cmd/ipam-node/app/app.go b/cmd/ipam-node/app/app.go index a1b49f1..1b18d09 100644 --- a/cmd/ipam-node/app/app.go +++ b/cmd/ipam-node/app/app.go @@ -31,8 +31,6 @@ import ( "github.com/google/renameio/v2" "github.com/spf13/cobra" "google.golang.org/grpc" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -40,7 +38,6 @@ import ( "k8s.io/component-base/term" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" // register json format for logger @@ -50,13 +47,14 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" nodev1 "github.com/Mellanox/nvidia-k8s-ipam/api/grpc/nvidia/ipam/node/v1" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-node/app/options" "github.com/Mellanox/nvidia-k8s-ipam/pkg/cmdutils" cniTypes "github.com/Mellanox/nvidia-k8s-ipam/pkg/cni/types" "github.com/Mellanox/nvidia-k8s-ipam/pkg/common" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/allocator" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/cleaner" - nodectrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/controllers/node" + ippoolctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/controllers/ippool" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/grpc/middleware" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/handlers" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/migrator" @@ -116,7 +114,8 @@ func RunNodeDaemon(ctx context.Context, config *rest.Config, opts *options.Optio ctrl.SetLogger(logger) logger.Info("start IPAM node daemon", - "version", version.GetVersionString(), "node", opts.NodeName) + "version", version.GetVersionString(), "node", opts.NodeName, + "IPPools Namespace", opts.PoolsNamespace) if err := deployShimCNI(logger, opts); err != nil { return err @@ -129,15 +128,16 @@ func RunNodeDaemon(ctx context.Context, config *rest.Config, opts *options.Optio return err } + if err := ipamv1alpha1.AddToScheme(scheme); err != nil { + logger.Error(err, "failed to register ipamv1alpha1 scheme") + return err + } + poolManager := poolPkg.NewManager() mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, - NewCache: cache.BuilderWithOptions(cache.Options{ - SelectorsByObject: cache.SelectorsByObject{&corev1.Node{}: cache.ObjectSelector{ - Field: fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", opts.NodeName)), - }}, - }), + Scheme: scheme, + Namespace: opts.PoolsNamespace, MetricsBindAddress: opts.MetricsAddr, Port: 9443, HealthProbeBindAddress: opts.ProbeAddr, @@ -146,12 +146,14 @@ func RunNodeDaemon(ctx context.Context, config *rest.Config, opts *options.Optio logger.Error(err, "unable to initialize manager") return err } - if err = (&nodectrl.NodeReconciler{ + + if err = (&ippoolctrl.IPPoolReconciler{ PoolManager: poolManager, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + NodeName: opts.NodeName, }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "Node") + logger.Error(err, "unable to create controller", "controller", "IPPool") return err } diff --git a/cmd/ipam-node/app/app_suite_test.go b/cmd/ipam-node/app/app_suite_test.go index 7d88376..8de4ff0 100644 --- a/cmd/ipam-node/app/app_suite_test.go +++ b/cmd/ipam-node/app/app_suite_test.go @@ -19,9 +19,12 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" ) var ( @@ -39,17 +42,25 @@ func TestApp(t *testing.T) { var _ = BeforeSuite(func() { By("bootstrapping test environment") - testEnv = &envtest.Environment{} + var err error + err = ipamv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{"../../../deploy/crds"}, + CRDInstallOptions: envtest.CRDInstallOptions{ + ErrorIfPathMissing: true, + }, + } ctx, cFunc = context.WithCancel(context.Background()) - var err error // cfg is defined in this file globally. cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - k8sClient, err = client.New(cfg, client.Options{}) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) }) diff --git a/cmd/ipam-node/app/app_test.go b/cmd/ipam-node/app/app_test.go index 1457550..cab85f5 100644 --- a/cmd/ipam-node/app/app_test.go +++ b/cmd/ipam-node/app/app_test.go @@ -27,12 +27,13 @@ import ( "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" nodev1 "github.com/Mellanox/nvidia-k8s-ipam/api/grpc/nvidia/ipam/node/v1" + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-node/app" "github.com/Mellanox/nvidia-k8s-ipam/cmd/ipam-node/app/options" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) const ( @@ -43,27 +44,67 @@ const ( testNamespace = "default" ) -func createTestNode() *corev1.Node { - nodeObj := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: testNodeName}, - } - ExpectWithOffset(1, pool.SetIPBlockAnnotation(nodeObj, map[string]*pool.IPPool{ - testPoolName1: { - Name: testPoolName1, - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.2", - EndIP: "192.168.0.254", - Gateway: "192.168.0.1", +func createTestPools() { + pool1 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: testPoolName1, Namespace: testNamespace}, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 252, + Gateway: "192.168.0.1", }, - testPoolName2: {Name: testPoolName2, - Subnet: "10.100.0.0/16", - StartIP: "10.100.0.2", - EndIP: "10.100.0.254", - Gateway: "10.100.0.1", + } + ExpectWithOffset(1, k8sClient.Create(ctx, pool1)) + + pool2 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: testPoolName2, Namespace: testNamespace}, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "10.100.0.0/16", + PerNodeBlockSize: 252, + Gateway: "10.100.0.1", }, - })).NotTo(HaveOccurred()) - ExpectWithOffset(1, k8sClient.Create(ctx, nodeObj)) - return nodeObj + } + ExpectWithOffset(1, k8sClient.Create(ctx, pool2)) + + // Update statuses with range allocation + Eventually(func(g Gomega) error { + status := ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + { + NodeName: testNodeName, + StartIP: "192.168.0.2", + EndIP: "192.168.0.254", + }, + }, + } + return updatePoolStatus(testPoolName1, status) + }, 30, 5).Should(Not(HaveOccurred())) + + Eventually(func(g Gomega) error { + status := ipamv1alpha1.IPPoolStatus{ + Allocations: []ipamv1alpha1.Allocation{ + { + NodeName: testNodeName, + StartIP: "10.100.0.2", + EndIP: "10.100.0.254", + }, + }, + } + return updatePoolStatus(testPoolName2, status) + }, 30, 5).Should(Not(HaveOccurred())) +} + +func updatePoolStatus(poolName string, status ipamv1alpha1.IPPoolStatus) error { + pool := &ipamv1alpha1.IPPool{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: poolName, Namespace: testNamespace}, pool) + if err != nil { + return err + } + pool.Status = status + err = k8sClient.Status().Update(ctx, pool) + if err != nil { + return err + } + return nil } func createTestPod() *corev1.Pod { @@ -98,6 +139,7 @@ func getOptions(testDir string) *options.Options { opts.CNIBinDir = cniBinDir opts.CNIConfDir = cniConfDir opts.CNIDaemonSocket = daemonSocket + opts.PoolsNamespace = testNamespace return opts } @@ -122,7 +164,7 @@ var _ = Describe("IPAM Node daemon", func() { testDir := GinkgoT().TempDir() opts := getOptions(testDir) - createTestNode() + createTestPools() pod := createTestPod() ctx = logr.NewContext(ctx, klog.NewKlogr()) diff --git a/cmd/ipam-node/app/options/options.go b/cmd/ipam-node/app/options/options.go index 8e4ed1f..8194262 100644 --- a/cmd/ipam-node/app/options/options.go +++ b/cmd/ipam-node/app/options/options.go @@ -36,12 +36,13 @@ const ( // New initialize and return new Options object func New() *Options { return &Options{ - Options: *cmdoptions.New(), - MetricsAddr: ":8080", - ProbeAddr: ":8081", - NodeName: "", - BindAddress: DefaultBindAddress, - StoreFile: DefaultStoreFile, + Options: *cmdoptions.New(), + MetricsAddr: ":8080", + ProbeAddr: ":8081", + NodeName: "", + BindAddress: DefaultBindAddress, + StoreFile: DefaultStoreFile, + PoolsNamespace: "kube-system", // shim CNI parameters CNIBinDir: "/opt/cni/bin", CNIBinFile: "/nv-ipam", @@ -58,11 +59,12 @@ func New() *Options { // Options holds command line options for controller type Options struct { cmdoptions.Options - MetricsAddr string - ProbeAddr string - NodeName string - BindAddress string - StoreFile string + MetricsAddr string + ProbeAddr string + NodeName string + PoolsNamespace string + BindAddress string + StoreFile string // shim CNI parameters CNIBinDir string CNIBinFile string @@ -91,6 +93,8 @@ func (o *Options) AddNamedFlagSets(sharedFS *cliflag.NamedFlagSets) { o.ProbeAddr, "The address the probe endpoint binds to.") daemonFS.StringVar(&o.NodeName, "node-name", o.NodeName, "The name of the Node on which the daemon runs") + daemonFS.StringVar(&o.PoolsNamespace, "ippools-namespace", + o.PoolsNamespace, "The name of the namespace to watch for IPPools CRs") daemonFS.StringVar(&o.BindAddress, "bind-address", o.BindAddress, "GPRC server bind address. e.g.: tcp://127.0.0.1:9092, unix:///var/lib/foo") daemonFS.StringVar(&o.StoreFile, "store-file", o.StoreFile, @@ -122,6 +126,9 @@ func (o *Options) Validate() error { if len(o.NodeName) == 0 { return fmt.Errorf("node-name is required parameter") } + if len(o.PoolsNamespace) == 0 { + return fmt.Errorf("ippools-namespace is required parameter") + } _, _, err := ParseBindAddress(o.BindAddress) if err != nil { return fmt.Errorf("bind-address is invalid: %v", err) diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml index 21380a7..507300f 100644 --- a/deploy/nv-ipam.yaml +++ b/deploy/nv-ipam.yaml @@ -7,12 +7,19 @@ rules: - apiGroups: - "" resources: - - nodes - pods verbs: - get - list - watch + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -70,6 +77,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: IPPOOLS_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace command: [ "/ipam-node" ] args: - --node-name=$(NODE_NAME) @@ -83,6 +94,7 @@ spec: - --cni-conf-dir=/etc/cni/net.d/nv-ipam.d - --cni-log-file=/var/log/nv-ipam-cni.log - --cni-log-level=info # log level for shim CNI + - --ippools-namespace=$(IPPOOLS_NAMESPACE) resources: requests: cpu: "100m" diff --git a/pkg/ipam-node/controllers/ippool/ippool.go b/pkg/ipam-node/controllers/ippool/ippool.go new file mode 100644 index 0000000..ddf310f --- /dev/null +++ b/pkg/ipam-node/controllers/ippool/ippool.go @@ -0,0 +1,86 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package controllers + +import ( + "context" + + apiErrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" +) + +// IPPoolReconciler reconciles Node objects +type IPPoolReconciler struct { + PoolManager pool.Manager + client.Client + Scheme *runtime.Scheme + NodeName string +} + +// Reconcile contains logic to sync IPPool objects +func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reqLog := log.FromContext(ctx) + ipPool := &ipamv1alpha1.IPPool{} + err := r.Client.Get(ctx, req.NamespacedName, ipPool) + if err != nil { + if apiErrors.IsNotFound(err) { + reqLog.Info("Pool not found, removing from PoolManager") + r.PoolManager.RemovePool(req.Name) + return ctrl.Result{}, nil + } + reqLog.Error(err, "failed to get Pool object from the cache") + return ctrl.Result{}, err + } +<<<<<<< Updated upstream + reqLog.Info("Notification on Pool", "name", ipPool.Name) +======= + reqLog.Info("Notification on IPPool", "name", ipPool.Name) + found := false +>>>>>>> Stashed changes + for _, alloc := range ipPool.Status.Allocations { + if alloc.NodeName == r.NodeName { + pool := &pool.IPPool{ + Name: ipPool.Name, + Subnet: ipPool.Spec.Subnet, + Gateway: ipPool.Spec.Gateway, + StartIP: alloc.StartIP, + EndIP: alloc.EndIP, + } +<<<<<<< Updated upstream + r.PoolManager.UpdatePool(pool) +======= + r.PoolManager.UpdatePool(ipPool) + found = true +>>>>>>> Stashed changes + break + } + } + if !found { + r.PoolManager.RemovePool(req.Name) + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *IPPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&ipamv1alpha1.IPPool{}). + Complete(r) +} diff --git a/pkg/ipam-node/controllers/node/node.go b/pkg/ipam-node/controllers/node/node.go deleted file mode 100644 index 13df8d7..0000000 --- a/pkg/ipam-node/controllers/node/node.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" -) - -// NodeReconciler reconciles Node objects -type NodeReconciler struct { - PoolManager pool.Manager - client.Client - Scheme *runtime.Scheme -} - -// Reconcile contains logic to sync Node objects -func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - reqLog := log.FromContext(ctx) - node := &corev1.Node{} - err := r.Client.Get(ctx, req.NamespacedName, node) - if err != nil { - if apiErrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - if err := r.PoolManager.Update(node); err != nil { - reqLog.Info("pool config from the node object is not updated, reset pool config", - "reason", err.Error()) - r.PoolManager.Reset() - } else { - reqLog.Info("pools configuration updated", "data", r.PoolManager.GetPools()) - } - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Node{}). - WithEventFilter(predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - if e.ObjectOld == nil || e.ObjectNew == nil { - return true - } - return e.ObjectOld.GetAnnotations()[pool.IPBlocksAnnotation] != - e.ObjectNew.GetAnnotations()[pool.IPBlocksAnnotation] - }, - }). - Complete(r) -} diff --git a/pkg/pool/manager.go b/pkg/pool/manager.go index 37ea6fa..995d13c 100644 --- a/pkg/pool/manager.go +++ b/pkg/pool/manager.go @@ -13,70 +13,53 @@ package pool -import ( - "fmt" - "sync" - - corev1 "k8s.io/api/core/v1" -) +import "sync" // Manager provide access to pools configuration // //go:generate mockery --name Manager type Manager interface { ConfigReader - // Update Pool's configs from node object, - // returns an error if node object doesn't contain valid config - Update(node *corev1.Node) error - // Reset clean Pool config which is cached in memory - Reset() + // Update Pool's config from IPPool CR + UpdatePool(pool *IPPool) + // Remove Pool's config + RemovePool(poolName string) } // NewManager create and initialize new manager instance func NewManager() Manager { - return &manager{} + return &manager{ + poolByName: make(map[string]*IPPool), + } } type manager struct { - lock sync.Mutex - reader ConfigReader + lock sync.Mutex + poolByName map[string]*IPPool } -// GetPoolByName is the Manager interface implementation for the manager -func (m *manager) GetPoolByName(name string) *IPPool { +func (m *manager) UpdatePool(pool *IPPool) { m.lock.Lock() defer m.lock.Unlock() - if m.reader == nil { - return nil - } - return m.reader.GetPoolByName(name) + m.poolByName[pool.Name] = pool } -// GetPools is the Manager interface implementation for the manager -func (m *manager) GetPools() map[string]*IPPool { +func (m *manager) RemovePool(poolName string) { m.lock.Lock() defer m.lock.Unlock() - if m.reader == nil { - return nil - } - return m.reader.GetPools() + delete(m.poolByName, poolName) } -// Update is the Manager interface implementation for the manager -func (m *manager) Update(node *corev1.Node) error { +// GetPoolByName is the Manager interface implementation for the manager +func (m *manager) GetPoolByName(name string) *IPPool { m.lock.Lock() defer m.lock.Unlock() - r, err := NewConfigReader(node) - if err != nil { - return fmt.Errorf("failed to update pools configuration from the node object: %v", err) - } - m.reader = r - return nil + return m.poolByName[name] } -// Reset is the Manager interface implementation for the manager -func (m *manager) Reset() { +// GetPools is the Manager interface implementation for the manager +func (m *manager) GetPools() map[string]*IPPool { m.lock.Lock() defer m.lock.Unlock() - m.reader = nil + return m.poolByName } diff --git a/pkg/pool/manager_test.go b/pkg/pool/manager_test.go index 585c316..af5e7b7 100644 --- a/pkg/pool/manager_test.go +++ b/pkg/pool/manager_test.go @@ -17,31 +17,25 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) var _ = Describe("Manager", func() { It("Update pool data", func() { - testPools := make(map[string]*pool.IPPool) testPoolName := "my-pool-1" - testPools[testPoolName] = &pool.IPPool{ - Name: "my-pool-1", + testPool := &pool.IPPool{ + Name: testPoolName, Subnet: "192.168.0.0/16", StartIP: "192.168.0.2", EndIP: "192.168.0.254", Gateway: "192.168.0.1", } - node := &corev1.Node{} - Expect(pool.SetIPBlockAnnotation(node, testPools)).NotTo(HaveOccurred()) - mgr := pool.NewManager() Expect(mgr.GetPoolByName(testPoolName)).To(BeNil()) - Expect(mgr.Update(node)).NotTo(HaveOccurred()) + mgr.UpdatePool(testPool) Expect(mgr.GetPoolByName(testPoolName)).NotTo(BeNil()) Expect(mgr.GetPools()).To(HaveLen(1)) - mgr.Reset() + mgr.RemovePool(testPoolName) Expect(mgr.GetPoolByName(testPoolName)).To(BeNil()) }) }) From d6513cd60dde13c5cbe6184678128a57973b0729 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Tue, 15 Aug 2023 10:36:44 +0300 Subject: [PATCH 4/5] Migration from cm to CRs In case the IP configuration ConfigMap exists: - Create IPPools CR according to config spec - Read Nodes ranges annotation - Populate the IPpools Status allocations - Clear Nodes ranges annotation - Delete ConfigMap In case an issue is preventing the migration flow, it can be skipped by setting the env var 'MIGRATOR_DISABLE_MIGRATION'. Signed-off-by: Fred Rolland --- cmd/ipam-controller/app/app.go | 16 ++ deploy/nv-ipam.yaml | 2 + pkg/ipam-controller/migrator/migrator.go | 252 ++++++++++++++++++ .../migrator/migrator_suite_test.go | 82 ++++++ pkg/ipam-controller/migrator/migrator_test.go | 234 ++++++++++++++++ pkg/ipam-node/controllers/ippool/ippool.go | 16 +- 6 files changed, 590 insertions(+), 12 deletions(-) create mode 100644 pkg/ipam-controller/migrator/migrator.go create mode 100644 pkg/ipam-controller/migrator/migrator_suite_test.go create mode 100644 pkg/ipam-controller/migrator/migrator_test.go diff --git a/cmd/ipam-controller/app/app.go b/cmd/ipam-controller/app/app.go index f668679..fc9a7fa 100644 --- a/cmd/ipam-controller/app/app.go +++ b/cmd/ipam-controller/app/app.go @@ -18,6 +18,7 @@ package app import ( "context" "fmt" + "os" "github.com/go-logr/logr" "github.com/spf13/cobra" @@ -28,6 +29,7 @@ import ( "k8s.io/component-base/term" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -42,6 +44,7 @@ import ( "github.com/Mellanox/nvidia-k8s-ipam/pkg/common" poolctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/ippool" nodectrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/node" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/migrator" "github.com/Mellanox/nvidia-k8s-ipam/pkg/version" ) @@ -125,6 +128,19 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio return err } + k8sClient, err := client.New(config, + client.Options{Scheme: mgr.GetScheme(), Mapper: mgr.GetRESTMapper()}) + if err != nil { + logger.Error(err, "failed to create k8sClient client") + os.Exit(1) + } + + if err := migrator.Migrate(ctx, k8sClient, opts.IPPoolsNamespace); err != nil { + logger.Error(err, fmt.Sprintf("failed to migrate NV-IPAM config from ConfigMap, "+ + "set %s env variable to disable migration", migrator.EnvDisableMigration)) + return err + } + nodeEventCH := make(chan event.GenericEvent, 1) if err = (&nodectrl.NodeReconciler{ diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml index 507300f..95c0f34 100644 --- a/deploy/nv-ipam.yaml +++ b/deploy/nv-ipam.yaml @@ -147,6 +147,7 @@ rules: - get - list - watch + - delete - apiGroups: - nv-ipam.nvidia.com resources: @@ -155,6 +156,7 @@ rules: - get - list - watch + - create - apiGroups: - nv-ipam.nvidia.com resources: diff --git a/pkg/ipam-controller/migrator/migrator.go b/pkg/ipam-controller/migrator/migrator.go new file mode 100644 index 0000000..34faa97 --- /dev/null +++ b/pkg/ipam-controller/migrator/migrator.go @@ -0,0 +1,252 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package migrator + +import ( + "context" + "encoding/json" + "fmt" + "net" + "os" + "reflect" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" +) + +const ( + // EnvDisableMigration contains the name of the environment variable which can be used + // to disable migration + EnvDisableMigration = "MIGRATOR_DISABLE_MIGRATION" + // EnvConfigMapName contains the name of the environment variable which can be used + // to specify the ConfigMap name containing the configuration to migrate from + EnvConfigMapName = "CONFIGMAP_NAME" + // EnvConfigMapNamespace contains the name of the environment variable which can be used + // to specify the namespace of the ConfigMap containing the configuration to migrate from + EnvConfigMapNamespace = "CONFIGMAP_NAMESPACE" + // DefaultConfigMapName is the default ConfigMap name used to read the configuration to + // migrate from + DefaultConfigMapName = "nvidia-k8s-ipam-config" +) + +// Migrate reads the ConfigMap with the IPAM configuration, reads the allocations +// from the Nodes annotation, create IPPool CRs and delete the ConfigMap and annotations +func Migrate(ctx context.Context, c client.Client, poolNamespace string) error { + logger := logr.FromContextOrDiscard(ctx).WithName("migrator") + if os.Getenv(EnvDisableMigration) != "" { + logger.Info(fmt.Sprintf("%s set, skip controller migration", EnvDisableMigration)) + return nil + } + + cmName := DefaultConfigMapName + if os.Getenv(EnvConfigMapName) != "" { + cmName = os.Getenv(EnvConfigMapName) + } + cmNamespace := poolNamespace + if os.Getenv(EnvConfigMapNamespace) != "" { + cmNamespace = os.Getenv(EnvConfigMapNamespace) + } + + cfg := &corev1.ConfigMap{} + key := types.NamespacedName{ + Name: cmName, + Namespace: cmNamespace, + } + err := c.Get(ctx, key, cfg) + if err != nil { + if apiErrors.IsNotFound(err) { + logger.Info("ConfigMap not found, skipping migration") + return nil + } + logger.Error(err, "failed to read ConfigMap object") + return err + } + + confData, exist := cfg.Data[config.ConfigMapKey] + if !exist { + err = fmt.Errorf("invalid configuration: ConfigMap %s doesn't contain %s key", + key, config.ConfigMapKey) + logger.Error(err, "Invalid config, no data") + return err + } + controllerConfig := &config.Config{} + if err := json.Unmarshal([]byte(confData), controllerConfig); err != nil { + logger.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid JSON", + config.ConfigMapKey)) + return err + } + if err := controllerConfig.Validate(); err != nil { + logger.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid config", + config.ConfigMapKey)) + return err + } + + pools := buildIPPools(controllerConfig, poolNamespace) + + for name, p := range pools { + err = c.Create(ctx, p) + logger.Info(fmt.Sprintf("Creating IPPool: %v", p)) + if apiErrors.IsAlreadyExists(err) { + existingPool := &ipamv1alpha1.IPPool{} + err = c.Get(ctx, client.ObjectKeyFromObject(p), existingPool) + if err != nil { + logger.Info("fail to get existing pool", "pool name", name) + return err + } + if !reflect.DeepEqual(existingPool.Spec, p.Spec) { + logger.Info("existing pool has different spec than config map setting", "pool name", name) + return fmt.Errorf("existing pool has different spec than config map setting") + } + } else if err != nil { + logger.Info("fail to create pool", "pool name", name) + return err + } + } + + err = updateAllocations(ctx, c, logger, pools, poolNamespace) + if err != nil { + return err + } + + err = c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: cmNamespace, Name: cmName}}) + if err != nil { + logger.Info("fail to delete nv-ipam config map") + return err + } + + return nil +} + +func updateAllocations(ctx context.Context, c client.Client, + logger logr.Logger, pools map[string]*ipamv1alpha1.IPPool, poolNamespace string) error { + nodeList := &corev1.NodeList{} + if err := c.List(ctx, nodeList); err != nil { + logger.Error(err, "failed to list nodes") + return err + } + nodesToClearAnnotation := sets.New[string]() + for poolName := range pools { + p := &ipamv1alpha1.IPPool{} + key := types.NamespacedName{ + Name: poolName, + Namespace: poolNamespace, + } + err := c.Get(ctx, key, p) + if err != nil { + logger.Info("fail getting IPPool", "reason", err.Error()) + return err + } + if len(p.Status.Allocations) > 0 { + logger.Info("skipping migration for IPPool, already has allocation", "ippool", poolName) + continue + } + allocs := make([]ipamv1alpha1.Allocation, 0) + for i := range nodeList.Items { + node := nodeList.Items[i] + nodeLog := logger.WithValues("node", node.Name) + poolCfg, err := pool.NewConfigReader(&node) + if err != nil { + nodeLog.Info("skip loading data from the node", "reason", err.Error()) + continue + } + nodesToClearAnnotation.Insert(node.Name) + nodeIPPoolConfig := poolCfg.GetPoolByName(poolName) + if nodeIPPoolConfig == nil { + nodeLog.Info("skip loading data for pool from the node, pool not configured", "node", node.Name, "pool", poolName) + continue + } + alloc := ipamv1alpha1.Allocation{ + NodeName: node.Name, + StartIP: nodeIPPoolConfig.StartIP, + EndIP: nodeIPPoolConfig.EndIP, + } + allocs = append(allocs, alloc) + } + if len(allocs) != 0 { + p.Status.Allocations = allocs + logger.Info(fmt.Sprintf("Updating IPPool status: %v", p)) + err = c.Status().Update(ctx, p) + if err != nil { + logger.Info("fail to update pool allocation from node", "reason", err.Error()) + return err + } + } + } + + for _, nodeName := range sets.List(nodesToClearAnnotation) { + logger.Info("clear IPBlocksAnnotation from node", "name", nodeName) + fmtKey := strings.ReplaceAll(pool.IPBlocksAnnotation, "/", "~1") + patch := []byte(fmt.Sprintf("[{\"op\": \"remove\", \"path\": \"/metadata/annotations/%s\"}]", fmtKey)) + err := c.Patch(ctx, &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + }, client.RawPatch(types.JSONPatchType, patch)) + if err != nil { + logger.Info("fail to remove IPBlocksAnnotation from node", "name", nodeName, "reason", err.Error()) + return err + } + } + return nil +} + +func buildIPPools(controllerConfig *config.Config, poolNamespace string) map[string]*ipamv1alpha1.IPPool { + var nodeSelector *corev1.NodeSelector + if len(controllerConfig.NodeSelector) > 0 { + nodeSelector = &corev1.NodeSelector{} + selectorsItems := make([]corev1.NodeSelectorTerm, 0, len(controllerConfig.NodeSelector)) + for k, v := range controllerConfig.NodeSelector { + selector := corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: k, + Operator: corev1.NodeSelectorOpIn, + Values: []string{v}, + }, + }, + MatchFields: nil, + } + selectorsItems = append(selectorsItems, selector) + } + nodeSelector.NodeSelectorTerms = selectorsItems + } + pools := make(map[string]*ipamv1alpha1.IPPool) + for pName, p := range controllerConfig.Pools { + // already validated by Validate function + _, subnet, _ := net.ParseCIDR(p.Subnet) + pools[pName] = &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: pName, + Namespace: poolNamespace, + }, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: subnet.String(), + Gateway: p.Gateway, + PerNodeBlockSize: p.PerNodeBlockSize, + NodeSelector: nodeSelector, + }, + } + } + return pools +} diff --git a/pkg/ipam-controller/migrator/migrator_suite_test.go b/pkg/ipam-controller/migrator/migrator_suite_test.go new file mode 100644 index 0000000..883e135 --- /dev/null +++ b/pkg/ipam-controller/migrator/migrator_suite_test.go @@ -0,0 +1,82 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package migrator_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" +) + +const ( + TestNamespace = "test-ns" + TestConfigMapName = "nvidia-k8s-ipam-config" +) + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + cFunc context.CancelFunc + ctx context.Context +) + +func TestMigrator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "IPAM Controller Migrator Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{"../../../deploy/crds"}, + CRDInstallOptions: envtest.CRDInstallOptions{ + ErrorIfPathMissing: true, + }, + } + + ctx, cFunc = context.WithCancel(context.Background()) + + var err error + err = ipamv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: TestNamespace}})).To(BeNil()) +}) + +var _ = AfterSuite(func() { + cFunc() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/pkg/ipam-controller/migrator/migrator_test.go b/pkg/ipam-controller/migrator/migrator_test.go new file mode 100644 index 0000000..ce6b85a --- /dev/null +++ b/pkg/ipam-controller/migrator/migrator_test.go @@ -0,0 +1,234 @@ +/* + Copyright 2023, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package migrator_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/migrator" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" + + ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" +) + +const ( + pool1Name = "pool1" + pool2Name = "pool2" +) + +func updateConfigMap(data string) { + d := map[string]string{config.ConfigMapKey: data} + err := k8sClient.Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: TestConfigMapName, Namespace: TestNamespace}, + Data: d, + }) + if err == nil { + return + } + if apiErrors.IsAlreadyExists(err) { + configMap := &corev1.ConfigMap{} + Expect(k8sClient.Get( + ctx, types.NamespacedName{Name: TestConfigMapName, Namespace: TestNamespace}, configMap)).NotTo(HaveOccurred()) + configMap.Data = d + Expect(k8sClient.Update( + ctx, configMap)).NotTo(HaveOccurred()) + } else { + Expect(err).NotTo(HaveOccurred()) + } +} + +var validConfig = fmt.Sprintf(` + { + "pools": { + "%s": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 10 , "gateway": "192.168.0.1"}, + "%s": { "subnet": "172.16.0.0/16", "perNodeBlockSize": 50 , "gateway": "172.16.0.1"} + }, + "nodeSelector": {"foo": "bar"} + } +`, pool1Name, pool2Name) + +func createNode(name string) *corev1.Node { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}} + Expect(k8sClient.Create(ctx, node)).NotTo(HaveOccurred()) + return node +} + +func getNode(name string) *corev1.Node { + node := &corev1.Node{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, node)).NotTo(HaveOccurred()) + return node +} + +func updateNode(node *corev1.Node) *corev1.Node { + Expect(k8sClient.Update(ctx, node)).NotTo(HaveOccurred()) + return node +} + +func getRangeFromNode(nodeName string) map[string]*pool.IPPool { + node := getNode(nodeName) + poolCfg, err := pool.NewConfigReader(node) + if err != nil { + return nil + } + return poolCfg.GetPools() +} + +var _ = Describe("Controller Migrator", func() { + + AfterEach(func() { + k8sClient.DeleteAllOf(ctx, &corev1.Node{}) + k8sClient.DeleteAllOf(ctx, &ipamv1alpha1.IPPool{}, client.InNamespace(TestNamespace)) + }) + + It("Basic tests", func() { + testNode1 := "node1" + testNode2 := "node2" + + By("Create valid cfg1") + updateConfigMap(validConfig) + + By("Set annotation with valid ranges for node1") + node1 := createNode(testNode1) + node1InitialRanges := map[string]*pool.IPPool{pool1Name: { + Name: pool1Name, + Subnet: "192.168.0.0/16", + StartIP: "192.168.0.11", + EndIP: "192.168.0.20", + Gateway: "192.168.0.1", + }, pool2Name: { + Name: pool2Name, + Subnet: "172.16.0.0/16", + StartIP: "172.16.0.1", + EndIP: "172.16.0.50", + Gateway: "172.16.0.1", + }} + Expect(pool.SetIPBlockAnnotation(node1, node1InitialRanges)).NotTo(HaveOccurred()) + Expect(updateNode(node1)) + + By("Set annotation with valid ranges for node2") + node2 := createNode(testNode2) + node2InitialRanges := map[string]*pool.IPPool{pool1Name: { + Name: pool1Name, + Subnet: "192.168.0.0/16", + StartIP: "192.168.0.21", + EndIP: "192.168.0.30", + Gateway: "192.168.0.1", + }, pool2Name: { + Name: pool2Name, + Subnet: "172.16.0.0/16", + StartIP: "172.16.0.51", + EndIP: "172.16.0.100", + Gateway: "172.16.0.1", + }} + Expect(pool.SetIPBlockAnnotation(node2, node2InitialRanges)).NotTo(HaveOccurred()) + Expect(updateNode(node2)) + + By("Run migrator") + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).NotTo(HaveOccurred()) + + By("Verify Pool1 Spec") + pool1 := &ipamv1alpha1.IPPool{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: TestNamespace, Name: pool1Name}, pool1)).NotTo(HaveOccurred()) + Expect(pool1.Spec.Gateway == "192.168.0.1" && + pool1.Spec.Subnet == "192.168.0.0/16" && pool1.Spec.PerNodeBlockSize == 10).To(BeTrue()) + Expect(pool1.Spec.NodeSelector).NotTo(BeNil()) + + By("Verify Pool1 Allocations") + expectedAllocations := []ipamv1alpha1.Allocation{{NodeName: testNode1, StartIP: "192.168.0.11", EndIP: "192.168.0.20"}, + {NodeName: testNode2, StartIP: "192.168.0.21", EndIP: "192.168.0.30"}} + Expect(expectedAllocations).To(BeEquivalentTo(pool1.Status.Allocations)) + + By("Verify Pool2 Spec") + pool2 := &ipamv1alpha1.IPPool{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: TestNamespace, Name: pool2Name}, pool2)).NotTo(HaveOccurred()) + Expect(pool2.Spec.Gateway == "172.16.0.1" && + pool2.Spec.Subnet == "172.16.0.0/16" && pool2.Spec.PerNodeBlockSize == 50).To(BeTrue()) + Expect(pool2.Spec.NodeSelector).NotTo(BeNil()) + + By("Verify Pool2 Allocations") + expectedAllocations = []ipamv1alpha1.Allocation{{NodeName: testNode1, StartIP: "192.168.0.11", EndIP: "192.168.0.20"}, + {NodeName: testNode2, StartIP: "192.168.0.21", EndIP: "192.168.0.30"}} + Expect(expectedAllocations).To(BeEquivalentTo(pool1.Status.Allocations)) + + By("Verify Nodes annotations are removed") + Expect(getRangeFromNode(testNode1)).To(BeEmpty()) + Expect(getRangeFromNode(testNode2)).To(BeEmpty()) + + By("Verify Config Map is deleted") + configMap := &corev1.ConfigMap{} + Expect(k8sClient.Get( + ctx, types.NamespacedName{Name: TestConfigMapName, Namespace: TestNamespace}, configMap)).To(HaveOccurred()) + }) + + It("No ConfigMap", func() { + By("Run migrator") + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).NotTo(HaveOccurred()) + }) + + Context("Negative flows", func() { + It("Invalid ConfigMap", func() { + By("Create invalid cfg - no data") + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: TestConfigMapName, + Namespace: TestNamespace, + }, + } + Expect(k8sClient.Create(ctx, cm)).NotTo(HaveOccurred()) + By("Run migrator - should fail") + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).To(HaveOccurred()) + + By("Create invalid cfg - not a json data") + updateConfigMap("{{") + By("Run migrator - should fail") + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).To(HaveOccurred()) + + By("Create invalid cfg - Gateway not in subnet") + var inValidConfig = ` + { + "pools": { + "pool-1": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 10 , "gateway": "172.20.0.1"} + } + }` + updateConfigMap(inValidConfig) + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).To(HaveOccurred()) + + By("Create valid cfg - IPPool exists with different spec") + updateConfigMap(validConfig) + pool1 := &ipamv1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: pool1Name, + Namespace: TestNamespace, + }, + Spec: ipamv1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 50, + Gateway: "192.168.0.1", + }, + } + Expect(k8sClient.Create(ctx, pool1)).NotTo(HaveOccurred()) + Expect(migrator.Migrate(ctx, k8sClient, TestNamespace)).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/ipam-node/controllers/ippool/ippool.go b/pkg/ipam-node/controllers/ippool/ippool.go index ddf310f..139ab9e 100644 --- a/pkg/ipam-node/controllers/ippool/ippool.go +++ b/pkg/ipam-node/controllers/ippool/ippool.go @@ -26,7 +26,7 @@ import ( "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) -// IPPoolReconciler reconciles Node objects +// IPPoolReconciler reconciles IPPool objects type IPPoolReconciler struct { PoolManager pool.Manager client.Client @@ -41,34 +41,26 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr err := r.Client.Get(ctx, req.NamespacedName, ipPool) if err != nil { if apiErrors.IsNotFound(err) { - reqLog.Info("Pool not found, removing from PoolManager") + reqLog.Info("IPPool not found, removing from PoolManager") r.PoolManager.RemovePool(req.Name) return ctrl.Result{}, nil } - reqLog.Error(err, "failed to get Pool object from the cache") + reqLog.Error(err, "failed to get IPPool object from the cache") return ctrl.Result{}, err } -<<<<<<< Updated upstream - reqLog.Info("Notification on Pool", "name", ipPool.Name) -======= reqLog.Info("Notification on IPPool", "name", ipPool.Name) found := false ->>>>>>> Stashed changes for _, alloc := range ipPool.Status.Allocations { if alloc.NodeName == r.NodeName { - pool := &pool.IPPool{ + ipPool := &pool.IPPool{ Name: ipPool.Name, Subnet: ipPool.Spec.Subnet, Gateway: ipPool.Spec.Gateway, StartIP: alloc.StartIP, EndIP: alloc.EndIP, } -<<<<<<< Updated upstream - r.PoolManager.UpdatePool(pool) -======= r.PoolManager.UpdatePool(ipPool) found = true ->>>>>>> Stashed changes break } } From 251cb7ec3519d43f9066e018b1d0f6fd707be403 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Wed, 23 Aug 2023 12:39:45 +0300 Subject: [PATCH 5/5] Update README for IPPool CRD Signed-off-by: Fred Rolland --- README.md | 160 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 5e96f6d..0b0da21 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,8 @@ NVIDIA IPAM plugin consists of 3 main components: ### ipam-controller -A Kubernetes(K8s) controller that Watches on a predefined K8s ConfigMap for defined IP Pools. -It then proceeds in reconciling K8s Node objects by assiging each node via `ipam.nvidia.com/ip-blocks` -annotation a cluster unique range of IPs of the defined IP Pools. +A Kubernetes(K8s) controller that Watches on IPPools CRs in a predefined Namespace. +It then proceeds by assiging each node via IPPools Status a cluster unique range of IPs of the defined IP Pools. ### ipam-node @@ -43,7 +42,7 @@ The daemon is responsible for: A node daemon provides GRPC service, which nv-ipam CNI plugin uses to request IP address allocation/deallocation. IPs are allocated from the provided IP Block assigned by ipam-controller for the node. To determine the cluster unique IP Block for the defined IP Pool, ipam-node watches K8s API -for the Node object and extracts IP Block information from node annotation. +for the IPPool objects and extracts IP Block information from IPPool Status. ### nv-ipam @@ -53,30 +52,65 @@ To allocate/deallocate IP address nv-ipam calls GRPC API of ipam-node daemon. ### IP allocation flow 1. User (cluster administrator) defines a set of named IP Pools to be used for IP allocation -of container interfaces via Kubernetes ConfigMap (more information in [Configuration](#configuration) section) +of container interfaces via IPPool CRD (more information in [Configuration](#configuration) section) _Example_: -```json -{ - "pools": { - "my-pool": {"subnet": "192.188.0.0/16", "perNodeBlockSize": 24, "gateway": "192.168.0.1"} - }, - "nodeSelector": { - "kubernetes.io/os": "linux" - } -} +```yaml +apiVersion: nv-ipam.nvidia.com/v1alpha1 +kind: IPPool +metadata: + name: pool1 + namespace: kube-system +spec: + subnet: 192.168.0.0/16 + perNodeBlockSize: 24 + gateway: 192.168.0.1 + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: In + values: + - worker ``` -2. ipam-controller calculates and assigns unique IP Blocks for each Node via annotation +2. ipam-controller calculates and assigns unique IP Blocks for each Node via IPPool Status: _Example_: ```yaml -annotations: - ipam.nvidia.com/ip-blocks: '{ - "my-pool": {"startIP": "192.168.0.2", "endIP": "192.168.0.25", "gateway": "192.168.0.1", "subnet": "192.168.0.0/16"} - }' +apiVersion: nv-ipam.nvidia.com/v1alpha1 +kind: IPPool +metadata: + name: pool1 + namespace: kube-system +spec: + gateway: 192.168.0.1 + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: In + values: + - worker + perNodeBlockSize: 24 + subnet: 192.168.0.0/16 +status: + allocations: + - endIP: 192.168.0.24 + nodeName: host-a + startIP: 192.168.0.1 + - endIP: 192.168.0.48 + nodeName: host-b + startIP: 192.168.0.25 + - endIP: 192.168.0.72 + nodeName: host-c + startIP: 192.168.0.49 + - endIP: 192.168.0.96 + nodeName: k8s-master + startIP: 192.168.0.73 + ``` 3. User specifies nv-ipam as IPAM plugin in CNI configuration @@ -103,7 +137,7 @@ corresponding IP Pool that was allocated for the node ### ipam-controller configuration -ipam-controller accepts configuration using command line flags and K8s configMap +ipam-controller accepts configuration using command line flags and IPPools CRs. #### Flags @@ -140,10 +174,6 @@ Common flags: Controller flags: - --config-name string - The name of the ConfigMap which holds controller configuration (default "nvidia-k8s-ipam-config") - --config-namespace string - The name of the namespace where ConfigMap with controller configuration exist (default "kube-system") --health-probe-bind-address string The address the probe endpoint binds to. (default ":8081") --kubeconfig string @@ -154,37 +184,43 @@ Controller flags: Determines the namespace in which the leader election resource will be created. (default "kube-system") --metrics-bind-address string The address the metric endpoint binds to. (default ":8080") + --ippools-namespace string + The name of the namespace to watch for IPPools CRs. (default "kube-system") ``` -#### ConfigMap +#### IPPool CR -ipam-controller accepts IP Pool configuration via configMap with a pre-defined key named `config` +ipam-controller accepts IP Pools configuration via IPPool CRs. +Multiple IPPool CRs can be created, with different NodeSelectors. ```yaml -apiVersion: v1 -kind: ConfigMap +apiVersion: nv-ipam.nvidia.com/v1alpha1 +kind: IPPool metadata: - name: nvidia-k8s-ipam-config + name: my-pool namespace: kube-system -data: - config: | - { - "pools": { - "my-pool": {"subnet": "192.168.0.0/16", "perNodeBlockSize": 100 , "gateway": "192.168.0.1"} - }, - "nodeSelector": {"kubernetes.io/os": "linux"} - } +spec: + subnet: 192.168.0.0/16 + perNodeBlockSize: 100 + gateway: 192.168.0.1 + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: In + values: + - worker ``` -* `pools`: contains a set of named IP Pools keyed by name +* `spec`: contains the IP pool configuration * `subnet`: IP Subnet of the pool * `gateway`: Gateway IP of the subnet * `perNodeBlockSize`: the number of IPs of IP Blocks allocated to Nodes. -* `nodeSelector`: a map of node selector labels, only nodes that match the provided labels will get assigned IP Blocks for the defined pools + * `nodeSelector`: A list of node selector terms. The terms are ORed. Each term can have a list of matchExpressions that are ANDed. Only the nodes that match the provided labels will get assigned IP Blocks for the defined pool. > __Notes:__ > -> * pool name is composed of alphanumeric letters separated by dots(`.`) undersocres(`_`) or hyphens(`-`) +> * pool name is composed of alphanumeric letters separated by dots(`.`) underscores(`_`) or hyphens(`-`) > * `perNodeBlockSize` minimum size is 2 > * `subnet` must be large enough to accommodate at least one `perNodeBlockSize` block of IPs @@ -238,6 +274,8 @@ Node daemon flags: The name of the Node on which the daemon runs --store-file string Path of the file which used to store allocations (default "/var/lib/cni/nv-ipam/store") + --ippools-namespace string + The name of the namespace to watch for IPPools CRs. (default "kube-system") Shim CNI Configuration flags: @@ -296,26 +334,30 @@ interface should have two IP addresses: one IPv4 and one IPv6. (default: network > _NOTE:_ This command will deploy latest dev build with default configuration ```shell +kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/crds/nv-ipam.nvidia.com_ippools.yaml kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/nv-ipam.yaml ``` -### Create ipam-controller config +### Create IPPool CR ```shell cat < Start IP: 192.168.0.1 End IP: 192.168.0.24 + host-b => Start IP: 192.168.0.25 End IP: 192.168.0.48 + host-c => Start IP: 192.168.0.49 End IP: 192.168.0.72 + k8s-master => Start IP: 192.168.0.73 End IP: 192.168.0.96 + +pool2 + host-a => Start IP: 172.16.0.1 End IP: 172.16.0.50 + host-b => Start IP: 172.16.0.51 End IP: 172.16.0.100 + host-c => Start IP: 172.16.0.101 End IP: 172.16.0.150 + k8s-master => Start IP: 172.16.0.151 End IP: 172.16.0.200 ``` View network status of pods: