Add Mofed termination grace period param

Adding to NicClusterPolicy CRD the possibility to specify the termination grace period for the Mofed container. The default is 300 seconds. It is also configurable in Helm values file. Signed-off-by: Fred Rolland <frolland@nvidia.com>
Mellanox · Jan 3, 2023 · db07f6e · db07f6e
1 parent 6f86918
commit db07f6e
Show file tree

Hide file tree

Showing 16 changed files with 46 additions and 66 deletions.
diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -79,6 +79,12 @@ type OFEDDriverSpec struct {
 	CertConfig *ConfigMapNameReference `json:"certConfig,omitempty"`
 	// Optional: Custom package repository configuration for OFED container
 	RepoConfig *ConfigMapNameReference `json:"repoConfig,omitempty"`
+	// TerminationGracePeriodSeconds specifies the length of time in seconds
+	// to wait before killing the OFED pod on termination
+	// +optional
+	// +kubebuilder:default:=300
+	// +kubebuilder:validation:Minimum:=0
+	TerminationGracePeriodSeconds int `json:"terminationGracePeriodSeconds,omitempty"`
 }
 
 // NVPeerDriverSpec describes configuration options for NV Peer Memory driver

diff --git a/bundle/manifests/nvidia-network-operator.clusterserviceversion.yaml b/bundle/manifests/nvidia-network-operator.clusterserviceversion.yaml
@@ -64,6 +64,7 @@ metadata:
                 "initialDelaySeconds": 10,
                 "periodSeconds": 20
               },
+              "terminationGracePeriodSeconds": 300,
               "upgradePolicy": {
                 "autoUpgrade": false,
                 "drain": {

diff --git a/config/crd/bases/mellanox.com_hostdevicenetworks.yaml b/config/crd/bases/mellanox.com_hostdevicenetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: hostdevicenetworks.mellanox.com
 spec:
@@ -98,9 +97,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/config/crd/bases/mellanox.com_ipoibnetworks.yaml b/config/crd/bases/mellanox.com_ipoibnetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: ipoibnetworks.mellanox.com
 spec:
@@ -78,9 +77,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/config/crd/bases/mellanox.com_macvlannetworks.yaml b/config/crd/bases/mellanox.com_macvlannetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: macvlannetworks.mellanox.com
 spec:
@@ -92,9 +91,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: nicclusterpolicies.mellanox.com
 spec:
@@ -173,6 +172,7 @@ spec:
                                 type: object
                               type: array
                           type: object
+                          x-kubernetes-map-type: atomic
                         weight:
                           description: Weight associated with matching the corresponding
                             nodeSelectorTerm, in the range 1-100.
@@ -268,10 +268,12 @@ spec:
                                 type: object
                               type: array
                           type: object
+                          x-kubernetes-map-type: atomic
                         type: array
                     required:
                     - nodeSelectorTerms
                     type: object
+                    x-kubernetes-map-type: atomic
                 type: object
               nvPeerDriver:
                 description: NVPeerDriverSpec describes configuration options for
@@ -353,6 +355,7 @@ spec:
                               required:
                               - key
                               type: object
+                              x-kubernetes-map-type: atomic
                             fieldRef:
                               description: 'Selects a field of the pod: supports metadata.name,
                                 metadata.namespace, `metadata.labels[''<KEY>'']`,
@@ -371,6 +374,7 @@ spec:
                               required:
                               - fieldPath
                               type: object
+                              x-kubernetes-map-type: atomic
                             resourceFieldRef:
                               description: 'Selects a resource of the container: only
                                 resources limits and requests (limits.cpu, limits.memory,
@@ -395,6 +399,7 @@ spec:
                               required:
                               - resource
                               type: object
+                              x-kubernetes-map-type: atomic
                             secretKeyRef:
                               description: Selects a key of a secret in the pod's
                                 namespace
@@ -415,6 +420,7 @@ spec:
                               required:
                               - key
                               type: object
+                              x-kubernetes-map-type: atomic
                           type: object
                       required:
                       - name
@@ -470,6 +476,12 @@ spec:
                     - initialDelaySeconds
                     - periodSeconds
                     type: object
+                  terminationGracePeriodSeconds:
+                    default: 300
+                    description: TerminationGracePeriodSeconds specifies the length
+                      of time in seconds to wait before killing the OFED pod on termination
+                    minimum: 0
+                    type: integer
                   upgradePolicy:
                     description: Ofed auto-upgrade settings
                     properties:
@@ -727,9 +739,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -1,4 +1,3 @@
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/config/samples/mellanox.com_v1alpha1_nicclusterpolicy.yaml b/config/samples/mellanox.com_v1alpha1_nicclusterpolicy.yaml
@@ -20,6 +20,7 @@ spec:
     image: mofed
     repository: mellanox
     version: 5.5-1.0.3.2
+    terminationGracePeriodSeconds: 300
     startupProbe:
       initialDelaySeconds: 10
       periodSeconds: 20

diff --git a/deployment/network-operator/README.md b/deployment/network-operator/README.md
@@ -368,6 +368,7 @@ imagePullSecrets:
 | `ofedDriver.env` | list | `[]` | An optional list of [environment variables](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#envvar-v1-core) passed to the Mellanox OFED driver image |
 | `ofedDriver.repoConfig.name` | string | `` | Private mirror repository configuration configMap name |
 | `ofedDriver.certConfig.name` | string | `` | Custom TLS key/certificate configuration configMap name |
+| `ofedDriver.terminationGracePeriodSeconds` | int | 300 | Mellanox OFED termination grace periods in seconds|
 | `ofedDriver.startupProbe.initialDelaySeconds` | int | 10 | Mellanox OFED startup probe initial delay                                                                                                                                 |
 | `ofedDriver.startupProbe.periodSeconds` | int | 20 | Mellanox OFED startup probe interval                                                                                                                                      |
 | `ofedDriver.livenessProbe.initialDelaySeconds` | int | 30 | Mellanox OFED liveness probe initial delay                                                                                                                                |

diff --git a/deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml b/deployment/network-operator/crds/mellanox.com_hostdevicenetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: hostdevicenetworks.mellanox.com
 spec:
@@ -98,9 +97,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/deployment/network-operator/crds/mellanox.com_ipoibnetworks.yaml b/deployment/network-operator/crds/mellanox.com_ipoibnetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: ipoibnetworks.mellanox.com
 spec:
@@ -78,9 +77,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/deployment/network-operator/crds/mellanox.com_macvlannetworks.yaml b/deployment/network-operator/crds/mellanox.com_macvlannetworks.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: macvlannetworks.mellanox.com
 spec:
@@ -92,9 +91,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.10.0
   creationTimestamp: null
   name: nicclusterpolicies.mellanox.com
 spec:
@@ -173,6 +172,7 @@ spec:
                                 type: object
                               type: array
                           type: object
+                          x-kubernetes-map-type: atomic
                         weight:
                           description: Weight associated with matching the corresponding
                             nodeSelectorTerm, in the range 1-100.
@@ -268,10 +268,12 @@ spec:
                                 type: object
                               type: array
                           type: object
+                          x-kubernetes-map-type: atomic
                         type: array
                     required:
                     - nodeSelectorTerms
                     type: object
+                    x-kubernetes-map-type: atomic
                 type: object
               nvPeerDriver:
                 description: NVPeerDriverSpec describes configuration options for
@@ -353,6 +355,7 @@ spec:
                               required:
                               - key
                               type: object
+                              x-kubernetes-map-type: atomic
                             fieldRef:
                               description: 'Selects a field of the pod: supports metadata.name,
                                 metadata.namespace, `metadata.labels[''<KEY>'']`,
@@ -371,6 +374,7 @@ spec:
                               required:
                               - fieldPath
                               type: object
+                              x-kubernetes-map-type: atomic
                             resourceFieldRef:
                               description: 'Selects a resource of the container: only
                                 resources limits and requests (limits.cpu, limits.memory,
@@ -395,6 +399,7 @@ spec:
                               required:
                               - resource
                               type: object
+                              x-kubernetes-map-type: atomic
                             secretKeyRef:
                               description: Selects a key of a secret in the pod's
                                 namespace
@@ -415,6 +420,7 @@ spec:
                               required:
                               - key
                               type: object
+                              x-kubernetes-map-type: atomic
                           type: object
                       required:
                       - name
@@ -470,6 +476,12 @@ spec:
                     - initialDelaySeconds
                     - periodSeconds
                     type: object
+                  terminationGracePeriodSeconds:
+                    default: 300
+                    description: TerminationGracePeriodSeconds specifies the length
+                      of time in seconds to wait before killing the OFED pod on termination
+                    minimum: 0
+                    type: integer
                   upgradePolicy:
                     description: Ofed auto-upgrade settings
                     properties:
@@ -727,9 +739,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []
diff --git a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml
@@ -41,6 +41,7 @@ spec:
       name: {{ .Values.ofedDriver.repoConfig.name }}
     {{- end }}
     imagePullSecrets: {{ include "network-operator.ofed.imagePullSecrets" . | nindent 4 }}
+    terminationGracePeriodSeconds: {{ .Values.ofedDriver.terminationGracePeriodSeconds }}
     startupProbe:
       initialDelaySeconds: {{ .Values.ofedDriver.startupProbe.initialDelaySeconds }}
       periodSeconds: {{ .Values.ofedDriver.startupProbe.periodSeconds }}

diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml
@@ -148,6 +148,7 @@ ofedDriver:
   # env:
   #   - name: EXAMPLE_ENV_VAR
   #     value: example_env_var_value
+  terminationGracePeriodSeconds: 300
   # Private mirror repository configuration
   repoConfig:
     name: ""

diff --git a/manifests/stage-ofed-driver/0050_ofed-driver-ds.yaml b/manifests/stage-ofed-driver/0050_ofed-driver-ds.yaml
@@ -113,7 +113,7 @@ spec:
             failureThreshold: 1
             periodSeconds: {{ .CrSpec.ReadinessProbe.PeriodSeconds }}
       # unloading OFED modules can take more time than default terminationGracePeriod (30 sec)
-      terminationGracePeriodSeconds: 120
+      terminationGracePeriodSeconds: {{ .CrSpec.TerminationGracePeriodSeconds }}
       volumes:
         - name: run-mlnx-ofed
           hostPath: