Mellanox · e0ne · Jul 4, 2023 · May 22, 2023 · May 22, 2023 · May 22, 2023
diff --git a/README.md b/README.md
@@ -28,6 +28,7 @@
   - [Docker image](#docker-image)
   - [Driver Containers](#driver-containers)
   - [Upgrade](#upgrade)
+  - [Externally Provided Configurations For Network Operator Sub-Components](#externally-provided-configurations-for-network-operator-sub-components)
 
 <small><i><a href='http://ecotrust-canada.github.io/markdown-toc/'>Table of contents generated with markdown-toc</a></i></small>
 
@@ -477,8 +478,95 @@ While this approach may seem odd. It provides a way to deliver drivers to immuta
 [Mellanox OFED and NV Peer Memory driver container](https://github.com/Mellanox/ofed-docker)
 
 ## Upgrade
-Network operator provides limited upgrade capabilities which require additional
-manual actions if a containerized OFED driver is used.
-Future releases of the network operator will provide automatic upgrade flow for the containerized driver.
-
 Check [Upgrade section in Helm Chart documentation](deployment/network-operator/README.md#upgrade) for details.
+
+## Externally Provided Configurations For Network Operator Sub-Components
+
+In most cases, Network Operator will be deployed together with the related configurations
+for the various sub-components it deploys e.g. Nvidia k8s IPAM plugin, RDMA shared device plugin
+or SR-IOV Network device plugin.
+
+Specifying configuration either via Helm values when installing NVIDIA
+network operator, or by specifying them when directly creating NicClusterPolicy CR.
+These configurations eventually trigger the creation of a ConfigMap object in K8s.
+
+As an example, NVIDIA K8s IPAM plugin configuration is specified either via:
+
+__Helm values:__
+
+```yaml
+deployCR: true
+nvIpam:
+  deploy: true
+  image: nvidia-k8s-ipam
+  repository: ghcr.io/mellanox
+  version: v0.0.3
+  config: |-
+    {
+    "pools":  {
+      "rdma-pool": {"subnet": "192.168.0.0/16", "perNodeBlockSize": 100, "gateway": "192.168.0.1"}
+      }
+    }
+```
+
+__NicClusterPolicy CR:__
+
+```yaml
+apiVersion: mellanox.com/v1alpha1
+kind: NicClusterPolicy
+metadata:
+  name: nic-cluster-policy
+spec:
+  nvIpam:
+    image: nvidia-k8s-ipam
+    repository: ghcr.io/mellanox
+    version: v0.0.3
+    config: '{
+      "pools":  {
+        "my-pool": {"subnet": "192.168.0.0/16", "perNodeBlockSize": 100, "gateway": "192.168.0.1"}
+      }
+    }'
+```
+
+The configuration is then processed by the operator, eventually rendering and creating a _ConfigMap_, `nvidia-k8s-ipam-config`, within the
+namespace the operator was deployed. It contains the configuration for _nvidia k8s IPAM plugin_.
+
+For some advanced use-cases, it is desirable to provide such configurations at a later time.
+(e.g if network configuration is not known during Network Operator deployment time)
+
+To support this, it is possible to explicitly set such configuration to `nil` in Helm values
+or omit the `config` field of the relevant component while creating NicClusterPolicy CR.
+This will prevent Network Operator from
+creating such ConfigMaps, allowing the user to provide its own.
+
+Example (omitting nvidia k8s ipam config):
+
+__Helm values:__
+
+```yaml
+deployCR: true
+nvIpam:
+  deploy: true
+  image: nvidia-k8s-ipam
+  repository: ghcr.io/mellanox
+  version: v0.0.3
+  config: null
+```
+
+__NicClusterPolicy CR:__
+
+```yaml
+apiVersion: mellanox.com/v1alpha1
+kind: NicClusterPolicy
+metadata:
+  name: nic-cluster-policy
+spec:
+  nvIpam:
+    image: nvidia-k8s-ipam
+    repository: ghcr.io/mellanox
+    version: v0.0.3
+```
+
+> __Note__: It is the responsibility of the user to delete any existing configurations (ConfigMaps) if
+> they were already created by the Network Operator as well as deleting his own configuration when they
+> are no longer required.
diff --git a/api/v1alpha1/nicclusterpolicy_types.go b/api/v1alpha1/nicclusterpolicy_types.go
@@ -50,6 +50,12 @@ type ImageSpec struct {
 	ImagePullSecrets []string `json:"imagePullSecrets"`
 }
 
+// ImageSpecWithConfig Contains ImageSpec and optional configuration
+type ImageSpecWithConfig struct {
+	ImageSpec `json:""`
+	Config    *string `json:"config,omitempty"`
+}
+
 type PodProbeSpec struct {
 	InitialDelaySeconds int `json:"initialDelaySeconds"`
 	PeriodSeconds       int `json:"periodSeconds"`
@@ -155,20 +161,18 @@ type NVPeerDriverSpec struct {
 }
 
 // DevicePluginSpec describes configuration options for device plugin
+// 1. Image information for device plugin
+// 2. Device plugin configuration
 type DevicePluginSpec struct {
-	// Image information for device plugin
-	ImageSpec `json:""`
-	// Device plugin configuration
-	Config string `json:"config"`
+	ImageSpecWithConfig `json:""`
 }
 
 // MultusSpec describes configuration options for Multus CNI
+//  1. Image information for Multus CNI
+//  2. Multus CNI config if config is missing or empty then multus config will be automatically generated from the CNI
+//     configuration file of the master plugin (the first file in lexicographical order in cni-conf-dir)
 type MultusSpec struct {
-	// Image information for device plugin
-	ImageSpec `json:""`
-	// Multus CNI config if config is missing or empty then multus config will be automatically generated from the CNI
-	// configuration file of the master plugin (the first file in lexicographical order in cni-conf-dir)
-	Config string `json:"config,omitempty"`
+	ImageSpecWithConfig `json:""`
 }
 
 // SecondaryNetwork describes configuration options for secondary network
@@ -208,11 +212,11 @@ type IBKubernetesSpec struct {
 	UfmSecret string `json:"ufmSecret,omitempty"`
 }
 
+// NVIPAMSpec describes configuration options for nv-ipam
+// 1. Image information for nv-ipam
+// 2. Configuration for nv-ipam
 type NVIPAMSpec struct {
-	// Image information for nv-ipam
-	ImageSpec `json:""`
-	// Config for nv-ipam in JSON format
-	Config string `json:"config,omitempty"`
+	ImageSpecWithConfig `json:""`
 }
 
 // NicClusterPolicySpec defines the desired state of NicClusterPolicy

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/mellanox.com_nicclusterpolicies.yaml b/config/crd/bases/mellanox.com_nicclusterpolicies.yaml
@@ -276,9 +276,10 @@ spec:
                     x-kubernetes-map-type: atomic
                 type: object
               nvIpam:
+                description: NVIPAMSpec describes configuration options for nv-ipam
+                  1. Image information for nv-ipam 2. Configuration for nv-ipam
                 properties:
                   config:
-                    description: Config for nv-ipam in JSON format
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -591,10 +592,10 @@ spec:
                 type: object
               rdmaSharedDevicePlugin:
                 description: DevicePluginSpec describes configuration options for
-                  device plugin
+                  device plugin 1. Image information for device plugin 2. Device plugin
+                  configuration
                 properties:
                   config:
-                    description: Device plugin configuration
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -610,7 +611,6 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
-                - config
                 - image
                 - repository
                 - version
@@ -686,10 +686,6 @@ spec:
                     description: Image and configuration information for multus
                     properties:
                       config:
-                        description: Multus CNI config if config is missing or empty
-                          then multus config will be automatically generated from
-                          the CNI configuration file of the master plugin (the first
-                          file in lexicographical order in cni-conf-dir)
                         type: string
                       image:
                         pattern: '[a-zA-Z0-9\-]+'
@@ -712,10 +708,10 @@ spec:
                 type: object
               sriovDevicePlugin:
                 description: DevicePluginSpec describes configuration options for
-                  device plugin
+                  device plugin 1. Image information for device plugin 2. Device plugin
+                  configuration
                 properties:
                   config:
-                    description: Device plugin configuration
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -731,7 +727,6 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
-                - config
                 - image
                 - repository
                 - version

diff --git a/deployment/network-operator/README.md b/deployment/network-operator/README.md
@@ -530,7 +530,7 @@ optionally deployed components:
 | `multus.repository` | string | `ghcr.io/k8snetworkplumbingwg` | Multus image repository  |
 | `multus.version` | string | `v3.8` | Multus image version  |
 | `multus.imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling any of the Multus image |
-| `multus.config` | string | `` | Multus CNI config, if empty then config will be automatically generated from the CNI configuration file of the master plugin (the first file in lexicographical order in cni-conf-dir)  |
+| `multus.config` | string | nil | Multus CNI config, if not specified or empty then config will be automatically generated from the CNI configuration file of the master plugin (the first file in lexicographical order in cni-conf-dir)  |
 
 ##### IPoIB CNI
 
@@ -554,14 +554,14 @@ optionally deployed components:
 
 #### NVIDIA IPAM Plugin
 
-| Name                      | Type   | Default            | description                                                                            |
-| ------------------------- | ------ |--------------------| -------------------------------------------------------------------------------------- |
-| `nvIpam.deploy`           | bool   | `false`            | Deploy NVIDIA IPAM Plugin                                                              |
-| `nvIpam.image`            | string | `nvidia-k8s-ipam`  | NVIDIA IPAM Plugin image name                                                          |
-| `nvIpam.repository`       | string | `ghcr.io/mellanox` | NVIDIA IPAM Plugin image repository                                                    |
-| `nvIpam.version`          | string | `v0.0.3`           | NVIDIA IPAM Plugin image version                                                       |
-| `nvIpam.imagePullSecrets` | list   | `[]`               | An optional list of references to secrets to use for pulling any of the Plugin image   |
-| `nvIpam.config`           | string | `''`               | Network pool configuration as described in https://github.com/Mellanox/nvidia-k8s-ipam |
+| Name                      | Type   | Default            | description                                                                              |
+| ------------------------- | ------ |--------------------| -----------------------------------------------------------------------------------------|
+| `nvIpam.deploy`           | bool   | `false`            | Deploy NVIDIA IPAM Plugin                                                                |
+| `nvIpam.image`            | string | `nvidia-k8s-ipam`  | NVIDIA IPAM Plugin image name                                                            |
+| `nvIpam.repository`       | string | `ghcr.io/mellanox` | NVIDIA IPAM Plugin image repository                                                      |
+| `nvIpam.version`          | string | `v0.0.3`           | NVIDIA IPAM Plugin image version                                                         |
+| `nvIpam.imagePullSecrets` | list   | `[]`               | An optional list of references to secrets to use for pulling any of the Plugin image     |
+| `nvIpam.config`           | string | `"{"pools": {"rdma-pool": {"subnet": "192.168.0.0/16", "perNodeBlockSize": 100, "gateway": "192.168.0.1"}}}"` | Network pool configuration as described in [nvidia-k8s-ipam](https://github.com/Mellanox/nvidia-k8s-ipam), the default defines a single IP Pool named `"rdma-pool"`|
 
 ## Deployment Examples
 

diff --git a/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/deployment/network-operator/crds/mellanox.com_nicclusterpolicies.yaml
@@ -276,9 +276,10 @@ spec:
                     x-kubernetes-map-type: atomic
                 type: object
               nvIpam:
+                description: NVIPAMSpec describes configuration options for nv-ipam
+                  1. Image information for nv-ipam 2. Configuration for nv-ipam
                 properties:
                   config:
-                    description: Config for nv-ipam in JSON format
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -591,10 +592,10 @@ spec:
                 type: object
               rdmaSharedDevicePlugin:
                 description: DevicePluginSpec describes configuration options for
-                  device plugin
+                  device plugin 1. Image information for device plugin 2. Device plugin
+                  configuration
                 properties:
                   config:
-                    description: Device plugin configuration
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -610,7 +611,6 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
-                - config
                 - image
                 - repository
                 - version
@@ -686,10 +686,6 @@ spec:
                     description: Image and configuration information for multus
                     properties:
                       config:
-                        description: Multus CNI config if config is missing or empty
-                          then multus config will be automatically generated from
-                          the CNI configuration file of the master plugin (the first
-                          file in lexicographical order in cni-conf-dir)
                         type: string
                       image:
                         pattern: '[a-zA-Z0-9\-]+'
@@ -712,10 +708,10 @@ spec:
                 type: object
               sriovDevicePlugin:
                 description: DevicePluginSpec describes configuration options for
-                  device plugin
+                  device plugin 1. Image information for device plugin 2. Device plugin
+                  configuration
                 properties:
                   config:
-                    description: Device plugin configuration
                     type: string
                   image:
                     pattern: '[a-zA-Z0-9\-]+'
@@ -731,7 +727,6 @@ spec:
                     pattern: '[a-zA-Z0-9\.-]+'
                     type: string
                 required:
-                - config
                 - image
                 - repository
                 - version

diff --git a/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml b/deployment/network-operator/templates/mellanox.com_v1alpha1_nicclusterpolicy_cr.yaml
@@ -184,6 +184,8 @@ spec:
     repository: {{ .Values.nvIpam.repository }}
     version: {{ .Values.nvIpam.version }}
     imagePullSecrets: {{ include "network-operator.nvIpam.imagePullSecrets" . }}
+    {{- if .Values.nvIpam.config | empty | not }}
     config: {{ .Values.nvIpam.config | quote }}
+    {{- end }}
   {{- end }}
 {{ end }}