Removed config maps and replaced with rados omaps

Existing config maps are now replaced with rados omaps that help
store information regarding the requested volume names and the rbd
image names backing the same.

Further to detect cluster, pool and which image a volume ID refers
to, changes to volume ID encoding has been done as per provided
design specification in the stateless ceph-csi proposal.

Additional changes and updates,
- Updated documentation
- Updated manifests
- Updated Helm chart
- Addressed a few csi-test failures

Signed-off-by: ShyamsundarR <srangana@redhat.com>

cmd/cephcsi.go

@@ -38,20 +38,20 @@ const (
 
 var (
 	// common flags
-	vtype           = flag.String("type", "", "driver type [rbd|cephfs]")
-	endpoint        = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
-	driverName      = flag.String("drivername", "", "name of the driver")
-	nodeID          = flag.String("nodeid", "", "node id")
-	metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
+	vtype      = flag.String("type", "", "driver type [rbd|cephfs]")
+	endpoint   = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
+	driverName = flag.String("drivername", "", "name of the driver")
+	nodeID     = flag.String("nodeid", "", "node id")
 
 	// rbd related flags
 	containerized = flag.Bool("containerized", true, "whether run as containerized")
-	configRoot    = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+
-		" cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets")
+	instanceID    = flag.String("instanceid", "", "Unique ID distinguishing this instance of Ceph CSI among other"+
+		" instances, when sharing Ceph clusters across CSI instances for provisioning")
 
 	// cephfs related flags
-	volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
-	mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir")
+	volumeMounter   = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
+	mountCacheDir   = flag.String("mountcachedir", "", "mount info cache save dir")
+	metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
 )
 
 func init() {
@@ -107,13 +107,8 @@ func main() {
 	switch driverType {
 	case rbdType:
 		rbd.PluginFolder = rbd.PluginFolder + dname
-		cp, err := util.CreatePersistanceStorage(
-			rbd.PluginFolder, *metadataStorage, dname)
-		if err != nil {
-			os.Exit(1)
-		}
 		driver := rbd.NewDriver()
-		driver.Run(dname, *nodeID, *endpoint, *configRoot, *containerized, cp)
+		driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized)
 
 	case cephfsType:
 		cephfs.PluginFolder = cephfs.PluginFolder + dname

deploy/cephfs/helm/Chart.yaml

@@ -4,7 +4,7 @@ appVersion: "1.0.0"
 description: "Container Storage Interface (CSI) driver,
 provisioner, and attacher for Ceph cephfs"
 name: ceph-csi-cephfs
-version: 0.6.0
+version: 0.7.0
 keywords:
   - ceph
   - cephfs

deploy/rbd/helm/Chart.yaml

@@ -4,7 +4,7 @@ appVersion: "1.0.0"
 description: "Container Storage Interface (CSI) driver,
 provisioner, snapshotter, and attacher for Ceph RBD"
 name: ceph-csi-rbd
-version: 0.6.0
+version: 0.7.0
 keywords:
   - ceph
   - rbd

deploy/rbd/helm/templates/csiplugin-configmap.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.configMapName | quote }}
+  labels:
+    app: {{ include "ceph-csi-rbd.name" . }}
+    chart: {{ include "ceph-csi-rbd.chart" . }}
+    component: {{ .Values.nodeplugin.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  config.json: |-
+    []

deploy/rbd/helm/templates/nodeplugin-daemonset.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }}
       hostNetwork: true
-      hostPID: true      
+      hostPID: true
       # to use e.g. Rook orchestrated cluster, and mons' FQDN is
       # resolved through k8s service, set dns policy to cluster first
       dnsPolicy: ClusterFirstWithHostNet
@@ -70,7 +70,6 @@ spec:
             - "--v=5"
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
-            - "--metadatastorage=k8s_configmap"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"
@@ -95,12 +94,14 @@ spec:
             - mountPath: /dev
               name: host-dev
             - mountPath: /rootfs
-              name: host-rootfs            
+              name: host-rootfs
             - mountPath: /sys
               name: host-sys
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
           resources:
 {{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
       volumes:
@@ -125,13 +126,16 @@ spec:
             path: /dev
         - name: host-rootfs
           hostPath:
-            path: /            
+            path: /
         - name: host-sys
           hostPath:
             path: /sys
         - name: lib-modules
           hostPath:
             path: /lib/modules
+        - name: ceph-csi-config
+          configMap:
+            name: {{ .Values.configMapName | quote }}
     {{- if .Values.nodeplugin.affinity -}}
       affinity:
 {{ toYaml .Values.nodeplugin.affinity . | indent 8 }}

deploy/rbd/helm/templates/provisioner-statefulset.yaml

@@ -85,7 +85,6 @@ spec:
             - "--v=5"
             - "--drivername=$(DRIVER_NAME)"
             - "--containerized=true"
-            - "--metadatastorage=k8s_configmap"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"
@@ -103,6 +102,8 @@ spec:
               mountPath: {{ .Values.socketDir }}
             - name: host-rootfs
               mountPath: "/rootfs"
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
           resources:
 {{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
       volumes:
@@ -112,6 +113,9 @@ spec:
         - name: host-rootfs
           hostPath:
             path: /
+        - name: ceph-csi-config
+          configMap:
+            name: {{ .Values.configMapName | quote }}
     {{- if .Values.provisioner.affinity -}}
       affinity:
 {{ toYaml .Values.provisioner.affinity . | indent 8 }}

deploy/rbd/helm/values.yaml

@@ -18,6 +18,7 @@ socketFile: csi.sock
 registrationDir: /var/lib/kubelet/plugins_registry
 volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices
 driverName: rbd.csi.ceph.com
+configMapName: ceph-csi-config
 
 attacher:
   name: attacher

deploy/rbd/kubernetes/csi-config-map.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+data:
+  config.json: |-
+    []
+metadata:
+  name: ceph-csi-config

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -78,19 +78,13 @@ spec:
             - "--v=5"
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
-            - "--metadatastorage=k8s_configmap"
-            - "--configroot=k8s_objects"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
             - name: CSI_ENDPOINT
               value: unix:///csi/csi-provisioner.sock
           imagePullPolicy: "IfNotPresent"
@@ -106,6 +100,8 @@ spec:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
       volumes:
         - name: host-dev
           hostPath:
@@ -123,3 +119,6 @@ spec:
           hostPath:
             path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
             type: DirectoryOrCreate
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -56,19 +56,13 @@ spec:
             - "--v=5"
             - "--drivername=rbd.csi.ceph.com"
             - "--containerized=true"
-            - "--metadatastorage=k8s_configmap"
-            - "--configroot=k8s_objects"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
           imagePullPolicy: "IfNotPresent"
@@ -90,6 +84,8 @@ spec:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
       volumes:
         - name: plugin-dir
           hostPath:
@@ -119,3 +115,6 @@ spec:
         - name: lib-modules
           hostPath:
             path: /lib/modules
+        - name: ceph-csi-config
+          configMap:
+            name: ceph-csi-config

docs/deploy-rbd.md

@@ -32,52 +32,39 @@ Option | Default value | Description
 `--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
 `--nodeid` | _empty_ | This node's ID
 `--containerized` | true | Whether running in containerized mode
-`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
-`--configroot` | `/etc/csi-config` | Directory in which CSI specific Ceph cluster configurations are present, OR the value `k8s_objects` if present as kubernetes secrets"
+`--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning
 
 **Available environmental variables:**
 
 `HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.
 
-`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify
-the path of your k8s config file (if not specified, the plugin will assume
-you're running it inside a k8s cluster and find the config itself).
-
-`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store,
-`POD_NAMESPACE` is used to define in which namespace you want
-the configmaps to be stored
-
 **Available volume parameters:**
 
 Parameter | Required | Description
 --------- | -------- | -----------
-`monitors` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
-`monValueFromSecret` | one of `monitors`, `clusterID` or and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors.
-`clusterID` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
+`clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
 `pool` | yes | Ceph pool into which the RBD image shall be created
 `imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)
 `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)
-`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-publish-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
-`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | for Kubernetes | namespaces of the above Secret objects
+`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-publish-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
+`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects
 `mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images
 
-NOTE: If `clusterID` parameter is used, then an accompanying Ceph cluster
-configuration secret or config files needs to be provided to the running pods.
-Refer to [Cluster ID based configuration](../examples/README.md#cluster-id-based-configuration)
-for more information. A suggested way to populate the clusterID is to use the
-output of `ceph fsid` of the Ceph cluster to be used for provisioning.
+**NOTE:** An accompanying CSI configuration file, needs to be provided to the
+running pods. Refer to [Creating CSI configuration for RBD based
+provisioning](../examples/README.md#creating-csi-configuration-for-rbd-based-provisioning)
+for more information.
+
+**NOTE:** A suggested way to populate and retain uniquness of the clusterID is
+to use the output of `ceph fsid` of the Ceph cluster to be used for
+provisioning.
 
 **Required secrets:**
 
 Admin credentials are required for provisioning new RBD images `ADMIN_NAME`:
 `ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the
 client with admin privileges, and the value is its password
 
-If clusterID is specified, then a secret with various keys and values as
-specified in `examples/rbd/template-ceph-cluster-ID-secret.yaml` needs to be
-created, with the secret name matching the string value provided as the
-`clusterID`.
-
 ## Deployment with Kubernetes
 
 Requires Kubernetes 1.11
@@ -101,6 +88,18 @@ Those manifests deploy service accounts, cluster roles and cluster role
 bindings. These are shared for both RBD and CephFS CSI plugins, as they require
 the same permissions.
 
+**Deploy ConfigMap for CSI plugins:**
+
+```bash
+kubectl create -f csi-config-map.yaml
+```
+
+The config map deploys an empty CSI configuration that is mounted as a volume
+within the Ceph CSI plugin pods. To add a specific Ceph clusters configuration
+details, refer to [Creating CSI configuration for RBD based
+provisioning](../examples/README.md#creating-csi-configuration-for-rbd-based-provisioning)
+for more information.
+
 **Deploy CSI sidecar containers:**
 
 ```bash
@@ -134,7 +133,10 @@ service/csi-rbdplugin-provisioner   ClusterIP   10.104.2.130   <none>        123
 ...
 ```
 
-You can try deploying a demo pod from `examples/rbd` to test the deployment further.
+Once the CSI plugin configuration is updated with details from a Ceph cluster of
+choice, you can try deploying a demo pod from examples/rbd using the
+instructions [provided](../examples/README.md#deploying-the-storage-class) to
+test the deployment further.
 
 ## Deployment with Helm