[WIP] collect metadata in k8s cm

Gopkg.toml

@@ -33,3 +33,7 @@
 [[override]]
   name = "github.com/golang/protobuf"
   version = "1.1.0"
+
+[[constraint]]
+  name = "k8s.io/client-go"
+  version = "7.0.0"

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -50,14 +50,19 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v=5"
             - "--drivername=csi-rbdplugin"
-            - "--containerized=true"            
+            - "--containerized=true"
+            - "--persistmetadata=false"
           env:
             - name: HOST_ROOTFS
               value: "/rootfs"
             - name: NODE_ID
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: CSI_ENDPOINT
               value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
           imagePullPolicy: "IfNotPresent"

docs/deploy-rbd.md

@@ -26,7 +26,7 @@ Option | Default value | Description
 `--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
 `--nodeid` | _empty_ | This node's ID
 `--containerized` | true | Whether running in containerized mode
-
+`--persistmetadata` | false | Whether should volume and snapshot metadata be persisted in a k8s configmap
 
 **Available environmental variables:**
 `HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.

pkg/rbd/controllerserver.go

@@ -40,6 +40,7 @@ const (
 
 type controllerServer struct {
 	*csicommon.DefaultControllerServer
+	persistMetadata bool
 }
 
 func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
@@ -118,7 +119,11 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 			}
 
 			rbdSnap := &rbdSnapshot{}
-			if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
+			sm, err := newSnapMeta(cs.persistMetadata)
+			if err != nil {
+				return nil, err
+			}
+			if err := sm.loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
 				return nil, err
 			}
 
@@ -137,11 +142,14 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 			glog.V(4).Infof("create volume %s", volName)
 		}
 	}
-
-	// Storing volInfo into a persistent file.
-	if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
-		glog.Warningf("rbd: failed to store volInfo with error: %v", err)
+	vm, err := newVolMeta(cs.persistMetadata)
+	if err != nil {
+		return nil, err
 	}
+	if err := vm.persistVolInfo(volumeID, path.Join(PluginFolder, "controller-snap"), rbdVol); err != nil {
+		return nil, err
+	}
+
 	rbdVolumes[volumeID] = rbdVol
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
@@ -162,9 +170,13 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 	volumeIDMutex.LockKey(volumeID)
 	defer volumeIDMutex.UnlockKey(volumeID)
 	rbdVol := &rbdVolume{}
-	if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
+
+	vm, err := newVolMeta(cs.persistMetadata)
+	if err != nil {
+		return nil, err
+	}
+	if err := vm.loadVolInfo(volumeID, path.Join(PluginFolder, "controller-snap"), rbdVol); err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
-			// Must have been deleted already. This is not an error (idempotency!).
 			return &csi.DeleteVolumeResponse{}, nil
 		}
 		return nil, err
@@ -177,8 +189,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err)
 		return nil, err
 	}
-	// Removing persistent storage file for the unmapped volume
-	if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
+	if err := vm.deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
 		return nil, err
 	}
 
@@ -296,24 +307,26 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
 
 	rbdSnap.CreatedAt = time.Now().UnixNano()
 
-	// Storing snapInfo into a persistent file.
-	if err := persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
-		glog.Warningf("rbd: failed to store snapInfo with error: %v", err)
 
+	sm, err := newSnapMeta(cs.persistMetadata)
+	if err != nil {
+		return nil, err
+	}
+	if err := sm.persistSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
+		glog.Warningf("rbd: failed to store snapInfo with error: %v", err)
 		// Unprotect snapshot
 		err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets())
 		if err != nil {
 			return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
 		}
-
 		// Deleting snapshot
 		glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName)
 		if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()); err != nil {
 			return nil, status.Error(codes.Unknown, fmt.Sprintf("This Snapshot should be removed but failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
 		}
-
 		return nil, err
 	}
+
 	rbdSnapshots[snapshotID] = rbdSnap
 	return &csi.CreateSnapshotResponse{
 		Snapshot: &csi.Snapshot{
@@ -342,12 +355,16 @@ func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
 	defer snapshotIDMutex.UnlockKey(snapshotID)
 
 	rbdSnap := &rbdSnapshot{}
-	if err := loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
+	sm, err := newSnapMeta(cs.persistMetadata)
+	if err != nil {
+		return nil, err
+	}
+	if err := sm.loadSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap"), rbdSnap); err != nil {
 		return nil, err
 	}
 
 	// Unprotect snapshot
-	err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets())
+	err = unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets())
 	if err != nil {
 		return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
 	}
@@ -358,8 +375,7 @@ func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
 		return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err))
 	}
 
-	// Removing persistent storage file for the unmapped snapshot
-	if err := deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil {
+	if err := sm.deleteSnapInfo(snapshotID, path.Join(PluginFolder, "controller-snap")); err != nil {
 		return nil, err
 	}
 

pkg/rbd/rbd.go

@@ -84,6 +84,9 @@ func init() {
 		// there might be some snapshots left, they must be re-inserted into rbdSnapshots map
 		loadExSnapshots()
 	}
+	if cm, err := getMetadataCM(); err == nil {
+		loadExDataFromCM(cm)
+	}
 }
 
 // loadExSnapshots check for any *.json files in the  PluginFolder/controller-snap folder
@@ -154,9 +157,10 @@ func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
 	}
 }
 
-func NewControllerServer(d *csicommon.CSIDriver) *controllerServer {
+func NewControllerServer(d *csicommon.CSIDriver, persistMetadata bool) *controllerServer {
 	return &controllerServer{
 		DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
+		persistMetadata: persistMetadata,
 	}
 }
 
@@ -175,7 +179,7 @@ func NewNodeServer(d *csicommon.CSIDriver, containerized bool) (*nodeServer, err
 	}, nil
 }
 
-func (rbd *rbd) Run(driverName, nodeID, endpoint string, containerized bool) {
+func (rbd *rbd) Run(driverName, nodeID, endpoint string, containerized bool, persistMetadata bool) {
 	var err error
 	glog.Infof("Driver: %v version: %v", driverName, version)
 
@@ -198,7 +202,7 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string, containerized bool) {
 	if err != nil {
 		glog.Fatalln("failed to start node server, err %v", err)
 	}
-	rbd.cs = NewControllerServer(rbd.driver)
+	rbd.cs = NewControllerServer(rbd.driver, persistMetadata)
 	s := csicommon.NewNonBlockingGRPCServer()
 	s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)
 	s.Wait()