feat(attach): avoid creation of volumeattachment object

k8s is very slow in attaching the volumes when dealing with the large number of volume attachment object. (k8s issue kubernetes/kubernetes#84169) The volumeattachment is not required for ZFSPV, so avoid creation of attachment object, also removed the csi-attacher container as this is also not needed as it acts on volumeattachment object. Signed-off-by: Pawan <pawan@mayadata.io>
openebs · Apr 20, 2020 · 40b9c33 · 40b9c33
1 parent 280949c
commit 40b9c33
Show file tree

Hide file tree

Showing 5 changed files with 113 additions and 211 deletions.
diff --git a/deploy/yamls/zfs-driver.yaml b/deploy/yamls/zfs-driver.yaml
@@ -1,6 +1,19 @@
 
 ---
 
+# Create the CSI Driver object
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: zfs.csi.openebs.io
+spec:
+  # do not require volumeattachment
+  attachRequired: false
+  podInfoOnMount: false
+  volumeLifecycleModes:
+  - Persistent
+---
+
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -463,6 +476,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["*"]
     resources: ["zfsvolumes", "zfssnapshots"]
     verbs: ["*"]
@@ -561,31 +577,6 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-        - name: csi-attacher
-          image: quay.io/k8scsi/csi-attacher:v2.0.0
-          imagePullPolicy: IfNotPresent
-          args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election"
-          env:
-            - name: ADDRESS
-              value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/csi/sockets/pluginproxy/
-        - name: csi-cluster-driver-registrar
-          image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
-          args:
-            - "--v=5"
-            - "--driver-requires-attachment=false"
-            - "--csi-address=$(ADDRESS)"
-          env:
-            - name: ADDRESS
-              value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: openebs-zfs-plugin
           image: quay.io/openebs/zfs-driver:ci
           imagePullPolicy: IfNotPresent
@@ -611,45 +602,6 @@ spec:
           emptyDir: {}
 ---
 
-############################## CSI- Attacher #######################
-# Attacher must be able to work with PVs, nodes and VolumeAttachments
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-attacher-role
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["csi.storage.k8s.io"]
-    resources: ["csinodeinfos"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments", "csinodes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-attacher-binding
-subjects:
-  - kind: ServiceAccount
-    name: openebs-zfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: openebs-zfs-attacher-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -705,32 +657,6 @@ roleRef:
 
 ---
 
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-cluster-driver-registrar-role
-rules:
-  - apiGroups: ["csi.storage.k8s.io"]
-    resources: ["csidrivers"]
-    verbs: ["create", "delete"]
-
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-cluster-driver-registrar-binding
-subjects:
-  - kind: ServiceAccount
-    name: openebs-zfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: openebs-zfs-cluster-driver-registrar-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-
 ########################################
 ###########                 ############
 ###########   Node plugin   ############

diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml
@@ -458,6 +458,19 @@ status:
 
 ---
 
+# Create the CSI Driver object
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: zfs.csi.openebs.io
+spec:
+  # do not require volumeattachment
+  attachRequired: false
+  podInfoOnMount: false
+  volumeLifecycleModes:
+  - Persistent
+---
+
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -920,6 +933,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["*"]
     resources: ["zfsvolumes", "zfssnapshots"]
     verbs: ["*"]
@@ -1018,31 +1034,6 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-        - name: csi-attacher
-          image: quay.io/k8scsi/csi-attacher:v2.0.0
-          imagePullPolicy: IfNotPresent
-          args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election"
-          env:
-            - name: ADDRESS
-              value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/csi/sockets/pluginproxy/
-        - name: csi-cluster-driver-registrar
-          image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
-          args:
-            - "--v=5"
-            - "--driver-requires-attachment=false"
-            - "--csi-address=$(ADDRESS)"
-          env:
-            - name: ADDRESS
-              value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: openebs-zfs-plugin
           image: quay.io/openebs/zfs-driver:ci
           imagePullPolicy: IfNotPresent
@@ -1068,45 +1059,6 @@ spec:
           emptyDir: {}
 ---
 
-############################## CSI- Attacher #######################
-# Attacher must be able to work with PVs, nodes and VolumeAttachments
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-attacher-role
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["csi.storage.k8s.io"]
-    resources: ["csinodeinfos"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments", "csinodes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-attacher-binding
-subjects:
-  - kind: ServiceAccount
-    name: openebs-zfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: openebs-zfs-attacher-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -1162,32 +1114,6 @@ roleRef:
 
 ---
 
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-cluster-driver-registrar-role
-rules:
-  - apiGroups: ["csi.storage.k8s.io"]
-    resources: ["csidrivers"]
-    verbs: ["create", "delete"]
-
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: openebs-zfs-cluster-driver-registrar-binding
-subjects:
-  - kind: ServiceAccount
-    name: openebs-zfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: openebs-zfs-cluster-driver-registrar-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-
 ########################################
 ###########                 ############
 ###########   Node plugin   ############

diff --git a/upgrade/README.md b/upgrade/README.md
@@ -1,4 +1,4 @@
-From zfs-driver:v0.6 version ZFS-LocalPV related CRs are now grouped together in its own group called `zfs.openebs.io`. Here steps are mentioned for how to upgrade for refactoring the CRDs.
+From zfs-driver:v0.6 version ZFS-LocalPV related CRs are now grouped together in its own group called `zfs.openebs.io`. Here steps are mentioned for how to upgrade for refactoring the CRDs. Please do not provision/deprovision any volume during the upgrade.
 
 steps to upgrade:-
 
@@ -21,7 +21,9 @@ zfsvolume.zfs.openebs.io/pvc-82368c44-eee8-47ee-85a6-633a8023faa8 created
 zfssnapshot.zfs.openebs.io/snapshot-dc61a056-f495-482b-8e6e-e7ddc4c13f47 created
 zfssnapshot.zfs.openebs.io/snapshot-f9db91ea-529e-4dac-b2b8-ead045c612da created
 ```
-`
+Please note that if you have modified the OPENEBS_NAMESPACE env in the driver's deployment to other namespace. Then you have to pass the namespace as an argument to the upgrade.sh script `sh upgrade/upgrash.sh [namespace]`.
+
+
 3. upgrade the driver to v0.6
 
 ```
@@ -55,4 +57,9 @@ zfssnapshot.openebs.io/snapshot-f9db91ea-529e-4dac-b2b8-ead045c612da configured
 zfssnapshot.openebs.io "snapshot-dc61a056-f495-482b-8e6e-e7ddc4c13f47" deleted
 zfssnapshot.openebs.io "snapshot-f9db91ea-529e-4dac-b2b8-ead045c612da" deleted
 customresourcedefinition.apiextensions.k8s.io "zfssnapshots.openebs.io" deleted
-```
+```
+
+Please note that if you have modified the OPENEBS_NAMESPACE env in the driver's deployment to other namespace. Then you have to pass the namespace as an argument to the cleanup.sh script `sh upgrade/cleanup.sh [namespace]`.
+
+5. restart kube-controller [optional]
+kube-controller-manager might be using stale volumeattachment resources, it might get flooded with the error logs. Restarting kube-controller will fix it.
diff --git a/upgrade/cleanup.sh b/upgrade/cleanup.sh
@@ -2,27 +2,53 @@
 
 set -e
 
-kubectl get zfsvolumes.openebs.io -n openebs -oyaml > volumes.yaml
+if [ -z $1 ]; then
+	# default namespace is openebs when all the custom resources are created
+	ZFSPV_NAMESPACE="openebs"
+else
+	ZFSPV_NAMESPACE=$1
+fi
 
-# remove the finalizer from the old CR
-sed -i "/zfs.openebs.io\/finalizer/d" volumes.yaml
-kubectl apply -f volumes.yaml
+echo "Fetching ZFS Volumes"
+numVol=`kubectl get zfsvolumes.openebs.io -n $ZFSPV_NAMESPACE | wc -l`
 
-# delete the old CR
-kubectl delete -f volumes.yaml
+if [ $numVol -gt 1 ]; then # numVol includes header also, so it should be greater than 1
+	echo "Cleaning the ZFS Volumes(`expr $numVol - 1`)"
+	kubectl get zfsvolumes.openebs.io -n $ZFSPV_NAMESPACE -oyaml > volumes.yaml
 
-# delete the CRD definition
+	# remove the finalizer from the old CR
+	sed -i "/zfs.openebs.io\/finalizer/d" volumes.yaml
+	kubectl apply -f volumes.yaml
+
+	# delete the old CR
+	kubectl delete -f volumes.yaml
+fi
+
+# delete the ZFSVolume CRD definition
 kubectl delete crd zfsvolumes.openebs.io
 
+numAttach=`kubectl get volumeattachment | grep zfs.csi.openebs.io | wc -l`
+
+if [ $numAttach -gt 0 ]; then
+	echo "Cleaning the volumeattachment(`expr $numAttach - 1`)"
+	# delete the volumeattachment object
+	kubectl delete volumeattachment --all
+fi
+
+echo "Fetching ZFS Snapshots"
+numSnap=`kubectl get zfssnapshots.openebs.io -n $ZFSPV_NAMESPACE | wc -l`
 
-kubectl get zfssnapshots.openebs.io -n openebs -oyaml > snapshots.yaml
+if [ $numSnap -gt 1 ]; then # numSnap includes header also, so it should be greater than 1
+	echo "Cleaning the ZFS Snapshot(`expr $numSnap - 1`)"
+	kubectl get zfssnapshots.openebs.io -n $ZFSPV_NAMESPACE -oyaml > snapshots.yaml
 
-# remove the finalizer from the old CR
-sed -i "/zfs.openebs.io\/finalizer/d" snapshots.yaml
-kubectl apply -f snapshots.yaml
+	# remove the finalizer from the old CR
+	sed -i "/zfs.openebs.io\/finalizer/d" snapshots.yaml
+	kubectl apply -f snapshots.yaml
 
-# delete the old CR
-kubectl delete -f snapshots.yaml
+	# delete the old CR
+	kubectl delete -f snapshots.yaml
+fi
 
-# delete the CRD definition
+# delete the ZFSSnapshot CRD definition
 kubectl delete crd zfssnapshots.openebs.io