Impl OverprovisionControl using OpenShift's ClusterResourceQuota

api/v1/storagecluster_types.go

@@ -18,9 +18,11 @@ package v1
 
 import (
 	nbv1 "github.com/noobaa/noobaa-operator/v2/pkg/apis/noobaa/v1alpha1"
+	quotav1 "github.com/openshift/api/quota/v1"
 	conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1"
 	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -70,6 +72,9 @@ type StorageClusterSpec struct {
 	// Mirroring specifies data mirroring configuration for the storage cluster.
 	// This configuration will only be applied to resources managed by the operator.
 	Mirroring MirroringSpec `json:"mirroring,omitempty"`
+	// OverprovisionControl specifies the allowed hard-limit PVs overprovisioning relative to
+	// the effective usable storage capacity.
+	OverprovisionControl []OverprovisionControlSpec `json:"overprovisionControl,omitempty"`
 }
 
 // KeyManagementServiceSpec provides a way to enable KMS
@@ -395,3 +400,14 @@ type ArbiterSpec struct {
 func init() {
 	SchemeBuilder.Register(&StorageCluster{}, &StorageClusterList{})
 }
+
+// OverprovisionControlSpec defines the allowed overprovisioning PVC consumption from the underlying cluster.
+// This may be an absolute value or as a percentage of the overall effective capacity.
+// One, and only one of those two (Capacity and Percentage) may be defined.
+type OverprovisionControlSpec struct {
+	StorageClassName string                               `json:"storageClassName,omitempty"`
+	QuotaName        string                               `json:"quotaName,omitempty"`
+	Capacity         *resource.Quantity                   `json:"capacity,omitempty"`
+	Percentage       uint                                 `json:"percentage,omitempty"`
+	Selector         quotav1.ClusterResourceQuotaSelector `json:"selector,omitempty"`
+}

config/crd/bases/ocs.openshift.io_storageclusters.yaml

@@ -780,6 +780,91 @@ spec:
                     nullable: true
                     type: object
                 type: object
+              overprovisionControl:
+                description: OverprovisionControl specifies the allowed hard-limit
+                  PVs overprovisioning relative to the effective usable storage capacity.
+                items:
+                  description: OverprovisionControlSpec defines the allowed overprovisioning
+                    PVC consumption from the underlying cluster. This may be an absolute
+                    value or as a percentage of the overall effective capacity. One,
+                    and only one of those two (Capacity and Percentage) may be defined.
+                  properties:
+                    capacity:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    percentage:
+                      type: integer
+                    quotaName:
+                      type: string
+                    selector:
+                      description: ClusterResourceQuotaSelector is used to select
+                        projects.  At least one of LabelSelector or AnnotationSelector
+                        must present.  If only one is present, it is the only selection
+                        criteria.  If both are specified, the project must match both
+                        restrictions.
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          description: AnnotationSelector is used to select projects
+                            by annotation.
+                          nullable: true
+                          type: object
+                        labels:
+                          description: LabelSelector is used to select projects by
+                            label.
+                          nullable: true
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                      type: object
+                    storageClassName:
+                      type: string
+                  type: object
+                type: array
               placement:
                 additionalProperties:
                   description: Placement is the placement for an object

config/rbac/role.yaml

@@ -129,6 +129,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - quota.openshift.io
+  resources:
+  - clusterresourcequotas
+  verbs:
+  - '*'
 - apiGroups:
   - route.openshift.io
   resources:

controllers/storagecluster/cephcluster.go

@@ -585,23 +585,7 @@ func newStorageClassDeviceSets(sc *ocsv1.StorageCluster, serverVersion *version.
 			}
 		}
 
-		count := ds.Count
-		replica := ds.Replica
-		if replica == 0 {
-			replica = defaults.DeviceSetReplica
-
-			// This is a temporary hack in place due to limitations
-			// in the current implementation of the OCP console.
-			// The console is hardcoded to create a StorageCluster
-			// with a Count of 3, as made sense for the previous
-			// behavior, but it cannot be updated until the next
-			// z-stream release of OCP 4.2. This workaround is to
-			// enable the new behavior while the console is waiting
-			// to be updated.
-			// TODO: Remove this behavior when OCP console is updated
-			count = count / 3
-		}
-
+		count, replica := countAndReplicaOf(&ds)
 		for i := 0; i < replica; i++ {
 			placement := rookCephv1.Placement{}
 			preparePlacement := rookCephv1.Placement{}
@@ -715,6 +699,26 @@ func newStorageClassDeviceSets(sc *ocsv1.StorageCluster, serverVersion *version.
 	return storageClassDeviceSets
 }
 
+func countAndReplicaOf(ds *ocsv1.StorageDeviceSet) (int, int) {
+	count := ds.Count
+	replica := ds.Replica
+	if replica == 0 {
+		replica = defaults.DeviceSetReplica
+
+		// This is a temporary hack in place due to limitations
+		// in the current implementation of the OCP console.
+		// The console is hardcoded to create a StorageCluster
+		// with a Count of 3, as made sense for the previous
+		// behavior, but it cannot be updated until the next
+		// z-stream release of OCP 4.2. This workaround is to
+		// enable the new behavior while the console is waiting
+		// to be updated.
+		// TODO: Remove this behavior when OCP console is updated
+		count = count / 3
+	}
+	return count, replica
+}
+
 func newCephDaemonResources(sc *ocsv1.StorageCluster) map[string]corev1.ResourceRequirements {
 
 	custom := sc.Spec.Resources

controllers/storagecluster/generate.go

@@ -74,3 +74,8 @@ func generateCephReplicatedSpec(initData *ocsv1.StorageCluster, poolType string)
 
 	return crs
 }
+
+// generateStorageQuotaName function generates a name for ClusterResourceQuota
+func generateStorageQuotaName(storageClassName, quotaName string) string {
+	return fmt.Sprintf("ocs-%s-%s", storageClassName, quotaName)
+}

controllers/storagecluster/reconcile.go

@@ -136,6 +136,7 @@ var validTopologyLabelKeys = []string{
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=*
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update
 // +kubebuilder:rbac:groups=operators.coreos.com,resources=operatorconditions,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=quota.openshift.io,resources=clusterresourcequotas,verbs=*
 
 // Reconcile reads that state of the cluster for a StorageCluster object and makes changes based on the state read
 // and what is in the StorageCluster.Spec
@@ -240,6 +241,18 @@ func (r *StorageClusterReconciler) validateStorageClusterSpec(instance *ocsv1.St
 		}
 		return err
 	}
+
+	if err := validateOverprovisionControlSpec(instance, r.Log); err != nil {
+		r.Log.Error(err, "Failed to validate OverprovisionControlSpec.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name))
+		r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, statusutil.EventReasonValidationFailed, err.Error())
+		instance.Status.Phase = statusutil.PhaseError
+		if updateErr := r.Client.Status().Update(context.TODO(), instance); updateErr != nil {
+			r.Log.Error(updateErr, "Could not update StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name))
+			return updateErr
+		}
+		return err
+	}
+
 	return nil
 }
 
@@ -344,6 +357,7 @@ func (r *StorageClusterReconciler) reconcilePhases(
 		// list of default ensure functions
 		objs = []resourceManager{
 			&ocsStorageClass{},
+			&ocsStorageQuota{},
 			&ocsSnapshotClass{},
 			&ocsCephObjectStores{},
 			&ocsCephObjectStoreUsers{},
@@ -682,3 +696,25 @@ func validateArbiterSpec(sc *ocsv1.StorageCluster, reqLogger logr.Logger) error
 	}
 	return nil
 }
+
+func validateOverprovisionControlSpec(sc *ocsv1.StorageCluster, reqLogger logr.Logger) error {
+	for _, opc := range sc.Spec.OverprovisionControl {
+		if opc.Capacity == nil && opc.Percentage == 0 {
+			return fmt.Errorf("missing either capacity or percentage for overprovision")
+		}
+		if opc.Capacity != nil && opc.Percentage > 0 {
+			return fmt.Errorf("can not define both capacity and percentage for overprovision")
+		}
+		if opc.StorageClassName == "" {
+			return fmt.Errorf("missing storageclassname")
+		}
+		if opc.StorageClassName != sc.Name+"-"+cephFsStorageClassName &&
+			opc.StorageClassName != sc.Name+"-"+cephRbdStorageClassName {
+			return fmt.Errorf("unsupported storageclassname: %s", opc.StorageClassName)
+		}
+		if opc.QuotaName == "" {
+			return fmt.Errorf("missing quotaname")
+		}
+	}
+	return nil
+}

controllers/storagecluster/storagecluster_controller_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/blang/semver"
 	"github.com/noobaa/noobaa-operator/v2/pkg/apis/noobaa/v1alpha1"
 	configv1 "github.com/openshift/api/config/v1"
+	quotav1 "github.com/openshift/api/quota/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	openshiftv1 "github.com/openshift/api/template/v1"
 	conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1"
@@ -999,6 +1000,11 @@ func createFakeScheme(t *testing.T) *runtime.Scheme {
 		assert.Fail(t, "failed to add appsv1 scheme")
 	}
 
+	err = quotav1.AddToScheme(scheme)
+	if err != nil {
+		assert.Fail(t, "failed to add quotav1 scheme")
+	}
+
 	return scheme
 }