Merge pull request #188 from projectsyn/feat/extra-rbd-storage-classes

Add support for configuring additional storage classes for RBD blockpools

.cruft.json

@@ -1,13 +1,13 @@
 {
   "template": "https://github.com/projectsyn/commodore-component-template.git",
-  "commit": "8840f87d25d97ce0d4bfed75d40173caaf4100fc",
+  "commit": "ba0ed2c8ff147199a748e9ff12cadce32c167fc0",
   "checkout": "main",
   "context": {
     "cookiecutter": {
       "name": "Rook Ceph",
       "slug": "rook-ceph",
       "parameter_key": "rook_ceph",
-      "test_cases": "defaults openshift4 cephfs",
+      "test_cases": "defaults openshift4 cephfs rbd-extra-storageclass",
       "add_lib": "n",
       "add_pp": "y",
       "add_golden": "y",

.github/workflows/test.yaml

@@ -35,6 +35,7 @@ jobs:
           - defaults
           - openshift4
           - cephfs
+          - rbd-extra-storageclass
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -52,6 +53,7 @@ jobs:
           - defaults
           - openshift4
           - cephfs
+          - rbd-extra-storageclass
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -57,4 +57,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/openshift4.yml tests/cephfs.yml
+test_instances = tests/defaults.yml tests/openshift4.yml tests/cephfs.yml tests/rbd-extra-storageclass.yml

component/rbd.libsonnet

@@ -29,6 +29,12 @@ local rbd_blockpools = [
 local rbd_storageclasses = [
   sp.configure_storageclass('rbd', name)
   for name in std.objectFields(rbd_params)
+] + [
+  sp.configure_storageclass('rbd', name, suffix=suffix)
+  for name in std.objectFields(rbd_params)
+  for suffix in std.objectFields(
+    std.get(rbd_params[name], 'extra_storage_classes', {})
+  )
 ];
 
 local rbd_snapclass = [

component/storagepool.libsonnet

@@ -20,11 +20,12 @@ local load_storageclass(type) =
   // return storageclass
   sc[0];
 
-local get_sc_config(type, pool) =
+local get_sc_config(type, pool, extra_class) =
   assert std.objectHas(params.ceph_cluster.storage_pools, type);
   assert std.objectHas(
     params.ceph_cluster.storage_pools[type], pool
   );
+  // we always use the pool-wide mount options
   local param_mount_options = com.getValueOrDefault(
     params.ceph_cluster.storage_pools[type][pool],
     'mount_options',
@@ -42,22 +43,44 @@ local get_sc_config(type, pool) =
       for opt in std.objectFields(param_mount_options)
     ]
   );
-  com.makeMergeable(com.getValueOrDefault(
-    params.ceph_cluster.storage_pools[type][pool],
-    'storage_class_config',
-    {}
-  )) +
-  {
+
+  // get default storage class config for pool
+  local sc_config =
+    com.makeMergeable(std.get(
+      params.ceph_cluster.storage_pools[type][pool],
+      'storage_class_config',
+      {}
+    ));
+
+  // get extra storage class config for additional storageclasses for pool
+  local sc_extra_config =
+    if extra_class != null then
+      com.makeMergeable(std.get(
+        std.get(
+          params.ceph_cluster.storage_pools[type][pool],
+          'extra_storage_classes',
+          {}
+        ),
+        extra_class
+      ))
+    else
+      {};
+
+  // all storage classes for the pool use the default sc config + potential
+  // extra config. This allows users to only specify the differences to the
+  // default class for extra classes.
+  sc_config + sc_extra_config + {
     mountOptions+: sc_mount_options,
   };
 
 // subpool used for CephFS
-local configure_sc(type, pool, subpool=null) =
+local configure_sc(type, pool, subpool=null, suffix=null) =
   local obj = load_storageclass(type);
-  local sc_config = get_sc_config(type, pool);
+  local sc_config = get_sc_config(type, pool, suffix);
   com.makeMergeable(obj) +
   sc.storageClass(
-    '%s-%s-%s' % [ type, pool, params.ceph_cluster.name ]
+    '%s-%s-%s' % [ type, pool, params.ceph_cluster.name ] +
+    if suffix != null then '-%s' % suffix else ''
   ) +
   sc_config +
   {

docs/modules/ROOT/pages/references/parameters.adoc

@@ -234,10 +234,12 @@ IMPORTANT: Storage class device sets added by `cephClusterSpec.storage.storageCl
 [horizontal]
 type:: dict
 keys:: Names of `CephBlockPool` resources
-values:: dicts with keys `config` and `mount_options`, and `storage_class_config`
+values:: dicts with keys `config` and `mount_options`, `storage_class_config`, and `extra_storage_classes`
 
 In this parameter `CephBlockPool` resources are configured.
-The component creates exactly one storageclass and volumesnapshotclass per block pool.
+The component creates one storageclass and volumesnapshotclass per block pool.
+
+NOTE: The component will create additional storage classes if field `extra_storage_classes` is provided.
 
 By default the parameter holds the following configuration:
 
@@ -263,7 +265,7 @@ This configuration results in
 * A storage class which creates PVs on this block pool, uses the `ext4` filesystem, supports volume expansion and configures PVs to be mounted with `-o discard`.
 * A `VolumeSnapshotClass` associated with the storage class
 
-See https://rook.io/docs/rook/v1.9/ceph-pool-crd.html[the Rook.io `CephBlockPool` CRD documentation] for all possible configurations in key `config`.
+See https://rook.github.io/docs/rook/v1.14/CRDs/Block-Storage/ceph-block-pool-crd/[the Rook.io `CephBlockPool` CRD documentation] for all possible configurations in key `config`.
 
 The values in key `storage_class_config` are merged into the `StorageClass` resource.
 
@@ -275,6 +277,35 @@ Providing a key with value `false` or `null` will result in the key not being ad
 See the filesystem documentation for the set of supported mount options.
 For example, see the list of supported mount options for `ext4` in the https://man7.org/linux/man-pages/man5/ext4.5.html#Mount_options_for_ext4[man page].
 
+==== `storage_pools.rbd.<poolname>.extra_storage_classes`
+
+[horizontal]
+type:: dictionary
+default:: `{}`
+
+.Example
+[source,yaml]
+----
+storagepool:
+  extra_storage_classes:
+    small-files:
+      parameters:
+        csi.storage.k8s.io/fstype: ext4 <1>
+        mkfsOptions: -m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1 -i1024 <2>
+----
+<1> When providing `mkfsOptions` it makes sense to explicitly specify the fstype, since mismatched fstype and mkfsOptions will likely result in errors.
+Note that provinding `fstype` isn't strictly necessary, if the same fstype is set in `storage_pools.rbd.<poolname>.storage_class_config`.
+<2> `-m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1` are the default `mkfsOptions` for ext4 in ceph-csi.
+
+This parameter allows users to configure additional storage classes for an RBD blockpool.
+The component will generate a storage class for each key-value pair in the parameter.
+
+The key will be used as a suffix for the storage class name.
+The resulting storage class name will have the form `rbd-<poolname>-<ceph cluster name>-<key>`.
+
+The component will use the configurations provided in `storage_pools.rbd.<poolname>.mount_options` and `storage_pools.rbd.<poolname>.storage_class_config` as the base for any storage classes defined here.
+The provided value will be merged into the base storage class definition.
+
 === `storage_pools.cephfs`
 
 [horizontal]

tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/00_namespaces.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/node-selector: ''
+  labels:
+    app.kubernetes.io/component: rook-ceph
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: syn-rook-ceph-operator
+    name: syn-rook-ceph-operator
+    openshift.io/cluster-monitoring: 'true'
+  name: syn-rook-ceph-operator
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/node-selector: ''
+  labels:
+    app.kubernetes.io/component: rook-ceph
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: syn-rook-ceph-cluster
+    name: syn-rook-ceph-cluster
+    openshift.io/cluster-monitoring: 'true'
+  name: syn-rook-ceph-cluster

tests/golden/rbd-extra-storageclass/rook-ceph/rook-ceph/01_aggregated_rbac.yaml

@@ -0,0 +1,115 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/component: rook-ceph
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: rook-ceph-view
+    name: rook-ceph-view
+    rbac.authorization.k8s.io/aggregate-to-admin: 'true'
+    rbac.authorization.k8s.io/aggregate-to-edit: 'true'
+    rbac.authorization.k8s.io/aggregate-to-view: 'true'
+  name: rook-ceph-view
+rules:
+  - apiGroups:
+      - ceph.rook.io
+    resources:
+      - cephblockpoolradosnamespaces
+      - cephblockpools
+      - cephbucketnotifications
+      - cephbuckettopics
+      - cephclients
+      - cephclusters
+      - cephfilesystemmirrors
+      - cephfilesystems
+      - cephfilesystemsubvolumegroups
+      - cephnfss
+      - cephobjectrealms
+      - cephobjectstores
+      - cephobjectstoreusers
+      - cephobjectzonegroups
+      - cephobjectzones
+      - cephrbdmirrors
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - objectbucket.io
+    resources:
+      - objectbucketclaims
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/component: rook-ceph
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: rook-ceph-edit
+    name: rook-ceph-edit
+    rbac.authorization.k8s.io/aggregate-to-admin: 'true'
+    rbac.authorization.k8s.io/aggregate-to-edit: 'true'
+  name: rook-ceph-edit
+rules:
+  - apiGroups:
+      - ceph.rook.io
+    resources:
+      - cephblockpoolradosnamespaces
+      - cephblockpools
+      - cephbucketnotifications
+      - cephbuckettopics
+      - cephclients
+      - cephclusters
+      - cephfilesystemmirrors
+      - cephfilesystems
+      - cephfilesystemsubvolumegroups
+      - cephnfss
+      - cephobjectrealms
+      - cephobjectstores
+      - cephobjectstoreusers
+      - cephobjectzonegroups
+      - cephobjectzones
+      - cephrbdmirrors
+    verbs:
+      - create
+      - delete
+      - deletecollection
+      - patch
+      - update
+  - apiGroups:
+      - objectbucket.io
+    resources:
+      - objectbucketclaims
+    verbs:
+      - create
+      - delete
+      - deletecollection
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations: {}
+  labels:
+    app.kubernetes.io/component: rook-ceph
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: rook-ceph-cluster-reader
+    name: rook-ceph-cluster-reader
+    rbac.authorization.k8s.io/aggregate-to-cluster-reader: 'true'
+  name: rook-ceph-cluster-reader
+rules:
+  - apiGroups:
+      - objectbucket.io
+    resources:
+      - objectbuckets
+    verbs:
+      - get
+      - list
+      - watch