deploy: support for read affinity options per cluster

Implemented the capability to include read affinity options
for individual clusters within the ceph-csi-config ConfigMap.
This allows users to configure the crush location for each
cluster separately. The read affinity options specified in
the ConfigMap will supersede those provided via command line arguments.

Signed-off-by: Praveen M <m.praveen@ibm.com>

charts/ceph-csi-cephfs/values.yaml

@@ -28,6 +28,11 @@ serviceAccounts:
 #     cephFS:
 #       subvolumeGroup: "csi"
 #       netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net"
+#     readAffinity:
+#       enabled: true
+#       crushLocationLabels:
+#         - topology.kubernetes.io/region
+#         - topology.kubernetes.io/zone
 csiConfig: []
 
 # Labels to apply to all resources

charts/ceph-csi-rbd/values.yaml

@@ -27,6 +27,11 @@ serviceAccounts:
 #       - "<MONValue2>"
 #     rbd:
 #       netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net"
+#     readAffinity:
+#       enabled: true
+#       crushLocationLabels:
+#         - topology.kubernetes.io/region
+#         - topology.kubernetes.io/zone
 csiConfig: []
 
 # Configuration details of clusterID,PoolID and FscID mapping

deploy/csi-config-map-sample.yaml

@@ -66,6 +66,15 @@ data:
         }
         "nfs": {
           "netNamespaceFilePath": "<kubeletRootPath>/plugins/nfs.csi.ceph.com/net",
+        },
+        "readAffinity": {
+          "enabled": "false",
+          "crushLocationLabels": [
+            "<Label1>",
+            "<Label2>"
+            ...
+            "<Label3>"
+          ]
         }
       }
     ]

internal/csi-common/driver.go

@@ -28,7 +28,7 @@ import (
 // CSIDriver stores driver information.
 type CSIDriver struct {
 	name    string
-	nodeID  string
+	NodeID  string
 	version string
 	// topology constraints that this nodeserver will advertise
 	topology     map[string]string
@@ -61,7 +61,7 @@ func NewCSIDriver(name, v, nodeID string) *CSIDriver {
 	driver := CSIDriver{
 		name:    name,
 		version: v,
-		nodeID:  nodeID,
+		NodeID:  nodeID,
 	}
 
 	return &driver

internal/csi-common/nodeserver-default.go

@@ -55,7 +55,7 @@ func (ns *DefaultNodeServer) NodeGetInfo(
 	}
 
 	return &csi.NodeGetInfoResponse{
-		NodeId:             ns.Driver.nodeID,
+		NodeId:             ns.Driver.NodeID,
 		AccessibleTopology: csiTopology,
 	}, nil
 }

internal/rbd/nodeserver.go

@@ -280,14 +280,23 @@ func (ns *NodeServer) populateRbdVol(
 
 // appendReadAffinityMapOptions appends readAffinityMapOptions to mapOptions
 // if mounter is rbdDefaultMounter and readAffinityMapOptions is not empty.
+// readAffinityMapOptions is read from csi config file if present else read
+// from the command line argument.
 func (ns NodeServer) appendReadAffinityMapOptions(rv *rbdVolume) {
+
+	var readAffinityMapOptions string
+	readAffinityMapOptions = ns.getReadAffinityMapOptionFromConfigMap(rv)
+	if readAffinityMapOptions == "" {
+		readAffinityMapOptions = ns.readAffinityMapOptions
+	}
+
 	switch {
-	case ns.readAffinityMapOptions == "" || rv.Mounter != rbdDefaultMounter:
+	case readAffinityMapOptions == "" || rv.Mounter != rbdDefaultMounter:
 		return
 	case rv.MapOptions != "":
-		rv.MapOptions += "," + ns.readAffinityMapOptions
+		rv.MapOptions += "," + readAffinityMapOptions
 	default:
-		rv.MapOptions = ns.readAffinityMapOptions
+		rv.MapOptions = readAffinityMapOptions
 	}
 }
 
@@ -1397,8 +1406,28 @@ func getDeviceSize(ctx context.Context, devicePath string) (uint64, error) {
 }
 
 func (ns *NodeServer) SetReadAffinityMapOptions(crushLocationMap map[string]string) {
+	ns.readAffinityMapOptions = constructReadAffinityMapOption(crushLocationMap)
+}
+
+func (ns *NodeServer) getReadAffinityMapOptionFromConfigMap(rv *rbdVolume) string {
+	crushLocationLabels, err  := util.GetReadAffinityOptions(util.CsiConfigFile, rv.ClusterID)
+	if err != nil {
+		log.FatalLogMsg(err.Error())
+	}
+
+	crushLocationMap, err := util.GetCrushLocationMap(crushLocationLabels, ns.Driver.NodeID)
+	if err != nil {
+		log.FatalLogMsg(err.Error())
+	}
+
+	readAffinityMapOptions := constructReadAffinityMapOption(crushLocationMap)
+
+	return readAffinityMapOptions
+}
+
+func constructReadAffinityMapOption(crushLocationMap map[string]string) string {
 	if len(crushLocationMap) == 0 {
-		return
+		return ""
 	}
 
 	var b strings.Builder
@@ -1412,5 +1441,6 @@ func (ns *NodeServer) SetReadAffinityMapOptions(crushLocationMap map[string]stri
 			b.WriteString(fmt.Sprintf("|%s:%s", key, val))
 		}
 	}
-	ns.readAffinityMapOptions = b.String()
+
+	return b.String()
 }

internal/util/csiconfig.go

@@ -64,6 +64,11 @@ type ClusterInfo struct {
 		// symlink filepath for the network namespace where we need to execute commands.
 		NetNamespaceFilePath string `json:"netNamespaceFilePath"`
 	} `json:"nfs"`
+	// Read affinity map options
+	ReadAffinity struct {
+		Enabled string `json:"enabled"`
+		CrushLocationLabels []string `json:"crushLocationLabels"`
+	}
 }
 
 // Expected JSON structure in the passed in config file is,
@@ -209,3 +214,24 @@ func GetNFSNetNamespaceFilePath(pathToConfig, clusterID string) (string, error)
 
 	return cluster.NFS.NetNamespaceFilePath, nil
 }
+
+// GetReadAffinityOptions returns the crushLocationLabels from csi config for the given clusterID
+// If `readAffinity.enabled` is set to true
+func GetReadAffinityOptions(pathToConfig, clusterId string) (string, error) {
+	cluster, err := readClusterInfo(pathToConfig, clusterId)
+	if err != nil {
+		return "", err
+	}
+
+	if cluster.ReadAffinity.Enabled == "false" {
+		return "", nil
+	}
+
+	if len(cluster.ReadAffinity.CrushLocationLabels) == 0 {
+		return "", fmt.Errorf("empty crush loction labels list for cluster ID (%s) in config", clusterId)
+	}
+
+	crushLocationLabels := strings.Join(cluster.ReadAffinity.CrushLocationLabels, ",")
+
+	return crushLocationLabels, nil
+}