rbd: add capability to automatically enable read affinity

This commit makes use of crush location labels from node
labels to supply `crush_location` and `read_from_replica=localize`
options during rbd map cmd. Using these options, ceph
will be able to redirect reads to the closest OSD,
improving performance.

Signed-off-by: Rakshith R <rar@redhat.com>

internal/rbd/nodeserver.go

@@ -147,6 +147,7 @@ func populateRbdVol(
 	ctx context.Context,
 	req *csi.NodeStageVolumeRequest,
 	cr *util.Credentials,
+	crushLocationMap map[string]string,
 ) (*rbdVolume, error) {
 	var err error
 	volID := req.GetVolumeId()
@@ -246,6 +247,7 @@ func populateRbdVol(
 		rv.Mounter = rbdNbdMounter
 	}
 
+	rv.CrushLocationMap = crushLocationMap
 	err = getMapOptions(req, rv)
 	if err != nil {
 		return nil, err
@@ -318,7 +320,7 @@ func (ns *NodeServer) NodeStageVolume(
 	}
 
 	isStaticVol := parseBoolOption(ctx, req.GetVolumeContext(), staticVol, false)
-	rv, err := populateRbdVol(ctx, req, cr)
+	rv, err := populateRbdVol(ctx, req, cr, ns.Driver.CrushLocationMap)
 	if err != nil {
 		return nil, err
 	}

internal/rbd/rbd_attach.go

@@ -306,7 +306,7 @@ func getMapOptions(req *csi.NodeStageVolumeRequest, rv *rbdVolume) error {
 		return err
 	}
 	if rv.Mounter == rbdDefaultMounter {
-		rv.MapOptions = krbdMapOptions
+		rv.MapOptions = appendCruchLocationMapOptions(krbdMapOptions, rv.CrushLocationMap)
 		rv.UnmapOptions = krbdUnmapOptions
 	} else if rv.Mounter == rbdNbdMounter {
 		rv.MapOptions = nbdMapOptions
@@ -388,6 +388,35 @@ func appendKRbdDeviceTypeAndOptions(cmdArgs []string, userOptions string) []stri
 	return cmdArgs
 }
 
+// appendCruchLocationMapOptions parses crushLocationMap to the format
+// "--read_from_replica=localize,--crush_location=key1:value1|key3:value2"
+// and appends it to mapOptions.
+func appendCruchLocationMapOptions(mapOptions string, crushLocationMap map[string]string) string {
+	if crushLocationMap == nil {
+		return mapOptions
+	}
+
+	crushLocation := ""
+	for key, val := range crushLocationMap {
+		if crushLocation != "" {
+			crushLocation = crushLocation + fmt.Sprintf("|%s:%s", key, val)
+		} else {
+			crushLocation = fmt.Sprintf("%s:%s", key, val)
+		}
+	}
+	if crushLocation == "" {
+		// if no crush locations are found, return the mapOptions as it is.
+		return mapOptions
+	}
+
+	newMapOptions := fmt.Sprintf("read_from_replica=localize,crush_location=%s", crushLocation)
+	if mapOptions != "" {
+		newMapOptions = mapOptions + "," + newMapOptions
+	}
+
+	return newMapOptions
+}
+
 // appendRbdNbdCliOptions append mandatory options and convert list of useroptions
 // provided for rbd integrated cli to rbd-nbd cli format specific.
 func appendRbdNbdCliOptions(cmdArgs []string, userOptions, cookie string) []string {

internal/rbd/rbd_attach_test.go

@@ -19,6 +19,8 @@ package rbd
 import (
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestParseMapOptions(t *testing.T) {
@@ -104,3 +106,57 @@ func TestParseMapOptions(t *testing.T) {
 		})
 	}
 }
+
+func Test_appendCruchLocationMapOptions(t *testing.T) {
+	type args struct {
+		mapOptions       string
+		crushLocationMap map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "both empty mapOptions and crushLocationMap",
+			args: args{
+				mapOptions:       "",
+				crushLocationMap: map[string]string{},
+			},
+			want: "",
+		},
+		{
+			name: "empty mapOptions and filled crushLocationMap",
+			args: args{
+				mapOptions: "",
+				crushLocationMap: map[string]string{
+					"region": "west",
+				},
+			},
+			want: "--read_from_replica=localize,--crush_location=region:west",
+		},
+		{
+			name: "filled mapOptions and crushLocationMap",
+			args: args{
+				mapOptions: "--readonly=localize",
+				crushLocationMap: map[string]string{
+					"region": "west",
+				},
+			},
+			want: "--readonly=true,--read_from_replica=localize,--crush_location=region:west|region:east",
+		},
+		{
+			name: "filled mapOptions and empty crushLocationMap",
+			args: args{
+				mapOptions:       "--readonly=true",
+				crushLocationMap: map[string]string{},
+			},
+			want: "--readonly=true",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, appendCruchLocationMapOptions(tt.args.mapOptions, tt.args.crushLocationMap))
+		})
+	}
+}

internal/rbd/rbd_util.go

@@ -177,6 +177,8 @@ type rbdVolume struct {
 	RequestedVolSize   int64
 	DisableInUseChecks bool
 	readOnly           bool
+	// CrushLocationMap contains details required to enable read affinity.
+	CrushLocationMap map[string]string
 }
 
 // rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics.