diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index 675b657c6ab5..b5d135cdfdfd 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -68,6 +68,15 @@ spec: # and pass the label names below, for CSI to consume and advertise # its equivalent topology domain # - "--domainlabels=failure-domain/region,failure-domain/zone" + # + # Options to enable read affinity. + # If enabled Ceph CSI will fetch labels from kubernetes node and + # pass `read_from_replica=localize,crush_location=type:value` during + # rbd map command. refer: + # https://docs.ceph.com/en/latest/man/8/rbd/#kernel-rbd-krbd-options + # for more details. + # - "--enable-read-affinity=true" + # - "--crush-location-labels=topology.io/zone,topology.io/rack" env: - name: POD_IP valueFrom: diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index fce51b09e8f8..8236bd8164c7 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -48,6 +48,8 @@ make image-cephcsi | `--skipforceflatten` | `false` | skip image flattening on kernel < 5.2 which support mapping of rbd images which has the deep-flatten feature | | `--maxsnapshotsonimage` | `450` | Maximum number of snapshots allowed on rbd image without flattening | | `--setmetadata` | `false` | Set metadata on volume | +| `--enable-read-affinity` | `false` | enable read affinity | +| `--crush-location-labels`| _empty_ | Kubernetes node labels that determines the CRUSH location the node belongs to, separated by ',' | **Available volume parameters:** @@ -196,6 +198,28 @@ The Helm chart is located in `charts/ceph-csi-rbd`. [See the Helm chart readme for installation instructions.](../charts/ceph-csi-rbd/README.md) +## Read Affinity using crush locations for RBD volumes + +Ceph CSI supports mapping RBD volumes with krbd options +`"read_from_replica=localize,crush_location=type1:value1|type2:value2"` to +optimize for reads from the most local serving OSD (OSD assigned in the +same ceph CRUSH map location i.e.,region,zone,rack,etc). +Refer [krbd-options](https://docs.ceph.com/en/latest/man/8/rbd/#kernel-rbd-krbd-options) +for more details. + +This can be enabled by adding labels to Kubernetes nodes in the format `"topology.io/zone=east"` +and passing cmdline arguments `"--enable-read-affinity=true"` and +`"--crush-location-labels=topology.io/zone,topology.io/region"` +to Ceph CSI RBD daemonset pod "csi-rbdplugin" container, resulting in Ceph CSI adding +`"--options read_from_replica=localize,crush_location=zone:east"` krbd options during +rbd map operation. +If enabled, this option will be added to all RBD volumes mapped by Ceph CSI. +Well known labels can be found +[here](https://kubernetes.io/docs/reference/labels-annotations-taints/). + +>Note: Label values will have all its dots `"."` normalized with dashes `"-"` +in order for it to work with ceph CRUSH map. + ## Encryption for RBD volumes > Enabling encryption on volumes created without encryption is **not supported**