diff --git a/.mergify.yml b/.mergify.yml index d74b5199e2c..544e2391fb2 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,7 +3,7 @@ pull_request_rules: - name: automatic merge conditions: - label!=DNM - - '#approved-reviews-by>=1' + - '#approved-reviews-by>=2' - 'status-success=continuous-integration/travis-ci/pr' actions: merge: diff --git a/Makefile b/Makefile index 11b1a67fc84..b0d756a1aac 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2018 The Kubernetes Authors. +# Copyright 2018 The Ceph-CSI Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 007b22e9f0a..ba091500639 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # Ceph CSI 1.0.0 +[![Go Report +Card](https://goreportcard.com/badge/github.com/ceph/ceph-csi)](https://goreportcard.com/report/github.com/ceph/ceph-csi) +[![Build +Status](https://travis-ci.org/ceph/ceph-csi.svg?branch=master)](https://travis-ci.org/ceph/ceph-csi) + [Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS. diff --git a/cmd/cephfs/main.go b/cmd/cephfs/main.go index a4d01a5bcd6..cbcb4a64108 100644 --- a/cmd/cephfs/main.go +++ b/cmd/cephfs/main.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/rbd/main.go b/cmd/rbd/main.go index bc4ddf53e9d..b5a4b18aba4 100644 --- a/cmd/rbd/main.go +++ b/cmd/rbd/main.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,13 +26,14 @@ import ( ) var ( - endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - driverName = flag.String("drivername", "rbd.csi.ceph.com", "name of the driver") - nodeID = flag.String("nodeid", "", "node id") - containerized = flag.Bool("containerized", true, "whether run as containerized") - metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]") - configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+ + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver") + nodeID = flag.String("nodeid", "", "node id") + containerized = flag.Bool("containerized", true, "whether run as containerized") + configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+ " cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets") + instanceID = flag.String("instanceid", "", "Unique ID distinguishing this instance of Ceph CSI among other"+ + " instances, when sharing Ceph clusters across CSI instances for provisioning") ) func init() { @@ -44,7 +45,6 @@ func init() { } func main() { - err := util.ValidateDriverName(*driverName) if err != nil { klog.Fatalln(err) @@ -52,13 +52,8 @@ func main() { //update plugin name rbd.PluginFolder = rbd.PluginFolder + *driverName - cp, err := util.CreatePersistanceStorage(rbd.PluginFolder, *metadataStorage, *driverName) - if err != nil { - os.Exit(1) - } - driver := rbd.NewDriver() - driver.Run(*driverName, *nodeID, *endpoint, *configRoot, *containerized, cp) + driver.Run(*driverName, *nodeID, *endpoint, *configRoot, *instanceID, *containerized) os.Exit(0) } diff --git a/deploy/cephfs/helm/Chart.yaml b/deploy/cephfs/helm/Chart.yaml index f324e7f576a..293abe9bffc 100644 --- a/deploy/cephfs/helm/Chart.yaml +++ b/deploy/cephfs/helm/Chart.yaml @@ -4,7 +4,7 @@ appVersion: "1.0.0" description: "Container Storage Interface (CSI) driver, provisioner, and attacher for Ceph cephfs" name: ceph-csi-cephfs -version: 0.5.1 +version: 0.5.2 keywords: - ceph - cephfs diff --git a/deploy/cephfs/helm/templates/_helpers.tpl b/deploy/cephfs/helm/templates/_helpers.tpl index e604150ae3a..635cca67b55 100644 --- a/deploy/cephfs/helm/templates/_helpers.tpl +++ b/deploy/cephfs/helm/templates/_helpers.tpl @@ -24,24 +24,6 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ceph-csi-cephfs.attacher.fullname" -}} -{{- if .Values.attacher.fullnameOverride -}} -{{- .Values.attacher.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). @@ -85,17 +67,6 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{/* -Create the name of the service account to use -*/}} -{{- define "ceph-csi-cephfs.serviceAccountName.attacher" -}} -{{- if .Values.serviceAccounts.attacher.create -}} - {{ default (include "ceph-csi-cephfs.attacher.fullname" .) .Values.serviceAccounts.attacher.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.attacher.name }} -{{- end -}} -{{- end -}} - {{/* Create the name of the service account to use */}} diff --git a/deploy/cephfs/helm/templates/attacher-clusterrole.yaml b/deploy/cephfs/helm/templates/attacher-clusterrole.yaml deleted file mode 100644 index a6625650094..00000000000 --- a/deploy/cephfs/helm/templates/attacher-clusterrole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.rbac.create -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] -{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml b/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml deleted file mode 100644 index 832e23dec2d..00000000000 --- a/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.rbac.create -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - apiGroup: rbac.authorization.k8s.io -{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-service.yaml b/deploy/cephfs/helm/templates/attacher-service.yaml deleted file mode 100644 index 379830d530c..00000000000 --- a/deploy/cephfs/helm/templates/attacher-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - app: {{ include "ceph-csi-cephfs.name" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - ports: - - name: dummy - port: 12345 diff --git a/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml b/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml deleted file mode 100644 index dbb70ccc222..00000000000 --- a/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.attacher.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-statefulset.yaml b/deploy/cephfs/helm/templates/attacher-statefulset.yaml deleted file mode 100644 index 88514d0626d..00000000000 --- a/deploy/cephfs/helm/templates/attacher-statefulset.yaml +++ /dev/null @@ -1,60 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1beta1 -metadata: - name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - serviceName: {{ include "ceph-csi-cephfs.attacher.fullname" . }} - replicas: {{ .Values.attacher.replicas }} - selector: - matchLabels: - app: {{ include "ceph-csi-cephfs.name" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - spec: - serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} - containers: - - name: csi-cephfsplugin-attacher - image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" - imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: {{ .Values.socketDir }} - resources: -{{ toYaml .Values.attacher.resources | indent 12 }} - volumes: - - name: socket-dir - hostPath: - path: {{ .Values.socketDir }} - type: DirectoryOrCreate - {{- if .Values.attacher.affinity -}} - affinity: -{{ toYaml .Values.attacher.affinity . | indent 8 }} - {{- end -}} - {{- if .Values.attacher.nodeSelector -}} - nodeSelector: -{{ toYaml .Values.attacher.nodeSelector | indent 8 }} - {{- end -}} - {{- if .Values.attacher.tolerations -}} - tolerations: -{{ toYaml .Values.attacher.tolerations | indent 8 }} - {{- end -}} diff --git a/deploy/cephfs/helm/templates/csidriver-crd.yaml b/deploy/cephfs/helm/templates/csidriver-crd.yaml new file mode 100644 index 00000000000..4c5021a7224 --- /dev/null +++ b/deploy/cephfs/helm/templates/csidriver-crd.yaml @@ -0,0 +1,10 @@ +--- +{{ if not .Values.attacher.enabled }} +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: false + podInfoOnMount: false +{{ end }} diff --git a/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml b/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml index 07e35a98e5a..e4b6be4b3a7 100644 --- a/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml +++ b/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml @@ -28,10 +28,12 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: ["csi.storage.k8s.io"] resources: ["csinodeinfos"] verbs: ["get", "list", "watch"] + {{ if .Values.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + {{ end }} {{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml index fe4fc642849..2bef74612e1 100644 --- a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml +++ b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml @@ -41,6 +41,20 @@ spec: mountPath: {{ .Values.socketDir }} resources: {{ toYaml .Values.provisioner.resources | indent 12 }} + {{ if .Values.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + {{ end }} - name: csi-cephfsplugin securityContext: privileged: true @@ -79,7 +93,7 @@ spec: #FIXME this seems way too much. Why is it needed at all for this? - name: host-rootfs hostPath: - path: / + path: / {{- if .Values.provisioner.affinity -}} affinity: {{ toYaml .Values.provisioner.affinity . | indent 8 }} diff --git a/deploy/cephfs/helm/values.yaml b/deploy/cephfs/helm/values.yaml index b31c9733e70..cfc64fe5440 100644 --- a/deploy/cephfs/helm/values.yaml +++ b/deploy/cephfs/helm/values.yaml @@ -20,7 +20,7 @@ volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices driverName: cephfs.csi.ceph.com attacher: name: attacher - + enabled: true replicaCount: 1 image: diff --git a/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml b/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml deleted file mode 100644 index 3b16a8ea29b..00000000000 --- a/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cephfs-csi-attacher - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-attacher-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-attacher-role -subjects: - - kind: ServiceAccount - name: cephfs-csi-attacher - namespace: default -roleRef: - kind: ClusterRole - name: cephfs-external-attacher-runner - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml deleted file mode 100644 index 1cd97126ba7..00000000000 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -kind: Service -apiVersion: v1 -metadata: - name: csi-cephfsplugin-attacher - labels: - app: csi-cephfsplugin-attacher -spec: - selector: - app: csi-cephfsplugin-attacher - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1beta1 -metadata: - name: csi-cephfsplugin-attacher -spec: - serviceName: "csi-cephfsplugin-attacher" - replicas: 1 - template: - metadata: - labels: - app: csi-cephfsplugin-attacher - spec: - serviceAccount: cephfs-csi-attacher - containers: - - name: csi-cephfsplugin-attacher - image: quay.io/k8scsi/csi-attacher:v1.0.1 - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com - type: DirectoryOrCreate diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index af5962933ac..473b493b903 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -39,6 +39,18 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-cephfsplugin-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-cephfsplugin securityContext: privileged: true diff --git a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml index 80ef301a951..b8d3ad8da16 100644 --- a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml @@ -31,6 +31,9 @@ rules: - apiGroups: ["csi.storage.k8s.io"] resources: ["csinodeinfos"] verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] --- kind: ClusterRoleBinding diff --git a/deploy/rbd/helm/Chart.yaml b/deploy/rbd/helm/Chart.yaml index 30585b9d8b2..dd46766a3cd 100644 --- a/deploy/rbd/helm/Chart.yaml +++ b/deploy/rbd/helm/Chart.yaml @@ -4,7 +4,7 @@ appVersion: "1.0.0" description: "Container Storage Interface (CSI) driver, provisioner, snapshotter, and attacher for Ceph RBD" name: ceph-csi-rbd -version: 0.5.1 +version: 0.5.2 keywords: - ceph - rbd diff --git a/deploy/rbd/helm/templates/_helpers.tpl b/deploy/rbd/helm/templates/_helpers.tpl index 3a9750303eb..0a2613d63e0 100644 --- a/deploy/rbd/helm/templates/_helpers.tpl +++ b/deploy/rbd/helm/templates/_helpers.tpl @@ -24,24 +24,6 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{- end -}} -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ceph-csi-rbd.attacher.fullname" -}} -{{- if .Values.attacher.fullnameOverride -}} -{{- .Values.attacher.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). @@ -85,17 +67,6 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{/* -Create the name of the service account to use -*/}} -{{- define "ceph-csi-rbd.serviceAccountName.attacher" -}} -{{- if .Values.serviceAccounts.attacher.create -}} - {{ default (include "ceph-csi-rbd.attacher.fullname" .) .Values.serviceAccounts.attacher.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.attacher.name }} -{{- end -}} -{{- end -}} - {{/* Create the name of the service account to use */}} diff --git a/deploy/rbd/helm/templates/attacher-clusterrole.yaml b/deploy/rbd/helm/templates/attacher-clusterrole.yaml deleted file mode 100644 index 59507abc34e..00000000000 --- a/deploy/rbd/helm/templates/attacher-clusterrole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.rbac.create -}} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-rbd.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] -{{- end -}} diff --git a/deploy/rbd/helm/templates/attacher-clusterrolebinding.yaml b/deploy/rbd/helm/templates/attacher-clusterrolebinding.yaml deleted file mode 100644 index e573d554f32..00000000000 --- a/deploy/rbd/helm/templates/attacher-clusterrolebinding.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.rbac.create -}} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-rbd.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "ceph-csi-rbd.serviceAccountName.attacher" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "ceph-csi-rbd.attacher.fullname" . }} - apiGroup: rbac.authorization.k8s.io -{{- end -}} diff --git a/deploy/rbd/helm/templates/attacher-service.yaml b/deploy/rbd/helm/templates/attacher-service.yaml deleted file mode 100644 index 87160b17de9..00000000000 --- a/deploy/rbd/helm/templates/attacher-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ include "ceph-csi-rbd.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - app: {{ include "ceph-csi-rbd.name" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - ports: - - name: dummy - port: 12345 diff --git a/deploy/rbd/helm/templates/attacher-serviceaccount.yaml b/deploy/rbd/helm/templates/attacher-serviceaccount.yaml deleted file mode 100644 index 7817df9287e..00000000000 --- a/deploy/rbd/helm/templates/attacher-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.attacher.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ceph-csi-rbd.serviceAccountName.attacher" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -{{- end -}} diff --git a/deploy/rbd/helm/templates/attacher-statefulset.yaml b/deploy/rbd/helm/templates/attacher-statefulset.yaml deleted file mode 100644 index 78e9a02db93..00000000000 --- a/deploy/rbd/helm/templates/attacher-statefulset.yaml +++ /dev/null @@ -1,60 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1beta1 -metadata: - name: {{ include "ceph-csi-rbd.attacher.fullname" . }} - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - serviceName: {{ include "ceph-csi-rbd.attacher.fullname" . }} - replicas: {{ .Values.attacher.replicas }} - selector: - matchLabels: - app: {{ include "ceph-csi-rbd.name" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ include "ceph-csi-rbd.name" . }} - chart: {{ include "ceph-csi-rbd.chart" . }} - component: {{ .Values.attacher.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - spec: - serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.attacher" . }} - containers: - - name: csi-rbdplugin-attacher - image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" - imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} - volumeMounts: - - name: socket-dir - mountPath: {{ .Values.socketDir }} - resources: -{{ toYaml .Values.attacher.resources | indent 12 }} - volumes: - - name: socket-dir - hostPath: - path: {{ .Values.socketDir }} - type: DirectoryOrCreate - {{- if .Values.attacher.affinity -}} - affinity: -{{ toYaml .Values.attacher.affinity . | indent 8 }} - {{- end -}} - {{- if .Values.attacher.nodeSelector -}} - nodeSelector: -{{ toYaml .Values.attacher.nodeSelector | indent 8 }} - {{- end -}} - {{- if .Values.attacher.tolerations -}} - tolerations: -{{ toYaml .Values.attacher.tolerations | indent 8 }} - {{- end -}} diff --git a/deploy/rbd/helm/templates/csidriver-crd.yaml b/deploy/rbd/helm/templates/csidriver-crd.yaml new file mode 100644 index 00000000000..4c5021a7224 --- /dev/null +++ b/deploy/rbd/helm/templates/csidriver-crd.yaml @@ -0,0 +1,10 @@ +--- +{{ if not .Values.attacher.enabled }} +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: {{ .Values.driverName }} +spec: + attachRequired: false + podInfoOnMount: false +{{ end }} diff --git a/deploy/rbd/helm/templates/provisioner-clusterrole.yaml b/deploy/rbd/helm/templates/provisioner-clusterrole.yaml index d324e455b86..898011baf03 100644 --- a/deploy/rbd/helm/templates/provisioner-clusterrole.yaml +++ b/deploy/rbd/helm/templates/provisioner-clusterrole.yaml @@ -34,9 +34,11 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] + {{ if .Values.attacher.enabled }} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + {{ end }} - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] diff --git a/deploy/rbd/helm/templates/provisioner-statefulset.yaml b/deploy/rbd/helm/templates/provisioner-statefulset.yaml index 269cb0a44b7..6ebdfc3883b 100644 --- a/deploy/rbd/helm/templates/provisioner-statefulset.yaml +++ b/deploy/rbd/helm/templates/provisioner-statefulset.yaml @@ -58,6 +58,20 @@ spec: mountPath: {{ .Values.socketDir }} resources: {{ toYaml .Values.snapshotter.resources | indent 12 }} + {{ if .Values.attacher.enabled }} + - name: csi-attacher + image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + {{ end }} - name: csi-rbdplugin securityContext: privileged: true @@ -97,7 +111,7 @@ spec: #FIXME this seems way too much. Why is it needed at all for this? - name: host-rootfs hostPath: - path: / + path: / {{- if .Values.provisioner.affinity -}} affinity: {{ toYaml .Values.provisioner.affinity . | indent 8 }} diff --git a/deploy/rbd/helm/values.yaml b/deploy/rbd/helm/values.yaml index fdeb5d6d57b..9d14fa19ef8 100644 --- a/deploy/rbd/helm/values.yaml +++ b/deploy/rbd/helm/values.yaml @@ -21,7 +21,7 @@ driverName: rbd.csi.ceph.com attacher: name: attacher - + enabled: true replicaCount: 1 image: diff --git a/deploy/rbd/kubernetes/csi-attacher-rbac.yaml b/deploy/rbd/kubernetes/csi-attacher-rbac.yaml deleted file mode 100644 index e502da5c99a..00000000000 --- a/deploy/rbd/kubernetes/csi-attacher-rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rbd-csi-attacher - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-attacher-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-attacher-role -subjects: - - kind: ServiceAccount - name: rbd-csi-attacher - namespace: default -roleRef: - kind: ClusterRole - name: rbd-external-attacher-runner - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml index 75615b054c0..79cd200a0e1 100644 --- a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml @@ -43,6 +43,9 @@ rules: - apiGroups: ["csi.storage.k8s.io"] resources: ["csinodeinfos"] verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] --- kind: ClusterRoleBinding diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml deleted file mode 100644 index 81029b7338e..00000000000 --- a/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -kind: Service -apiVersion: v1 -metadata: - name: csi-rbdplugin-attacher - labels: - app: csi-rbdplugin-attacher -spec: - selector: - app: csi-rbdplugin-attacher - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1beta1 -metadata: - name: csi-rbdplugin-attacher -spec: - serviceName: "csi-rbdplugin-attacher" - replicas: 1 - template: - metadata: - labels: - app: csi-rbdplugin-attacher - spec: - serviceAccount: rbd-csi-attacher - containers: - - name: csi-rbdplugin-attacher - image: quay.io/k8scsi/csi-attacher:v1.0.1 - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: unix:///csi/csi-attacher.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/rbd.csi.ceph.com - type: DirectoryOrCreate diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index bd14fa363aa..33256b4d01f 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -54,6 +54,18 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi-provisioner.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: csi-rbdplugin securityContext: privileged: true @@ -66,7 +78,6 @@ spec: - "--v=5" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - - "--metadatastorage=k8s_configmap" - "--configroot=k8s_objects" env: - name: HOST_ROOTFS diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index 95bd27b7d2e..8ba17319527 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -56,7 +56,6 @@ spec: - "--v=5" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - - "--metadatastorage=k8s_configmap" - "--configroot=k8s_objects" env: - name: HOST_ROOTFS diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index 6266107e73c..65c4f91dfb6 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -90,7 +90,6 @@ YAML manifests are located in `deploy/cephfs/kubernetes`. **Deploy RBACs for sidecar containers and node plugins:** ```bash -kubectl create -f csi-attacher-rbac.yaml kubectl create -f csi-provisioner-rbac.yaml kubectl create -f csi-nodeplugin-rbac.yaml ``` @@ -102,12 +101,11 @@ the same permissions. **Deploy CSI sidecar containers:** ```bash -kubectl create -f csi-cephfsplugin-attacher.yaml kubectl create -f csi-cephfsplugin-provisioner.yaml ``` -Deploys stateful sets for external-attacher and external-provisioner -sidecar containers for CSI CephFS. +Deploys stateful set of provision which includes external-provisioner +,external-attacher for CSI CephFS. **Deploy CSI CephFS driver:** @@ -115,7 +113,7 @@ sidecar containers for CSI CephFS. kubectl create -f csi-cephfsplugin.yaml ``` -Deploys a daemon set with two containers: CSI driver-registrar and +Deploys a daemon set with two containers: CSI node-driver-registrar and the CSI CephFS driver. ## Verifying the deployment in Kubernetes @@ -125,14 +123,11 @@ After successfully completing the steps above, you should see output similar to ```bash $ kubectl get all NAME READY STATUS RESTARTS AGE -pod/csi-cephfsplugin-attacher-0 1/1 Running 0 26s -pod/csi-cephfsplugin-provisioner-0 1/1 Running 0 25s +pod/csi-cephfsplugin-provisioner-0 3/3 Running 0 25s pod/csi-cephfsplugin-rljcv 2/2 Running 0 24s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/csi-cephfsplugin-attacher ClusterIP 10.104.116.218 12345/TCP 27s service/csi-cephfsplugin-provisioner ClusterIP 10.101.78.75 12345/TCP 26s - ... ``` diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index 50bdc532518..347602a749c 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -93,7 +93,6 @@ YAML manifests are located in `deploy/rbd/kubernetes`. **Deploy RBACs for sidecar containers and node plugins:** ```bash -kubectl create -f csi-attacher-rbac.yaml kubectl create -f csi-provisioner-rbac.yaml kubectl create -f csi-nodeplugin-rbac.yaml ``` @@ -105,12 +104,11 @@ the same permissions. **Deploy CSI sidecar containers:** ```bash -kubectl create -f csi-rbdplugin-attacher.yaml kubectl create -f csi-rbdplugin-provisioner.yaml ``` -Deploys stateful sets for external-attacher and external-provisioner -sidecar containers for CSI RBD. +Deploys stateful set of provision which includes external-provisioner +,external-attacher,csi-snapshotter sidecar containers and CSI RBD plugin. **Deploy RBD CSI driver:** @@ -118,7 +116,8 @@ sidecar containers for CSI RBD. kubectl create -f csi-rbdplugin.yaml ``` -Deploys a daemon set with two containers: CSI driver-registrar and the CSI RBD driver. +Deploys a daemon set with two containers: CSI node-driver-registrar and the CSI +RBD driver. ## Verifying the deployment in Kubernetes @@ -127,14 +126,11 @@ After successfully completing the steps above, you should see output similar to ```bash $ kubectl get all NAME READY STATUS RESTARTS AGE -pod/csi-rbdplugin-attacher-0 1/1 Running 0 23s pod/csi-rbdplugin-fptqr 2/2 Running 0 21s -pod/csi-rbdplugin-provisioner-0 1/1 Running 0 22s +pod/csi-rbdplugin-provisioner-0 4/4 Running 0 22s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/csi-rbdplugin-attacher ClusterIP 10.109.15.54 12345/TCP 26s service/csi-rbdplugin-provisioner ClusterIP 10.104.2.130 12345/TCP 23s - ... ``` diff --git a/examples/cephfs/plugin-deploy.sh b/examples/cephfs/plugin-deploy.sh index 3a2c028deba..d678629b837 100755 --- a/examples/cephfs/plugin-deploy.sh +++ b/examples/cephfs/plugin-deploy.sh @@ -8,7 +8,7 @@ fi cd "$deployment_base" || exit 1 -objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin) +objects=(csi-provisioner-rbac csi-nodeplugin-rbac csi-cephfsplugin-provisioner csi-cephfsplugin) for obj in "${objects[@]}"; do kubectl create -f "./$obj.yaml" diff --git a/examples/cephfs/plugin-teardown.sh b/examples/cephfs/plugin-teardown.sh index fc710d7e73a..ebe68072be8 100755 --- a/examples/cephfs/plugin-teardown.sh +++ b/examples/cephfs/plugin-teardown.sh @@ -8,7 +8,7 @@ fi cd "$deployment_base" || exit 1 -objects=(csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac) +objects=(csi-cephfsplugin-provisioner csi-cephfsplugin csi-provisioner-rbac csi-nodeplugin-rbac) for obj in "${objects[@]}"; do kubectl delete -f "./$obj.yaml" diff --git a/examples/rbd/plugin-deploy.sh b/examples/rbd/plugin-deploy.sh index f638c47da73..57398ee7837 100755 --- a/examples/rbd/plugin-deploy.sh +++ b/examples/rbd/plugin-deploy.sh @@ -8,7 +8,7 @@ fi cd "$deployment_base" || exit 1 -objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin) +objects=(csi-provisioner-rbac csi-nodeplugin-rbac csi-rbdplugin-provisioner csi-rbdplugin) for obj in "${objects[@]}"; do kubectl create -f "./$obj.yaml" diff --git a/examples/rbd/plugin-teardown.sh b/examples/rbd/plugin-teardown.sh index 5fa3b664988..2ee04be1ca3 100755 --- a/examples/rbd/plugin-teardown.sh +++ b/examples/rbd/plugin-teardown.sh @@ -8,7 +8,7 @@ fi cd "$deployment_base" || exit 1 -objects=(csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac) +objects=(csi-rbdplugin-provisioner csi-rbdplugin csi-provisioner-rbac csi-nodeplugin-rbac) for obj in "${objects[@]}"; do kubectl delete -f "./$obj.yaml" diff --git a/pkg/cephfs/cephuser.go b/pkg/cephfs/cephuser.go index 113933019db..aecc552f0b1 100644 --- a/pkg/cephfs/cephuser.go +++ b/pkg/cephfs/cephuser.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package cephfs import ( "fmt" + + "github.com/ceph/ceph-csi/pkg/util" ) const ( @@ -72,7 +74,7 @@ func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID "-m", volOptions.Monitors, "-n", adminID, "--key="+adminCr.key, - "-c", cephConfigPath, + "-c", util.CephConfigPath, "-f", "json", "auth", "get", userID, ) @@ -85,7 +87,7 @@ func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volum "-m", volOptions.Monitors, "-n", adminID, "--key="+adminCr.key, - "-c", cephConfigPath, + "-c", util.CephConfigPath, "-f", "json", "auth", "get-or-create", userID, // User capabilities @@ -102,7 +104,7 @@ func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volum "-m", volOptions.Monitors, "-n", adminID, "--key="+adminCr.key, - "-c", cephConfigPath, + "-c", util.CephConfigPath, "auth", "rm", userID, ) } diff --git a/pkg/cephfs/controllerserver.go b/pkg/cephfs/controllerserver.go index c2e596cbf26..99261c9075b 100644 --- a/pkg/cephfs/controllerserver.go +++ b/pkg/cephfs/controllerserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/credentials.go b/pkg/cephfs/credentials.go index 0cf866ce521..1273c08b0b2 100644 --- a/pkg/cephfs/credentials.go +++ b/pkg/cephfs/credentials.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/driver.go b/pkg/cephfs/driver.go index 7c5dfc61e03..8017b02b355 100644 --- a/pkg/cephfs/driver.go +++ b/pkg/cephfs/driver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -101,7 +101,7 @@ func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir klog.Infof("cephfs: setting default volume mounter to %s", DefaultVolumeMounter) - if err := writeCephConfig(); err != nil { + if err := util.WriteCephConfig(); err != nil { klog.Fatalf("failed to write ceph configuration file: %v", err) } diff --git a/pkg/cephfs/identityserver.go b/pkg/cephfs/identityserver.go index c8d5edc5230..6a929c29d18 100644 --- a/pkg/cephfs/identityserver.go +++ b/pkg/cephfs/identityserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index 273471e0588..d2351edc2c1 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/util.go b/pkg/cephfs/util.go index 19928c0fa64..8b4426958d4 100644 --- a/pkg/cephfs/util.go +++ b/pkg/cephfs/util.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/volume.go b/pkg/cephfs/volume.go index a4d6f197d68..9d7f1692837 100644 --- a/pkg/cephfs/volume.go +++ b/pkg/cephfs/volume.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/cephfs/volumemounter.go b/pkg/cephfs/volumemounter.go index 6a91afafcb2..f76b2f096ce 100644 --- a/pkg/cephfs/volumemounter.go +++ b/pkg/cephfs/volumemounter.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import ( "strconv" "sync" + "github.com/ceph/ceph-csi/pkg/util" "k8s.io/klog" ) @@ -115,7 +116,7 @@ func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) er args := [...]string{ mountPoint, "-m", volOptions.Monitors, - "-c", cephConfigPath, + "-c", util.CephConfigPath, "-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key, "-r", volOptions.RootPath, "-o", "nonempty", diff --git a/pkg/cephfs/volumeoptions.go b/pkg/cephfs/volumeoptions.go index eb092accfc2..49809fd1fc1 100644 --- a/pkg/cephfs/volumeoptions.go +++ b/pkg/cephfs/volumeoptions.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 27c407fdf8c..9902cfc4bab 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,19 +18,12 @@ package rbd import ( "fmt" - "os/exec" - "sort" - "strconv" - "syscall" csicommon "github.com/ceph/ceph-csi/pkg/csi-common" "github.com/ceph/ceph-csi/pkg/util" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/protobuf/ptypes" - "github.com/golang/protobuf/ptypes/timestamp" "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" - "github.com/pborman/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc/codes" @@ -46,33 +39,6 @@ const ( // controller server spec. type ControllerServer struct { *csicommon.DefaultControllerServer - MetadataStore util.CachePersister -} - -var ( - rbdVolumes = map[string]*rbdVolume{} - rbdSnapshots = map[string]*rbdSnapshot{} -) - -// LoadExDataFromMetadataStore loads the rbd volume and snapshot -// info from metadata store -func (cs *ControllerServer) LoadExDataFromMetadataStore() error { - vol := &rbdVolume{} - // nolint - cs.MetadataStore.ForAll("csi-rbd-vol-", vol, func(identifier string) error { - rbdVolumes[identifier] = vol - return nil - }) - - snap := &rbdSnapshot{} - // nolint - cs.MetadataStore.ForAll("csi-rbd-(.*)-snap-", snap, func(identifier string) error { - rbdSnapshots[identifier] = snap - return nil - }) - - klog.Infof("Loaded %d volumes and %d snapshots from metadata store", len(rbdVolumes), len(rbdSnapshots)) - return nil } func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) error { @@ -87,10 +53,17 @@ func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) erro if req.VolumeCapabilities == nil { return status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty") } + options := req.GetParameters() + if value, ok := options["clusterID"]; !ok || len(value) == 0 { + return status.Error(codes.InvalidArgument, "Missing or empty cluster ID to provision volume from") + } + if value, ok := options["pool"]; !ok || len(value) == 0 { + return status.Error(codes.InvalidArgument, "Missing or empty pool name to provision volume from") + } return nil } -func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { +func (cs *ControllerServer) genVolFromCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { // TODO (sbezverk) Last check for not exceeding total storage capacity isMultiNode := false @@ -111,43 +84,33 @@ func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { } // if it's NOT SINGLE_NODE_WRITER and it's BLOCK we'll set the parameter to ignore the in-use checks - rbdVol, err := getRBDVolumeOptions(req.GetParameters(), (isMultiNode && isBlock)) + rbdVol, err := genVolFromVolumeOptions(req.GetParameters(), (isMultiNode && isBlock)) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - // Generating Volume Name and Volume ID, as according to CSI spec they MUST be different - volName := req.GetName() - uniqueID := uuid.NewUUID().String() - rbdVol.VolName = volName - volumeID := "csi-rbd-vol-" + uniqueID - rbdVol.VolID = volumeID + rbdVol.RequestName = req.GetName() + // Volume Size - Default is 1 GiB volSizeBytes := int64(oneGB) if req.GetCapacityRange() != nil { volSizeBytes = req.GetCapacityRange().GetRequiredBytes() } - rbdVol.VolSize = util.RoundUpToMiB(volSizeBytes) + // always round up the request size in bytes to the nearest MiB + rbdVol.VolSize = util.MiB * util.RoundUpToMiB(volSizeBytes) + // NOTE: rbdVol does not contain VolID and RbdImageName populated, everything + // else is populated post create request parsing return rbdVol, nil } -func storeVolumeMetadata(vol *rbdVolume, cp util.CachePersister) error { - if err := cp.Create(vol.VolID, vol); err != nil { - klog.Errorf("failed to store metadata for volume %s: %v", vol.VolID, err) - return err - } - - return nil -} - -// CreateVolume creates the volume in backend and store the volume metadata +// CreateVolume creates the volume in backend func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - if err := cs.validateVolumeReq(req); err != nil { return nil, err } + volumeNameMutex.LockKey(req.GetName()) defer func() { if err := volumeNameMutex.UnlockKey(req.GetName()); err != nil { @@ -155,50 +118,46 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } }() - // Need to check for already existing volume name, and if found - // check for the requested capacity and already allocated capacity - if exVol, err := getRBDVolumeByName(req.GetName()); err == nil { - // Since err is nil, it means the volume with the same name already exists - // need to check if the size of existing volume is the same as in new - // request - if exVol.VolSize >= req.GetCapacityRange().GetRequiredBytes() { - // existing volume is compatible with new request and should be reused. - - if err = storeVolumeMetadata(exVol, cs.MetadataStore); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } + rbdVol, err := cs.genVolFromCreateRequest(req) + if err != nil { + return nil, err + } - // TODO (sbezverk) Do I need to make sure that RBD volume still exists? - return &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: exVol.VolID, - CapacityBytes: exVol.VolSize, - VolumeContext: req.GetParameters(), - }, - }, nil + found, err := checkVolExists(rbdVol, req.GetSecrets()) + if err != nil { + if _, ok := err.(ErrVolNameConflict); ok { + return nil, status.Error(codes.AlreadyExists, err.Error()) } - return nil, status.Errorf(codes.AlreadyExists, "Volume with the same name: %s but with different size already exist", req.GetName()) + + return nil, status.Error(codes.Internal, err.Error()) + } + if found { + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: rbdVol.VolID, + CapacityBytes: rbdVol.VolSize, + VolumeContext: req.GetParameters(), + }, + }, nil } - rbdVol, err := parseVolCreateRequest(req) + err = reserveVol(rbdVol, req.GetSecrets()) if err != nil { - return nil, err + return nil, status.Error(codes.Internal, err.Error()) } + defer func() { + if err != nil { + errDefer := unreserveVol(rbdVol, req.GetSecrets()) + if errDefer != nil { + klog.Warningf("failed undoing reservation of volume: %s", req.GetName()) + } + } + }() - // Check if there is already RBD image with requested name - err = cs.checkRBDStatus(rbdVol, req, int(rbdVol.VolSize)) + err = cs.createBackingImage(rbdVol, req, util.RoundUpToMiB(rbdVol.VolSize)) if err != nil { return nil, err } - // store volume size in bytes (snapshot and check existing volume needs volume - // size in bytes) - rbdVol.VolSize = rbdVol.VolSize * util.MiB - - rbdVolumes[rbdVol.VolID] = rbdVol - - if err = storeVolumeMetadata(rbdVol, cs.MetadataStore); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ @@ -209,27 +168,24 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }, nil } -func (cs *ControllerServer) checkRBDStatus(rbdVol *rbdVolume, req *csi.CreateVolumeRequest, volSizeMiB int) error { +func (cs *ControllerServer) createBackingImage(rbdVol *rbdVolume, req *csi.CreateVolumeRequest, volSizeMiB int64) error { var err error - // Check if there is already RBD image with requested name - //nolint - found, _, _ := rbdStatus(rbdVol, rbdVol.UserID, req.GetSecrets()) - if !found { - // if VolumeContentSource is not nil, this request is for snapshot - if req.VolumeContentSource != nil { - if err = cs.checkSnapshot(req, rbdVol); err != nil { - return err - } - } else { - err = createRBDImage(rbdVol, volSizeMiB, rbdVol.AdminID, req.GetSecrets()) - if err != nil { - klog.Warningf("failed to create volume: %v", err) - return status.Error(codes.Internal, err.Error()) - } - klog.V(4).Infof("create volume %s", rbdVol.VolName) + // if VolumeContentSource is not nil, this request is for snapshot + if req.VolumeContentSource != nil { + if err = cs.checkSnapshot(req, rbdVol); err != nil { + return err + } + } else { + err = createImage(rbdVol, volSizeMiB, rbdVol.AdminID, req.GetSecrets()) + if err != nil { + klog.Warningf("failed to create volume: %v", err) + return status.Error(codes.Internal, err.Error()) } + + klog.V(4).Infof("created image %s", rbdVol.RbdImageName) } + return nil } func (cs *ControllerServer) checkSnapshot(req *csi.CreateVolumeRequest, rbdVol *rbdVolume) error { @@ -244,15 +200,18 @@ func (cs *ControllerServer) checkSnapshot(req *csi.CreateVolumeRequest, rbdVol * } rbdSnap := &rbdSnapshot{} - if err := cs.MetadataStore.Get(snapshotID, rbdSnap); err != nil { - return status.Error(codes.NotFound, err.Error()) + if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil { + if _, ok := err.(ErrSnapNotFound); !ok { + return status.Error(codes.Internal, err.Error()) + } + return status.Error(codes.InvalidArgument, "Missing requested Snapshot ID") } err := restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminID, req.GetSecrets()) if err != nil { return status.Error(codes.Internal, err.Error()) } - klog.V(4).Infof("create volume %s from snapshot %s", req.GetName(), rbdSnap.SnapName) + klog.V(4).Infof("create volume %s from snapshot %s", req.GetName(), rbdSnap.RbdSnapName) return nil } @@ -266,7 +225,6 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol // For now the image get unconditionally deleted, but here retention policy can be checked volumeID := req.GetVolumeId() volumeIDMutex.LockKey(volumeID) - defer func() { if err := volumeIDMutex.UnlockKey(volumeID); err != nil { klog.Warningf("failed to unlock mutex volume:%s %v", volumeID, err) @@ -274,79 +232,51 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol }() rbdVol := &rbdVolume{} - if err := cs.MetadataStore.Get(volumeID, rbdVol); err != nil { - if err, ok := err.(*util.CacheEntryNotFound); ok { - klog.V(3).Infof("metadata for volume %s not found, assuming the volume to be already deleted (%v)", volumeID, err) + if err := genVolFromVolID(rbdVol, volumeID, req.GetSecrets()); err != nil { + // If image key is missing, there can be no unreserve as request name is unknown, and also + // means there is no image, hence return success + if _, ok := err.(util.ErrKeyNotFound); !ok { return &csi.DeleteVolumeResponse{}, nil } - return nil, err - } - - volName := rbdVol.VolName - // Deleting rbd image - klog.V(4).Infof("deleting volume %s", volName) - if err := deleteRBDImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil { - // TODO: can we detect "already deleted" situations here and proceed? - klog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err) - return nil, status.Error(codes.Internal, err.Error()) - } - - if err := cs.MetadataStore.Delete(volumeID); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - delete(rbdVolumes, volumeID) - return &csi.DeleteVolumeResponse{}, nil -} - -// ListVolumes returns a list of volumes stored in memory -func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { - var startToken int - if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_VOLUMES); err != nil { - klog.Warningf("invalid list volume req: %v", req) - return nil, err - } - - //validate starting token if present - if len(req.GetStartingToken()) > 0 { - i, parseErr := strconv.ParseUint(req.StartingToken, 10, 32) - if parseErr != nil { - return nil, status.Errorf(codes.Aborted, "invalid starting token %s", parseErr.Error()) + if _, ok := err.(ErrImageNotFound); !ok { + return nil, status.Error(codes.Internal, err.Error()) } - //check starting Token is greater than list of rbd volumes - if len(rbdVolumes) < int(i) { - return nil, status.Errorf(codes.Aborted, "invalid starting token %s", parseErr.Error()) - } - startToken = int(i) - } - var entries []*csi.ListVolumesResponse_Entry + // If image is missing, then there was a key to point to it, it means we need to cleanup + // the keys and return success + volumeNameMutex.LockKey(rbdVol.RequestName) + defer func() { + if err := volumeNameMutex.UnlockKey(rbdVol.RequestName); err != nil { + klog.Warningf("failed to unlock mutex volume:%s %v", rbdVol.RequestName, err) + } + }() - keys := make([]string, 0) - for k := range rbdVolumes { - keys = append(keys, k) + if err := unreserveVol(rbdVol, req.GetSecrets()); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &csi.DeleteVolumeResponse{}, nil } - sort.Strings(keys) - for index, k := range keys { - if index < startToken { - continue + // lock out parallel create requests against the same volume name as we + // cleanup the image and associated omaps for the same + volumeNameMutex.LockKey(rbdVol.RequestName) + defer func() { + if err := volumeNameMutex.UnlockKey(rbdVol.RequestName); err != nil { + klog.Warningf("failed to unlock mutex volume:%s %v", rbdVol.RequestName, err) } - entries = append(entries, &csi.ListVolumesResponse_Entry{ - Volume: &csi.Volume{ - VolumeId: rbdVolumes[k].VolID, - CapacityBytes: rbdVolumes[k].VolSize, - VolumeContext: extractStoredVolOpt(rbdVolumes[k]), - }, - }) - } + }() - resp := &csi.ListVolumesResponse{ - Entries: entries, + // Deleting rbd image + klog.V(4).Infof("deleting image %s", rbdVol.RbdImageName) + if err := deleteImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil { + // TODO: can we detect "already deleted" situations here and proceed? + klog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", + rbdVol.Pool, rbdVol.RbdImageName, err) + return nil, status.Error(codes.Internal, err.Error()) } - return resp, nil + return &csi.DeleteVolumeResponse{}, nil } // ValidateVolumeCapabilities checks whether the volume capabilities requested @@ -364,114 +294,96 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req }, nil } -// ControllerUnpublishVolume returns success response -func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - return &csi.ControllerUnpublishVolumeResponse{}, nil -} - -// ControllerPublishVolume returns success response -func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - return &csi.ControllerPublishVolumeResponse{}, nil -} - // CreateSnapshot creates the snapshot in backend and stores metadata // in store // nolint: gocyclo func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + var rbdVol *rbdVolume if err := cs.validateSnapshotReq(req); err != nil { return nil, err } - snapshotNameMutex.LockKey(req.GetName()) + snapshotNameMutex.LockKey(req.GetName()) defer func() { if err := snapshotNameMutex.UnlockKey(req.GetName()); err != nil { klog.Warningf("failed to unlock mutex snapshot:%s %v", req.GetName(), err) } }() - // Need to check for already existing snapshot name, and if found - // check for the requested source volume id and already allocated source volume id - if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil { - if req.SourceVolumeId == exSnap.SourceVolumeID { - if err = storeSnapshotMetadata(exSnap, cs.MetadataStore); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - return &csi.CreateSnapshotResponse{ - Snapshot: &csi.Snapshot{ - SizeBytes: exSnap.SizeBytes, - SnapshotId: exSnap.SnapID, - SourceVolumeId: exSnap.SourceVolumeID, - CreationTime: ×tamp.Timestamp{ - Seconds: exSnap.CreatedAt, - }, - ReadyToUse: true, - }, - }, nil + // Fetch source volume information + rbdVol = new(rbdVolume) + err := genVolFromVolID(rbdVol, req.GetSourceVolumeId(), req.GetSecrets()) + if err != nil { + if _, ok := err.(ErrImageNotFound); ok { + return nil, status.Errorf(codes.NotFound, "Source Volume ID %s cannot found", req.GetSourceVolumeId()) } - return nil, status.Errorf(codes.AlreadyExists, "Snapshot with the same name: %s but with different source volume id already exist", req.GetName()) + return nil, status.Errorf(codes.Internal, err.Error()) + } + + // Check if source volume was created with required image features for snaps + if !hasSnapshotFeature(rbdVol.ImageFeatures) { + return nil, status.Errorf(codes.InvalidArgument, "volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId()) } - rbdSnap, err := getRBDSnapshotOptions(req.GetParameters()) + // Create snap volume + rbdSnap, err := genSnapFromOptions(req.GetParameters()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } + rbdSnap.RbdImageName = rbdVol.RbdImageName + rbdSnap.SizeBytes = rbdVol.VolSize + rbdSnap.SourceVolumeID = req.GetSourceVolumeId() + rbdSnap.RequestName = req.GetName() - // Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different - snapName := req.GetName() - uniqueID := uuid.NewUUID().String() - rbdVolume, err := getRBDVolumeByID(req.GetSourceVolumeId()) + // Need to check for already existing snapshot name, and if found + // check for the requested source volume id and already allocated source volume id + found, err := checkSnapExists(rbdSnap, req.GetSecrets()) if err != nil { - return nil, status.Errorf(codes.NotFound, "Source Volume ID %s cannot found", req.GetSourceVolumeId()) + return nil, status.Errorf(codes.Internal, err.Error()) } - if !hasSnapshotFeature(rbdVolume.ImageFeatures) { - return nil, status.Errorf(codes.InvalidArgument, "volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId()) + if found { + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SizeBytes: rbdSnap.SizeBytes, + SnapshotId: rbdSnap.SnapID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreationTime: rbdSnap.CreatedAt, + ReadyToUse: true, + }, + }, nil } - rbdSnap.VolName = rbdVolume.VolName - rbdSnap.SnapName = snapName - snapshotID := "csi-rbd-" + rbdVolume.VolName + "-snap-" + uniqueID - rbdSnap.SnapID = snapshotID - rbdSnap.SourceVolumeID = req.GetSourceVolumeId() - rbdSnap.SizeBytes = rbdVolume.VolSize - - err = cs.doSnapshot(rbdSnap, req.GetSecrets()) - // if we already have the snapshot, return the snapshot + err = reserveSnap(rbdSnap, req.GetSecrets()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + defer func() { + if err != nil { + klog.Warningf("snapshot failed, undoing reservation of snap name: %s", req.GetName()) + errDefer := unreserveSnap(rbdSnap, req.GetSecrets()) + if errDefer != nil { + klog.Warningf("failed undoing reservation of snapshot: %s", req.GetName()) + } + } + }() - rbdSnap.CreatedAt = ptypes.TimestampNow().GetSeconds() - - rbdSnapshots[snapshotID] = rbdSnap - - if err = storeSnapshotMetadata(rbdSnap, cs.MetadataStore); err != nil { + err = cs.doSnapshot(rbdSnap, req.GetSecrets()) + if err != nil { return nil, status.Error(codes.Internal, err.Error()) } return &csi.CreateSnapshotResponse{ Snapshot: &csi.Snapshot{ SizeBytes: rbdSnap.SizeBytes, - SnapshotId: snapshotID, + SnapshotId: rbdSnap.SnapID, SourceVolumeId: req.GetSourceVolumeId(), - CreationTime: ×tamp.Timestamp{ - Seconds: rbdSnap.CreatedAt, - }, - ReadyToUse: true, + CreationTime: rbdSnap.CreatedAt, + ReadyToUse: true, }, }, nil } -func storeSnapshotMetadata(rbdSnap *rbdSnapshot, cp util.CachePersister) error { - if err := cp.Create(rbdSnap.SnapID, rbdSnap); err != nil { - klog.Errorf("failed to store metadata for snapshot %s: %v", rbdSnap.SnapID, err) - return err - } - - return nil -} - func (cs *ControllerServer) validateSnapshotReq(req *csi.CreateSnapshotRequest) error { if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { klog.Warningf("invalid create snapshot req: %v", protosanitizer.StripSecrets(req)) @@ -485,41 +397,54 @@ func (cs *ControllerServer) validateSnapshotReq(req *csi.CreateSnapshotRequest) if len(req.SourceVolumeId) == 0 { return status.Error(codes.InvalidArgument, "Source Volume ID cannot be empty") } + options := req.GetParameters() + if value, ok := options["clusterID"]; !ok || len(value) == 0 { + return status.Error(codes.InvalidArgument, "Missing or empty cluster ID to snapshot volume from") + } + if value, ok := options["pool"]; !ok || len(value) == 0 { + return status.Error(codes.InvalidArgument, "Missing or empty pool name to snapshot volume from") + } return nil } -func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]string) error { - err := createSnapshot(rbdSnap, rbdSnap.AdminID, secret) - // if we already have the snapshot, return the snapshot +func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]string) (err error) { + err = createSnapshot(rbdSnap, rbdSnap.AdminID, secret) + // If snap creation fails, even due to snapname already used, fail, next attempt will get a new + // uuid for use as the snap name if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { - if status.ExitStatus() == int(syscall.EEXIST) { - klog.Warningf("Snapshot with the same name: %s, we return this.", rbdSnap.SnapName) - } else { - klog.Warningf("failed to create snapshot: %v", err) - return err - } - } else { - klog.Warningf("failed to create snapshot: %v", err) - return err + klog.Warningf("failed to create snapshot: %v", err) + return status.Error(codes.Internal, err.Error()) + } + defer func() { + if err != nil { + errDefer := deleteSnapshot(rbdSnap, rbdSnap.AdminID, secret) + if errDefer != nil { + err = fmt.Errorf("snapshot created but failed to delete snapshot due to"+ + " other failures: %v", err) } - } else { - klog.Warningf("failed to create snapshot: %v", err) - return err + err = status.Error(codes.Internal, err.Error()) } - } else { - klog.V(4).Infof("create snapshot %s", rbdSnap.SnapName) - err = protectSnapshot(rbdSnap, rbdSnap.AdminID, secret) + }() + err = protectSnapshot(rbdSnap, rbdSnap.AdminID, secret) + if err != nil { + return errors.New("snapshot created but failed to protect snapshot") + } + defer func() { if err != nil { - err = deleteSnapshot(rbdSnap, rbdSnap.AdminID, secret) - if err != nil { - return fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) + errDefer := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, secret) + if errDefer != nil { + err = fmt.Errorf("snapshot created but failed to unprotect and delete snapshot"+ + " due to other failures: %v", err) } - return errors.New("snapshot is created but failed to protect snapshot") } + }() + + err = getSnapshotMetadata(rbdSnap, rbdSnap.AdminID, secret) + if err != nil { + return err } + return nil } @@ -535,8 +460,8 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS if len(snapshotID) == 0 { return nil, status.Error(codes.InvalidArgument, "Snapshot ID cannot be empty") } - snapshotIDMutex.LockKey(snapshotID) + snapshotIDMutex.LockKey(snapshotID) defer func() { if err := snapshotIDMutex.UnlockKey(snapshotID); err != nil { klog.Warningf("failed to unlock mutex snapshot:%s %v", snapshotID, err) @@ -544,95 +469,41 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS }() rbdSnap := &rbdSnapshot{} - if err := cs.MetadataStore.Get(snapshotID, rbdSnap); err != nil { - if err, ok := err.(*util.CacheEntryNotFound); ok { - klog.V(3).Infof("metadata for snapshot %s not found, assuming the snapshot to be already deleted (%v)", snapshotID, err) - return &csi.DeleteSnapshotResponse{}, nil + if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil { + // Consider missing snap as already deleted, and proceed to remove the omap values + if _, ok := err.(ErrSnapNotFound); !ok { + return nil, status.Error(codes.Internal, err.Error()) } - - return nil, err + if err := unreserveSnap(rbdSnap, req.GetSecrets()); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &csi.DeleteSnapshotResponse{}, nil } + // lock out parallel create requests against the same snap name as we + // cleanup the image and associated omaps for the same + snapshotNameMutex.LockKey(rbdSnap.RequestName) + defer func() { + if err := snapshotNameMutex.UnlockKey(rbdSnap.RequestName); err != nil { + klog.Warningf("failed to unlock mutex snapshot:%s %v", rbdSnap.RequestName, err) + } + }() + // Unprotect snapshot err := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets()) if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err) + return nil, status.Errorf(codes.FailedPrecondition, + "failed to unprotect snapshot: %s/%s with error: %v", + rbdSnap.Pool, rbdSnap.RbdSnapName, err) } // Deleting snapshot - klog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) + klog.V(4).Infof("deleting Snaphot %s", rbdSnap.RbdSnapName) if err := deleteSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets()); err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err) + return nil, status.Errorf(codes.FailedPrecondition, + "failed to delete snapshot: %s/%s with error: %v", + rbdSnap.Pool, rbdSnap.RbdSnapName, err) } - if err := cs.MetadataStore.Delete(snapshotID); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - delete(rbdSnapshots, snapshotID) - return &csi.DeleteSnapshotResponse{}, nil } - -// ListSnapshots lists the snapshots in the store -func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { - if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS); err != nil { - klog.Warningf("invalid list snapshot req: %v", req) - return nil, err - } - - sourceVolumeID := req.GetSourceVolumeId() - - // TODO (sngchlko) list with token - // TODO (#94) protect concurrent access to global data structures - - // list only a specific snapshot which has snapshot ID - if snapshotID := req.GetSnapshotId(); len(snapshotID) != 0 { - if rbdSnap, ok := rbdSnapshots[snapshotID]; ok { - // if source volume ID also set, check source volume id on the cache. - if len(sourceVolumeID) != 0 && rbdSnap.SourceVolumeID != sourceVolumeID { - return nil, status.Errorf(codes.Unknown, "Requested Source Volume ID %s is different from %s", sourceVolumeID, rbdSnap.SourceVolumeID) - } - return &csi.ListSnapshotsResponse{ - Entries: []*csi.ListSnapshotsResponse_Entry{ - { - Snapshot: &csi.Snapshot{ - SizeBytes: rbdSnap.SizeBytes, - SnapshotId: rbdSnap.SnapID, - SourceVolumeId: rbdSnap.SourceVolumeID, - CreationTime: ×tamp.Timestamp{ - Seconds: rbdSnap.CreatedAt, - }, - ReadyToUse: true, - }, - }, - }, - }, nil - } - return nil, status.Errorf(codes.NotFound, "Snapshot ID %s cannot found", snapshotID) - - } - - entries := []*csi.ListSnapshotsResponse_Entry{} - for _, rbdSnap := range rbdSnapshots { - // if source volume ID also set, check source volume id on the cache. - if len(sourceVolumeID) != 0 && rbdSnap.SourceVolumeID != sourceVolumeID { - continue - } - entries = append(entries, &csi.ListSnapshotsResponse_Entry{ - Snapshot: &csi.Snapshot{ - SizeBytes: rbdSnap.SizeBytes, - SnapshotId: rbdSnap.SnapID, - SourceVolumeId: rbdSnap.SourceVolumeID, - CreationTime: ×tamp.Timestamp{ - Seconds: rbdSnap.CreatedAt, - }, - ReadyToUse: true, - }, - }) - } - - return &csi.ListSnapshotsResponse{ - Entries: entries, - }, nil -} diff --git a/pkg/rbd/identityserver.go b/pkg/rbd/identityserver.go index 891759f7cc4..da5b9e1dac4 100644 --- a/pkg/rbd/identityserver.go +++ b/pkg/rbd/identityserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index f31ec4cc5c2..22fa897bb46 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( "strings" "github.com/ceph/ceph-csi/pkg/csi-common" + "github.com/ceph/ceph-csi/pkg/util" "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" @@ -82,11 +83,11 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } } - volOptions, err := getRBDVolumeOptions(req.GetVolumeContext(), disableInUseChecks) + volOptions, err := genVolFromVolumeOptions(req.GetVolumeContext(), disableInUseChecks) if err != nil { return nil, err } - volOptions.VolName = volName + volOptions.RbdImageName = volName // Mapping RBD image devicePath, err := attachRBDImage(volOptions, volOptions.UserID, req.GetSecrets()) if err != nil { @@ -103,22 +104,15 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } func (ns *NodeServer) getVolumeName(req *csi.NodePublishVolumeRequest) (string, error) { - var volName string - isBlock := req.GetVolumeCapability().GetBlock() != nil - targetPath := req.GetTargetPath() - if isBlock { - // Get volName from targetPath - s := strings.Split(targetPath, "/") - volName = s[len(s)-1] - } else { - // Get volName from targetPath - if !strings.HasSuffix(targetPath, "/mount") { - return "", fmt.Errorf("rbd: malformed the value of target path: %s", targetPath) - } - s := strings.Split(strings.TrimSuffix(targetPath, "/mount"), "/") - volName = s[len(s)-1] + var vi util.CsiIdentifier + + err := vi.DecomposeCsiID(req.GetVolumeId()) + if err != nil { + klog.V(4).Infof("error decoding volume ID (%s) (%s)", err, req.GetVolumeId()) + return "", err } - return volName, nil + + return rbdImgNamePrefix + vi.ObjectUUID, nil } func (ns *NodeServer) mountVolume(req *csi.NodePublishVolumeRequest, devicePath string) error { diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index bcd5c6c90f4..2355cc7f4b7 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,10 +27,98 @@ import ( "k8s.io/utils/exec" ) -// PluginFolder defines the location of rbdplugin +/* +RADOS omaps usage: + +This note details how we preserve idempotent nature of create requests and retain the relationship +between orchestrator (CO) generated Names and plugin generated names for images and snapshots + +The implementation uses Ceph RADOS omaps to preserve the relationship between request name and +generated image (or snapshot) name. There are 4 types of omaps in use, +- A "csi.volumes.[csi-id]" (or "csi.volumes"+.+CsiInstanceID), we call this the csiVolsDirectory + - stores keys named using the CO generated names for volume requests + - keys are named "csi.volume."+[CO generated VolName] + - Key value contains the RBD image uuid that is created or will be created, for the CO provided + name + +- A "csi.snaps.[csi-id]" (or "csi.snaps"+.+CsiInstanceID), we refer to this as the csiSnapsDirectory + - stores keys named using the CO generated names for snapshot requests + - keys are named "csi.snap."+[CO generated SnapName] + - Key value contains the RBD snapshot uuid that is created or will be created, for the CO + provided name + +- A per image omap named "rbd.csi.volume."+[RBD image uuid], we refer to this as the rbdImageOMap + - stores a single key named "csi.volname", that has the value of the CO generated VolName that + this image refers to + +- A per snapshot omap named "rbd.csi.snap."+[RBD snapshot uuid], we refer to this as the snapOMap + - stores a key named "csi.snapname", that has the value of the CO generated SnapName that this + snapshot refers to + - also stores another key named "csi.source", that has the value of the image name that is the + source of the snapshot + +Creation of omaps: +When a volume create request is received (or a snapshot create, the snapshot is not detailed in this + comment further as the process is similar), +- The csiVolsDirectory is consulted to find if there is already a key with the CO VolName, and if present, +it is used to read its references to reach the RBD image that backs this VolName, to check if the +RBD image can satisfy the requirements for the request + - If during the process of checking the same, it is found that some linking information is stale + or missing, the corresponding keys upto the key in the csiVolsDirectory is cleaned up, to start afresh +- If the key with the CO VolName is not found, or was cleaned up, the request is treated as a +new create request, and an rbdImageOMap is created first with a generated uuid, this ensures that we +do not use a uuid that is already in use +- Next, a key with the VolName is created in the csiVolsDirectory, and its value is updated to store the +generated uuid +- This is followed by updating the rbdImageOMap with the VolName in the rbdImageCSIVolNameKey +- Finally, the image is created (or promoted from a snapshot, if content source was provided) using +the uuid and a corresponding image name prefix (rbdImgNamePrefix or rbdSnapNamePrefix) + +The entire operation is locked based on VolName hash, to ensure there is only ever a single entity +modifying the related omaps for a given VolName. + +This ensures idempotent nature of creates, as the same CO generated VolName would attempt to use +the same RBD image name to serve the request, as the relations are saved in the respective omaps. + +Deletion of omaps: +Delete requests would not contain the VolName, hence deletion uses the volume ID, which is encoded +with the image name in it, to find the image and the rbdImageOMap. The rbdImageOMap is read to get +the VolName that this image points to. This VolName can be further used to read and delete the key +from the csiVolsDirectory. + +As we trace back and find the VolName, we also take a hash based lock on the VolName before +proceeding with deleting the image and the related omap entries, to ensure there is only ever a +single entity modifying the related omaps for a given VolName. +*/ + const ( - rbdDefaultAdminID = "admin" - rbdDefaultUserID = rbdDefaultAdminID + // volIDVersion is the version number of volume ID encoding scheme + volIDVersion uint16 = 1 + rbdDefaultAdminID = "admin" + rbdDefaultUserID = rbdDefaultAdminID + + // CSI volume-name keyname prefix, for key in csiVolsDirectory, suffix is the CSI passed volume name + csiVolNameKeyPrefix = "csi.volume." + // Per RBD image object map name prefix, suffix is the RBD image uuid + rbdImageOMapPrefix = "csi.volume." + // CSI volume-name key in per RBD image object map, containing CSI volume-name for which the + // image was created + rbdImageCSIVolNameKey = "csi.volname" + // RBD image name prefix, suffix is a uuid generated per image + rbdImgNamePrefix = "csi-vol-" + + //CSI snap-name keyname prefix, for key in csiSnapsDirectory, suffix is the CSI passed snapshot name + csiSnapNameKeyPrefix = "csi.snap." + // Per RBD snapshot object map name prefix, suffix is the RBD image uuid + rbdSnapOMapPrefix = "csi.snap." + // CSI snap-name key in per RBD snapshot object map, containing CSI snapshot-name for which the + // snapshot was created + rbdSnapCSISnapNameKey = "csi.snapname" + // source image name key in per RBD snapshot object map, containing RBD source image name for + // which the snapshot was created + rbdSnapSourceImageKey = "csi.source" + // RBD snapshot name prefix, suffix is a uuid generated per snapshot + rbdSnapNamePrefix = "csi-snap-" ) // PluginFolder defines the location of ceph plugin @@ -49,6 +137,14 @@ var ( version = "1.0.0" // confStore is the global config store confStore *util.ConfigStore + // CsiInstanceID is the instance ID that is unique to an instance of CSI, used when sharing + // ceph clusters across CSI instances, to differentiate omap names per CSI instance + CsiInstanceID = "default" + // csiVolsDirectory is the name of the CSI volumes object map that contains CSI volume-name + // based keys + csiVolsDirectory = "csi.volumes" + // csiSnapsDirectory is the name of the CSI snapshots object map that contains CSI snapshot-name based keys + csiSnapsDirectory = "csi.snaps" ) // NewDriver returns new rbd driver @@ -64,10 +160,9 @@ func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer { } // NewControllerServer initialize a controller server for rbd CSI driver -func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer { +func NewControllerServer(d *csicommon.CSIDriver) *ControllerServer { return &ControllerServer{ DefaultControllerServer: csicommon.NewDefaultControllerServer(d), - MetadataStore: cachePersister, } } @@ -89,15 +184,28 @@ func NewNodeServer(d *csicommon.CSIDriver, containerized bool) (*NodeServer, err // Run start a non-blocking grpc controller,node and identityserver for // rbd CSI driver which can serve multiple parallel requests -func (r *Driver) Run(driverName, nodeID, endpoint, configRoot string, containerized bool, cachePersister util.CachePersister) { +func (r *Driver) Run(driverName, nodeID, endpoint, configRoot, instanceID string, containerized bool) { var err error + klog.Infof("Driver: %v version: %v", driverName, version) // Initialize config store confStore, err = util.NewConfigStore(configRoot) if err != nil { - klog.Fatalln("Failed to initialize config store.") + klog.Fatalln("Failed to initialize config store") + } + + // Create ceph.conf for use with CLI commands + if err = util.WriteCephConfig(); err != nil { + klog.Fatalf("failed to write ceph configuration file") + } + + // Use passed in instance ID, if provided for omap suffix naming + if instanceID != "" { + CsiInstanceID = instanceID } + csiVolsDirectory = csiVolsDirectory + "." + CsiInstanceID + csiSnapsDirectory = csiSnapsDirectory + "." + CsiInstanceID // Initialize default library driver r.cd = csicommon.NewCSIDriver(driverName, version, nodeID) @@ -106,10 +214,7 @@ func (r *Driver) Run(driverName, nodeID, endpoint, configRoot string, containeri } r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, - csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, }) @@ -128,11 +233,7 @@ func (r *Driver) Run(driverName, nodeID, endpoint, configRoot string, containeri klog.Fatalf("failed to start node server, err %v\n", err) } - r.cs = NewControllerServer(r.cd, cachePersister) - - if err = r.cs.LoadExDataFromMetadataStore(); err != nil { - klog.Fatalf("failed to load metadata from store, err %v\n", err) - } + r.cs = NewControllerServer(r.cd) s := csicommon.NewNonBlockingGRPCServer() s.Start(endpoint, r.ids, r.cs, r.ns) diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index d56ac74f3b5..8bc8285b9fa 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -227,7 +227,7 @@ func checkRbdNbdTools() bool { func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string]string) (string, error) { var err error - image := volOptions.VolName + image := volOptions.RbdImageName imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image) useNBD := false @@ -271,7 +271,7 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string } func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (string, error) { - image := volOpt.VolName + image := volOpt.RbdImageName imagePath := fmt.Sprintf("%s/%s", volOpt.Pool, image) mon, err := getMon(volOpt, creds) @@ -280,7 +280,7 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri } klog.V(5).Infof("rbd: map mon %s", mon) - key, err := getRBDKey(volOpt.ClusterID, userID, creds) + key, err := getKey(volOpt.ClusterID, userID, creds) if err != nil { return "", err } @@ -306,7 +306,7 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri } func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, credentials map[string]string) error { - image := volOptions.VolName + image := volOptions.RbdImageName imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image) err := wait.ExponentialBackoff(backoff, func() (bool, error) { diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 14fcc0ebfde..af1b8b3587f 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,11 +17,17 @@ limitations under the License. package rbd import ( + "encoding/json" "fmt" "os/exec" "strings" "time" + "github.com/ceph/ceph-csi/pkg/util" + + "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/timestamp" + "github.com/pborman/uuid" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" @@ -39,35 +45,47 @@ const ( rbdDefaultMounter = "rbd" ) +// rbdVolume represents a CSI volume and its RBD image specifics type rbdVolume struct { - VolName string `json:"volName"` - VolID string `json:"volID"` - Monitors string `json:"monitors"` - MonValueFromSecret string `json:"monValueFromSecret"` - Pool string `json:"pool"` - ImageFormat string `json:"imageFormat"` - ImageFeatures string `json:"imageFeatures"` - VolSize int64 `json:"volSize"` - AdminID string `json:"adminId"` - UserID string `json:"userId"` - Mounter string `json:"mounter"` - DisableInUseChecks bool `json:"disableInUseChecks"` - ClusterID string `json:"clusterId"` + // RbdImageName is the name of the RBD image backing this rbdVolume + // VolID is the volume ID that is exchanged with CSI drivers, identifying this rbdVol + // RequestName is the CSI generated volume name for the rbdVolume + RbdImageName string + VolID string + Monitors string + MonValueFromSecret string + Pool string + ImageFormat string + ImageFeatures string + VolSize int64 + AdminID string + UserID string + Mounter string + DisableInUseChecks bool + ClusterID string + RequestName string } +// rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics type rbdSnapshot struct { - SourceVolumeID string `json:"sourceVolumeID"` - VolName string `json:"volName"` - SnapName string `json:"snapName"` - SnapID string `json:"sanpID"` - Monitors string `json:"monitors"` - MonValueFromSecret string `json:"monValueFromSecret"` - Pool string `json:"pool"` - CreatedAt int64 `json:"createdAt"` - SizeBytes int64 `json:"sizeBytes"` - AdminID string `json:"adminId"` - UserID string `json:"userId"` - ClusterID string `json:"clusterId"` + // SourceVolumeID is the volume ID of RbdImageName, that is exchanged with CSI drivers + // RbdImageName is the name of the RBD image, that is this rbdSnapshot's source image + // RbdSnapName is the name of the RBD snapshot backing this rbdSnapshot + // SnapID is the snapshot ID that is exchanged with CSI drivers, identifying this rbdSnapshot + // RequestName is the CSI generated snapshot name for the rbdSnapshot + SourceVolumeID string + RbdImageName string + RbdSnapName string + SnapID string + Monitors string + MonValueFromSecret string + Pool string + CreatedAt *timestamp.Timestamp + SizeBytes int64 + AdminID string + UserID string + ClusterID string + RequestName string } var ( @@ -87,7 +105,7 @@ var ( supportedFeatures = sets.NewString("layering") ) -func getRBDKey(clusterid, id string, credentials map[string]string) (string, error) { +func getKey(clusterid, id string, credentials map[string]string) (string, error) { var ( ok bool err error @@ -126,8 +144,8 @@ func getMon(pOpts *rbdVolume, credentials map[string]string) (string, error) { return mon, nil } -// CreateImage creates a new ceph image with provision and volume options. -func createRBDImage(pOpts *rbdVolume, volSz int, adminID string, credentials map[string]string) error { +// createImage creates a new ceph image with provision and volume options. +func createImage(pOpts *rbdVolume, volSz int64, adminID string, credentials map[string]string) error { var output []byte mon, err := getMon(pOpts, credentials) @@ -135,10 +153,10 @@ func createRBDImage(pOpts *rbdVolume, volSz int, adminID string, credentials map return err } - image := pOpts.VolName + image := pOpts.RbdImageName volSzMiB := fmt.Sprintf("%dM", volSz) - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -166,10 +184,10 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) ( var output string var cmd []byte - image := pOpts.VolName + image := pOpts.RbdImageName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - key, err := getRBDKey(pOpts.ClusterID, userID, credentials) + key, err := getKey(pOpts.ClusterID, userID, credentials) if err != nil { return false, "", err } @@ -205,10 +223,479 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) ( return false, output, nil } -// DeleteImage deletes a ceph image with provision and volume options. -func deleteRBDImage(pOpts *rbdVolume, adminID string, credentials map[string]string) error { +/* +checkSnapExists, and its counterpart checkVolExists, function as checks to determine if passed +in rbdSnapshot or rbdVolume exists on the backend. + +**NOTE:** These functions manipulate the rados omaps that hold information regarding +volume names as requested by the CSI drivers. Hence, these need to be invoked only when the +respective CSI driver generated snapshot or volume name based locks are held, as otherwise racy +access to these omaps may end up leaving them in an inconsistent state. + +These functions need enough information about cluster and pool (ie, Monitors, Pool, IDs filled in) +to operate. They further require that the RequestName element of the structure have a valid value +to operate on and determine if the said RequestName already exists on the backend. + +These functions populate the snapshot or the image name, its attributes and the CSI snapshot/volume +ID for the same when succesful. + +These functions also cleanup omap reservations that are stale. I.e when omap entries exist and +backing images or snapshots are missing, or one of the omaps exist and the next is missing. This is +because, the order of omap creation and deletion are inverse of each other, and protected by the +request name lock, and hence any stale omaps are leftovers from incomplete transactions and are +hence safe to garbage collect. +*/ +func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (found bool, err error) { + // Structure members used to determine if provided rbdSnapshot exists, are checked here, to + // provide an easy way to check when this function can be reused + if rbdSnap.RequestName == "" || rbdSnap.Monitors == "" || rbdSnap.AdminID == "" || + rbdSnap.Pool == "" || rbdSnap.RbdImageName == "" || rbdSnap.ClusterID == "" { + return false, errors.New("missing information in rbdSnapshot to check for existence") + } + + key, err := getKey(rbdSnap.ClusterID, rbdSnap.AdminID, credentials) + if err != nil { + return false, err + } + + // check if request name is already part of the snaps omap + snapUUID, err := util.GetOMapValue(rbdSnap.Monitors, rbdSnap.AdminID, + key, rbdSnap.Pool, csiSnapsDirectory, csiSnapNameKeyPrefix+rbdSnap.RequestName) + if err != nil { + return false, nil + } + + rbdSnap.RbdSnapName = rbdSnapNamePrefix + snapUUID + + // TODO: use listomapvals to dump all keys instead of reading them one-by-one + // check if the snapshot image omap is present + savedSnapName, err := util.GetOMapValue(rbdSnap.Monitors, rbdSnap.AdminID, + key, rbdSnap.Pool, rbdSnapOMapPrefix+snapUUID, rbdSnapCSISnapNameKey) + if err != nil { + if _, ok := err.(util.ErrKeyNotFound); ok { + err = unreserveSnap(rbdSnap, credentials) + } + return false, err + } + + // check if snapshot image omap points back to the request name + if savedSnapName != rbdSnap.RequestName { + // NOTE: This should never be possible, hence no cleanup, but log error + // and return, as cleanup may need to occur manually! + return false, fmt.Errorf("internal state inconsistent, omap snap"+ + " names disagree, request name (%s) snap name (%s) image omap"+ + " snap name (%s)", rbdSnap.RequestName, rbdSnap.RbdSnapName, savedSnapName) + } + + // check if the snapshot source image omap is present + savedVolName, err := util.GetOMapValue(rbdSnap.Monitors, rbdSnap.AdminID, + key, rbdSnap.Pool, rbdSnapOMapPrefix+snapUUID, rbdSnapSourceImageKey) + if err != nil { + if _, ok := err.(util.ErrKeyNotFound); ok { + err = unreserveSnap(rbdSnap, credentials) + } + return false, err + } + + // check if snapshot source image omap points back to the source volume passed in + if savedVolName != rbdSnap.RbdImageName { + // NOTE: This should never be possible, hence no cleanup, but log error + // and return, as cleanup may need to occur manually! + return false, fmt.Errorf("internal state inconsistent, omap volume"+ + " names disagree, request name (%s) image name (%s) image omap"+ + " volume name (%s)", rbdSnap.RequestName, rbdSnap.RbdImageName, savedVolName) + } + + // Fetch on-disk image attributes + err = updateSnapWithImageInfo(rbdSnap, credentials) + if err != nil { + if _, ok := err.(ErrSnapNotFound); ok { + err = unreserveSnap(rbdSnap, credentials) + return false, err + } + + return false, err + } + + // found a snapshot already available, process and return its information + poolID, err := util.GetPoolID(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool) + if err != nil { + klog.Errorf("internal error fetching pool ID (%s)", err) + return false, err + } + + vi := util.CsiIdentifier{ + PoolID: poolID, + EncodingVersion: volIDVersion, + ClusterID: rbdSnap.ClusterID, + ObjectUUID: snapUUID, + } + rbdSnap.SnapID, err = vi.ComposeCsiID() + if err != nil { + return false, err + } + + klog.V(4).Infof("Found existing snap (%s) with snap name (%s) for request (%s)", + rbdSnap.SnapID, rbdSnap.RbdSnapName, rbdSnap.RequestName) + + return true, nil +} + +// ErrVolNameConflict is generated when a requested CSI volume name already exists on RBD but with +// different properties, and hence is in conflict with the passed in CSI volume name +type ErrVolNameConflict struct { + requestName string + err error +} + +func (e ErrVolNameConflict) Error() string { + return e.err.Error() +} + +/* +Check comment on checkSnapExists, to understand how this function behaves + +**NOTE:** These functions manipulate the rados omaps that hold information regarding +volume names as requested by the CSI drivers. Hence, these need to be invoked only when the +respective CSI snapshot or volume name based locks are held, as otherwise racy access to these +omaps may end up leaving the ompas in an inconsistent state. +*/ +func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (found bool, err error) { + var vi util.CsiIdentifier + + // Structure members used to determine if provided rbdVolume exists, are checked here, to + // provide an easy way to check when this function can be reused + if rbdVol.RequestName == "" || rbdVol.Monitors == "" || rbdVol.AdminID == "" || + rbdVol.Pool == "" || rbdVol.ClusterID == "" || rbdVol.VolSize == 0 { + return false, errors.New("missing information in rbdVolume to check for existence") + } + + key, err := getKey(rbdVol.ClusterID, rbdVol.AdminID, credentials) + if err != nil { + return false, err + } + + // check if request name is already part of the volumes omap + imageUUID, err := util.GetOMapValue(rbdVol.Monitors, rbdVol.AdminID, + key, rbdVol.Pool, csiVolsDirectory, csiVolNameKeyPrefix+rbdVol.RequestName) + if err != nil { + return false, nil + } + + rbdVol.RbdImageName = rbdImgNamePrefix + imageUUID + + // check if the image omap is present + savedVolName, err := util.GetOMapValue(rbdVol.Monitors, rbdVol.AdminID, + key, rbdVol.Pool, rbdImageOMapPrefix+imageUUID, rbdImageCSIVolNameKey) + if err != nil { + if _, ok := err.(util.ErrKeyNotFound); ok { + err = unreserveVol(rbdVol, credentials) + } + return false, err + } + + // check if image omap points back to the request name + if savedVolName != rbdVol.RequestName { + // NOTE: This should never be possible, hence no cleanup, but log error + // and return, as cleanup may need to occur manually! + return false, fmt.Errorf("internal state inconsistent, omap volume"+ + " names disagree, request name (%s) image name (%s) image omap"+ + " volume name (%s)", rbdVol.RequestName, rbdVol.RbdImageName, savedVolName) + } + + // NOTE: Return volsize should be on-disk volsize, not request vol size, so + // save it for size checks before fetching image data + requestSize := rbdVol.VolSize + // Fetch on-disk image attributes and compare against request + err = updateVolWithImageInfo(rbdVol, credentials) + if err != nil { + if _, ok := err.(ErrImageNotFound); ok { + err = unreserveVol(rbdVol, credentials) + return false, err + } + + return false, err + } + + // size checks + if rbdVol.VolSize < requestSize { + err = fmt.Errorf("image with the same name (%s) but with different size already exists", + rbdVol.RbdImageName) + return false, ErrVolNameConflict{rbdVol.RbdImageName, err} + } + // TODO: We should also ensure image features and format is the same + + // found a volume already available, process and return it! + poolID, err := util.GetPoolID(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool) + if err != nil { + klog.Errorf("internal error fetching pool ID (%s)", err) + return false, err + } + + vi = util.CsiIdentifier{ + PoolID: poolID, + EncodingVersion: volIDVersion, + ClusterID: rbdVol.ClusterID, + ObjectUUID: imageUUID, + } + rbdVol.VolID, err = vi.ComposeCsiID() + if err != nil { + return false, err + } + + klog.V(4).Infof("Found existng volume (%s) with image name (%s) for request (%s)", + rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName) + + return true, nil +} + +/* +unreserveSnap and unreserveVol remove omaps associated with the snapshot and the image name, +and also remove the corresponding request name key in the snaps or volumes omaps respectively. + +This is performed within the request name lock, to ensure that requests with the same name do not +manipulate the omap entries concurrently. +*/ +func unreserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error { + key, err := getKey(rbdSnap.ClusterID, rbdSnap.AdminID, credentials) + if err != nil { + return err + } + + // delete snap image omap (first, inverse of create order) + snapUUID := strings.TrimPrefix(rbdSnap.RbdSnapName, rbdSnapNamePrefix) + err = util.RemoveObject(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, rbdSnapOMapPrefix+snapUUID) + if err != nil { + if _, ok := err.(util.ErrObjectNotFound); !ok { + klog.V(4).Infof("failed removing oMap %s (%s)", rbdSnapOMapPrefix+snapUUID, err) + return err + } + } + + // delete the request name omap key (last, inverse of create order) + err = util.RemoveOMapKey(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + csiSnapsDirectory, csiSnapNameKeyPrefix+rbdSnap.RequestName) + if err != nil { + klog.V(4).Infof("failed removing oMap key %s (%s)", csiSnapNameKeyPrefix+rbdSnap.RequestName, err) + return err + } + + return nil +} + +func unreserveVol(rbdVol *rbdVolume, credentials map[string]string) error { + key, err := getKey(rbdVol.ClusterID, rbdVol.AdminID, credentials) + if err != nil { + return err + } + + // delete image omap (first, inverse of create order) + imageUUID := strings.TrimPrefix(rbdVol.RbdImageName, rbdImgNamePrefix) + err = util.RemoveObject(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, rbdImageOMapPrefix+imageUUID) + if err != nil { + if _, ok := err.(util.ErrObjectNotFound); !ok { + klog.V(4).Infof("failed removing oMap %s (%s)", rbdImageOMapPrefix+imageUUID, err) + return err + } + } + + // delete the request name omap key (last, inverse of create order) + err = util.RemoveOMapKey(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + csiVolsDirectory, csiVolNameKeyPrefix+rbdVol.RequestName) + if err != nil { + klog.V(4).Infof("failed removing oMap key %s (%s)", csiVolNameKeyPrefix+rbdVol.RequestName, err) + return err + } + + return nil +} + +// reserveOMapName creates an omap with passed in oMapNameSuffix and a generated . +// It ensures generated omap name does not already exist and if conflicts are detected, a set +// number of retires with newer uuids are attempted before returning an error +func reserveOMapName(monitors, adminID, key, poolName, oMapNameSuffix string) (string, error) { + var iterUUID string + + maxAttempts := 5 + attempt := 1 + for attempt <= maxAttempts { + // generate a uuid for the image name + iterUUID = uuid.NewUUID().String() + + err := util.CreateObject(monitors, adminID, key, poolName, oMapNameSuffix+iterUUID) + if err != nil { + if _, ok := err.(util.ErrObjectExists); ok { + attempt++ + // try again with a different uuid, for maxAttempts tries + klog.V(4).Infof("uuid (%s) conflict detected, retrying (attempt %d of %d)", + iterUUID, attempt, maxAttempts) + continue + } + + return "", err + } + + break + } + + if attempt > maxAttempts { + return "", errors.New("uuid conflicts exceeds retry threshold") + } + + return iterUUID, nil +} + +/* +reserveSnap and reserveVol add respective entries to the volumes and snapshots omaps, post +generating a target snapshot or image name for use. Further, these functions create the snapshot or +image name omaps, to store back pointers to the CSI generated request names. + +This is performed within the request name lock, to ensure that requests with the same name do not +manipulate the omap entries concurrently. +*/ +func reserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error { + var vi util.CsiIdentifier + + key, err := getKey(rbdSnap.ClusterID, rbdSnap.AdminID, credentials) + if err != nil { + return err + } + + poolID, err := util.GetPoolID(rbdSnap.Monitors, rbdSnap.AdminID, key, + rbdSnap.Pool) + if err != nil { + klog.Errorf("Error fetching pool ID (%s)", err) + return err + } + + // Create the snapUUID based omap first, to reserve the same and avoid conflicts + // NOTE: If any service loss occurs post creation of the snap omap, and before + // setting the omap key (rbdSnapCSISnapNameKey) to point back to the snaps omap, the + // snap omap key will leak + snapUUID, err := reserveOMapName(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + rbdSnapOMapPrefix) + if err != nil { + return err + } + + // Create request snapUUID key in csi snaps omap and store the uuid based + // snap name into it + err = util.SetOMapKeyValue(rbdSnap.Monitors, rbdSnap.AdminID, key, + rbdSnap.Pool, csiSnapsDirectory, csiSnapNameKeyPrefix+rbdSnap.RequestName, snapUUID) + if err != nil { + return err + } + defer func() { + if err != nil { + klog.Warningf("reservation failed for volume: %s", rbdSnap.RequestName) + errDefer := unreserveSnap(rbdSnap, credentials) + if errDefer != nil { + klog.Warningf("failed undoing reservation of snapshot: %s", rbdSnap.RequestName) + } + } + }() + + // Create snap name based omap and store CSI request name key and source information + err = util.SetOMapKeyValue(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + rbdSnapOMapPrefix+snapUUID, rbdSnapCSISnapNameKey, rbdSnap.RequestName) + if err != nil { + return err + } + err = util.SetOMapKeyValue(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + rbdSnapOMapPrefix+snapUUID, rbdSnapSourceImageKey, rbdSnap.RbdImageName) + if err != nil { + return err + } + + // generate the volume ID to return to the CO system + vi = util.CsiIdentifier{ + PoolID: poolID, + EncodingVersion: volIDVersion, + ClusterID: rbdSnap.ClusterID, + ObjectUUID: snapUUID, + } + rbdSnap.SnapID, err = vi.ComposeCsiID() + if err != nil { + return err + } + rbdSnap.RbdSnapName = rbdSnapNamePrefix + snapUUID + klog.V(4).Infof("Generated Volume ID (%s) and image name (%s) for request name (%s)", + rbdSnap.SnapID, rbdSnap.RbdImageName, rbdSnap.RequestName) + + return nil +} + +func reserveVol(rbdVol *rbdVolume, credentials map[string]string) error { + var vi util.CsiIdentifier + + key, err := getKey(rbdVol.ClusterID, rbdVol.AdminID, credentials) + if err != nil { + return err + } + + poolID, err := util.GetPoolID(rbdVol.Monitors, rbdVol.AdminID, key, + rbdVol.Pool) + if err != nil { + klog.Errorf("Error fetching pool ID (%s)", err) + return err + } + + // Create the imageUUID based omap first, to reserve the same and avoid conflicts + // NOTE: If any service loss occurs post creation of the image omap, and before + // setting the omap key (rbdImageCSIVolNameKey) to point back to the volumes omap, + // the image omap key will leak + imageUUID, err := reserveOMapName(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, rbdImageOMapPrefix) + if err != nil { + return err + } + + // Create request volName key in csi volumes omap and store the uuid based + // image name into it + err = util.SetOMapKeyValue(rbdVol.Monitors, rbdVol.AdminID, key, + rbdVol.Pool, csiVolsDirectory, csiVolNameKeyPrefix+rbdVol.RequestName, imageUUID) + if err != nil { + return err + } + defer func() { + if err != nil { + klog.Warningf("reservation failed for volume: %s", rbdVol.RequestName) + errDefer := unreserveVol(rbdVol, credentials) + if errDefer != nil { + klog.Warningf("failed undoing reservation of volume: %s", rbdVol.RequestName) + } + } + }() + + // Create image name based omap and store CSI request volume name key and data + err = util.SetOMapKeyValue(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + rbdImageOMapPrefix+imageUUID, rbdImageCSIVolNameKey, rbdVol.RequestName) + if err != nil { + return err + } + + // generate the volume ID to return to the CO system + vi = util.CsiIdentifier{ + PoolID: poolID, + EncodingVersion: volIDVersion, + ClusterID: rbdVol.ClusterID, + ObjectUUID: imageUUID, + } + rbdVol.VolID, err = vi.ComposeCsiID() + if err != nil { + return err + } + rbdVol.RbdImageName = rbdImgNamePrefix + imageUUID + klog.V(4).Infof("Generated Volume ID (%s) and image name (%s) for request name (%s)", + rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName) + + return nil +} + +// deleteImage deletes a ceph image with provision and volume options. +func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string) error { var output []byte - image := pOpts.VolName + + image := pOpts.RbdImageName found, _, err := rbdStatus(pOpts, adminID, credentials) if err != nil { return err @@ -217,7 +704,7 @@ func deleteRBDImage(pOpts *rbdVolume, adminID string, credentials map[string]str klog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -229,13 +716,200 @@ func deleteRBDImage(pOpts *rbdVolume, adminID string, credentials map[string]str klog.V(4).Infof("rbd: rm %s using mon %s, pool %s", image, mon, pOpts.Pool) args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) - if err == nil { - return nil + if err != nil { + klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) + return err + } + + err = unreserveVol(pOpts, credentials) + if err != nil { + klog.Errorf("failed to remove reservation for volume (%s) with backing image (%s) (%s)", + pOpts.RequestName, pOpts.RbdImageName, err) + err = nil } - klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) + return err } +// updateSnapWithImageInfo updates provided rbdSnapshot with information from on-disk data +// regarding the same +func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, credentials map[string]string) error { + key, err := getKey(rbdSnap.ClusterID, rbdSnap.AdminID, credentials) + if err != nil { + return err + } + + snapInfo, err := getSnapInfo(rbdSnap.Monitors, rbdSnap.AdminID, key, + rbdSnap.Pool, rbdSnap.RbdImageName, rbdSnap.RbdSnapName) + if err != nil { + return err + } + + rbdSnap.SizeBytes = snapInfo.Size + + tm, err := time.Parse(time.ANSIC, snapInfo.TimeStamp) + if err != nil { + return err + } + + rbdSnap.CreatedAt, err = ptypes.TimestampProto(tm) + if err != nil { + return err + } + + return nil +} + +// updateVolWithImageInfo updates provided rbdVolume with information from on-disk data +// regarding the same +func updateVolWithImageInfo(rbdVol *rbdVolume, credentials map[string]string) error { + key, err := getKey(rbdVol.ClusterID, rbdVol.AdminID, credentials) + if err != nil { + return err + } + + imageInfo, err := getImageInfo(rbdVol.Monitors, rbdVol.AdminID, key, + rbdVol.Pool, rbdVol.RbdImageName) + if err != nil { + return err + } + + switch imageInfo.Format { + case 2: + rbdVol.ImageFormat = rbdImageFormat2 + default: + return fmt.Errorf("unknown or unsupported image format (%d) returned for image (%s)", + imageInfo.Format, rbdVol.RbdImageName) + } + + rbdVol.VolSize = imageInfo.Size + rbdVol.ImageFeatures = strings.Join(imageInfo.Features, ",") + + return nil +} + +// genSnapFromSnapID generates a rbdSnapshot structure from the provided identifier, updating +// the structure with elements from on-disk snapshot metadata as well +func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, credentials map[string]string) error { + var ( + options map[string]string + vi util.CsiIdentifier + ) + options = make(map[string]string) + + rbdSnap.SnapID = snapshotID + + err := vi.DecomposeCsiID(rbdSnap.SnapID) + if err != nil { + klog.V(4).Infof("error decoding snapshot ID (%s) (%s)", err, rbdSnap.SnapID) + return err + } + + rbdSnap.ClusterID = vi.ClusterID + options["clusterID"] = rbdSnap.ClusterID + rbdSnap.RbdSnapName = rbdSnapNamePrefix + vi.ObjectUUID + + rbdSnap.Monitors, _, _, err = getMonsAndClusterID(options) + if err != nil { + return err + } + + rbdSnap.AdminID, rbdSnap.UserID, err = getIDs(options, rbdSnap.ClusterID) + if err != nil { + return err + } + + key, err := getKey(rbdSnap.ClusterID, rbdSnap.AdminID, credentials) + if err != nil { + return err + } + + rbdSnap.Pool, err = util.GetPoolName(rbdSnap.Monitors, rbdSnap.AdminID, key, vi.PoolID) + if err != nil { + return err + } + + // TODO: fetch all omap vals in one call, than make multiple listomapvals + snapUUID := strings.TrimPrefix(rbdSnap.RbdSnapName, rbdSnapNamePrefix) + rbdSnap.RequestName, err = util.GetOMapValue(rbdSnap.Monitors, rbdSnap.AdminID, + key, rbdSnap.Pool, rbdSnapOMapPrefix+snapUUID, rbdSnapCSISnapNameKey) + if err != nil { + return err + } + + rbdSnap.RbdImageName, err = util.GetOMapValue(rbdSnap.Monitors, rbdSnap.AdminID, + key, rbdSnap.Pool, rbdSnapOMapPrefix+snapUUID, rbdSnapSourceImageKey) + if err != nil { + return err + } + + err = updateSnapWithImageInfo(rbdSnap, credentials) + if err != nil { + return err + } + + return nil +} + +// genVolFromVolID generates a rbdVolume structure from the provided identifier, updating +// the structure with elements from on-disk image metadata as well +func genVolFromVolID(rbdVol *rbdVolume, volumeID string, credentials map[string]string) error { + var ( + options map[string]string + vi util.CsiIdentifier + ) + options = make(map[string]string) + + // rbdVolume fields that are not filled up in this function are: + // Mounter, MultiNodeWritable + rbdVol.VolID = volumeID + + err := vi.DecomposeCsiID(rbdVol.VolID) + if err != nil { + klog.V(4).Infof("error decoding volume ID (%s) (%s)", err, rbdVol.VolID) + return err + } + + rbdVol.ClusterID = vi.ClusterID + options["clusterID"] = rbdVol.ClusterID + rbdVol.RbdImageName = rbdImgNamePrefix + vi.ObjectUUID + + rbdVol.Monitors, _, _, err = getMonsAndClusterID(options) + if err != nil { + return err + } + + rbdVol.AdminID, rbdVol.UserID, err = getIDs(options, rbdVol.ClusterID) + if err != nil { + return err + } + + key, err := getKey(rbdVol.ClusterID, rbdVol.AdminID, credentials) + if err != nil { + return err + } + + rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, rbdVol.AdminID, key, + vi.PoolID) + if err != nil { + return err + } + + imageUUID := strings.TrimPrefix(rbdVol.RbdImageName, rbdImgNamePrefix) + rbdVol.RequestName, err = util.GetOMapValue(rbdVol.Monitors, rbdVol.AdminID, + key, rbdVol.Pool, rbdImageOMapPrefix+imageUUID, rbdImageCSIVolNameKey) + if err != nil { + return err + } + + err = updateVolWithImageInfo(rbdVol, credentials) + if err != nil { + return err + } + + return nil +} + func execCommand(command string, args []string) ([]byte, error) { // #nosec cmd := exec.Command(command, args...) @@ -296,7 +970,7 @@ func getIDs(options map[string]string, clusterID string) (adminID, userID string return adminID, userID, err } -func getRBDVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) { +func genVolFromVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) { var ( ok bool err error @@ -326,7 +1000,8 @@ func getRBDVolumeOptions(volOptions map[string]string, disableInUseChecks bool) arr := strings.Split(imageFeatures, ",") for _, f := range arr { if !supportedFeatures.Has(f) { - return nil, fmt.Errorf("invalid feature %q for volume csi-rbdplugin, supported features are: %v", f, supportedFeatures) + return nil, fmt.Errorf("invalid feature %q for volume csi-rbdplugin, supported"+ + " features are: %v", f, supportedFeatures) } } rbdVol.ImageFeatures = imageFeatures @@ -364,7 +1039,7 @@ func getCredsFromVol(rbdVol *rbdVolume, volOptions map[string]string) error { return err } -func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) { +func genSnapFromOptions(snapOptions map[string]string) (*rbdSnapshot, error) { var ( ok bool err error @@ -398,31 +1073,6 @@ func hasSnapshotFeature(imageFeatures string) bool { return false } -func getRBDVolumeByID(volumeID string) (*rbdVolume, error) { - if rbdVol, ok := rbdVolumes[volumeID]; ok { - return rbdVol, nil - } - return nil, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID) -} - -func getRBDVolumeByName(volName string) (*rbdVolume, error) { - for _, rbdVol := range rbdVolumes { - if rbdVol.VolName == volName { - return rbdVol, nil - } - } - return nil, fmt.Errorf("volume name %s does not exit in the volumes list", volName) -} - -func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) { - for _, rbdSnap := range rbdSnapshots { - if rbdSnap.SnapName == snapName { - return rbdSnap, nil - } - } - return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) -} - func getSnapMon(pOpts *rbdSnapshot, credentials map[string]string) (string, error) { mon := pOpts.Monitors if len(mon) == 0 { @@ -443,10 +1093,10 @@ func getSnapMon(pOpts *rbdSnapshot, credentials map[string]string) (string, erro func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { var output []byte - image := pOpts.VolName - snapID := pOpts.SnapID + image := pOpts.RbdImageName + snapName := pOpts.RbdSnapName - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -456,7 +1106,7 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string] } klog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s ", image, mon, pOpts.Pool) - args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key} + args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -467,37 +1117,6 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string] return nil } -func extractStoredVolOpt(r *rbdVolume) map[string]string { - volOptions := make(map[string]string) - volOptions["pool"] = r.Pool - - if len(r.Monitors) > 0 { - volOptions["monitors"] = r.Monitors - } - - if len(r.MonValueFromSecret) > 0 { - volOptions["monValueFromSecret"] = r.MonValueFromSecret - } - - volOptions["imageFormat"] = r.ImageFormat - - if len(r.ImageFeatures) > 0 { - volOptions["imageFeatures"] = r.ImageFeatures - } - - if len(r.AdminID) > 0 { - volOptions["adminId"] = r.AdminID - } - - if len(r.UserID) > 0 { - volOptions["userId"] = r.UserID - } - if len(r.Mounter) > 0 { - volOptions["mounter"] = r.Mounter - } - return volOptions -} - func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { var output []byte @@ -506,15 +1125,15 @@ func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s return err } - image := pOpts.VolName - snapID := pOpts.SnapID + image := pOpts.RbdImageName + snapName := pOpts.RbdSnapName - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } klog.V(4).Infof("rbd: snap create %s using mon %s, pool %s", image, mon, pOpts.Pool) - args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key} + args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -533,15 +1152,15 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[strin return err } - image := pOpts.VolName - snapID := pOpts.SnapID + image := pOpts.RbdImageName + snapName := pOpts.RbdSnapName - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } klog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s", image, mon, pOpts.Pool) - args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key} + args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -560,15 +1179,15 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s return err } - image := pOpts.VolName - snapID := pOpts.SnapID + image := pOpts.RbdImageName + snapName := pOpts.RbdSnapName - key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) + key, err := getKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } klog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s", image, mon, pOpts.Pool) - args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminID, "-m", mon, "--key=" + key} + args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -576,6 +1195,11 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s return errors.Wrapf(err, "failed to delete snapshot, command output: %s", string(output)) } + if err := unreserveSnap(pOpts, credentials); err != nil { + klog.Errorf("failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)", + pOpts.RequestName, pOpts.RbdSnapName, pOpts.RbdImageName, err) + } + return nil } @@ -587,15 +1211,16 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string return err } - image := pVolOpts.VolName - snapID := pSnapOpts.SnapID + image := pVolOpts.RbdImageName + snapName := pSnapOpts.RbdSnapName - key, err := getRBDKey(pVolOpts.ClusterID, adminID, credentials) + key, err := getKey(pVolOpts.ClusterID, adminID, credentials) if err != nil { return err } klog.V(4).Infof("rbd: clone %s using mon %s, pool %s", image, mon, pVolOpts.Pool) - args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminID, "-m", mon, "--key=" + key} + args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.RbdImageName + "@" + snapName, + pVolOpts.Pool + "/" + image, "--id", adminID, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -605,3 +1230,161 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string return nil } + +// getSnapshotMetadata fetches on-disk metadata about the snapshot and populates the passed in +// rbdSnapshot structure +func getSnapshotMetadata(pSnapOpts *rbdSnapshot, adminID string, credentials map[string]string) error { + mon, err := getSnapMon(pSnapOpts, credentials) + if err != nil { + return err + } + + imageName := pSnapOpts.RbdImageName + snapName := pSnapOpts.RbdSnapName + + key, err := getKey(pSnapOpts.ClusterID, adminID, credentials) + if err != nil { + return err + } + + snapInfo, err := getSnapInfo(mon, adminID, key, pSnapOpts.Pool, imageName, snapName) + if err != nil { + return err + } + + pSnapOpts.SizeBytes = snapInfo.Size + + tm, err := time.Parse(time.ANSIC, snapInfo.TimeStamp) + if err != nil { + return err + } + + pSnapOpts.CreatedAt, err = ptypes.TimestampProto(tm) + if err != nil { + return err + } + + return nil +} + +// imageInfo strongly typed JSON spec for image info +type imageInfo struct { + ObjectUUID string `json:"name"` + Size int64 `json:"size"` + Format int64 `json:"format"` + Features []string `json:"features"` + CreatedAt string `json:"create_timestamp"` +} + +// ErrImageNotFound is returned when image name is not found in the cluster on the given pool +type ErrImageNotFound struct { + imageName string + err error +} + +func (e ErrImageNotFound) Error() string { + return e.err.Error() +} + +// getImageInfo queries rbd about the given image and returns its metadata, and returns +// ErrImageNotFound if provided image is not found +func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo, error) { + // rbd --format=json info [image-spec | snap-spec] + + var imgInfo imageInfo + + stdout, _, err := util.ExecCommand( + "rbd", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", util.CephConfigPath, + "--format="+"json", + "info", poolName+"/"+imageName) + if err != nil { + klog.Errorf("failed getting information for image (%s): (%s)", poolName+"/"+imageName, err) + if strings.Contains(string(stdout), "rbd: error opening image "+imageName+ + ": (2) No such file or directory") { + return imgInfo, ErrImageNotFound{imageName, err} + } + return imgInfo, err + } + + err = json.Unmarshal(stdout, &imgInfo) + if err != nil { + klog.Errorf("failed to parse JSON output of image info (%s): (%s)", + poolName+"/"+imageName, err) + return imgInfo, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", + err, string(stdout)) + } + + return imgInfo, nil +} + +// snapInfo strongly typed JSON spec for snap ls rbd output +type snapInfo struct { + ID int64 `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + TimeStamp string `json:"timestamp"` +} + +// ErrSnapNotFound is returned when snap name passed is not found in the list of snapshots for the +// given image +type ErrSnapNotFound struct { + snapName string + err error +} + +func (e ErrSnapNotFound) Error() string { + return e.err.Error() +} + +/* +getSnapInfo queries rbd about the snapshots of the given image and returns its metadata, and +returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if povided snap +is not found in the images snapshot list +*/ +func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (snapInfo, error) { + // rbd --format=json snap ls [image-spec] + + var ( + snpInfo snapInfo + snaps []snapInfo + ) + + stdout, _, err := util.ExecCommand( + "rbd", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", util.CephConfigPath, + "--format="+"json", + "snap", "ls", poolName+"/"+imageName) + if err != nil { + klog.Errorf("failed getting snap (%s) information from image (%s): (%s)", + snapName, poolName+"/"+imageName, err) + if strings.Contains(string(stdout), "rbd: error opening image "+imageName+ + ": (2) No such file or directory") { + return snpInfo, ErrImageNotFound{imageName, err} + } + return snpInfo, err + } + + err = json.Unmarshal(stdout, &snaps) + if err != nil { + klog.Errorf("failed to parse JSON output of image snap list (%s): (%s)", + poolName+"/"+imageName, err) + return snpInfo, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", + err, string(stdout)) + } + + for _, snap := range snaps { + if snap.Name == snapName { + return snap, nil + } + } + + return snpInfo, ErrSnapNotFound{snapName, fmt.Errorf("snap (%s) for image (%s) not found", + snapName, poolName+"/"+imageName)} +} diff --git a/pkg/util/cachepersister.go b/pkg/util/cachepersister.go index 4faf6366c34..eb2eea7c5bb 100644 --- a/pkg/util/cachepersister.go +++ b/pkg/util/cachepersister.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/cephcmds.go b/pkg/util/cephcmds.go new file mode 100644 index 00000000000..dc08b5bf1db --- /dev/null +++ b/pkg/util/cephcmds.go @@ -0,0 +1,278 @@ +/* +Copyright 2019 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "k8s.io/klog" + "os" + "os/exec" + "strings" +) + +// ExecCommand executes passed in program with args and returns seperate stdout and stderr streams +func ExecCommand(program string, args ...string) (stdout, stderr []byte, err error) { + var ( + cmd = exec.Command(program, args...) // nolint: gosec + stdoutBuf bytes.Buffer + stderrBuf bytes.Buffer + ) + + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + + if err := cmd.Run(); err != nil { + return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+ + " occurred while running %s", err, program) + } + + return stdoutBuf.Bytes(), nil, nil +} + +// cephStoragePoolSummary strongly typed JSON spec for osd ls pools output +type cephStoragePoolSummary struct { + Name string `json:"poolname"` + Number int64 `json:"poolnum"` +} + +// GetPoolID searches a list of pools in a cluster and returns the ID of the pool that matches +// the passed in poolName parameter +func GetPoolID(monitors string, adminID string, key string, poolName string) (int64, error) { + // ceph -f json osd lspools + // JSON out: [{"poolnum":,"poolname":}] + + stdout, _, err := ExecCommand( + "ceph", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-f", "json", + "osd", "lspools") + if err != nil { + klog.Errorf("failed getting pool list from cluster (%s)", err) + return 0, err + } + + var pools []cephStoragePoolSummary + err = json.Unmarshal(stdout, &pools) + if err != nil { + klog.Errorf("failed to parse JSON output of pool list from cluster (%s)", err) + return 0, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", err, string(stdout)) + } + + for _, p := range pools { + if poolName == p.Name { + return p.Number, nil + } + } + + return 0, fmt.Errorf("pool (%s) not found in Ceph cluster", poolName) +} + +// GetPoolName lists all pools in a ceph cluster, and matches the pool whose pool ID is equal to +// the requested poolID parameter +func GetPoolName(monitors string, adminID string, key string, poolID int64) (string, error) { + // ceph -f json osd lspools + // [{"poolnum":1,"poolname":"replicapool"}] + + stdout, _, err := ExecCommand( + "ceph", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-f", "json", + "osd", "lspools") + if err != nil { + klog.Errorf("failed getting pool list from cluster (%s)", err) + return "", err + } + + var pools []cephStoragePoolSummary + err = json.Unmarshal(stdout, &pools) + if err != nil { + klog.Errorf("failed to parse JSON output of pool list from cluster (%s)", err) + return "", fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", err, string(stdout)) + } + + for _, p := range pools { + if poolID == p.Number { + return p.Name, nil + } + } + + return "", fmt.Errorf("pool ID (%d) not found in Ceph cluster", poolID) +} + +// SetOMapKeyValue sets the given key and value into the provided Ceph omap name +func SetOMapKeyValue(monitors, adminID, key, poolName, oMapName, oMapKey, keyValue string) error { + // Command: "rados setomapval oMapName oMapKey keyValue" + + _, _, err := ExecCommand( + "rados", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-p", poolName, + "setomapval", oMapName, oMapKey, keyValue) + if err != nil { + klog.Errorf("failed adding key (%s with value %s), to omap (%s) in "+ + "pool (%s): (%v)", oMapKey, keyValue, oMapName, poolName, err) + return err + } + + return nil +} + +// ErrKeyNotFound is returned when requested key in omap is not found +type ErrKeyNotFound struct { + keyName string + err error +} + +func (e ErrKeyNotFound) Error() string { + return e.err.Error() +} + +// GetOMapValue gets the value for the given key from the named omap +func GetOMapValue(monitors, adminID, key, poolName, oMapName, oMapKey string) (string, error) { + // Command: "rados getomapval oMapName oMapKey " + // No such key: replicapool/csi.volumes.directory.default/csi.volname + tmpFile, err := ioutil.TempFile("", "omap-get-") + if err != nil { + klog.Errorf("failed creating a temporary file for key contents") + return "", err + } + defer tmpFile.Close() + defer os.Remove(tmpFile.Name()) + + stdout, _, err := ExecCommand( + "rados", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-p", poolName, + "getomapval", oMapName, oMapKey, tmpFile.Name()) + if err != nil { + // no logs, as attempting to check for key/value is done even on regular call sequences + if strings.Contains(string(stdout), "No such key: "+poolName+"/"+oMapName+"/"+oMapKey) { + return "", ErrKeyNotFound{poolName + "/" + oMapName + "/" + oMapKey, err} + } + return "", err + } + + keyValue, err := ioutil.ReadAll(tmpFile) + return string(keyValue), err +} + +// RemoveOMapKey removes the omap key from the given omap name +func RemoveOMapKey(monitors, adminID, key, poolName, oMapName, oMapKey string) error { + // Command: "rados rmomapkey oMapName oMapKey" + + _, _, err := ExecCommand( + "rados", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-p", poolName, + "rmomapkey", oMapName, oMapKey) + if err != nil { + // NOTE: Missing omap key removal does not return an error + klog.Errorf("failed removing key (%s), from omap (%s) in "+ + "pool (%s): (%v)", oMapKey, oMapName, poolName, err) + return err + } + + return nil +} + +// ErrObjectExists is returned when named omap is already present in rados +type ErrObjectExists struct { + objectName string + err error +} + +func (e ErrObjectExists) Error() string { + return e.err.Error() +} + +// CreateObject creates the object name passed in and returns ErrObjectExists if the provided object +// is already present in rados +func CreateObject(monitors, adminID, key, poolName, objectName string) error { + // Command: "rados create objectName" + + stdout, _, err := ExecCommand( + "rados", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-p", poolName, + "create", objectName) + if err != nil { + klog.Errorf("failed creating omap (%s) in pool (%s): (%v)", objectName, poolName, err) + if strings.Contains(string(stdout), "error creating "+poolName+"/"+objectName+ + ": (17) File exists") { + return ErrObjectExists{objectName, err} + } + return err + } + + return nil +} + +// ErrObjectNotFound is returned when named omap is not found in rados +type ErrObjectNotFound struct { + oMapName string + err error +} + +func (e ErrObjectNotFound) Error() string { + return e.err.Error() +} + +// RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap +// is not found in rados +func RemoveObject(monitors, adminID, key, poolName, oMapName string) error { + // Command: "rados rm oMapName" + + stdout, _, err := ExecCommand( + "rados", + "-m", monitors, + "--id", adminID, + "--key="+key, + "-c", CephConfigPath, + "-p", poolName, + "rm", oMapName) + if err != nil { + klog.Errorf("failed removing omap (%s) in pool (%s): (%v)", oMapName, poolName, err) + if strings.Contains(string(stdout), "error removing "+poolName+">"+oMapName+ + ": (2) No such file or directory") { + return ErrObjectNotFound{oMapName, err} + } + return err + } + + return nil +} diff --git a/pkg/cephfs/cephconf.go b/pkg/util/cephconf.go similarity index 77% rename from pkg/cephfs/cephconf.go rename to pkg/util/cephconf.go index dac9dc9018e..2a0d8bea107 100644 --- a/pkg/cephfs/cephconf.go +++ b/pkg/util/cephconf.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cephfs +package util import ( "io/ioutil" @@ -32,17 +32,19 @@ fuse_set_user_groups = false const ( cephConfigRoot = "/etc/ceph" - cephConfigPath = "/etc/ceph/ceph.conf" + CephConfigPath = "/etc/ceph/ceph.conf" ) func createCephConfigRoot() error { return os.MkdirAll(cephConfigRoot, 0755) // #nosec } -func writeCephConfig() error { +// WriteCephConfig writes out a basic ceph.conf file, making it easy to use +// ceph related CLIs +func WriteCephConfig() error { if err := createCephConfigRoot(); err != nil { return err } - return ioutil.WriteFile(cephConfigPath, cephConfig, 0640) + return ioutil.WriteFile(CephConfigPath, cephConfig, 0640) } diff --git a/pkg/util/k8scmcache.go b/pkg/util/k8scmcache.go index 5982602e26d..90bcd09b0ae 100644 --- a/pkg/util/k8scmcache.go +++ b/pkg/util/k8scmcache.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/nodecache.go b/pkg/util/nodecache.go index bcb94debceb..52710bfd8a9 100644 --- a/pkg/util/nodecache.go +++ b/pkg/util/nodecache.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2018 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/stripsecrets.go b/pkg/util/stripsecrets.go index 8b818de9d6b..3edce0fcee7 100644 --- a/pkg/util/stripsecrets.go +++ b/pkg/util/stripsecrets.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2019 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/util.go b/pkg/util/util.go index 6c298bc99dd..ea8ec1fd61e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2019 The Ceph-CSI Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/util/volid.go b/pkg/util/volid.go new file mode 100644 index 00000000000..8c989cdbb97 --- /dev/null +++ b/pkg/util/volid.go @@ -0,0 +1,151 @@ +/* +Copyright 2019 Ceph-CSI authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "strings" +) + +/* +CsiIdentifier contains the elements that form a CSI ID to be returned by the CSI plugin, and +contains enough information to decompose and extract required cluster and pool information to locate +the volume that relates to the CSI ID. + +The CSI identifier is composed as elaborated in the comment against ComposeCsiID and thus, +DecomposeCsiID is the inverse of the same function. + +The CsiIdentifier structure carries the following fields, +- PoolID: 64 bit integer of the pool that the volume belongs to, where the ID comes from Ceph pool + identifier for the corresponding pool name. +- EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID, + and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain + ability to parse backward compatible encodings. +- ClusterID: Is a unique ID per cluster that the CSI instance is serving and is restricted to + lengths that can be accommodated in the encoding scheme. +- ObjectUUID: Is the on-disk uuid of the object (image/snapshot) name, for the CSI volume that + corresponds to this CSI ID. +*/ +type CsiIdentifier struct { + PoolID int64 // TODO: Name appropriately when reused for CephFS + EncodingVersion uint16 + ClusterID string + ObjectUUID string +} + +// This maximum comes from the CSI spec on max bytes allowed in the various CSI ID fields +const maxVolIDLen = 128 + +/* +ComposeCsiID composes a CSI ID from passed in parameters. +Version 1 of the encoding scheme is as follows, + [csi_id_version=1:4byte] + [-:1byte] + [length of clusterID=1:4byte] + [-:1byte] + [clusterID:36bytes (MAX)] + [-:1byte] + [poolID:16bytes] + [-:1byte] + [ObjectUUID:36bytes] + + Total of constant field lengths, including '-' field separators would hence be, + 4+1+4+1+1+16+1+36 = 64 +*/ +const ( + knownFieldSize = 64 + uuidSize = 36 +) + +func (ci CsiIdentifier) ComposeCsiID() (string, error) { + buf16 := make([]byte, 2) + buf64 := make([]byte, 8) + + if (knownFieldSize + len(ci.ClusterID)) > maxVolIDLen { + return "", errors.New("CSI ID encoding length overflow") + } + + if len(ci.ObjectUUID) != uuidSize { + return "", errors.New("CSI ID invalid object uuid") + } + + binary.BigEndian.PutUint16(buf16, ci.EncodingVersion) + versionEncodedHex := hex.EncodeToString(buf16) + + binary.BigEndian.PutUint16(buf16, uint16(len(ci.ClusterID))) + clusterIDLength := hex.EncodeToString(buf16) + + binary.BigEndian.PutUint64(buf64, uint64(ci.PoolID)) + poolIDEncodedHex := hex.EncodeToString(buf64) + + return strings.Join([]string{versionEncodedHex, clusterIDLength, ci.ClusterID, + poolIDEncodedHex, ci.ObjectUUID}, "-"), nil +} + +/* +DecomposeCsiID composes a CsiIdentifier from passed in string +*/ +func (ci *CsiIdentifier) DecomposeCsiID(composedCsiID string) (err error) { + bytesToProcess := uint16(len(composedCsiID)) + + // if length is less that expected constant elements, then bail out! + if bytesToProcess < knownFieldSize { + return errors.New("failed to decode CSI identifier, string underflow") + } + + buf16, err := hex.DecodeString(composedCsiID[0:4]) + if err != nil { + return err + } + ci.EncodingVersion = binary.BigEndian.Uint16(buf16) + // 4 for version encoding and 1 for '-' separator + bytesToProcess -= 5 + + buf16, err = hex.DecodeString(composedCsiID[5:9]) + if err != nil { + return err + } + clusterIDLength := binary.BigEndian.Uint16(buf16) + // 4 for length encoding and 1 for '-' separator + bytesToProcess -= 5 + + if bytesToProcess < (clusterIDLength + 1) { + return errors.New("failed to decode CSI identifier, string underflow") + } + ci.ClusterID = composedCsiID[10 : 10+clusterIDLength] + // additional 1 for '-' separator + bytesToProcess -= (clusterIDLength + 1) + nextFieldStartIdx := 10 + clusterIDLength + 1 + + if bytesToProcess < 17 { + return errors.New("failed to decode CSI identifier, string underflow") + } + buf64, err := hex.DecodeString(composedCsiID[nextFieldStartIdx : nextFieldStartIdx+16]) + if err != nil { + return err + } + ci.PoolID = int64(binary.BigEndian.Uint64(buf64)) + // 16 for poolID encoding and 1 for '-' separator + bytesToProcess -= 17 + nextFieldStartIdx = nextFieldStartIdx + 17 + + // has to be an exact match + if bytesToProcess != uuidSize { + return errors.New("failed to decode CSI identifier, string size mismatch") + } + ci.ObjectUUID = composedCsiID[nextFieldStartIdx : nextFieldStartIdx+uuidSize] + + return err +} diff --git a/pkg/util/volid_test.go b/pkg/util/volid_test.go new file mode 100644 index 00000000000..7ae34662f2a --- /dev/null +++ b/pkg/util/volid_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2019 Ceph-CSI authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" +) + +type testTuple struct { + vID CsiIdentifier + composedVolID string + wantEnc bool + wantEncError bool + wantDec bool + wantDecError bool +} + +// TODO: Add more test tuples to test out other edge conditions +var testData = []testTuple{ + { + vID: CsiIdentifier{ + PoolID: 0xffff, + EncodingVersion: 0xffff, + ClusterID: "01616094-9d93-4178-bf45-c7eac19e8b15", + ObjectUUID: "00000000-1111-2222-bbbb-cacacacacaca", + }, + composedVolID: "ffff-0024-01616094-9d93-4178-bf45-c7eac19e8b15-000000000000ffff-00000000-1111-2222-bbbb-cacacacacaca", + wantEnc: true, + wantEncError: false, + wantDec: true, + wantDecError: false, + }, +} + +func TestComposeDecomposeID(t *testing.T) { + var ( + err error + viDecompose CsiIdentifier + composedVolID string + ) + + for _, test := range testData { + if test.wantEnc { + composedVolID, err = test.vID.ComposeCsiID() + + if err != nil && !test.wantEncError { + t.Errorf("Composing failed: want (%#v), got (%#v %#v)", + test, composedVolID, err) + } + + if err == nil && test.wantEncError { + t.Errorf("Composing failed: want (%#v), got (%#v %#v)", + test, composedVolID, err) + } + + if !test.wantEncError && err == nil && composedVolID != test.composedVolID { + t.Errorf("Composing failed: want (%#v), got (%#v %#v)", + test, composedVolID, err) + } + } + + if test.wantDec { + err = viDecompose.DecomposeCsiID(test.composedVolID) + + if err != nil && !test.wantDecError { + t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)", + test, viDecompose, err) + } + + if err == nil && test.wantDecError { + t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)", + test, viDecompose, err) + } + + if !test.wantDecError && err == nil && viDecompose != test.vID { + t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)", + test, viDecompose, err) + } + } + } +}