diff --git a/glide.lock b/glide.lock index 060ecbfcc0..fce3cc7619 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 7bde4a309512fa8bc317673f9fb93bdfa24b62df67755afe7d34f96e6ad261f0 -updated: 2019-02-21T23:12:00.943302286Z +hash: d0cf74cfb00fb63c14f0864b1812e0e4ba0e215fbb9834387152b75e91f04855 +updated: 2019-03-07T21:00:01.993836682Z imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -163,6 +163,15 @@ imports: version: 8eab2debe79d12b7bd3d10653910df25fa9552ba - name: github.com/json-iterator/go version: ab8a2e0c74be9d3be70b3184d9acc634935ded82 +- name: github.com/kubernetes-csi/external-snapshotter + version: 0d2cdd7bde5ea9d41ae47ef4184ca20ebe0435e1 + subpackages: + - pkg/apis/volumesnapshot/v1alpha1 + - pkg/client/clientset/versioned + - pkg/client/clientset/versioned/fake + - pkg/client/clientset/versioned/scheme + - pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1 + - pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1/fake - name: github.com/mailru/easyjson version: 2f5df55504ebc322e4d52d34df6a1f5b503bf26d subpackages: @@ -181,6 +190,8 @@ imports: version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 - name: github.com/modern-go/reflect2 version: 94122c33edd36123c84d5368cfb2b69df93a0ec8 +- name: github.com/pborman/uuid + version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 - name: github.com/peterbourgon/diskv version: 5f041e8faa004a95c88a202771f4cc3e991971e6 - name: github.com/pkg/errors @@ -366,6 +377,7 @@ imports: - pkg/util/runtime - pkg/util/sets - pkg/util/strategicpatch + - pkg/util/uuid - pkg/util/validation - pkg/util/validation/field - pkg/util/wait diff --git a/glide.yaml b/glide.yaml index 366e482240..49cfaa1bda 100644 --- a/glide.yaml +++ b/glide.yaml @@ -16,6 +16,8 @@ import: repo: https://github.com/kastenhq/ibmcloud-storage-volume-lib.git - package: github.com/jpillora/backoff version: 1.0.0 +- package: github.com/kubernetes-csi/external-snapshotter + version: v1.0.1 - package: github.com/Masterminds/sprig version: v2.15.0 - package: github.com/mitchellh/mapstructure diff --git a/pkg/blockstorage/zone/zone.go b/pkg/blockstorage/zone/zone.go index 5f541f4164..683493a0ea 100644 --- a/pkg/blockstorage/zone/zone.go +++ b/pkg/blockstorage/zone/zone.go @@ -12,6 +12,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/kanisterio/kanister/pkg/kube" + kubevolume "github.com/kanisterio/kanister/pkg/kube/volume" ) type ( @@ -124,7 +125,7 @@ func nodeZones(ctx context.Context) (map[string]struct{}, error) { } zoneSet := make(map[string]struct{}, len(ns.Items)) for _, n := range ns.Items { - if v, ok := n.Labels[kube.PVZoneLabelName]; ok { + if v, ok := n.Labels[kubevolume.PVZoneLabelName]; ok { zoneSet[v] = struct{}{} } } diff --git a/pkg/function/create_volume_from_snapshot.go b/pkg/function/create_volume_from_snapshot.go index c2e62585e6..741966a095 100644 --- a/pkg/function/create_volume_from_snapshot.go +++ b/pkg/function/create_volume_from_snapshot.go @@ -14,6 +14,7 @@ import ( "github.com/kanisterio/kanister/pkg/blockstorage/awsebs" "github.com/kanisterio/kanister/pkg/blockstorage/getter" "github.com/kanisterio/kanister/pkg/kube" + kubevolume "github.com/kanisterio/kanister/pkg/kube/volume" "github.com/kanisterio/kanister/pkg/param" ) @@ -72,7 +73,7 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam } _, err = cli.Core().PersistentVolumeClaims(namespace).Get(pvcName, metav1.GetOptions{}) if err == nil { - if err = kube.DeletePVC(cli, namespace, pvcName); err != nil { + if err = kubevolume.DeletePVC(cli, namespace, pvcName); err != nil { return nil, err } } @@ -93,11 +94,11 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam } annotations := map[string]string{} - pvc, err := kube.CreatePVC(ctx, cli, namespace, pvcName, vol.Size, vol.ID, annotations) + pvc, err := kubevolume.CreatePVC(ctx, cli, namespace, pvcName, vol.Size, vol.ID, annotations) if err != nil { return nil, errors.Wrapf(err, "Unable to create PVC for volume %v", *vol) } - pv, err := kube.CreatePV(ctx, cli, vol, vol.Type, annotations) + pv, err := kubevolume.CreatePV(ctx, cli, vol, vol.Type, annotations) if err != nil { return nil, errors.Wrapf(err, "Unable to create PV for volume %v", *vol) } diff --git a/pkg/function/create_volume_snapshot.go b/pkg/function/create_volume_snapshot.go index 70a4dfbf69..680208c0a4 100644 --- a/pkg/function/create_volume_snapshot.go +++ b/pkg/function/create_volume_snapshot.go @@ -19,6 +19,7 @@ import ( "github.com/kanisterio/kanister/pkg/blockstorage/awsebs" "github.com/kanisterio/kanister/pkg/blockstorage/getter" "github.com/kanisterio/kanister/pkg/kube" + kubevolume "github.com/kanisterio/kanister/pkg/kube/volume" "github.com/kanisterio/kanister/pkg/param" ) @@ -189,7 +190,7 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str return nil, errors.Wrap(err, "Profile validation failed") } // Get Region from PV label or EC2 metadata - if pvRegion, ok := pvLabels[kube.PVRegionLabelName]; ok { + if pvRegion, ok := pvLabels[kubevolume.PVRegionLabelName]; ok { region = pvRegion } else { region, err = awsebs.GetRegionFromEC2Metadata() @@ -197,7 +198,7 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str return nil, err } } - if pvZone, ok := pvLabels[kube.PVZoneLabelName]; ok { + if pvZone, ok := pvLabels[kubevolume.PVZoneLabelName]; ok { config[awsebs.ConfigRegion] = region config[awsebs.AccessKeyID] = tp.Profile.Credential.KeyPair.ID config[awsebs.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret @@ -215,7 +216,7 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str if err = ValidateProfile(tp.Profile, blockstorage.TypeGPD); err != nil { return nil, errors.Wrap(err, "Profile validation failed") } - if pvZone, ok := pvLabels[kube.PVZoneLabelName]; ok { + if pvZone, ok := pvLabels[kubevolume.PVZoneLabelName]; ok { config[blockstorage.GoogleProjectID] = tp.Profile.Credential.KeyPair.ID config[blockstorage.GoogleServiceKey] = tp.Profile.Credential.KeyPair.Secret provider, err = getter.Get(blockstorage.TypeGPD, config) diff --git a/pkg/function/create_volume_snapshot_test.go b/pkg/function/create_volume_snapshot_test.go index 50eecc31e4..7298f291a9 100644 --- a/pkg/function/create_volume_snapshot_test.go +++ b/pkg/function/create_volume_snapshot_test.go @@ -11,7 +11,7 @@ import ( crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" "github.com/kanisterio/kanister/pkg/blockstorage" - "github.com/kanisterio/kanister/pkg/kube" + kubevolume "github.com/kanisterio/kanister/pkg/kube/volume" "github.com/kanisterio/kanister/pkg/param" "github.com/kanisterio/kanister/pkg/testutil/mockblockstorage" ) @@ -53,8 +53,8 @@ func (s *CreateVolumeSnapshotTestSuite) TestGetPVCInfo(c *C) { ObjectMeta: metav1.ObjectMeta{ Name: "pv-test-1", Labels: map[string]string{ - kube.PVZoneLabelName: "us-west-2a", - kube.PVRegionLabelName: "us-west-2", + kubevolume.PVZoneLabelName: "us-west-2a", + kubevolume.PVRegionLabelName: "us-west-2", }, }, Spec: v1.PersistentVolumeSpec{ @@ -105,7 +105,7 @@ func (s *CreateVolumeSnapshotTestSuite) TestGetPVCInfo(c *C) { ObjectMeta: metav1.ObjectMeta{ Name: "pv-test-3", Labels: map[string]string{ - kube.PVZoneLabelName: "us-west-2a", + kubevolume.PVZoneLabelName: "us-west-2a", }, }, Spec: v1.PersistentVolumeSpec{ diff --git a/pkg/kube/client.go b/pkg/kube/client.go index d32c3f7ec8..509df72a8b 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -1,9 +1,9 @@ package kube import ( + snapshot "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - // Load the GCP plugin - required to authenticate against + "k8s.io/client-go/kubernetes" // Load the GCP plugin - required to authenticate against // GKE clusters _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" @@ -50,3 +50,18 @@ func NewClient() (kubernetes.Interface, error) { } return clientset, nil } + +// NewClientSnapshot returns a VolumeSnapshot client configured by the Kanister environment. +func NewSnapshotClient() (snapshot.Interface, error) { + config, err := LoadConfig() + if err != nil { + return nil, err + } + + // creates the clientset + clientset, err := snapshot.NewForConfig(config) + if err != nil { + return nil, err + } + return clientset, nil +} diff --git a/pkg/kube/snapshot/snapshot.go b/pkg/kube/snapshot/snapshot.go new file mode 100644 index 0000000000..3c25c1711d --- /dev/null +++ b/pkg/kube/snapshot/snapshot.go @@ -0,0 +1,199 @@ +package snapshot + +import ( + "context" + + snapshot "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" + snapshotclient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes" + + "github.com/kanisterio/kanister/pkg/poll" +) + +const ( + snapshotKind = "VolumeSnapshot" + pvcKind = "PersistentVolumeClaim" +) + +var snapshotAPIGroup = "snapshot.storage.k8s.io" + +// Create creates a VolumeSnapshot and returns it or any error happened meanwhile. +// +// 'name' is the name of the VolumeSnapshot. +// 'namespace' is namespace of the PVC. VolumeSnapshot will be crated in the same namespace. +// 'volumeName' is the name of the PVC of which we will take snapshot. It must be in the same namespace 'ns'. +// 'waitForReady' will block the caller until the snapshot status is 'ReadyToUse'. +// or 'ctx.Done()' is signalled. Otherwise it will return immediately after the snapshot is cut. +func Create(ctx context.Context, kubeCli kubernetes.Interface, snapCli snapshotclient.Interface, name, namespace, volumeName string, snapshotClass *string, waitForReady bool) error { + if _, err := kubeCli.CoreV1().PersistentVolumeClaims(namespace).Get(volumeName, metav1.GetOptions{}); err != nil { + if k8errors.IsNotFound(err) { + return errors.Errorf("Failed to find PVC %s, Namespace %s", volumeName, namespace) + } + return errors.Errorf("Failed to query PVC %s, Namespace %s: %v", volumeName, namespace, err) + } + + snap := &snapshot.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: snapshot.VolumeSnapshotSpec{ + Source: &corev1.TypedLocalObjectReference{ + Kind: pvcKind, + Name: volumeName, + }, + VolumeSnapshotClassName: snapshotClass, + }, + } + + snap, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Create(snap) + if err != nil { + return err + } + + if !waitForReady { + return nil + } + + err = WaitOnReadyToUse(ctx, snapCli, name, namespace) + if err != nil { + return err + } + + _, err = snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Get(name, metav1.GetOptions{}) + return err +} + +// Get will return the VolumeSnapshot in the namespace 'namespace' with given 'name'. +// +// 'name' is the name of the VolumeSnapshot that will be returned. +// 'namespace' is the namespace of the VolumeSnapshot that will be returned. +func Get(ctx context.Context, snapCli snapshotclient.Interface, name, namespace string) (*snapshot.VolumeSnapshot, error) { + return snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Get(name, metav1.GetOptions{}) +} + +// Delete will delete the VolumeSnapshot and returns any error as a result. +// +// 'name' is the name of the VolumeSnapshot that will be deleted. +// 'namespace' is the namespace of the VolumeSnapshot that will be deleted. +func Delete(ctx context.Context, snapCli snapshotclient.Interface, name, namespace string) error { + return snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Delete(name, &metav1.DeleteOptions{}) +} + +// Clone will clone the VolumeSnapshot to namespace 'cloneNamespace'. +// Underlying VolumeSnapshotContent will be cloned with a different name. +// +// 'name' is the name of the VolumeSnapshot that will be cloned. +// 'namespace' is the namespace of the VolumeSnapshot that will be cloned. +// 'cloneName' is name of the clone. +// 'cloneNamespace' is the namespace where the clone will be created. +// 'waitForReady' will make the function blocks until the clone's status is ready to use. +func Clone(ctx context.Context, snapCli snapshotclient.Interface, name, namespace, cloneName, cloneNamespace string, waitForReady bool) error { + snap, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return err + } + if !snap.Status.ReadyToUse { + return errors.Errorf("Original snapshot is not ready, VolumeSnapshot: %s, Namespace: %s", cloneName, cloneNamespace) + } + if snap.Spec.SnapshotContentName == "" { + return errors.Errorf("Original snapshot does not have content, VolumeSnapshot: %s, Namespace: %s", cloneName, cloneNamespace) + } + + _, err = snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(cloneNamespace).Get(cloneName, metav1.GetOptions{}) + if err == nil { + return errors.Errorf("Target snapshot already exists in target namespace, Volumesnapshot: %s, Namespace: %s", cloneName, cloneNamespace) + } + if !k8errors.IsNotFound(err) { + return errors.Errorf("Failed to query target Volumesnapshot: %s, Namespace: %s: %v", cloneName, cloneNamespace, err) + } + + content, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshotContents().Get(snap.Spec.SnapshotContentName, metav1.GetOptions{}) + if err != nil { + return errors.Errorf("Failed to get original snapshot content, VolumesnapshotContent: %s: %v", snap.Spec.SnapshotContentName, err) + } + + // Create a 'VolumeSnapshotContent' referenced by a 'VolumeSnapshot' 'cloneName' in namespace 'cloneNamespace'. + clonedSnap, clonedContent := cloneSnapshotAndContent(content, cloneName, cloneNamespace) + + // Create cloned VolumeSnapshotContent first, then cloned VolumeSnapshot. + // Snapshotter will check the binding and set the status of VolumeSnapshot. + _, err = snapCli.VolumesnapshotV1alpha1().VolumeSnapshotContents().Create(clonedContent) + if err != nil { + return errors.Errorf("Failed to create target snapshot content, VolumeSnapshotContent: %s: %v", clonedContent.Name, err) + } + + clonedSnap, err = snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(cloneNamespace).Create(clonedSnap) + if err != nil { + return errors.Errorf("Failed to create target snapshot, VolumeSnapshot: %s: %v", clonedSnap.Name, err) + } + + if !waitForReady { + return nil + } + + err = WaitOnReadyToUse(ctx, snapCli, clonedSnap.Name, clonedSnap.Namespace) + if err != nil { + return err + } + + return nil +} + +// WaitOnReadyToUse will block until the Volumesnapshot in namespace 'namespace' with name 'snapshotName' +// has status 'ReadyToUse' or 'ctx.Done()' is signalled. +func WaitOnReadyToUse(ctx context.Context, snapCli snapshotclient.Interface, snapshotName, namespace string) error { + return poll.Wait(ctx, func(context.Context) (bool, error) { + snap, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Get(snapshotName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return snap.Status.ReadyToUse, nil + }) +} + +// clonedSnapshotAndContent will return a 'VolumeSnapshot' and 'VolumeSnapshotContent' for a snapshot clone. +// Cloned 'VolumeSnapshotContent' will copy the value for 'Source.CSI.{Driver,SnapshotHandle}' +// since these are the minimum info for snapshotter to get information from the source. +// It needs to prepopulate 'VolumeSnapshotRef' field with the values from the 'VolumeSnapshot' +// that will also be returned. +func cloneSnapshotAndContent(content *snapshot.VolumeSnapshotContent, clonedName, clonedNamespace string) (*snapshot.VolumeSnapshot, *snapshot.VolumeSnapshotContent) { + clonedContent := &snapshot.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: clonedContentName(content.Name), + }, + Spec: snapshot.VolumeSnapshotContentSpec{ + VolumeSnapshotSource: snapshot.VolumeSnapshotSource{ + CSI: &snapshot.CSIVolumeSnapshotSource{ + Driver: content.Spec.VolumeSnapshotSource.CSI.Driver, + SnapshotHandle: content.Spec.VolumeSnapshotSource.CSI.SnapshotHandle, + }, + }, + VolumeSnapshotRef: &corev1.ObjectReference{ + Kind: snapshotKind, + Namespace: clonedNamespace, + Name: clonedName, + }, + VolumeSnapshotClassName: content.Spec.VolumeSnapshotClassName, + }, + } + clonedSnap := &snapshot.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: clonedName, + }, + Spec: snapshot.VolumeSnapshotSpec{ + SnapshotContentName: clonedContent.Name, + VolumeSnapshotClassName: content.Spec.VolumeSnapshotClassName, + }, + } + return clonedSnap, clonedContent +} + +func clonedContentName(contentName string) string { + return contentName + "-k10clone-" + string(uuid.NewUUID()) +} diff --git a/pkg/kube/snapshot/snapshot_test.go b/pkg/kube/snapshot/snapshot_test.go new file mode 100644 index 0000000000..88bdf91fc3 --- /dev/null +++ b/pkg/kube/snapshot/snapshot_test.go @@ -0,0 +1,302 @@ +package snapshot + +import ( + "context" + "strconv" + "strings" + "testing" + "time" + + snapshot "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" + snapshotclient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + snapshotfake "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/fake" + . "gopkg.in/check.v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + "github.com/kanisterio/kanister/pkg/kube" + "github.com/kanisterio/kanister/pkg/kube/volume" + "github.com/kanisterio/kanister/pkg/poll" +) + +func Test(t *testing.T) { TestingT(t) } + +type SnapshotTestSuite struct { + sourceNamespace string + targetNamespace string + cli kubernetes.Interface + snapCli snapshotclient.Interface + snapshotClass *string + storageClassCSI *string +} + +var _ = Suite(&SnapshotTestSuite{}) + +var ( + defaultNamespace = "default" + fakeClass = "fake-snapshotclass" + fakeDriver = "fake-driver" + fakeSnapshotHandle = "snapshot/csi/handle1" + + testTimeout = 5 * time.Minute + + volNamePrefix = "pvc-snapshot-test-" + snapshotNamePrefix = "snap-snapshot-test-" +) + +func (s *SnapshotTestSuite) SetUpSuite(c *C) { + suffix := strconv.Itoa(int(time.Now().UnixNano() % 100000)) + s.sourceNamespace = "snapshot-test-source-" + suffix + s.targetNamespace = "snapshot-test-target-" + suffix + + cli, err := kube.NewClient() + c.Assert(err, IsNil) + s.cli = cli + + sc, err := kube.NewSnapshotClient() + c.Assert(err, IsNil) + s.snapCli = sc + + vscs, err := sc.VolumesnapshotV1alpha1().VolumeSnapshotClasses().List(metav1.ListOptions{}) + c.Assert(err, IsNil) + if len(vscs.Items) != 0 { + s.snapshotClass = &vscs.Items[0].Name + } + + storageClasses, err := cli.StorageV1().StorageClasses().List(metav1.ListOptions{}) + c.Assert(err, IsNil) + for _, class := range storageClasses.Items { + if isCSIProvisioner(class.Provisioner) { + s.storageClassCSI = &class.Name + } + } + + _, err = cli.CoreV1().Namespaces().Create(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: s.sourceNamespace}}) + c.Assert(err, IsNil) + + _, err = cli.CoreV1().Namespaces().Create(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: s.targetNamespace}}) + c.Assert(err, IsNil) +} + +func (s *SnapshotTestSuite) TearDownSuite(c *C) { + s.cleanupNamespace(c, s.sourceNamespace) + s.cleanupNamespace(c, s.targetNamespace) +} + +func (s *SnapshotTestSuite) TestVolumeSnapshotFake(c *C) { + snapshotName := "snap-1-fake" + volName := "pvc-1-fake" + fakeCli := fake.NewSimpleClientset() + fakeSnapCli := snapshotfake.NewSimpleClientset() + + size, err := resource.ParseQuantity("1Gi") + c.Assert(err, IsNil) + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: volName, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: size, + }, + }, + }, + } + _, err = fakeCli.CoreV1().PersistentVolumeClaims(defaultNamespace).Create(pvc) + c.Assert(err, IsNil) + + err = Create(context.Background(), fakeCli, fakeSnapCli, snapshotName, defaultNamespace, volName, &fakeClass, false) + c.Assert(err, IsNil) + snap, err := Get(context.Background(), fakeSnapCli, snapshotName, defaultNamespace) + c.Assert(err, IsNil) + c.Assert(snap.Name, Equals, snapshotName) + + err = Create(context.Background(), fakeCli, fakeSnapCli, snapshotName, defaultNamespace, volName, &fakeClass, false) + c.Assert(err, NotNil) + err = Delete(context.Background(), fakeSnapCli, snap.Name, snap.Namespace) + c.Assert(err, IsNil) + err = Delete(context.Background(), fakeSnapCli, snap.Name, snap.Namespace) + c.Assert(err, NotNil) +} + +func (s *SnapshotTestSuite) TestVolumeSnapshotCloneFake(c *C) { + fakeSnapshotName := "snap-1-fake" + fakeContentName := "snapcontent-1-fake" + + content := &snapshot.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeContentName, + }, + Spec: snapshot.VolumeSnapshotContentSpec{ + VolumeSnapshotSource: snapshot.VolumeSnapshotSource{ + CSI: &snapshot.CSIVolumeSnapshotSource{ + Driver: fakeDriver, + SnapshotHandle: fakeSnapshotHandle, + }, + }, + VolumeSnapshotClassName: &fakeClass, + VolumeSnapshotRef: &corev1.ObjectReference{ + Name: fakeSnapshotName, + Namespace: defaultNamespace, + }, + }, + } + snapshot := &snapshot.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeSnapshotName, + Namespace: defaultNamespace, + }, + Spec: snapshot.VolumeSnapshotSpec{ + SnapshotContentName: fakeContentName, + VolumeSnapshotClassName: &fakeClass, + }, + Status: snapshot.VolumeSnapshotStatus{ + ReadyToUse: true, + }, + } + + snapCli := snapshotfake.NewSimpleClientset() + fakeTargetNamespace := "new-ns" + fakeClone := "clone-1" + + _, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(defaultNamespace).Create(snapshot) + c.Assert(err, IsNil) + _, err = snapCli.VolumesnapshotV1alpha1().VolumeSnapshotContents().Create(content) + c.Assert(err, IsNil) + + _, err = Get(context.Background(), snapCli, fakeSnapshotName, defaultNamespace) + c.Assert(err, IsNil) + + err = Clone(context.Background(), snapCli, fakeSnapshotName, defaultNamespace, fakeClone, fakeTargetNamespace, false) + c.Assert(err, IsNil) + + clone, err := Get(context.Background(), snapCli, fakeClone, fakeTargetNamespace) + c.Assert(err, IsNil) + + cloneContent, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshotContents().Get(clone.Spec.SnapshotContentName, metav1.GetOptions{}) + c.Assert(err, IsNil) + c.Assert(strings.HasPrefix(cloneContent.Name, fakeContentName), Equals, true) +} + +func (s *SnapshotTestSuite) TestVolumeSnapshot(c *C) { + if s.snapshotClass == nil { + c.Skip("No Volumesnapshotclass in the cluster, create a volumesnapshotclass in the cluster") + } + if s.storageClassCSI == nil { + c.Skip("No Storageclass with CSI provisioner, install CSI and create a storageclass for it") + } + c.Logf("VolumeSnapshot test - source namespace: %s - target namespace: %s", s.sourceNamespace, s.targetNamespace) + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + size, err := resource.ParseQuantity("200Gi") + c.Assert(err, IsNil) + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: volNamePrefix, + Namespace: s.sourceNamespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceStorage): size, + }, + }, + StorageClassName: s.storageClassCSI, + }, + } + pvc, err = s.cli.CoreV1().PersistentVolumeClaims(s.sourceNamespace).Create(pvc) + c.Assert(err, IsNil) + poll.Wait(ctx, func(ctx context.Context) (bool, error) { + pvc, err = s.cli.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return pvc.Status.Phase == corev1.ClaimBound, nil + }) + + snapshotName := snapshotNamePrefix + strconv.Itoa(int(time.Now().UnixNano())) + wait := true + err = Create(ctx, s.cli, s.snapCli, snapshotName, s.sourceNamespace, pvc.Name, s.snapshotClass, wait) + c.Assert(err, IsNil) + + snap, err := Get(ctx, s.snapCli, snapshotName, s.sourceNamespace) + c.Assert(err, IsNil) + c.Assert(snap.Name, Equals, snapshotName) + c.Assert(snap.Status.ReadyToUse, Equals, true) + + err = Create(ctx, s.cli, s.snapCli, snapshotName, s.sourceNamespace, pvc.Name, s.snapshotClass, wait) + c.Assert(err, NotNil) + + snapshotCloneName := snapshotName + "-clone" + volumeCloneName := pvc.Name + "-clone" + err = Clone(ctx, s.snapCli, snapshotName, s.sourceNamespace, snapshotCloneName, s.targetNamespace, wait) + c.Assert(err, IsNil) + + _, err = volume.CreatePVCFromSnapshot(ctx, s.cli, s.snapCli, s.targetNamespace, volumeCloneName, snapshotCloneName, nil) + c.Assert(err, IsNil) + poll.Wait(ctx, func(ctx context.Context) (bool, error) { + pvc, err = s.cli.CoreV1().PersistentVolumeClaims(s.targetNamespace).Get(volumeCloneName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return pvc.Status.Phase == corev1.ClaimBound, nil + }) + + err = Delete(ctx, s.snapCli, snap.Name, snap.Namespace) + c.Assert(err, IsNil) + + err = Delete(ctx, s.snapCli, snap.Name, snap.Namespace) + c.Assert(err, NotNil) + +} + +func isCSIProvisioner(provisioner string) bool { + for _, part := range strings.Split(provisioner, ".") { + if part == "csi" { + return true + } + } + return false +} + +func (s *SnapshotTestSuite) cleanupNamespace(c *C, ns string) { + pvcs, erra := s.cli.CoreV1().PersistentVolumeClaims(ns).List(metav1.ListOptions{}) + if erra != nil { + c.Logf("Failed to list PVCs, Namespace: %s, Error: %v", ns, erra) + } else { + for _, pvc := range pvcs.Items { + if err := s.cli.CoreV1().PersistentVolumeClaims(ns).Delete(pvc.Name, &metav1.DeleteOptions{}); err != nil { + erra = err + c.Logf("Failed to delete PVC, PVC: %s, Namespace: %s, Error: %v", pvc.Name, ns, err) + } + } + } + + vss, errb := s.snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(ns).List(metav1.ListOptions{}) + if errb != nil { + c.Logf("Failed to list snapshots, Namespace: %s, Error: %v", ns, errb) + } else { + for _, vs := range vss.Items { + if err := s.snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(ns).Delete(vs.Name, &metav1.DeleteOptions{}); err != nil { + errb = err + c.Logf("Failed to delete snapshot, Volumesnapshot: %s, Namespace %s, Error: %v", vs.Name, vs.Namespace, err) + } + } + } + + if erra != nil || errb != nil { + c.Logf("Skipping deleting the namespace, Namespace: %s", ns) + } + + err := s.cli.CoreV1().Namespaces().Delete(ns, &metav1.DeleteOptions{}) + if err != nil { + c.Logf("Failed to delete namespace, Namespace: %s, Error: %v", ns, err) + } +} diff --git a/pkg/kube/volume.go b/pkg/kube/volume/volume.go similarity index 73% rename from pkg/kube/volume.go rename to pkg/kube/volume/volume.go index c2d7d3e91b..4f23ca7565 100644 --- a/pkg/kube/volume.go +++ b/pkg/kube/volume/volume.go @@ -1,4 +1,4 @@ -package kube +package volume import ( "context" @@ -8,6 +8,7 @@ import ( "strings" "time" + snapshotclient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" "github.com/pkg/errors" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -61,8 +62,8 @@ func CreatePVC(ctx context.Context, kubeCli kubernetes.Interface, ns string, nam pvc.ObjectMeta.GenerateName = pvcGenerateName } - // If targetVolID is set, static provisioning is desired if targetVolID != "" { + // If targetVolID is set, static provisioning is desired pvc.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{pvMatchLabelName: filepath.Base(targetVolID)}, } @@ -79,6 +80,60 @@ func CreatePVC(ctx context.Context, kubeCli kubernetes.Interface, ns string, nam return createdPVC.Name, nil } +// CreatePVCFromSnapshot will restore a volume and returns the resulting +// PersistentVolumeClaim and any error that happened in the process. +// +// 'volumeName' is the name of the PVC that will be restored from the snapshot. +// 'snapshotName' is the name of the VolumeSnapshot that will be used for restoring. +// 'namespace' is the namespace of the VolumeSnapshot. The PVC will be restored to the same namepsace. +// 'restoreSize' will override existing restore size from snapshot content if provided. +func CreatePVCFromSnapshot(ctx context.Context, kubeCli kubernetes.Interface, snapCli snapshotclient.Interface, namespace, volumeName, snapshotName string, restoreSize *int) (string, error) { + snap, err := snapCli.VolumesnapshotV1alpha1().VolumeSnapshots(namespace).Get(snapshotName, metav1.GetOptions{}) + if err != nil { + return "", err + } + size := snap.Status.RestoreSize + if restoreSize != nil { + s := resource.MustParse(fmt.Sprintf("%dGi", restoreSize)) + size = &s + } + if size == nil { + return "", fmt.Errorf("Restore size is empty and no restore size argument given, Volumesnapshot: %s", snap.Name) + } + + snapshotKind := "VolumeSnapshot" + snapshotAPIGroup := "snapshot.storage.k8s.io" + pvc := &v1.PersistentVolumeClaim{ + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: snapshotKind, + Name: snapshotName, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: *size, + }, + }, + }, + } + if volumeName != "" { + pvc.ObjectMeta.Name = volumeName + } else { + pvc.ObjectMeta.GenerateName = pvcGenerateName + } + + pvc, err = kubeCli.CoreV1().PersistentVolumeClaims(namespace).Create(pvc) + if err != nil { + if volumeName != "" && apierrors.IsAlreadyExists(err) { + return volumeName, nil + } + return "", errors.Wrapf(err, "Unable to create PVC, PVC: %v", pvc) + } + return "", err +} + // CreatePV creates a PersistentVolume and returns its name // For retry idempotency, checks whether PV associated with volume already exists func CreatePV(ctx context.Context, kubeCli kubernetes.Interface, vol *blockstorage.Volume, volType blockstorage.Type, annotations map[string]string) (string, error) { diff --git a/pkg/kube/volume_test.go b/pkg/kube/volume/volume_test.go similarity index 95% rename from pkg/kube/volume_test.go rename to pkg/kube/volume/volume_test.go index 4aa913629f..c2b8d94c2c 100644 --- a/pkg/kube/volume_test.go +++ b/pkg/kube/volume/volume_test.go @@ -1,9 +1,10 @@ -package kube +package volume import ( "context" "path/filepath" "reflect" + "testing" . "gopkg.in/check.v1" "k8s.io/api/core/v1" @@ -11,6 +12,8 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +func Test(t *testing.T) { TestingT(t) } + type TestVolSuite struct{} var _ = Suite(&TestVolSuite{})