Skip to content

Commit

Permalink
Update kanister snapshot functions to include gcp support (#4804)
Browse files Browse the repository at this point in the history 
* copy k10/blockstorage/gcepd to kansiter

* Modify blockstorage for GCP Volume snapshot

* Use poll.Wait() instead of time.Sleep()

* Update snapshot func for gcp support

* nit fix

* Address review suggestions
  • Loading branch information
@SupriyaKasten
SupriyaKasten authored and Ilya Kislenko committed Jan 29, 2019
1 parent 87783f0 commit c379c15
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 54 deletions.
9 changes: 6 additions & 3 deletions pkg/function/create_volume_from_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,17 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam
pvcName = pvcNames[i]
}
config := make(map[string]string)
if err = ValidateProfile(profile, pvcInfo.Type); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
if err = ValidateProfile(profile); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
}
provider, err := getter.Get(pvcInfo.Type, config)
if err != nil {
Expand Down
56 changes: 40 additions & 16 deletions pkg/function/create_volume_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,25 +62,29 @@ type volumeInfo struct {
region string
}

func ValidateProfile(profile *param.Profile) error {
func ValidateProfile(profile *param.Profile, sType blockstorage.Type) error {
if profile == nil {
return errors.New("Profile must be non-nil")
}
if profile.Location.Type != crv1alpha1.LocationTypeS3Compliant {
return errors.New("Location type not supported")
}
if len(profile.Location.Region) == 0 {
return errors.New("Region is not set")
}

if profile.Credential.Type != param.CredentialTypeKeyPair {
return errors.New("Credential type not supported")
}
if len(profile.Credential.KeyPair.ID) == 0 {
return errors.New("AWS access key id is not set")
return errors.New("Access key ID is not set")
}
if len(profile.Credential.KeyPair.Secret) == 0 {
return errors.New("Secret access key is not set")
}
switch sType {
case blockstorage.TypeEBS:
if profile.Location.Type != crv1alpha1.LocationTypeS3Compliant {
return errors.New("Location type not supported")
}
if len(profile.Location.Region) == 0 {
return errors.New("Region is not set")
}
}
return nil
}

Expand Down Expand Up @@ -178,8 +182,10 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str
// Check to see which provider is the source. Spec mandates only one of the provider
// fields will be set
config := make(map[string]string)
if ebs := pv.Spec.AWSElasticBlockStore; ebs != nil {
if err = ValidateProfile(tp.Profile); err != nil {
switch {
case pv.Spec.AWSElasticBlockStore != nil:
ebs := pv.Spec.AWSElasticBlockStore
if err = ValidateProfile(tp.Profile, blockstorage.TypeEBS); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
// Get Region from PV label or EC2 metadata
Expand All @@ -190,20 +196,38 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str
if err != nil {
return nil, err
}

if pvZone, ok := pvLabels[kube.PVZoneLabelName]; ok {
config[awsebs.ConfigRegion] = region
config[awsebs.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
provider, err = getter.Get(blockstorage.TypeEBS, config)
if err != nil {
return nil, errors.Wrap(err, "Could not get storage provider")
}
return &volumeInfo{provider: provider, volumeID: filepath.Base(ebs.VolumeID), sType: blockstorage.TypeEBS, volZone: pvZone, pvc: name, size: size, region: region}, nil
}
return nil, errors.Errorf("PV zone label is empty, pvName: %s, namespace: %s", pvName, namespace)
}

case pv.Spec.GCEPersistentDisk != nil:
gpd := pv.Spec.GCEPersistentDisk
region = ""
if err = ValidateProfile(tp.Profile, blockstorage.TypeGPD); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
if pvZone, ok := pvLabels[kube.PVZoneLabelName]; ok {
config[awsebs.ConfigRegion] = region
config[awsebs.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
provider, err = getter.Get(blockstorage.TypeEBS, config)
config[blockstorage.GoogleProjectID] = tp.Profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = tp.Profile.Credential.KeyPair.Secret
provider, err = getter.Get(blockstorage.TypeGPD, config)
if err != nil {
return nil, errors.Wrap(err, "Could not get storage provider")
}
return &volumeInfo{provider: provider, volumeID: filepath.Base(ebs.VolumeID), sType: blockstorage.TypeEBS, volZone: pvZone, pvc: name, size: size, region: region}, nil
return &volumeInfo{provider: provider, volumeID: filepath.Base(gpd.PDName), sType: blockstorage.TypeGPD, volZone: pvZone, pvc: name, size: size, region: region}, nil
}
return nil, errors.Errorf("PV zone label is empty, pvName: %s, namespace: %s", pvName, namespace)
}
return nil, errors.New("Storage type not supported")
return nil, errors.New("Storage type not supported!")
}

func getPVCList(tp param.TemplateParams) ([]string, error) {
Expand Down
9 changes: 6 additions & 3 deletions pkg/function/delete_volume_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,17 @@ func deleteVolumeSnapshot(ctx context.Context, cli kubernetes.Interface, namespa
providerList := make(map[string]blockstorage.Provider)
for _, pvcInfo := range PVCData {
config := make(map[string]string)
if err = ValidateProfile(profile, pvcInfo.Type); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
if err = ValidateProfile(profile); err != nil {
return nil, errors.Wrap(err, "Profile validation failed")
}
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
}
provider, err := getter.Get(pvcInfo.Type, config)
if err != nil {
Expand Down
102 changes: 73 additions & 29 deletions pkg/function/e2e_volume_snapshot_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package function

import (
"context"
"io/ioutil"
"os"
"strings"

Expand All @@ -14,14 +15,13 @@ import (

kanister "github.com/kanisterio/kanister/pkg"
crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
"github.com/kanisterio/kanister/pkg/client/clientset/versioned"
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/param"
"github.com/kanisterio/kanister/pkg/resource"
"github.com/kanisterio/kanister/pkg/testutil"
"github.com/kanisterio/kanister/pkg/testutil/mockblockstorage"
)

const (
Expand All @@ -33,11 +33,10 @@ const (
)

type VolumeSnapshotTestSuite struct {
cli kubernetes.Interface
crCli versioned.Interface
namespace string
mockGetter getter.Getter
tp *param.TemplateParams
cli kubernetes.Interface
crCli versioned.Interface
namespace string
tp *param.TemplateParams
}

var _ = Suite(&VolumeSnapshotTestSuite{})
Expand All @@ -63,20 +62,31 @@ func (s *VolumeSnapshotTestSuite) SetUpTest(c *C) {
c.Assert(err, IsNil)
s.namespace = cns.GetName()

sec := NewTestProfileSecret()
sec, err = s.cli.Core().Secrets(s.namespace).Create(sec)
ctx := context.Background()
ss, err := s.cli.AppsV1().StatefulSets(s.namespace).Create(newStatefulSet(s.namespace))
c.Assert(err, IsNil)

p := NewTestProfile(s.namespace, sec.GetName())
_, err = s.crCli.CrV1alpha1().Profiles(s.namespace).Create(p)
err = kube.WaitOnStatefulSetReady(ctx, s.cli, ss.GetNamespace(), ss.GetName())
c.Assert(err, IsNil)

s.mockGetter = mockblockstorage.NewGetter()
pods, _, err := kube.FetchPods(s.cli, s.namespace, ss.UID)
c.Assert(err, IsNil)
volToPvc := kube.StatefulSetVolumes(s.cli, ss, &pods[0])
pvc, _ := volToPvc[pods[0].Spec.Containers[0].VolumeMounts[0].Name]
c.Assert(len(pvc) > 0, Equals, true)
id, secret, err := s.getCreds(c, s.cli, s.namespace, pvc)
c.Assert(err, IsNil)
if id == "" || secret == "" {
c.Skip("Skipping the test since storage type not supported")
}

ctx := context.Background()
ss, err := s.cli.AppsV1().StatefulSets(s.namespace).Create(newStatefulSet(s.namespace))
serviceKey, err := getServiceKey(c)
c.Assert(err, IsNil)
err = kube.WaitOnStatefulSetReady(ctx, s.cli, ss.GetNamespace(), ss.GetName())
sec := NewTestProfileSecret(serviceKey, id, secret)
sec, err = s.cli.Core().Secrets(s.namespace).Create(sec)
c.Assert(err, IsNil)

p := NewTestProfile(s.namespace, sec.GetName())
_, err = s.crCli.CrV1alpha1().Profiles(s.namespace).Create(p)
c.Assert(err, IsNil)

as := crv1alpha1.ActionSpec{
Expand All @@ -86,7 +96,7 @@ func (s *VolumeSnapshotTestSuite) SetUpTest(c *C) {
Namespace: s.namespace,
},
Profile: &crv1alpha1.ObjectReference{
Name: testutil.TestProfileName,
Name: p.GetName(),
Namespace: s.namespace,
},
}
Expand All @@ -98,14 +108,14 @@ func (s *VolumeSnapshotTestSuite) SetUpTest(c *C) {
}

// NewTestProfileSecret function returns a pointer to a new Secret test object.
func NewTestProfileSecret() *v1.Secret {
func NewTestProfileSecret(serviceKey string, id string, secret string) *v1.Secret {
return &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "test-secret-",
},
StringData: map[string]string{
"id": os.Getenv(awsebs.AccessKeyID),
"secret": os.Getenv(awsebs.SecretAccessKey),
"id": id,
"secret": secret,
},
}
}
Expand Down Expand Up @@ -279,15 +289,6 @@ func newStatefulSet(namespace string) *appsv1.StatefulSet {
}

func (s *VolumeSnapshotTestSuite) TestVolumeSnapshot(c *C) {
if len(os.Getenv(AWSRegion)) == 0 {
c.Skip("Skipping the test since env variable AWS_REGION is not set")
}
if len(os.Getenv(awsebs.AccessKeyID)) == 0 {
c.Skip("Skipping the test since env variable AWS_ACCESS_KEY_ID is not set")
}
if len(os.Getenv(awsebs.SecretAccessKey)) == 0 {
c.Skip("Skipping the test since env variable AWS_SECRET_ACCESS_KEY is not set")
}
ctx := context.Background()
actions := []string{"backup", "restore", "delete"}
bp := newVolumeSnapshotBlueprint()
Expand All @@ -310,3 +311,46 @@ func (s *VolumeSnapshotTestSuite) TestVolumeSnapshot(c *C) {
}
}
}

func (s *VolumeSnapshotTestSuite) getCreds(c *C, cli kubernetes.Interface, namespace string, pvcname string) (string, string, error) {
pvc, err := cli.Core().PersistentVolumeClaims(namespace).Get(pvcname, metav1.GetOptions{})
if err != nil {
return "", "", err
}
pvName := pvc.Spec.VolumeName
pv, err := cli.Core().PersistentVolumes().Get(pvName, metav1.GetOptions{})
if err != nil {
return "", "", err
}
switch {
case pv.Spec.AWSElasticBlockStore != nil:
_ = GetEnvOrSkip(c, AWSRegion)
return GetEnvOrSkip(c, awsebs.AccessKeyID), GetEnvOrSkip(c, awsebs.SecretAccessKey), nil

case pv.Spec.GCEPersistentDisk != nil:
serviceKey, err := getServiceKey(c)
if err != nil {
return "", "", err
}
return "test_project_id", serviceKey, nil
}
return "", "", nil
}

func getServiceKey(c *C) (string, error) {
filename := GetEnvOrSkip(c, blockstorage.GoogleCloudCreds)
b, err := ioutil.ReadFile(filename)
if err != nil {
return "", err
}
return string(b), nil
}

func GetEnvOrSkip(c *C, varName string) string {
v := os.Getenv(varName)
// Ensure the variable is set
if v == "" {
c.Skip("Required environment variable " + varName + " is not set")
}
return v
}
9 changes: 6 additions & 3 deletions pkg/function/wait_for_snapshot_completion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,17 @@ func waitForSnapshotsCompletion(ctx context.Context, snapshotinfo string, profil
}
for _, pvcInfo := range PVCData {
config := make(map[string]string)
if err = ValidateProfile(profile, pvcInfo.Type); err != nil {
return errors.Wrap(err, "Profile validation failed")
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
if err = ValidateProfile(profile); err != nil {
return errors.Wrap(err, "Profile validation failed")
}
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
default:
return errors.New("Storage provider not supported " + string(pvcInfo.Type))
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/kube/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ func CreatePV(ctx context.Context, kubeCli kubernetes.Interface, vol *blockstora
}
pv.ObjectMeta.Labels[PVZoneLabelName] = vol.Az
pv.ObjectMeta.Labels[PVRegionLabelName] = zoneToRegion(vol.Az)
case blockstorage.TypeGPD:
pv.Spec.PersistentVolumeSource.GCEPersistentDisk = &v1.GCEPersistentDiskVolumeSource{
PDName: vol.ID,
}
pv.ObjectMeta.Labels[PVZoneLabelName] = vol.Az
pv.ObjectMeta.Labels[PVRegionLabelName] = zoneToRegion(vol.Az)
default:
return "", errors.Errorf("Volume type %v(%T) not supported ", volType, volType)
}
Expand Down

0 comments on commit c379c15

Please sign in to comment.