Skip to content

Commit

Permalink
Using dynamic methods to fetch zones. (#1185)
Browse files Browse the repository at this point in the history 
* gce and start of aws

* calling aws Get regions

* changes to dynamically fetch zones

* ut update

* adding GetRegions methods

* UTs for gke and azure

* Additional messaging

* after Onkars review

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
@bathina2 @mergify
bathina2 and mergify[bot] committed Jan 22, 2022
1 parent e5a2867 commit 15ff2bb
Show file tree
Hide file tree
Showing 8 changed files with 217 additions and 552 deletions.
230 changes: 38 additions & 192 deletions pkg/blockstorage/awsebs/awsebs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ var _ zone.Mapper = (*EbsStorage)(nil)
type EbsStorage struct {
Ec2Cli *EC2
Role string
config *aws.Config
}

// EC2 is kasten's wrapper around ec2.EC2 structs
Expand Down Expand Up @@ -72,7 +73,7 @@ func NewProvider(ctx context.Context, config map[string]string) (blockstorage.Pr
if err != nil {
return nil, errors.Wrapf(err, "Could not get EC2 client")
}
return &EbsStorage{Ec2Cli: ec2Cli, Role: config[awsconfig.ConfigRole]}, nil
return &EbsStorage{Ec2Cli: ec2Cli, Role: config[awsconfig.ConfigRole], config: awsConfig}, nil
}

// newEC2Client returns ec2 client struct.
Expand All @@ -82,7 +83,7 @@ func newEC2Client(awsRegion string, config *aws.Config) (*EC2, error) {
}
s, err := session.NewSession(config)
if err != nil {
return nil, errors.Wrap(err, "Failed to create session for EFS")
return nil, errors.Wrap(err, "Failed to create session for EBS")
}
conf := config.WithMaxRetries(maxRetries).WithRegion(awsRegion).WithCredentials(config.Credentials)
return &EC2{EC2: ec2.New(s, conf)}, nil
Expand Down Expand Up @@ -618,8 +619,40 @@ func GetRegionFromEC2Metadata() (string, error) {

// FromRegion is part of zone.Mapper
func (s *EbsStorage) FromRegion(ctx context.Context, region string) ([]string, error) {
// Fall back to using a static map.
return staticRegionToZones(region)
ec2Cli, err := newEC2Client(region, s.config)
if err != nil {
return nil, errors.Wrapf(err, "Could not get EC2 client while fetching zones FromRegion (%s)", region)
}
trueBool := true
filterKey := "region-name"
zones, err := ec2Cli.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{
AllAvailabilityZones: &trueBool,
Filters: []*ec2.Filter{
{Name: &filterKey, Values: []*string{&region}},
},
})
if err != nil {
return nil, errors.Wrapf(err, "Failed to get availability zones for region %s", region)
}
zoneList := []string{}
for _, zone := range zones.AvailabilityZones {
zoneList = append(zoneList, *zone.ZoneName)
}
return zoneList, nil
}

func (s *EbsStorage) GetRegions(ctx context.Context) ([]string, error) {
trueBool := true
result, err := s.Ec2Cli.DescribeRegions(&ec2.DescribeRegionsInput{AllRegions: &trueBool})
if err != nil {
return nil, errors.Wrap(err, "Failed to describe regions")
}
regions := []string{}

for _, region := range result.Regions {
regions = append(regions, *region.RegionName)
}
return regions, nil
}

func (s *EbsStorage) queryRegionToZones(ctx context.Context, region string) ([]string, error) {
Expand Down Expand Up @@ -651,196 +684,9 @@ func (s *EbsStorage) SnapshotRestoreTargets(ctx context.Context, snapshot *block
return false, nil, errors.Errorf("Required volume fields not available, volumeType: %s, Az: %s, VolumeTags: %v", snapshot.Volume.VolumeType, snapshot.Volume.Az, snapshot.Volume.Tags)
}
// EBS snapshots can only be restored in their region
zl, err := staticRegionToZones(snapshot.Region)
zl, err := s.FromRegion(ctx, snapshot.Region)
if err != nil {
return false, nil, err
}
return false, map[string][]string{snapshot.Region: zl}, nil
}

func staticRegionToZones(region string) ([]string, error) {
switch region {
case "ap-south-1":
return []string{
"ap-south-1a",
"ap-south-1b",
"ap-south-1c",
}, nil
case "eu-west-3":
return []string{
"eu-west-3a",
"eu-west-3b",
"eu-west-3c",
}, nil
case "eu-north-1":
return []string{
"eu-north-1a",
"eu-north-1b",
"eu-north-1c",
}, nil
case "eu-west-2":
return []string{
"eu-west-2-wl1-lon-wlz-1",
"eu-west-2a",
"eu-west-2b",
"eu-west-2c",
}, nil
case "eu-west-1":
return []string{
"eu-west-1a",
"eu-west-1b",
"eu-west-1c",
}, nil
case "ap-northeast-2":
return []string{
"ap-northeast-2-wl1-cjj-wlz-1",
"ap-northeast-2a",
"ap-northeast-2b",
"ap-northeast-2c",
"ap-northeast-2d",
}, nil
case "ap-northeast-1":
return []string{
"ap-northeast-1-wl1-nrt-wlz-1",
"ap-northeast-1-wl1-kix-wlz-1",
"ap-northeast-1a",
"ap-northeast-1c",
"ap-northeast-1d",
}, nil
case "sa-east-1":
return []string{
"sa-east-1a",
"sa-east-1b",
"sa-east-1c",
}, nil
case "ca-central-1":
return []string{
"ca-central-1a",
"ca-central-1b",
"ca-central-1d",
}, nil
case "ap-southeast-1":
return []string{
"ap-southeast-1a",
"ap-southeast-1b",
"ap-southeast-1c",
}, nil
case "ap-southeast-2":
return []string{
"ap-southeast-2a",
"ap-southeast-2b",
"ap-southeast-2c",
}, nil
case "ap-southeast-3":
return []string{
"ap-southeast-3a",
"ap-southeast-3b",
"ap-southeast-3c",
}, nil
case "eu-central-1":
return []string{
"eu-central-1a",
"eu-central-1b",
"eu-central-1c",
"eu-central-1-wl1-ber-wlz-1",
"eu-central-1-wl1-dtm-wlz-1",
"eu-central-1-wl1-muc-wlz-1",
}, nil
case "us-east-1":
return []string{
"us-east-1a",
"us-east-1b",
"us-east-1c",
"us-east-1d",
"us-east-1e",
"us-east-1f",
"us-east-1-bos-1a",
"us-east-1-chi-1a",
"us-east-1-dfw-1a",
"us-east-1-iah-1a",
"us-east-1-mci-1a",
"us-east-1-mia-1a",
"us-east-1-msp-1a",
"us-east-1-nyc-1a",
"us-east-1-phl-1a",
"us-east-1-wl1-atl-wlz-1",
"us-east-1-wl1-bos-wlz-1",
"us-east-1-wl1-chi-wlz-1",
"us-east-1-wl1-dfw-wlz-1",
"us-east-1-wl1-iah-wlz-1",
"us-east-1-wl1-mia-wlz-1",
"us-east-1-wl1-nyc-wlz-1",
"us-east-1-wl1-was-wlz-1",
}, nil
case "us-east-2":
return []string{
"us-east-2a",
"us-east-2b",
"us-east-2c",
}, nil
case "us-west-1":
return []string{
"us-west-1a",
"us-west-1b",
}, nil
case "us-west-2":
return []string{
"us-west-2a",
"us-west-2b",
"us-west-2c",
"us-west-2d",
"us-west-2-den-1a",
"us-west-2-las-1a",
"us-west-2-lax-1a",
"us-west-2-lax-1b",
"us-west-2-pdx-1a",
"us-west-2-wl1-den-wlz-1",
"us-west-2-wl1-las-wlz-1",
"us-west-2-wl1-phx-wlz-1",
"us-west-2-wl1-sea-wlz-1",
"us-west-2-wl1-sfo-wlz-1",
}, nil
case "ap-east-1":
return []string{
"ap-east-1a",
"ap-east-1b",
"ap-east-1c",
}, nil
case "me-south-1":
return []string{
"me-south-1a",
"me-south-1b",
"me-south-1c",
}, nil
case "eu-south-1":
return []string{
"eu-south-1a",
"eu-south-1b",
"eu-south-1c",
}, nil
case "af-south-1":
return []string{
"af-south-1a",
"af-south-1b",
"af-south-1c",
}, nil
case "ap-northeast-3":
return []string{
"ap-northeast-3a",
"ap-northeast-3b",
"ap-northeast-3c",
}, nil
case "cn-north-1":
return []string{
"cn-north-1a",
"cn-north-1b",
}, nil
case "cn-northwest-1":
return []string{
"cn-northwest-1a",
"cn-northwest-1b",
"cn-northwest-1c",
}, nil
}
return nil, errors.New("Cannot get availability zones for region")
}
30 changes: 30 additions & 0 deletions pkg/blockstorage/awsebs/awsebs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package awsebs

import (
"context"
"strings"
"testing"
"time"

Expand All @@ -24,6 +25,9 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
. "gopkg.in/check.v1"

kaws "github.com/kanisterio/kanister/pkg/aws"
envconfig "github.com/kanisterio/kanister/pkg/config"

"github.com/kanisterio/kanister/pkg/blockstorage"
)

Expand Down Expand Up @@ -93,3 +97,29 @@ func (s AWSEBSSuite) TestVolumeParse(c *C) {
c.Check(volume.VolumeType, Equals, expected.VolumeType)
c.Check(volume.Attributes, DeepEquals, expected.Attributes)
}

func (s AWSEBSSuite) TestGetRegions(c *C) {
ctx := context.Background()
config := map[string]string{}

config[kaws.AccessKeyID] = envconfig.GetEnvOrSkip(c, kaws.AccessKeyID)
config[kaws.SecretAccessKey] = envconfig.GetEnvOrSkip(c, kaws.SecretAccessKey)

// create provider with region
config[kaws.ConfigRegion] = "us-west-2"
bsp, err := NewProvider(ctx, config)
c.Assert(err, IsNil)
ebsp := bsp.(*EbsStorage)

// get zones with other region
zones, err := ebsp.FromRegion(ctx, "us-east-1")
c.Assert(err, IsNil)
for _, zone := range zones {
c.Assert(strings.Contains(zone, "us-east-1"), Equals, true)
c.Assert(strings.Contains(zone, "us-west-2"), Equals, false)
}

regions, err := ebsp.GetRegions(ctx)
c.Assert(err, IsNil)
c.Assert(regions, NotNil)
}
11 changes: 0 additions & 11 deletions pkg/blockstorage/awsebs/zone_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
package awsebs

import (
"context"

. "gopkg.in/check.v1"

"github.com/kanisterio/kanister/pkg/blockstorage/zone"
Expand Down Expand Up @@ -53,12 +51,3 @@ func (s ZoneSuite) TestZoneWithUnknownNodeZones(c *C) {
c.Assert(z, DeepEquals, tc.out)
}
}

var _ zone.Mapper = (*ebsTest)(nil)

type ebsTest struct{}

func (et *ebsTest) FromRegion(ctx context.Context, region string) ([]string, error) {
// Fall back to using a static map.
return staticRegionToZones(region)
}

0 comments on commit 15ff2bb

Please sign in to comment.