Skip to content

Commit

Permalink
Merge pull request #215 from kanisterio/efs-assume-role
Browse files Browse the repository at this point in the history 
Add AssumeRole support for EFS provider.
  • Loading branch information
Hakan Memisoglu committed Aug 21, 2019
2 parents 809a5d0 + 22213ad commit 2210b2c
Show file tree
Hide file tree
Showing 12 changed files with 85 additions and 59 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ require (
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.10.0
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
google.golang.org/api v0.3.1
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ github.com/aws/aws-sdk-go v1.20.12 h1:xV7xfLSkiqd7JOnLlfER+Jz8kI98rAGJvtXssYkCRs
github.com/aws/aws-sdk-go v1.20.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.20 h1:OAR/GtjMOhenkp1NNKr1N1FgIP3mQXHeGbRhvVIAQp0=
github.com/aws/aws-sdk-go v1.20.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.4 h1:F6f/iQRhuSfrpUdy80q29898H0NYN27pX+95tkJ+BIY=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
Expand Down Expand Up @@ -452,6 +453,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
Expand Down
30 changes: 2 additions & 28 deletions pkg/blockstorage/awsebs/awsebs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
Expand All @@ -21,6 +20,7 @@ import (
"github.com/kanisterio/kanister/pkg/blockstorage"
ktags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
"github.com/kanisterio/kanister/pkg/blockstorage/zone"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
)

var _ blockstorage.Provider = (*ebsStorage)(nil)
Expand All @@ -38,14 +38,6 @@ type EC2 struct {

const (
maxRetries = 10
// ConfigRegion represents region key required in the map "config"
ConfigRegion = "region"
// AccessKeyID represents AWS Access key ID
AccessKeyID = "AWS_ACCESS_KEY_ID"
// SecretAccessKey represents AWS Secret Access Key
SecretAccessKey = "AWS_SECRET_ACCESS_KEY"
// SessionToken represents AWS Session Key
SessionToken = "AWS_SESSION_TOKEN"
)

func (s *ebsStorage) Type() blockstorage.Type {
Expand All @@ -54,7 +46,7 @@ func (s *ebsStorage) Type() blockstorage.Type {

// NewProvider returns a provider for the EBS storage type in the specified region
func NewProvider(config map[string]string) (blockstorage.Provider, error) {
awsConfig, region, err := GetConfig(config)
awsConfig, region, _, err := awsconfig.GetConfig(config)
if err != nil {
return nil, err
}
Expand All @@ -65,24 +57,6 @@ func NewProvider(config map[string]string) (blockstorage.Provider, error) {
return &ebsStorage{ec2Cli: ec2Cli}, nil
}

// GetConfig returns a configuration to establish AWS connection and the connected region name.
func GetConfig(config map[string]string) (*aws.Config, string, error) {
region, ok := config[ConfigRegion]
if !ok {
return nil, "", errors.New("region required for storage type EBS")
}
accessKey, ok := config[AccessKeyID]
if !ok {
return nil, "", errors.New("AWS_ACCESS_KEY_ID required for storage type EBS")
}
secretAccessKey, ok := config[SecretAccessKey]
if !ok {
return nil, "", errors.New("AWS_SECRET_ACCESS_KEY required for storage type EBS")
}
sessionToken := config[SessionToken]
return &aws.Config{Credentials: credentials.NewStaticCredentials(accessKey, secretAccessKey, sessionToken)}, region, nil
}

// newEC2Client returns ec2 client struct.
func newEC2Client(awsRegion string, config *aws.Config) (*EC2, error) {
httpClient := &http.Client{Transport: http.DefaultTransport}
Expand Down
16 changes: 11 additions & 5 deletions pkg/blockstorage/awsefs/awsefs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/backup"
awsefs "github.com/aws/aws-sdk-go/service/efs"
Expand All @@ -15,8 +16,8 @@ import (
"k8s.io/apimachinery/pkg/util/rand"

"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
kantags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
)

type efs struct {
Expand Down Expand Up @@ -44,7 +45,7 @@ const (

// NewEFSProvider retuns a blockstorage provider for AWS EFS.
func NewEFSProvider(config map[string]string) (blockstorage.Provider, error) {
awsConfig, region, err := awsebs.GetConfig(config)
awsConfig, region, role, err := awsconfig.GetConfig(config)
if err != nil {
return nil, errors.Wrap(err, "Failed to get configuration for EFS")
}
Expand All @@ -61,13 +62,18 @@ func NewEFSProvider(config map[string]string) (blockstorage.Provider, error) {
return nil, errors.New("Account ID is empty")
}
accountID := *user.Account
efsCli := awsefs.New(s, aws.NewConfig().WithRegion(region))
backupCli := backup.New(s, aws.NewConfig().WithRegion(region))
creds := awsConfig.Credentials
if role != "" {
creds = stscreds.NewCredentials(s, role)
}
efsCli := awsefs.New(s, aws.NewConfig().WithRegion(region).WithCredentials(creds))
backupCli := backup.New(s, aws.NewConfig().WithRegion(region).WithCredentials(creds))
return &efs{
EFS: efsCli,
Backup: backupCli,
region: region,
accountID: accountID}, nil
accountID: accountID,
}, nil
}

func (e *efs) Type() blockstorage.Type {
Expand Down
12 changes: 6 additions & 6 deletions pkg/blockstorage/blockstorage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
. "gopkg.in/check.v1"

"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
ktags "github.com/kanisterio/kanister/pkg/blockstorage/tags"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
)

const (
Expand Down Expand Up @@ -246,17 +246,17 @@ func (s *BlockStorageProviderSuite) getConfig(c *C, region string) map[string]st
config := make(map[string]string)
switch s.storageType {
case blockstorage.TypeEBS:
config[awsebs.ConfigRegion] = region
accessKey, ok := os.LookupEnv(awsebs.AccessKeyID)
config[awsconfig.ConfigRegion] = region
accessKey, ok := os.LookupEnv(awsconfig.AccessKeyID)
if !ok {
c.Skip("The necessary env variable AWS_ACCESS_KEY_ID is not set.")
}
secretAccessKey, ok := os.LookupEnv(awsebs.SecretAccessKey)
secretAccessKey, ok := os.LookupEnv(awsconfig.SecretAccessKey)
if !ok {
c.Skip("The necessary env variable AWS_SECRET_ACCESS_KEY is not set.")
}
config[awsebs.AccessKeyID] = accessKey
config[awsebs.SecretAccessKey] = secretAccessKey
config[awsconfig.AccessKeyID] = accessKey
config[awsconfig.SecretAccessKey] = secretAccessKey
case blockstorage.TypeGPD:
creds, ok := os.LookupEnv(blockstorage.GoogleCloudCreds)
if !ok {
Expand Down
41 changes: 41 additions & 0 deletions pkg/config/aws/aws.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package aws

import (
"errors"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
)

const (
// ConfigRegion represents region key required in the map "config"
ConfigRegion = "region"
// ConfigRole represents the key for the ARN of the role which can be assumed.
// It is optional.
ConfigRole = "role"
// AccessKeyID represents AWS Access key ID
AccessKeyID = "AWS_ACCESS_KEY_ID"
// SecretAccessKey represents AWS Secret Access Key
SecretAccessKey = "AWS_SECRET_ACCESS_KEY"
// SessionToken represents AWS Session Key
SessionToken = "AWS_SESSION_TOKEN"
)

// GetConfig returns a configuration to establish AWS connection, connected region name and the role to assume if it exists.
func GetConfig(config map[string]string) (awsConfig *aws.Config, region string, role string, err error) {
region, ok := config[ConfigRegion]
if !ok {
return nil, "", "", errors.New("region required for storage type EBS")
}
accessKey, ok := config[AccessKeyID]
if !ok {
return nil, "", "", errors.New("AWS_ACCESS_KEY_ID required for storage type EBS")
}
secretAccessKey, ok := config[SecretAccessKey]
if !ok {
return nil, "", "", errors.New("AWS_SECRET_ACCESS_KEY required for storage type EBS")
}
sessionToken := config[SessionToken]
role = config[ConfigRole]
return &aws.Config{Credentials: credentials.NewStaticCredentials(accessKey, secretAccessKey, sessionToken)}, region, role, nil
}
8 changes: 4 additions & 4 deletions pkg/function/create_volume_from_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (

kanister "github.com/kanisterio/kanister/pkg"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/kube"
kubevolume "github.com/kanisterio/kanister/pkg/kube/volume"
"github.com/kanisterio/kanister/pkg/param"
Expand Down Expand Up @@ -60,9 +60,9 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
config[awsconfig.ConfigRegion] = pvcInfo.Region
config[awsconfig.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsconfig.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
Expand Down
7 changes: 4 additions & 3 deletions pkg/function/create_volume_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/kube"
kubevolume "github.com/kanisterio/kanister/pkg/kube/volume"
"github.com/kanisterio/kanister/pkg/param"
Expand Down Expand Up @@ -203,9 +204,9 @@ func getPVCInfo(ctx context.Context, kubeCli kubernetes.Interface, namespace str
}
}
if pvZone, ok := pvLabels[kubevolume.PVZoneLabelName]; ok {
config[awsebs.ConfigRegion] = region
config[awsebs.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
config[awsconfig.ConfigRegion] = region
config[awsconfig.AccessKeyID] = tp.Profile.Credential.KeyPair.ID
config[awsconfig.SecretAccessKey] = tp.Profile.Credential.KeyPair.Secret
provider, err = getter.Get(blockstorage.TypeEBS, config)
if err != nil {
return nil, errors.Wrap(err, "Could not get storage provider")
Expand Down
8 changes: 4 additions & 4 deletions pkg/function/delete_volume_snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (

kanister "github.com/kanisterio/kanister/pkg"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/param"
)
Expand Down Expand Up @@ -52,9 +52,9 @@ func deleteVolumeSnapshot(ctx context.Context, cli kubernetes.Interface, namespa
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
config[awsconfig.ConfigRegion] = pvcInfo.Region
config[awsconfig.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsconfig.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
Expand Down
4 changes: 2 additions & 2 deletions pkg/function/e2e_volume_snapshot_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
kanister "github.com/kanisterio/kanister/pkg"
crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/client/clientset/versioned"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/param"
"github.com/kanisterio/kanister/pkg/resource"
Expand Down Expand Up @@ -327,7 +327,7 @@ func (s *VolumeSnapshotTestSuite) getCreds(c *C, cli kubernetes.Interface, names
switch {
case pv.Spec.AWSElasticBlockStore != nil:
_ = GetEnvOrSkip(c, AWSRegion)
return GetEnvOrSkip(c, awsebs.AccessKeyID), GetEnvOrSkip(c, awsebs.SecretAccessKey), crv1alpha1.LocationTypeS3Compliant, nil
return GetEnvOrSkip(c, awsconfig.AccessKeyID), GetEnvOrSkip(c, awsconfig.SecretAccessKey), crv1alpha1.LocationTypeS3Compliant, nil

case pv.Spec.GCEPersistentDisk != nil:
serviceKey, err := getServiceKey(c)
Expand Down
8 changes: 4 additions & 4 deletions pkg/function/wait_for_snapshot_completion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (

kanister "github.com/kanisterio/kanister/pkg"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
"github.com/kanisterio/kanister/pkg/blockstorage/getter"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/param"
)

Expand Down Expand Up @@ -56,9 +56,9 @@ func waitForSnapshotsCompletion(ctx context.Context, snapshotinfo string, profil
}
switch pvcInfo.Type {
case blockstorage.TypeEBS:
config[awsebs.ConfigRegion] = pvcInfo.Region
config[awsebs.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsebs.SecretAccessKey] = profile.Credential.KeyPair.Secret
config[awsconfig.ConfigRegion] = pvcInfo.Region
config[awsconfig.AccessKeyID] = profile.Credential.KeyPair.ID
config[awsconfig.SecretAccessKey] = profile.Credential.KeyPair.Secret
case blockstorage.TypeGPD:
config[blockstorage.GoogleProjectID] = profile.Credential.KeyPair.ID
config[blockstorage.GoogleServiceKey] = profile.Credential.KeyPair.Secret
Expand Down
6 changes: 3 additions & 3 deletions pkg/testutil/fixture.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/kanisterio/kanister/pkg/blockstorage"
"github.com/kanisterio/kanister/pkg/blockstorage/awsebs"
awsconfig "github.com/kanisterio/kanister/pkg/config/aws"
"github.com/kanisterio/kanister/pkg/objectstore"
"github.com/kanisterio/kanister/pkg/param"
)
Expand All @@ -26,8 +26,8 @@ func ObjectStoreProfileOrSkip(c *check.C, osType objectstore.ProviderType, locat

switch osType {
case objectstore.ProviderTypeS3:
key = GetEnvOrSkip(c, awsebs.AccessKeyID)
val = GetEnvOrSkip(c, awsebs.SecretAccessKey)
key = GetEnvOrSkip(c, awsconfig.AccessKeyID)
val = GetEnvOrSkip(c, awsconfig.SecretAccessKey)
case objectstore.ProviderTypeGCS:
GetEnvOrSkip(c, blockstorage.GoogleCloudCreds)
creds, err := google.FindDefaultCredentials(context.Background(), compute.ComputeScope)
Expand Down

0 comments on commit 2210b2c

Please sign in to comment.