Merge pull request #172 from kanisterio/sync

Import Validation; azure kanister profile; azure objectstore support;  Export func IsBucketNotFoundError;

helm/profile/templates/profile.yaml

@@ -13,6 +13,9 @@ data:
   {{- else if .Values.gcp.projectID }}
   project_id: {{ .Values.gcp.projectID | b64enc | quote }}
   service_key: {{ .Values.gcp.serviceKey | b64enc | quote }}
+  {{- else if .Values.azure.storageAccount }}
+  storage_account: {{ .Values.azure.storageAccount | b64enc | quote }}
+  storage_key: {{ .Values.azure.storageKey | b64enc | quote }}
   {{- end }}
 
 ---
@@ -38,6 +41,9 @@ credential:
     {{- else if .Values.gcp.projectID }}
     idField: project_id
     secretField: service_key
+    {{- else if .Values.azure.storageAccount }}
+    idField: storage_account
+    secretField: storage_key
     {{- end }}
     secret:
       apiVersion: v1

helm/profile/values.yaml

@@ -20,4 +20,8 @@ gcp:
   projectID:
   serviceKey:
 
+azure:
+  storageAccount:
+  storageKey:
+
 verifySSL: true

pkg/apis/cr/v1alpha1/types.go

@@ -230,6 +230,7 @@ type LocationType string
 const (
 	LocationTypeGCS         LocationType = "gcs"
 	LocationTypeS3Compliant LocationType = "s3Compliant"
+	LocationTypeAzure       LocationType = "azure"
 )
 
 // Location

pkg/blockstorage/awsebs/awsebs.go

@@ -617,6 +617,22 @@ func queryRegionToZones(ctx context.Context, region string) ([]string, error) {
 	return azs, nil
 }
 
+func (s *ebsStorage) SnapshotRestoreTargets(ctx context.Context, snapshot *blockstorage.Snapshot) (global bool, regionsAndZones map[string][]string, err error) {
+	// A few checks from VolumeCreateFromSnapshot
+	if snapshot.Volume == nil {
+		return false, nil, errors.New("Snapshot volume information not available")
+	}
+	if snapshot.Volume.VolumeType == "" || snapshot.Volume.Az == "" || snapshot.Volume.Tags == nil {
+		return false, nil, errors.Errorf("Required volume fields not available, volumeType: %s, Az: %s, VolumeTags: %v", snapshot.Volume.VolumeType, snapshot.Volume.Az, snapshot.Volume.Tags)
+	}
+	// EBS snapshots can only be restored in their region
+	zl, err := staticRegionToZones(snapshot.Region)
+	if err != nil {
+		return false, nil, err
+	}
+	return false, map[string][]string{snapshot.Region: zl}, nil
+}
+
 func staticRegionToZones(region string) ([]string, error) {
 	switch region {
 	case "ap-south-1":

pkg/blockstorage/blockstorage.go

@@ -24,3 +24,10 @@ type Provider interface {
 	VolumesList(ctx context.Context, tags map[string]string, zone string) ([]*Volume, error)
 	SnapshotsList(ctx context.Context, tags map[string]string) ([]*Snapshot, error)
 }
+
+// RestoreTargeter implements the SnapshotRestoreTargets method
+type RestoreTargeter interface {
+	// SnapshotRestoreTargets returns whether a snapshot can be restored globally.
+	// If not globally restorable, returns a map of the regions and zones to which snapshot can be restored.
+	SnapshotRestoreTargets(context.Context, *Snapshot) (global bool, regionsAndZones map[string][]string, err error)
+}

pkg/blockstorage/helpers.go

@@ -8,10 +8,12 @@ import (
 
 // Google Cloud environment variable names
 const (
-	GoogleCloudZone  = "CLOUDSDK_COMPUTE_ZONE"
-	GoogleCloudCreds = "GOOGLE_APPLICATION_CREDENTIALS"
-	GoogleProjectID  = "projectID"
-	GoogleServiceKey = "serviceKey"
+	GoogleCloudZone     = "CLOUDSDK_COMPUTE_ZONE"
+	GoogleCloudCreds    = "GOOGLE_APPLICATION_CREDENTIALS"
+	GoogleProjectID     = "projectID"
+	GoogleServiceKey    = "serviceKey"
+	AzureStorageAccount = "AZURE_STORAGE_ACCOUNT_NAME"
+	AzureStorageKey     = "AZURE_STORAGE_ACCOUNT_KEY"
 )
 
 // SanitizeTags are used to sanitize the tags

pkg/blockstorage/zone/zone.go

@@ -16,6 +16,7 @@ import (
 )
 
 type (
+	// Mapper interface indicates provider that supports FromRegion mapping to list of zones
 	Mapper interface {
 		FromRegion(ctx context.Context, region string) ([]string, error)
 	}
@@ -29,7 +30,7 @@ func FromSourceRegionZone(ctx context.Context, m Mapper, region string, sourceZo
 	newZones := make(map[string]struct{})
 	cli, err := kube.NewClient()
 	if err == nil {
-		nzs, rs, errzr := nodeZonesAndRegion(ctx, cli)
+		nzs, rs, errzr := NodeZonesAndRegion(ctx, cli)
 		if err != nil {
 			log.Errorf("Ignoring error getting Node availability zones. Error: %+v", errzr)
 		}
@@ -171,7 +172,8 @@ const (
 	nodeZonesErr = `Failed to get Node availability zones.`
 )
 
-func nodeZonesAndRegion(ctx context.Context, cli kubernetes.Interface) (map[string]struct{}, string, error) {
+// NodeZonesAndRegion returns cloud provider failure-domain region and zones as reported by K8s
+func NodeZonesAndRegion(ctx context.Context, cli kubernetes.Interface) (map[string]struct{}, string, error) {
 	if cli == nil {
 		return nil, "", errors.New(nodeZonesErr)
 	}

pkg/blockstorage/zone/zone_kube_test.go

@@ -17,7 +17,7 @@ func (s KubeTestZoneSuite) TestNodeZones(c *C) {
 	ctx := context.Background()
 	cli, err := kube.NewClient()
 	c.Assert(err, IsNil)
-	zones, _, err := nodeZonesAndRegion(ctx, cli)
+	zones, _, err := NodeZonesAndRegion(ctx, cli)
 	c.Assert(err, IsNil)
 	c.Assert(zones, Not(HasLen), 0)
 }

pkg/blockstorage/zone/zone_test.go

@@ -82,7 +82,7 @@ func (s ZoneSuite) TestNodeZoneAndRegionGCP(c *C) {
 	expectedZone["us-west2-b"] = struct{}{}
 	expectedZone["us-west2-c"] = struct{}{}
 	cli := fake.NewSimpleClientset(node1, node2, node3)
-	z, r, err := nodeZonesAndRegion(ctx, cli)
+	z, r, err := NodeZonesAndRegion(ctx, cli)
 	c.Assert(err, IsNil)
 	c.Assert(reflect.DeepEqual(z, expectedZone), Equals, true)
 	c.Assert(r, Equals, "us-west2")
@@ -113,7 +113,7 @@ func (s ZoneSuite) TestNodeZoneAndRegionEBS(c *C) {
 	expectedZone["us-west-2b"] = struct{}{}
 	expectedZone["us-west-2c"] = struct{}{}
 	cli := fake.NewSimpleClientset(node1, node2, node3)
-	z, r, err := nodeZonesAndRegion(ctx, cli)
+	z, r, err := NodeZonesAndRegion(ctx, cli)
 	c.Assert(err, IsNil)
 	c.Assert(reflect.DeepEqual(z, expectedZone), Equals, true)
 	c.Assert(r, Equals, "us-west-2")

pkg/function/restore_data.go

@@ -141,7 +141,7 @@ func restoreDataPodFunc(cli kubernetes.Interface, tp param.TemplateParams, names
 		format.Log(pod.Name, pod.Spec.Containers[0].Name, stdout)
 		format.Log(pod.Name, pod.Spec.Containers[0].Name, stderr)
 		if err != nil {
-			return nil, errors.Wrapf(err, "Failed to create and upload backup")
+			return nil, errors.Wrapf(err, "Failed to restore backup")
 		}
 		out, err := parseLogAndCreateOutput(stdout)
 		return out, errors.Wrap(err, "Failed to parse phase output")

pkg/kanctl/profile.go

@@ -23,14 +23,16 @@ import (
 )
 
 const (
-	bucketFlag        = "bucket"
-	endpointFlag      = "endpoint"
-	prefixFlag        = "prefix"
-	regionFlag        = "region"
-	awsAccessKeyFlag  = "access-key"
-	awsSecretKeyFlag  = "secret-key"
-	gcpProjectIDFlag  = "project-id"
-	gcpServiceKeyFlag = "service-key"
+	bucketFlag              = "bucket"
+	endpointFlag            = "endpoint"
+	prefixFlag              = "prefix"
+	regionFlag              = "region"
+	awsAccessKeyFlag        = "access-key"
+	awsSecretKeyFlag        = "secret-key"
+	gcpProjectIDFlag        = "project-id"
+	gcpServiceKeyFlag       = "service-key"
+	AzureStorageAccountFlag = "storage-account"
+	AzureStorageKeyFlag     = "storage-key"
 
 	idField           = "access_key_id"
 	secretField       = "secret_access_key"
@@ -64,6 +66,7 @@ func newProfileCommand() *cobra.Command {
 
 	cmd.AddCommand(newS3CompliantProfileCmd())
 	cmd.AddCommand(newGCPProfileCmd())
+	cmd.AddCommand(newAzureProfileCmd())
 	cmd.PersistentFlags().StringP(bucketFlag, "b", "", "object store bucket name")
 	cmd.PersistentFlags().StringP(endpointFlag, "e", "", "endpoint URL of the object store bucket")
 	cmd.PersistentFlags().StringP(prefixFlag, "p", "", "prefix URL of the object store bucket")
@@ -108,6 +111,24 @@ func newGCPProfileCmd() *cobra.Command {
 	return cmd
 }
 
+func newAzureProfileCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "azure",
+		Short: "Create new azure profile",
+		Args:  cobra.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return createNewProfile(cmd, args)
+		},
+	}
+
+	cmd.Flags().StringP(AzureStorageAccountFlag, "a", "", "Storage account name of the azure storage")
+	cmd.Flags().StringP(AzureStorageKeyFlag, "s", "", "Storage account key of the azure storage")
+
+	cmd.MarkFlagRequired(AzureStorageAccountFlag)
+	cmd.MarkFlagRequired(AzureStorageKeyFlag)
+	return cmd
+}
+
 func createNewProfile(cmd *cobra.Command, args []string) error {
 	if len(args) > 0 {
 		return newArgsLengthError("expected 0 args. Got %#v", args)
@@ -175,6 +196,9 @@ func getLocationParams(cmd *cobra.Command) (*locationParams, error) {
 	case "gcp":
 		lType = v1alpha1.LocationTypeGCS
 		profileName = "gcp-profile-"
+	case "azure":
+		lType = v1alpha1.LocationTypeAzure
+		profileName = "azure-profile-"
 	default:
 		return nil, errors.New("Profile type not supported: " + cmd.Name())
 	}
@@ -239,8 +263,13 @@ func constructSecret(ctx context.Context, lP *locationParams, cmd *cobra.Command
 		data[idField] = projectID
 		data[secretField] = serviceKey
 		secretname = "gcp"
+	case v1alpha1.LocationTypeAzure:
+		storageAccount, _ := cmd.Flags().GetString(AzureStorageAccountFlag)
+		storageKey, _ := cmd.Flags().GetString(AzureStorageKeyFlag)
+		data[idField] = storageAccount
+		data[secretField] = storageKey
+		secretname = "azure"
 	}
-
 	return &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf(secretFormat, secretname, randString(6)),

pkg/location/location.go

@@ -99,6 +99,8 @@ func getProviderType(lType crv1alpha1.LocationType) (objectstore.ProviderType, e
 		return objectstore.ProviderTypeS3, nil
 	case crv1alpha1.LocationTypeGCS:
 		return objectstore.ProviderTypeGCS, nil
+	case crv1alpha1.LocationTypeAzure:
+		return objectstore.ProviderTypeAzure, nil
 	default:
 		return "", errors.Errorf("Unsupported Location type: %s", lType)
 	}
@@ -136,6 +138,12 @@ func getOSSecret(pType objectstore.ProviderType, cred param.Credential) (*object
 			ProjectID:  cred.KeyPair.ID,
 			ServiceKey: cred.KeyPair.Secret,
 		}
+	case objectstore.ProviderTypeAzure:
+		secret.Type = objectstore.SecretTypeAzStorageAccount
+		secret.Azure = &objectstore.SecretAzure{
+			StorageAccount: cred.KeyPair.ID,
+			StorageKey:     cred.KeyPair.Secret,
+		}
 	default:
 		return nil, errors.Errorf("unknown or unsupported provider type '%s'", pType)
 	}

pkg/location/location_test.go

@@ -10,6 +10,7 @@ import (
 	. "gopkg.in/check.v1"
 
 	crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
+	"github.com/kanisterio/kanister/pkg/blockstorage"
 	"github.com/kanisterio/kanister/pkg/objectstore"
 	"github.com/kanisterio/kanister/pkg/param"
 	"github.com/kanisterio/kanister/pkg/testutil"
@@ -36,6 +37,7 @@ const (
 
 var _ = Suite(&LocationSuite{osType: objectstore.ProviderTypeS3, region: testRegionS3})
 var _ = Suite(&LocationSuite{osType: objectstore.ProviderTypeGCS, region: ""})
+var _ = Suite(&LocationSuite{osType: objectstore.ProviderTypeAzure, region: ""})
 
 func (s *LocationSuite) SetUpSuite(c *C) {
 	var location crv1alpha1.Location
@@ -45,19 +47,23 @@ func (s *LocationSuite) SetUpSuite(c *C) {
 		testutil.GetEnvOrSkip(c, AWSSecretAccessKey)
 		location = crv1alpha1.Location{
 			Type:   crv1alpha1.LocationTypeS3Compliant,
-			Bucket: testBucketName,
 			Region: s.region,
 		}
 	case objectstore.ProviderTypeGCS:
 		testutil.GetEnvOrSkip(c, GoogleCloudCreds)
 		location = crv1alpha1.Location{
-			Type:   crv1alpha1.LocationTypeGCS,
-			Bucket: testBucketName,
+			Type: crv1alpha1.LocationTypeGCS,
+		}
+	case objectstore.ProviderTypeAzure:
+		testutil.GetEnvOrSkip(c, blockstorage.AzureStorageAccount)
+		testutil.GetEnvOrSkip(c, blockstorage.AzureStorageKey)
+		location = crv1alpha1.Location{
+			Type: crv1alpha1.LocationTypeAzure,
 		}
 	default:
 		c.Fatalf("Unrecognized objectstore '%s'", s.osType)
 	}
-
+	location.Bucket = testBucketName
 	s.profile = *testutil.ObjectStoreProfileOrSkip(c, s.osType, location)
 	var err error
 	ctx := context.Background()

pkg/objectstore/aws.go

@@ -26,7 +26,7 @@ func config(region string) *aws.Config {
 	return c
 }
 
-func isBucketNotFoundError(err error) bool {
+func IsBucketNotFoundError(err error) bool {
 	if err == nil {
 		return false
 	}

pkg/objectstore/bucket.go

@@ -185,7 +185,7 @@ func (p *s3Provider) getRegionForBucket(ctx context.Context, bucketName string)
 
 func (p *s3Provider) getOrCreateBucket(ctx context.Context, bucketName, region string) (Bucket, error) {
 	d, err := p.GetBucket(ctx, bucketName)
-	if isBucketNotFoundError(err) {
+	if IsBucketNotFoundError(err) {
 		// Create bucket when it does not exist
 		return p.CreateBucket(ctx, bucketName, region)
 	}

pkg/objectstore/objectstore_test.go

@@ -131,7 +131,7 @@ func (s *ObjectStoreProviderSuite) TestGetNonExistingBucket(c *C) {
 	bucketName := s.createBucketName(c)
 	bucket, err := s.provider.GetBucket(ctx, bucketName)
 	c.Check(err, NotNil)
-	c.Assert(isBucketNotFoundError(err), Equals, true)
+	c.Assert(IsBucketNotFoundError(err), Equals, true)
 	c.Check(bucket, IsNil)
 }
 

pkg/restic/restic.go

@@ -59,7 +59,7 @@ func RestoreCommandByTag(profile *param.Profile, repository, tag, restorePath, e
 // SnapshotsCommand returns restic snapshots command
 func SnapshotsCommand(profile *param.Profile, repository, encryptionKey string) []string {
 	cmd := resticArgs(profile, repository, encryptionKey)
-	cmd = append(cmd, "snapshots")
+	cmd = append(cmd, "snapshots", "--json")
 	command := strings.Join(cmd, " ")
 	return shCommand(command)
 }

pkg/testutil/fixture.go

@@ -31,6 +31,9 @@ func ObjectStoreProfileOrSkip(c *check.C, osType objectstore.ProviderType, locat
 		c.Check(err, check.IsNil)
 		key = creds.ProjectID
 		val = string(creds.JSON)
+	case objectstore.ProviderTypeAzure:
+		key = GetEnvOrSkip(c, blockstorage.AzureStorageAccount)
+		val = GetEnvOrSkip(c, blockstorage.AzureStorageKey)
 	}
 	return &param.Profile{
 		Location: location,