*: add ABS support for backup and restore (#1842)

* add ABS support for backup and restore

Gopkg.toml

@@ -40,3 +40,7 @@
 
 [[constraint]]
   name = "golang.org/x/time"
+
+[[constraint]]
+  name = "github.com/Azure/azure-sdk-for-go"
+  version = "v11.3.0-beta"

pkg/apis/etcd/v1beta2/backup_types.go

@@ -17,10 +17,15 @@ package v1beta2
 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 const (
-	BackupStorageTypeS3 BackupStorageType = "S3"
+	// AWS S3 related consts
+	BackupStorageTypeS3          BackupStorageType = "S3"
+	AWSSecretCredentialsFileName                   = "credentials"
+	AWSSecretConfigFileName                        = "config"
 
-	AWSSecretCredentialsFileName = "credentials"
-	AWSSecretConfigFileName      = "config"
+	// Azure ABS related consts
+	BackupStorageTypeABS      BackupStorageType = "ABS"
+	AzureSecretStorageAccount                   = "storage-account"
+	AzureSecretStorageKey                       = "storage-key"
 )
 
 type BackupStorageType string
@@ -71,6 +76,9 @@ type BackupSpec struct {
 type BackupSource struct {
 	// S3 defines the S3 backup source spec.
 	S3 *S3BackupSource `json:"s3,omitempty"`
+
+	// ABS defines the ABS backup source spec.
+	ABS *ABSBackupSource `json:"abs,omitempty"`
 }
 
 // BackupStatus represents the status of the EtcdBackup Custom Resource.
@@ -104,3 +112,14 @@ type S3BackupSource struct {
 	// stores.
 	Endpoint string `json:"endpoint,omitempty"`
 }
+
+// ABSBackupSource provides the spec how to store backups on ABS.
+type ABSBackupSource struct {
+	// Path is the full abs path where the backup is saved.
+	// The format of the path must be: "<abs-container-name>/<path-to-backup-file>"
+	// e.g: "myabscontainer/etcd.backup"
+	Path string `json:"path"`
+
+	// The name of the secret object that stores the Azure storage credential
+	ABSSecret string `json:"absSecret"`
+}

pkg/apis/etcd/v1beta2/restore_types.go

@@ -63,6 +63,9 @@ type EtcdClusterRef struct {
 type RestoreSource struct {
 	// S3 tells where on S3 the backup is saved and how to fetch the backup.
 	S3 *S3RestoreSource `json:"s3,omitempty"`
+
+	// ABS tells where on ABS the backup is saved and how to fetch the backup.
+	ABS *ABSRestoreSource `json:"abs,omitempty"`
 }
 
 type S3RestoreSource struct {
@@ -84,6 +87,16 @@ type S3RestoreSource struct {
 	Endpoint string `json:"endpoint"`
 }
 
+type ABSRestoreSource struct {
+	// Path is the full abs path where the backup is saved.
+	// The format of the path must be: "<abs-container-name>/<path-to-backup-file>"
+	// e.g: "myabscontainer/etcd.backup"
+	Path string `json:"path"`
+
+	// The name of the secret object that stores the Azure Blob Storage credential.
+	ABSSecret string `json:"absSecret"`
+}
+
 // RestoreStatus reports the status of this restore operation.
 type RestoreStatus struct {
 	// Succeeded indicates if the backup has Succeeded.

pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go

@@ -32,6 +32,14 @@ import (
 // Deprecated: deepcopy registration will go away when static deepcopy is fully implemented.
 func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc {
 	return []conversion.GeneratedDeepCopyFunc{
+		{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*ABSBackupSource).DeepCopyInto(out.(*ABSBackupSource))
+			return nil
+		}, InType: reflect.TypeOf(&ABSBackupSource{})},
+		{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*ABSRestoreSource).DeepCopyInto(out.(*ABSRestoreSource))
+			return nil
+		}, InType: reflect.TypeOf(&ABSRestoreSource{})},
 		{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*BackupSource).DeepCopyInto(out.(*BackupSource))
 			return nil
@@ -131,6 +139,38 @@ func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc {
 	}
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ABSBackupSource) DeepCopyInto(out *ABSBackupSource) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ABSBackupSource.
+func (in *ABSBackupSource) DeepCopy() *ABSBackupSource {
+	if in == nil {
+		return nil
+	}
+	out := new(ABSBackupSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ABSRestoreSource) DeepCopyInto(out *ABSRestoreSource) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ABSRestoreSource.
+func (in *ABSRestoreSource) DeepCopy() *ABSRestoreSource {
+	if in == nil {
+		return nil
+	}
+	out := new(ABSRestoreSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BackupSource) DeepCopyInto(out *BackupSource) {
 	*out = *in
@@ -143,6 +183,15 @@ func (in *BackupSource) DeepCopyInto(out *BackupSource) {
 			**out = **in
 		}
 	}
+	if in.ABS != nil {
+		in, out := &in.ABS, &out.ABS
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(ABSBackupSource)
+			**out = **in
+		}
+	}
 	return
 }
 
@@ -597,6 +646,15 @@ func (in *RestoreSource) DeepCopyInto(out *RestoreSource) {
 			**out = **in
 		}
 	}
+	if in.ABS != nil {
+		in, out := &in.ABS, &out.ABS
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(ABSRestoreSource)
+			**out = **in
+		}
+	}
 	return
 }
 

pkg/backup/backupapi/http.go

@@ -19,6 +19,10 @@ import (
 	"path"
 )
 
+const (
+	APIV1 = "/v1"
+)
+
 // BackupURLForRestore creates a URL struct for retrieving an existing backup specified by a restore CR
 func BackupURLForRestore(scheme, host, restoreName string) *url.URL {
 	return &url.URL{

pkg/backup/reader/abs_reader.go

@@ -0,0 +1,58 @@
+// Copyright 2017 The etcd-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/coreos/etcd-operator/pkg/backup/util"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+)
+
+// ensure absReader satisfies reader interface.
+var _ Reader = &absReader{}
+
+// absReader provides Reader implementation for reading a file from ABS
+type absReader struct {
+	abs *storage.BlobStorageClient
+}
+
+// NewABSReader return a Reader implementation to read a file from ABS in the form of absReader
+func NewABSReader(abs *storage.BlobStorageClient) Reader {
+	return &absReader{abs}
+}
+
+// Open opens the file on path where path must be in the format "<abs-container-name>/<key>"
+func (absr *absReader) Open(path string) (io.ReadCloser, error) {
+	container, key, err := util.ParseBucketAndKey(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse abs container and key: %v", err)
+	}
+
+	containerRef := absr.abs.GetContainerReference(container)
+	containerExists, err := containerRef.Exists()
+	if err != nil {
+		return nil, err
+	}
+
+	if !containerExists {
+		return nil, fmt.Errorf("container %v does not exist", container)
+	}
+
+	blob := containerRef.GetBlobReference(key)
+	return blob.Get(&storage.GetBlobOptions{})
+}