Add basic implementation for remote state on azure (#7064)

* Add basic implementation for remote state on azure

* Don't auto-provision the container

* Fix compilation errors

* Add factory to the remote map

* Add documentation

* Add acceptance tests

command/remote_config.go

@@ -348,8 +348,8 @@ Usage: terraform remote config [options]
 Options:
 
   -backend=Atlas         Specifies the type of remote backend. Must be one
-                         of Atlas, Consul, Etcd, GCS, HTTP, S3, or Swift. Defaults
-                         to Atlas.
+                         of Atlas, Consul, Etcd, GCS, HTTP, MAS, S3, or Swift.
+                         Defaults to Atlas.
 
   -backend-config="k=v"  Specifies configuration for the remote storage
                          backend. This can be specified multiple times.

state/remote/mas.go

@@ -0,0 +1,178 @@
+package remote
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/Azure/azure-sdk-for-go/arm/storage"
+	mainStorage "github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
+	riviera "github.com/jen20/riviera/azure"
+)
+
+func masFactory(conf map[string]string) (Client, error) {
+	storageAccountName, ok := conf["storage_account_name"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'storage_account_name' configuration")
+	}
+	containerName, ok := conf["container_name"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'container_name' configuration")
+	}
+	keyName, ok := conf["key"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'key' configuration")
+	}
+
+	accessKey, ok := confOrEnv(conf, "access_key", "ARM_ACCESS_KEY")
+	if !ok {
+		resourceGroupName, ok := conf["resource_group_name"]
+		if !ok {
+			return nil, fmt.Errorf("missing 'resource_group' configuration")
+		}
+
+		var err error
+		accessKey, err = getStorageAccountAccessKey(conf, resourceGroupName, storageAccountName)
+		if err != nil {
+			return nil, fmt.Errorf("Couldn't read access key from storage account: %s.", err)
+		}
+	}
+
+	storageClient, err := mainStorage.NewBasicClient(storageAccountName, accessKey)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating storage client for storage account %q: %s", storageAccountName, err)
+	}
+
+	blobClient := storageClient.GetBlobService()
+
+	return &MASClient{
+		blobClient:    &blobClient,
+		containerName: containerName,
+		keyName:       keyName,
+	}, nil
+}
+
+func getStorageAccountAccessKey(conf map[string]string, resourceGroupName, storageAccountName string) (string, error) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		return "", err
+	}
+
+	oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(creds.TenantID)
+	if err != nil {
+		return "", err
+	}
+	if oauthConfig == nil {
+		return "", fmt.Errorf("Unable to configure OAuthConfig for tenant %s", creds.TenantID)
+	}
+
+	spt, err := azure.NewServicePrincipalToken(*oauthConfig, creds.ClientID, creds.ClientSecret, azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return "", err
+	}
+
+	accountsClient := storage.NewAccountsClient(creds.SubscriptionID)
+	accountsClient.Authorizer = spt
+
+	keys, err := accountsClient.ListKeys(resourceGroupName, storageAccountName)
+	if err != nil {
+		return "", fmt.Errorf("Error retrieving keys for storage account %q: %s", storageAccountName, err)
+	}
+
+	if keys.Key1 == nil {
+		return "", fmt.Errorf("Nil key returned for storage account %q", storageAccountName)
+	}
+
+	return *keys.Key1, nil
+}
+
+func getCredentialsFromConf(conf map[string]string) (*riviera.AzureResourceManagerCredentials, error) {
+	subscriptionID, ok := confOrEnv(conf, "arm_subscription_id", "ARM_SUBSCRIPTION_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_subscription_id' configuration")
+	}
+	clientID, ok := confOrEnv(conf, "arm_client_id", "ARM_CLIENT_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_client_id' configuration")
+	}
+	clientSecret, ok := confOrEnv(conf, "arm_client_secret", "ARM_CLIENT_SECRET")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_client_secret' configuration")
+	}
+	tenantID, ok := confOrEnv(conf, "arm_tenant_id", "ARM_TENANT_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_tenant_id' configuration")
+	}
+
+	return &riviera.AzureResourceManagerCredentials{
+		SubscriptionID: subscriptionID,
+		ClientID:       clientID,
+		ClientSecret:   clientSecret,
+		TenantID:       tenantID,
+	}, nil
+}
+
+func confOrEnv(conf map[string]string, confKey, envVar string) (string, bool) {
+	value, ok := conf[confKey]
+	if ok {
+		return value, true
+	}
+
+	value = os.Getenv(envVar)
+
+	return value, value != ""
+}
+
+type MASClient struct {
+	blobClient    *mainStorage.BlobStorageClient
+	containerName string
+	keyName       string
+}
+
+func (c *MASClient) Get() (*Payload, error) {
+	blob, err := c.blobClient.GetBlob(c.containerName, c.keyName)
+	if err != nil {
+		if storErr, ok := err.(mainStorage.AzureStorageServiceError); ok {
+			if storErr.Code == "BlobNotFound" {
+				return nil, nil
+			}
+		}
+		return nil, err
+	}
+
+	defer blob.Close()
+
+	data, err := ioutil.ReadAll(blob)
+	if err != nil {
+		return nil, err
+	}
+
+	payload := &Payload{
+		Data: data,
+	}
+
+	// If there was no data, then return nil
+	if len(payload.Data) == 0 {
+		return nil, nil
+	}
+
+	return payload, nil
+}
+
+func (c *MASClient) Put(data []byte) error {
+	return c.blobClient.CreateBlockBlobFromReader(
+		c.containerName,
+		c.keyName,
+		uint64(len(data)),
+		bytes.NewReader(data),
+		map[string]string{
+			"Content-Type": "application/json",
+		},
+	)
+}
+
+func (c *MASClient) Delete() error {
+	return c.blobClient.DeleteBlob(c.containerName, c.keyName, nil)
+}

state/remote/mas_test.go

@@ -0,0 +1,155 @@
+package remote
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	mainStorage "github.com/Azure/azure-sdk-for-go/storage"
+	riviera "github.com/jen20/riviera/azure"
+	"github.com/jen20/riviera/storage"
+)
+
+func TestMASClient_impl(t *testing.T) {
+	var _ Client = new(MASClient)
+}
+
+func TestMASClient(t *testing.T) {
+	// This test creates a bucket in MAS and populates it.
+	// It may incur costs, so it will only run if MAS credential environment
+	// variables are present.
+
+	config := map[string]string{
+		"arm_subscription_id": os.Getenv("ARM_SUBSCRIPTION_ID"),
+		"arm_client_id":       os.Getenv("ARM_CLIENT_ID"),
+		"arm_client_secret":   os.Getenv("ARM_CLIENT_SECRET"),
+		"arm_tenant_id":       os.Getenv("ARM_TENANT_ID"),
+	}
+
+	for k, v := range config {
+		if v == "" {
+			t.Skipf("skipping; %s must be set", strings.ToUpper(k))
+		}
+	}
+
+	config["resource_group_name"] = fmt.Sprintf("terraform-%x", time.Now().Unix())
+	config["storage_account_name"] = fmt.Sprintf("terraform%x", time.Now().Unix())
+	config["container_name"] = "terraform"
+	config["key"] = "test.tfstate"
+
+	setup(t, config)
+	defer teardown(t, config)
+
+	client, err := masFactory(config)
+	if err != nil {
+		t.Fatalf("Error for valid config: %v", err)
+	}
+
+	testClient(t, client)
+}
+
+func setup(t *testing.T, conf map[string]string) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		t.Fatalf("Error getting credentials from conf: %v", err)
+	}
+	rivieraClient, err := getRivieraClient(creds)
+	if err != nil {
+		t.Fatalf("Error instantiating the riviera client: %v", err)
+	}
+
+	// Create resource group
+	r := rivieraClient.NewRequest()
+	r.Command = riviera.CreateResourceGroup{
+		Name:     conf["resource_group_name"],
+		Location: riviera.WestUS,
+	}
+	response, err := r.Execute()
+	if err != nil {
+		t.Fatalf("Error creating a resource group: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error creating a resource group: %v", response.Error.Error())
+	}
+
+	// Create storage account
+	r = rivieraClient.NewRequest()
+	r.Command = storage.CreateStorageAccount{
+		ResourceGroupName: conf["resource_group_name"],
+		Name:              conf["storage_account_name"],
+		AccountType:       riviera.String("Standard_LRS"),
+		Location:          riviera.WestUS,
+	}
+	response, err = r.Execute()
+	if err != nil {
+		t.Fatalf("Error creating a storage account: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error creating a storage account: %v", response.Error.Error())
+	}
+
+	// Create container
+	accessKey, err := getStorageAccountAccessKey(conf, conf["resource_group_name"], conf["storage_account_name"])
+	if err != nil {
+		t.Fatalf("Error creating a storage account: %v", err)
+	}
+	storageClient, err := mainStorage.NewBasicClient(conf["storage_account_name"], accessKey)
+	if err != nil {
+		t.Fatalf("Error creating storage client for storage account %q: %s", conf["storage_account_name"], err)
+	}
+	blobClient := storageClient.GetBlobService()
+	_, err = blobClient.CreateContainerIfNotExists(conf["container_name"], mainStorage.ContainerAccessTypePrivate)
+	if err != nil {
+		t.Fatalf("Couldn't create container with name %s: %s.", conf["container_name"], err)
+	}
+}
+
+func teardown(t *testing.T, conf map[string]string) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		t.Fatalf("Error getting credentials from conf: %v", err)
+	}
+	rivieraClient, err := getRivieraClient(creds)
+	if err != nil {
+		t.Fatalf("Error instantiating the riviera client: %v", err)
+	}
+
+	r := rivieraClient.NewRequest()
+	r.Command = riviera.DeleteResourceGroup{
+		Name: conf["resource_group_name"],
+	}
+	response, err := r.Execute()
+	if err != nil {
+		t.Fatalf("Error deleting the resource group: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error deleting the resource group: %v", err)
+	}
+}
+
+func getRivieraClient(credentials *riviera.AzureResourceManagerCredentials) (*riviera.Client, error) {
+	rivieraClient, err := riviera.NewClient(credentials)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating Riviera client: %s", err)
+	}
+
+	request := rivieraClient.NewRequest()
+	request.Command = riviera.RegisterResourceProvider{
+		Namespace: "Microsoft.Storage",
+	}
+
+	response, err := request.Execute()
+	if err != nil {
+		return nil, fmt.Errorf("Cannot request provider registration for Azure Resource Manager: %s.", err)
+	}
+
+	if !response.IsSuccessful() {
+		return nil, fmt.Errorf("Credentials for acessing the Azure Resource Manager API are likely " +
+			"to be incorrect, or\n  the service principal does not have permission to use " +
+			"the Azure Service Management\n  API.")
+	}
+
+	return rivieraClient, nil
+}

state/remote/remote.go

@@ -41,6 +41,7 @@ var BuiltinClients = map[string]Factory{
 	"etcd":        etcdFactory,
 	"gcs":         gcsFactory,
 	"http":        httpFactory,
+	"mas":         masFactory,
 	"s3":          s3Factory,
 	"swift":       swiftFactory,
 	"artifactory": artifactoryFactory,