Skip to content

Commit

Permalink
Storage Data Plane upgrades
Browse files Browse the repository at this point in the history
- Use latest tombuildsstuff/giovanni which has switched to the go-azure-sdk base layer
- Support for sdk-level retries for known eventually consistent scenarios
- Support for AAD authentication for all data plane APIs (where the API supports it)
- `azurerm_storage_share_directory`: deprecate `share_name` and `storage_account_name` in favor of `storage_share_id`
- Refactor data plane client helpers to be operation-aware, so that supported authentication methods can be determined not just
  by endpoint but also by the operation (enables more granular support for preferred authentication methods).
- Some tidying of `provider` package - move helper functions into own source file
- Some tidying of `services/storage/client` package - move data plane client helpers into own source file
  • Loading branch information
manicminer committed Feb 20, 2024
1 parent 9dab447 commit b76ce67
Show file tree
Hide file tree
Showing 57 changed files with 1,622 additions and 1,359 deletions.
17 changes: 6 additions & 11 deletions internal/common/client_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/hashicorp/go-azure-helpers/sender"
"github.com/hashicorp/go-azure-sdk/sdk/auth"
"github.com/hashicorp/go-azure-sdk/sdk/client"
"github.com/hashicorp/go-azure-sdk/sdk/client/resourcemanager"
"github.com/hashicorp/go-azure-sdk/sdk/environments"
"github.com/hashicorp/terraform-plugin-sdk/v2/meta"
"github.com/hashicorp/terraform-provider-azurerm/internal/features"
Expand Down Expand Up @@ -66,24 +65,20 @@ type ClientOptions struct {
}

// Configure set up a resourcemanager.Client using an auth.Authorizer from hashicorp/go-azure-sdk
func (o ClientOptions) Configure(c *resourcemanager.Client, authorizer auth.Authorizer) {
c.Authorizer = authorizer
c.UserAgent = userAgent(c.UserAgent, o.TerraformVersion, o.PartnerId, o.DisableTerraformPartnerID)
func (o ClientOptions) Configure(c client.BaseClient, authorizer auth.Authorizer) {
c.SetAuthorizer(authorizer)
c.SetUserAgent(userAgent(c.GetUserAgent(), o.TerraformVersion, o.PartnerId, o.DisableTerraformPartnerID))

requestMiddlewares := make([]client.RequestMiddleware, 0)
if !o.DisableCorrelationRequestID {
id := o.CustomCorrelationRequestID
if id == "" {
id = correlationRequestID()
}
requestMiddlewares = append(requestMiddlewares, correlationRequestIDMiddleware(id))
c.AppendRequestMiddleware(correlationRequestIDMiddleware(id))
}
requestMiddlewares = append(requestMiddlewares, requestLoggerMiddleware("AzureRM"))
c.RequestMiddlewares = &requestMiddlewares

c.ResponseMiddlewares = &[]client.ResponseMiddleware{
responseLoggerMiddleware("AzureRM"),
}
c.AppendRequestMiddleware(requestLoggerMiddleware("AzureRM"))
c.AppendResponseMiddleware(responseLoggerMiddleware("AzureRM"))
}

// ConfigureClient sets up an autorest.Client using an autorest.Authorizer
Expand Down
128 changes: 128 additions & 0 deletions internal/provider/helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package provider

import (
"encoding/base64"
"fmt"
"os"
"strings"

"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
)

func decodeCertificate(clientCertificate string) ([]byte, error) {
var pfx []byte
if clientCertificate != "" {
out := make([]byte, base64.StdEncoding.DecodedLen(len(clientCertificate)))
n, err := base64.StdEncoding.Decode(out, []byte(clientCertificate))
if err != nil {
return pfx, fmt.Errorf("could not decode client certificate data: %v", err)
}
pfx = out[:n]
}
return pfx, nil
}

func getOidcToken(d *schema.ResourceData) (*string, error) {
idToken := strings.TrimSpace(d.Get("oidc_token").(string))

if path := d.Get("oidc_token_file_path").(string); path != "" {
fileTokenRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading OIDC Token from file %q: %v", path, err)
}

fileToken := strings.TrimSpace(string(fileTokenRaw))

if idToken != "" && idToken != fileToken {
return nil, fmt.Errorf("mismatch between supplied OIDC token and supplied OIDC token file contents - please either remove one or ensure they match")
}

idToken = fileToken
}

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" {
path := os.Getenv("AZURE_FEDERATED_TOKEN_FILE")
fileTokenRaw, err := os.ReadFile(os.Getenv("AZURE_FEDERATED_TOKEN_FILE"))

if err != nil {
return nil, fmt.Errorf("reading OIDC Token from file %q provided by AKS Workload Identity: %v", path, err)
}

fileToken := strings.TrimSpace(string(fileTokenRaw))

if idToken != "" && idToken != fileToken {
return nil, fmt.Errorf("mismatch between supplied OIDC token and OIDC token file contents provided by AKS Workload Identity - please either remove one, ensure they match, or disable use_aks_workload_identity")
}

idToken = fileToken
}

return &idToken, nil
}

func getClientId(d *schema.ResourceData) (*string, error) {
clientId := strings.TrimSpace(d.Get("client_id").(string))

if path := d.Get("client_id_file_path").(string); path != "" {
fileClientIdRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading Client ID from file %q: %v", path, err)
}

fileClientId := strings.TrimSpace(string(fileClientIdRaw))

if clientId != "" && clientId != fileClientId {
return nil, fmt.Errorf("mismatch between supplied Client ID and supplied Client ID file contents - please either remove one or ensure they match")
}

clientId = fileClientId
}

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_CLIENT_ID") != "" {
aksClientId := os.Getenv("AZURE_CLIENT_ID")
if clientId != "" && clientId != aksClientId {
return nil, fmt.Errorf("mismatch between supplied Client ID and that provided by AKS Workload Identity - please remove, ensure they match, or disable use_aks_workload_identity")
}
clientId = aksClientId
}

return &clientId, nil
}

func getClientSecret(d *schema.ResourceData) (*string, error) {
clientSecret := strings.TrimSpace(d.Get("client_secret").(string))

if path := d.Get("client_secret_file_path").(string); path != "" {
fileSecretRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading Client Secret from file %q: %v", path, err)
}

fileSecret := strings.TrimSpace(string(fileSecretRaw))

if clientSecret != "" && clientSecret != fileSecret {
return nil, fmt.Errorf("mismatch between supplied Client Secret and supplied Client Secret file contents - please either remove one or ensure they match")
}

clientSecret = fileSecret
}

return &clientSecret, nil
}

func getTenantId(d *schema.ResourceData) (*string, error) {
tenantId := strings.TrimSpace(d.Get("tenant_id").(string))

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_TENANT_ID") != "" {
aksTenantId := os.Getenv("AZURE_TENANT_ID")
if tenantId != "" && tenantId != aksTenantId {
return nil, fmt.Errorf("mismatch between supplied Tenant ID and that provided by AKS Workload Identity - please remove, ensure they match, or disable use_aks_workload_identity")
}
tenantId = aksTenantId
}

return &tenantId, nil
}
119 changes: 0 additions & 119 deletions internal/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package provider

import (
"context"
"encoding/base64"
"fmt"
"log"
"os"
Expand Down Expand Up @@ -510,124 +509,6 @@ func buildClient(ctx context.Context, p *schema.Provider, d *schema.ResourceData
return client, nil
}

func decodeCertificate(clientCertificate string) ([]byte, error) {
var pfx []byte
if clientCertificate != "" {
out := make([]byte, base64.StdEncoding.DecodedLen(len(clientCertificate)))
n, err := base64.StdEncoding.Decode(out, []byte(clientCertificate))
if err != nil {
return pfx, fmt.Errorf("could not decode client certificate data: %v", err)
}
pfx = out[:n]
}
return pfx, nil
}

func getOidcToken(d *schema.ResourceData) (*string, error) {
idToken := strings.TrimSpace(d.Get("oidc_token").(string))

if path := d.Get("oidc_token_file_path").(string); path != "" {
fileTokenRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading OIDC Token from file %q: %v", path, err)
}

fileToken := strings.TrimSpace(string(fileTokenRaw))

if idToken != "" && idToken != fileToken {
return nil, fmt.Errorf("mismatch between supplied OIDC token and supplied OIDC token file contents - please either remove one or ensure they match")
}

idToken = fileToken
}

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" {
path := os.Getenv("AZURE_FEDERATED_TOKEN_FILE")
fileTokenRaw, err := os.ReadFile(os.Getenv("AZURE_FEDERATED_TOKEN_FILE"))

if err != nil {
return nil, fmt.Errorf("reading OIDC Token from file %q provided by AKS Workload Identity: %v", path, err)
}

fileToken := strings.TrimSpace(string(fileTokenRaw))

if idToken != "" && idToken != fileToken {
return nil, fmt.Errorf("mismatch between supplied OIDC token and OIDC token file contents provided by AKS Workload Identity - please either remove one, ensure they match, or disable use_aks_workload_identity")
}

idToken = fileToken
}

return &idToken, nil
}

func getClientId(d *schema.ResourceData) (*string, error) {
clientId := strings.TrimSpace(d.Get("client_id").(string))

if path := d.Get("client_id_file_path").(string); path != "" {
fileClientIdRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading Client ID from file %q: %v", path, err)
}

fileClientId := strings.TrimSpace(string(fileClientIdRaw))

if clientId != "" && clientId != fileClientId {
return nil, fmt.Errorf("mismatch between supplied Client ID and supplied Client ID file contents - please either remove one or ensure they match")
}

clientId = fileClientId
}

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_CLIENT_ID") != "" {
aksClientId := os.Getenv("AZURE_CLIENT_ID")
if clientId != "" && clientId != aksClientId {
return nil, fmt.Errorf("mismatch between supplied Client ID and that provided by AKS Workload Identity - please remove, ensure they match, or disable use_aks_workload_identity")
}
clientId = aksClientId
}

return &clientId, nil
}

func getClientSecret(d *schema.ResourceData) (*string, error) {
clientSecret := strings.TrimSpace(d.Get("client_secret").(string))

if path := d.Get("client_secret_file_path").(string); path != "" {
fileSecretRaw, err := os.ReadFile(path)

if err != nil {
return nil, fmt.Errorf("reading Client Secret from file %q: %v", path, err)
}

fileSecret := strings.TrimSpace(string(fileSecretRaw))

if clientSecret != "" && clientSecret != fileSecret {
return nil, fmt.Errorf("mismatch between supplied Client Secret and supplied Client Secret file contents - please either remove one or ensure they match")
}

clientSecret = fileSecret
}

return &clientSecret, nil
}

func getTenantId(d *schema.ResourceData) (*string, error) {
tenantId := strings.TrimSpace(d.Get("tenant_id").(string))

if d.Get("use_aks_workload_identity").(bool) && os.Getenv("AZURE_TENANT_ID") != "" {
aksTenantId := os.Getenv("AZURE_TENANT_ID")
if tenantId != "" && tenantId != aksTenantId {
return nil, fmt.Errorf("mismatch between supplied Tenant ID and that provided by AKS Workload Identity - please remove, ensure they match, or disable use_aks_workload_identity")
}
tenantId = aksTenantId
}

return &tenantId, nil
}

const resourceProviderRegistrationErrorFmt = `Error ensuring Resource Providers are registered.
Terraform automatically attempts to register the Resource Providers it supports to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ func (r ArcKubernetesFluxConfigurationResource) Arguments() map[string]*pluginsd
"container_id": {
Type: pluginsdk.TypeString,
Required: true,
ValidateFunc: storageValidate.StorageContainerDataPlaneID,
ValidateFunc: storageValidate.StorageContainerDataPlaneIDForDomainSuffix(""), // TODO need to know the storage domain suffix at schema time!
},

"account_key": {
Expand Down Expand Up @@ -570,7 +570,7 @@ func (r ArcKubernetesFluxConfigurationResource) Create() sdk.ResourceFunc {
properties.Properties.Bucket, properties.Properties.ConfigurationProtectedSettings = expandBucketDefinitionModel(model.Bucket)
} else if _, exists = metadata.ResourceData.GetOk("blob_storage"); exists {
properties.Properties.SourceKind = pointer.To(fluxconfiguration.SourceKindTypeAzureBlob)
azureBlob, err := expandArcAzureBlobDefinitionModel(model.BlobStorage)
azureBlob, err := expandArcAzureBlobDefinitionModel(model.BlobStorage, metadata.Client.Storage.StorageDomainSuffix)
if err != nil {
return fmt.Errorf("expanding `blob_storage`: %+v", err)
}
Expand Down Expand Up @@ -624,7 +624,7 @@ func (r ArcKubernetesFluxConfigurationResource) Update() sdk.ResourceFunc {

properties.Properties.ConfigurationProtectedSettings = nil
if metadata.ResourceData.HasChange("blob_storage") {
azureBlob, err := expandArcAzureBlobDefinitionModel(model.BlobStorage)
azureBlob, err := expandArcAzureBlobDefinitionModel(model.BlobStorage, metadata.Client.Storage.StorageDomainSuffix)
if err != nil {
return fmt.Errorf("expanding `blob_storage`: %+v", err)
}
Expand Down Expand Up @@ -717,7 +717,7 @@ func (r ArcKubernetesFluxConfigurationResource) Read() sdk.ResourceFunc {

if model := resp.Model; model != nil {
if properties := model.Properties; properties != nil {
blobStorage, err := flattenArcAzureBlobDefinitionModel(properties.AzureBlob, configModel.BlobStorage)
blobStorage, err := flattenArcAzureBlobDefinitionModel(properties.AzureBlob, configModel.BlobStorage, metadata.Client.Storage.StorageDomainSuffix)
if err != nil {
return fmt.Errorf("flattening `blob_storage`: %+v", err)
}
Expand Down Expand Up @@ -762,7 +762,7 @@ func (r ArcKubernetesFluxConfigurationResource) Delete() sdk.ResourceFunc {
}
}

func expandArcAzureBlobDefinitionModel(inputList []AzureBlobDefinitionModel) (*fluxconfiguration.AzureBlobDefinition, error) {
func expandArcAzureBlobDefinitionModel(inputList []AzureBlobDefinitionModel, storageDomainSuffix string) (*fluxconfiguration.AzureBlobDefinition, error) {
if len(inputList) == 0 {
return nil, nil
}
Expand All @@ -778,7 +778,7 @@ func expandArcAzureBlobDefinitionModel(inputList []AzureBlobDefinitionModel) (*f
}

if input.ContainerID != "" {
id, err := parse.StorageContainerDataPlaneID(input.ContainerID)
id, err := parse.StorageContainerDataPlaneID(input.ContainerID, storageDomainSuffix)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -969,13 +969,13 @@ func expandRepositoryRefDefinitionModel(referenceType string, referenceValue str
return &output, nil
}

func flattenArcAzureBlobDefinitionModel(input *fluxconfiguration.AzureBlobDefinition, azureBlob []AzureBlobDefinitionModel) ([]AzureBlobDefinitionModel, error) {
func flattenArcAzureBlobDefinitionModel(input *fluxconfiguration.AzureBlobDefinition, azureBlob []AzureBlobDefinitionModel, storageDomainSuffix string) ([]AzureBlobDefinitionModel, error) {
outputList := make([]AzureBlobDefinitionModel, 0)
if input == nil {
return outputList, nil
}

id, err := parse.StorageContainerDataPlaneID(fmt.Sprintf("%s/%s", pointer.From(input.Url), pointer.From(input.ContainerName)))
id, err := parse.StorageContainerDataPlaneID(fmt.Sprintf("%s/%s", pointer.From(input.Url), pointer.From(input.ContainerName)), storageDomainSuffix)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit b76ce67

Please sign in to comment.