Skip to content

Commit

Permalink
Add batching for IAM binding/member/audit config changes
Browse files Browse the repository at this point in the history
Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
emilymye authored and modular-magician committed Aug 13, 2019
1 parent 6c23c8f commit e0f41f0
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 48 deletions.
13 changes: 9 additions & 4 deletions google/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,9 +156,8 @@ type Config struct {
ServiceManagementBasePath string
clientServiceMan *servicemanagement.APIService

ServiceUsageBasePath string
clientServiceUsage *serviceusage.Service
requestBatcherServiceUsage *RequestBatcher
ServiceUsageBasePath string
clientServiceUsage *serviceusage.Service

BigQueryBasePath string
clientBigQuery *bigquery.Service
Expand All @@ -185,6 +184,9 @@ type Config struct {
// we expose those directly instead of providing the `Service` object
// as a factory.
clientBigtableProjectsInstances *bigtableadmin.ProjectsInstancesService

requestBatcherServiceUsage *RequestBatcher
requestBatcherIam *RequestBatcher
}

var defaultClientScopes = []string{
Expand Down Expand Up @@ -397,7 +399,6 @@ func (c *Config) LoadAndValidate() error {
}
c.clientServiceUsage.UserAgent = userAgent
c.clientServiceUsage.BasePath = serviceUsageClientBasePath
c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", context, c.BatchingConfig)

cloudBillingClientBasePath := removeBasePathVersion(c.CloudBillingBasePath)
log.Printf("[INFO] Instantiating Google Cloud Billing client for path %s", cloudBillingClientBasePath)
Expand Down Expand Up @@ -542,6 +543,10 @@ func (c *Config) LoadAndValidate() error {
c.clientStorageTransfer.BasePath = storageTransferClientBasePath

c.Region = GetRegionFromRegionSelfLink(c.Region)

c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", context, c.BatchingConfig)
c.requestBatcherIam = NewRequestBatcher("IAM", context, c.BatchingConfig)

return nil
}

Expand Down
60 changes: 60 additions & 0 deletions google/iam_batching.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package google

import (
"fmt"
"google.golang.org/api/cloudresourcemanager/v1"
"time"
)

const (
batchKeyTmplModifyIamPolicy = "%s modifyIamPolicy"

IamBatchingEnabled = true
IamBatchingDisabled = false
)

func batchRequestModifyIamPolicy(updater ResourceIamUpdater, modify iamPolicyModifyFunc, config *Config, reqDesc string) error {
batchKey := fmt.Sprintf(batchKeyTmplModifyIamPolicy, updater.GetMutexKey())

request := &BatchRequest{
ResourceName: updater.GetResourceId(),
Body: []iamPolicyModifyFunc{modify},
CombineF: combineBatchIamPolicyModifiers,
SendF: sendBatchModifyIamPolicy(updater),
DebugId: reqDesc,
}

_, err := config.requestBatcherIam.SendRequestWithTimeout(batchKey, request, time.Minute*30)
return err
}

func combineBatchIamPolicyModifiers(currV interface{}, toAddV interface{}) (interface{}, error) {
currModifiers, ok := currV.([]iamPolicyModifyFunc)
if !ok {
return nil, fmt.Errorf("provider error in batch combiner: expected data to be type []iamPolicyModifyFunc, got %v with type %T", currV, currV)
}

newModifiers, ok := toAddV.([]iamPolicyModifyFunc)
if !ok {
return nil, fmt.Errorf("provider error in batch combiner: expected data to be type []iamPolicyModifyFunc, got %v with type %T", currV, currV)
}

return append(currModifiers, newModifiers...), nil
}

func sendBatchModifyIamPolicy(updater ResourceIamUpdater) batcherSendFunc {
return func(resourceName string, body interface{}) (interface{}, error) {
modifiers, ok := body.([]iamPolicyModifyFunc)
if !ok {
return nil, fmt.Errorf("provider error: expected data to be type []iamPolicyModifyFunc, got %v with type %T", body, body)
}
return nil, iamPolicyReadModifyWrite(updater, func(policy *cloudresourcemanager.Policy) error {
for _, modifyF := range modifiers {
if err := modifyF(policy); err != nil {
return err
}
}
return nil
})
}
}
6 changes: 3 additions & 3 deletions google/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,9 @@ func ResourceMapWithErrors() (map[string]*schema.Resource, error) {
"google_organization_policy": resourceGoogleOrganizationPolicy(),
"google_project": resourceGoogleProject(),
"google_project_iam_policy": resourceGoogleProjectIamPolicy(),
"google_project_iam_binding": ResourceIamBinding(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
"google_project_iam_member": ResourceIamMember(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
"google_project_iam_audit_config": ResourceIamAuditConfig(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
"google_project_iam_binding": ResourceIamBindingWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
"google_project_iam_member": ResourceIamMemberWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
"google_project_iam_audit_config": ResourceIamAuditConfigWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
"google_project_service": resourceGoogleProjectService(),
"google_project_iam_custom_role": resourceGoogleProjectIamCustomRole(),
"google_project_organization_policy": resourceGoogleProjectOrganizationPolicy(),
Expand Down
59 changes: 41 additions & 18 deletions google/resource_iam_audit_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,35 +39,46 @@ var iamAuditConfigSchema = map[string]*schema.Schema{
}

func ResourceIamAuditConfig(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc) *schema.Resource {
return ResourceIamAuditConfigWithBatching(parentSpecificSchema, newUpdaterFunc, resourceIdParser, IamBatchingDisabled)
}

func ResourceIamAuditConfigWithBatching(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc, enableBatching bool) *schema.Resource {
return &schema.Resource{
Create: resourceIamAuditConfigCreate(newUpdaterFunc),
Create: resourceIamAuditConfigCreate(newUpdaterFunc, enableBatching),
Read: resourceIamAuditConfigRead(newUpdaterFunc),
Update: resourceIamAuditConfigUpdate(newUpdaterFunc),
Delete: resourceIamAuditConfigDelete(newUpdaterFunc),
Update: resourceIamAuditConfigUpdate(newUpdaterFunc, enableBatching),
Delete: resourceIamAuditConfigDelete(newUpdaterFunc, enableBatching),
Schema: mergeSchemas(iamAuditConfigSchema, parentSpecificSchema),
Importer: &schema.ResourceImporter{
State: iamAuditConfigImport(resourceIdParser),
},
}
}

func resourceIamAuditConfigCreate(newUpdaterFunc newResourceIamUpdaterFunc) schema.CreateFunc {
func resourceIamAuditConfigCreate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.CreateFunc {
return func(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
updater, err := newUpdaterFunc(d, config)
if err != nil {
return err
}

p := getResourceIamAuditConfig(d)
err = iamPolicyReadModifyWrite(updater, func(ep *cloudresourcemanager.Policy) error {
ep.AuditConfigs = mergeAuditConfigs(append(ep.AuditConfigs, p))
ac := getResourceIamAuditConfig(d)
modifyF := func(ep *cloudresourcemanager.Policy) error {
ep.AuditConfigs = mergeAuditConfigs(append(ep.AuditConfigs, ac))
return nil
})
}

if enableBatching {
err = batchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
"Add audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
} else {
err = iamPolicyReadModifyWrite(updater, modifyF)
}
if err != nil {
return err
}
d.SetId(updater.GetResourceId() + "/audit_config/" + p.Service)
d.SetId(updater.GetResourceId() + "/audit_config/" + ac.Service)
return resourceIamAuditConfigRead(newUpdaterFunc)(d, meta)
}
}
Expand Down Expand Up @@ -139,7 +150,7 @@ func iamAuditConfigImport(resourceIdParser resourceIdParserFunc) schema.StateFun
}
}

func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) schema.UpdateFunc {
func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.UpdateFunc {
return func(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
updater, err := newUpdaterFunc(d, config)
Expand All @@ -148,11 +159,17 @@ func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) sche
}

ac := getResourceIamAuditConfig(d)
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
cleaned := removeAllAuditConfigsWithService(p.AuditConfigs, ac.Service)
p.AuditConfigs = append(cleaned, ac)
modifyF := func(ep *cloudresourcemanager.Policy) error {
cleaned := removeAllAuditConfigsWithService(ep.AuditConfigs, ac.Service)
ep.AuditConfigs = append(cleaned, ac)
return nil
})
}
if enableBatching {
err = batchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
"Overwrite audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
} else {
err = iamPolicyReadModifyWrite(updater, modifyF)
}
if err != nil {
return err
}
Expand All @@ -161,7 +178,7 @@ func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) sche
}
}

func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.DeleteFunc {
func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.DeleteFunc {
return func(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
updater, err := newUpdaterFunc(d, config)
Expand All @@ -170,10 +187,16 @@ func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc) sche
}

ac := getResourceIamAuditConfig(d)
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
p.AuditConfigs = removeAllAuditConfigsWithService(p.AuditConfigs, ac.Service)
modifyF := func(ep *cloudresourcemanager.Policy) error {
ep.AuditConfigs = removeAllAuditConfigsWithService(ep.AuditConfigs, ac.Service)
return nil
})
}
if enableBatching {
err = batchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
"Delete audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
} else {
err = iamPolicyReadModifyWrite(updater, modifyF)
}
if err != nil {
return handleNotFoundError(err, d, fmt.Sprintf("Resource %s with IAM audit config %q", updater.DescribeResource(), d.Id()))
}
Expand Down
40 changes: 31 additions & 9 deletions google/resource_iam_binding.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,24 @@ var iamBindingSchema = map[string]*schema.Schema{
}

func ResourceIamBinding(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc) *schema.Resource {
return ResourceIamBindingWithBatching(parentSpecificSchema, newUpdaterFunc, resourceIdParser, IamBatchingDisabled)
}

// Resource that batches requests to the same IAM policy across multiple IAM fine-grained resources
func ResourceIamBindingWithBatching(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc, enableBatching bool) *schema.Resource {
return &schema.Resource{
Create: resourceIamBindingCreateUpdate(newUpdaterFunc),
Create: resourceIamBindingCreateUpdate(newUpdaterFunc, enableBatching),
Read: resourceIamBindingRead(newUpdaterFunc),
Update: resourceIamBindingCreateUpdate(newUpdaterFunc),
Delete: resourceIamBindingDelete(newUpdaterFunc),
Update: resourceIamBindingCreateUpdate(newUpdaterFunc, enableBatching),
Delete: resourceIamBindingDelete(newUpdaterFunc, enableBatching),
Schema: mergeSchemas(iamBindingSchema, parentSpecificSchema),
Importer: &schema.ResourceImporter{
State: iamBindingImport(resourceIdParser),
},
}
}

func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc) func(*schema.ResourceData, interface{}) error {
func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) func(*schema.ResourceData, interface{}) error {
return func(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
updater, err := newUpdaterFunc(d, config)
Expand All @@ -55,11 +60,18 @@ func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc) fu
}

binding := getResourceIamBinding(d)
err = iamPolicyReadModifyWrite(updater, func(ep *cloudresourcemanager.Policy) error {
modifyF := func(ep *cloudresourcemanager.Policy) error {
cleaned := removeAllBindingsWithRole(ep.Bindings, binding.Role)
ep.Bindings = append(cleaned, binding)
return nil
})
}

if enableBatching {
err = batchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
"Set IAM Binding for role %q on %q", binding.Role, updater.DescribeResource()))
} else {
err = iamPolicyReadModifyWrite(updater, modifyF)
}
if err != nil {
return err
}
Expand Down Expand Up @@ -142,7 +154,7 @@ func iamBindingImport(resourceIdParser resourceIdParserFunc) schema.StateFunc {
}
}

func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.DeleteFunc {
func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.DeleteFunc {
return func(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
updater, err := newUpdaterFunc(d, config)
Expand All @@ -151,10 +163,20 @@ func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.D
}

binding := getResourceIamBinding(d)
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
modifyF := func(p *cloudresourcemanager.Policy) error {
p.Bindings = removeAllBindingsWithRole(p.Bindings, binding.Role)
return nil
})
}

if enableBatching {
err = batchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
"Delete IAM Binding for role %q on %q", binding.Role, updater.DescribeResource()))
} else {
err = iamPolicyReadModifyWrite(updater, modifyF)
}
if err != nil {
return err
}
if err != nil {
return handleNotFoundError(err, d, fmt.Sprintf("Resource %q for IAM binding with role %q", updater.DescribeResource(), binding.Role))
}
Expand Down
Loading

0 comments on commit e0f41f0

Please sign in to comment.