Add retry for storage bucket 412

.changelog/3434.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+storage: Added retries for `google_storage_bucket_iam_*` on 412 (precondition not met) errors for eventually consistent bucket creation.
+```

google-beta/error_retry_predicates.go

@@ -215,3 +215,10 @@ func isNotFoundRetryableError(opType string) RetryErrorPredicateFunc {
 		return false, ""
 	}
 }
+
+func isStoragePreconditionError(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 412 {
+		return true, fmt.Sprintf("Retry on storage precondition not met")
+	}
+	return false, ""
+}

google-beta/iam_storage_bucket.go

@@ -102,7 +102,7 @@ func (u *StorageBucketIamUpdater) GetResourceIamPolicy() (*cloudresourcemanager.
 		return nil, err
 	}
 
-	policy, err := sendRequest(u.Config, "GET", "", url, obj)
+	policy, err := sendRequest(u.Config, "GET", "", url, obj, isStoragePreconditionError)
 	if err != nil {
 		return nil, errwrap.Wrapf(fmt.Sprintf("Error retrieving IAM policy for %s: {{err}}", u.DescribeResource()), err)
 	}
@@ -129,7 +129,7 @@ func (u *StorageBucketIamUpdater) SetResourceIamPolicy(policy *cloudresourcemana
 		return err
 	}
 
-	_, err = sendRequestWithTimeout(u.Config, "PUT", "", url, obj, u.d.Timeout(schema.TimeoutCreate))
+	_, err = sendRequestWithTimeout(u.Config, "PUT", "", url, obj, u.d.Timeout(schema.TimeoutCreate), isStoragePreconditionError)
 	if err != nil {
 		return errwrap.Wrapf(fmt.Sprintf("Error setting IAM policy for %s: {{err}}", u.DescribeResource()), err)
 	}