Add retries to datastore index when under contention. (#3615) (#2154)

Signed-off-by: Modular Magician <magic-modules@google.com>
hashicorp · Jun 10, 2020 · 4e268e1 · 4e268e1
1 parent 953ef47
commit 4e268e1
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 5 deletions.
diff --git a/.changelog/3615.txt b/.changelog/3615.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+datastore: Added retries to `google_datastore_index` requests when under contention.
+```
diff --git a/google-beta/datastore_operation.go b/google-beta/datastore_operation.go
@@ -31,7 +31,7 @@ func (w *DatastoreOperationWaiter) QueryOp() (interface{}, error) {
 	}
 	// Returns the proper get.
 	url := fmt.Sprintf("https://datastore.googleapis.com/v1/%s", w.CommonOperationWaiter.Op.Name)
-	return sendRequest(w.Config, "GET", w.Project, url, nil)
+	return sendRequest(w.Config, "GET", w.Project, url, nil, datastoreIndex409Contention)
 }
 
 func createDatastoreWaiter(config *Config, op map[string]interface{}, project, activity string) (*DatastoreOperationWaiter, error) {

diff --git a/google-beta/error_retry_predicates.go b/google-beta/error_retry_predicates.go
@@ -266,3 +266,12 @@ func isCloudFunctionsSourceCodeError(err error) (bool, string) {
 	}
 	return false, ""
 }
+
+func datastoreIndex409Contention(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		if gerr.Code == 409 && strings.Contains(gerr.Body, "too much contention") {
+			return true, "too much contention - waiting for less activity"
+		}
+	}
+	return false, ""
+}
diff --git a/google-beta/resource_datastore_index.go b/google-beta/resource_datastore_index.go
@@ -126,7 +126,7 @@ func resourceDatastoreIndexCreate(d *schema.ResourceData, meta interface{}) erro
 	if err != nil {
 		return err
 	}
-	res, err := sendRequestWithTimeout(config, "POST", project, url, obj, d.Timeout(schema.TimeoutCreate))
+	res, err := sendRequestWithTimeout(config, "POST", project, url, obj, d.Timeout(schema.TimeoutCreate), datastoreIndex409Contention)
 	if err != nil {
 		return fmt.Errorf("Error creating Index: %s", err)
 	}
@@ -178,7 +178,7 @@ func resourceDatastoreIndexRead(d *schema.ResourceData, meta interface{}) error
 	if err != nil {
 		return err
 	}
-	res, err := sendRequest(config, "GET", project, url, nil)
+	res, err := sendRequest(config, "GET", project, url, nil, datastoreIndex409Contention)
 	if err != nil {
 		return handleNotFoundError(err, d, fmt.Sprintf("DatastoreIndex %q", d.Id()))
 	}
@@ -219,7 +219,7 @@ func resourceDatastoreIndexDelete(d *schema.ResourceData, meta interface{}) erro
 	var obj map[string]interface{}
 	log.Printf("[DEBUG] Deleting Index %q", d.Id())
 
-	res, err := sendRequestWithTimeout(config, "DELETE", project, url, obj, d.Timeout(schema.TimeoutDelete))
+	res, err := sendRequestWithTimeout(config, "DELETE", project, url, obj, d.Timeout(schema.TimeoutDelete), datastoreIndex409Contention)
 	if err != nil {
 		return handleNotFoundError(err, d, "Index")
 	}

diff --git a/google-beta/resource_datastore_index_generated_test.go b/google-beta/resource_datastore_index_generated_test.go
@@ -80,7 +80,7 @@ func testAccCheckDatastoreIndexDestroyProducer(t *testing.T) func(s *terraform.S
 				return err
 			}
 
-			_, err = sendRequest(config, "GET", "", url, nil)
+			_, err = sendRequest(config, "GET", "", url, nil, datastoreIndex409Contention)
 			if err == nil {
 				return fmt.Errorf("DatastoreIndex still exists at %s", url)
 			}