Support for concurrent nodepool CRUD operations

converters/google/resources/mutexkv.go

@@ -13,7 +13,7 @@ import (
 // their access to individual security groups based on SG ID.
 type MutexKV struct {
 	lock  sync.Mutex
-	store map[string]*sync.Mutex
+	store map[string]*sync.RWMutex
 }
 
 // Locks the mutex for the given key. Caller is responsible for calling Unlock
@@ -31,13 +31,28 @@ func (m *MutexKV) Unlock(key string) {
 	log.Printf("[DEBUG] Unlocked %q", key)
 }
 
+// Acquires a read-lock on the mutex for the given key. Caller is responsible for calling RUnlock
+// for the same key
+func (m *MutexKV) RLock(key string) {
+	log.Printf("[DEBUG] RLocking %q", key)
+	m.get(key).RLock()
+	log.Printf("[DEBUG] RLocked %q", key)
+}
+
+// Releases a read-lock on the mutex for the given key. Caller must have called RLock for the same key first
+func (m *MutexKV) RUnlock(key string) {
+	log.Printf("[DEBUG] RUnlocking %q", key)
+	m.get(key).RUnlock()
+	log.Printf("[DEBUG] RUnlocked %q", key)
+}
+
 // Returns a mutex for the given key, no guarantee of its lock status
-func (m *MutexKV) get(key string) *sync.Mutex {
+func (m *MutexKV) get(key string) *sync.RWMutex {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	mutex, ok := m.store[key]
 	if !ok {
-		mutex = &sync.Mutex{}
+		mutex = &sync.RWMutex{}
 		m.store[key] = mutex
 	}
 	return mutex
@@ -46,6 +61,6 @@ func (m *MutexKV) get(key string) *sync.Mutex {
 // Returns a properly initialized MutexKV
 func NewMutexKV() *MutexKV {
 	return &MutexKV{
-		store: make(map[string]*sync.Mutex),
+		store: make(map[string]*sync.RWMutex),
 	}
 }

converters/google/resources/utils.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
 	"google.golang.org/api/googleapi"
@@ -573,3 +574,19 @@ func checkGoogleIamPolicy(value string) error {
 	}
 	return nil
 }
+
+// Retries an operation while the canonical error code is FAILED_PRECONDTION
+// which indicates there is an incompatible operation already running on the
+// cluster. This error can be safely retried until the incompatible operation
+// completes, and the newly requested operation can begin.
+func retryWhileIncompatibleOperation(timeout time.Duration, lockKey string, f func() error) error {
+	return resource.Retry(timeout, func() *resource.RetryError {
+		if err := lockedCall(lockKey, f); err != nil {
+			if isFailedPreconditionError(err) {
+				return resource.RetryableError(err)
+			}
+			return resource.NonRetryableError(err)
+		}
+		return nil
+	})
+}