fix(provider/gce): Paginate instance template list in svg caching.

spinnaker · Oct 24, 2017 · ded22ee · ded22ee
1 parent 84ebd77
commit ded22ee
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 10 deletions.
diff --git a/.../spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy b/.../spinnaker/clouddriver/google/provider/agent/GoogleRegionalServerGroupCachingAgent.groovy
@@ -133,11 +133,7 @@ class GoogleRegionalServerGroupCachingAgent extends AbstractGoogleCachingAgent i
     BatchRequest instanceGroupsRequest = buildBatchRequest()
     BatchRequest autoscalerRequest = buildBatchRequest()
 
-    List<InstanceTemplate> instanceTemplates = timeExecute(
-        compute.instanceTemplates().list(project),
-        "compute.instanceTemplates.list",
-        TAG_SCOPE, SCOPE_GLOBAL)
-        .getItems()
+    List<InstanceTemplate> instanceTemplates = GoogleZonalServerGroupCachingAgent.fetchInstanceTemplates(compute, project)
 
     InstanceGroupManagerCallbacks instanceGroupManagerCallbacks = new InstanceGroupManagerCallbacks(
       providerCache: providerCache,

diff --git a/...lix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy b/...lix/spinnaker/clouddriver/google/provider/agent/GoogleZonalServerGroupCachingAgent.groovy
@@ -22,6 +22,7 @@ import com.google.api.client.googleapis.batch.BatchRequest
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback
 import com.google.api.client.googleapis.json.GoogleJsonError
 import com.google.api.client.http.HttpHeaders
+import com.google.api.services.compute.Compute
 import com.google.api.services.compute.model.*
 import com.netflix.frigga.Names
 import com.netflix.frigga.ami.AppVersion
@@ -42,6 +43,7 @@ import com.netflix.spinnaker.clouddriver.google.model.GoogleServerGroup
 import com.netflix.spinnaker.clouddriver.google.model.callbacks.Utils
 import com.netflix.spinnaker.clouddriver.google.model.loadbalancing.GoogleHttpLoadBalancingPolicy
 import com.netflix.spinnaker.clouddriver.google.security.GoogleNamedAccountCredentials
+import com.netflix.spinnaker.clouddriver.googlecommon.GoogleExecutor
 import groovy.transform.Canonical
 import groovy.util.logging.Slf4j
 
@@ -135,11 +137,7 @@ class GoogleZonalServerGroupCachingAgent extends AbstractGoogleCachingAgent impl
     BatchRequest instanceGroupsRequest = buildBatchRequest()
     BatchRequest autoscalerRequest = buildBatchRequest()
 
-    List<InstanceTemplate> instanceTemplates = timeExecute(
-        compute.instanceTemplates().list(project),
-        "compute.instanceTemplates.list",
-        TAG_SCOPE, SCOPE_GLOBAL)
-        .getItems()
+    List<InstanceTemplate> instanceTemplates = fetchInstanceTemplates(compute, project)
 
     zones?.each { String zone ->
       InstanceGroupManagerCallbacks instanceGroupManagerCallbacks = new InstanceGroupManagerCallbacks(
@@ -165,6 +163,25 @@ class GoogleZonalServerGroupCachingAgent extends AbstractGoogleCachingAgent impl
     serverGroups
   }
 
+  static List<InstanceTemplate> fetchInstanceTemplates(Compute compute, String project) {
+    Boolean executedAtLeastOnce = false
+    String nextPageToken = null
+    List<InstanceTemplate> instanceTemplates = []
+    while (!executedAtLeastOnce || nextPageToken) {
+      InstanceTemplateList instanceTemplateList = GoogleExecutor.timeExecute(
+          GoogleExecutor.getRegistry(),
+          compute.instanceTemplates().list(project).setPageToken(nextPageToken),
+          "google.api",
+          "compute.instanceTemplates.list",
+          GoogleExecutor.TAG_SCOPE, GoogleExecutor.SCOPE_GLOBAL)
+
+      executedAtLeastOnce = true
+      nextPageToken = instanceTemplateList.getNextPageToken()
+      instanceTemplates.addAll(instanceTemplateList.getItems())
+    }
+    return instanceTemplates
+  }
+
   @Override
   boolean handles(OnDemandAgent.OnDemandType type, String cloudProvider) {
     type == OnDemandAgent.OnDemandType.ServerGroup && cloudProvider == GoogleCloudProvider.ID