feat(provider/kubernetes): v2 generic on demand caching (#1884)

clouddriver-core/src/main/groovy/com/netflix/spinnaker/clouddriver/cache/OnDemandAgent.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2015 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.cache;
+
+import com.netflix.spinnaker.cats.agent.CacheResult;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public interface OnDemandAgent {
+  String getProviderName();
+
+  String getOnDemandAgentType();
+
+  // TODO(ttomsu): This seems like it should go in a different interface.
+  OnDemandMetricsSupport getMetricsSupport();
+
+  enum OnDemandType {
+    ServerGroup,
+    SecurityGroup,
+    LoadBalancer,
+    Job,
+    TargetGroup;
+
+    static OnDemandType fromString(String s) {
+      return Arrays.stream(values())
+          .filter(v -> v.toString().equalsIgnoreCase(s))
+          .findFirst()
+          .orElseThrow(() -> new IllegalArgumentException("Cannot create OnDemandType from '" + s + "'"));
+    }
+  }
+
+  boolean handles(OnDemandType type, String cloudProvider);
+
+  static class OnDemandResult {
+    String sourceAgentType;
+    Collection<String> authoritativeTypes = new ArrayList<>();
+    CacheResult cacheResult;
+    Map<String, Collection<String>> evictions = new HashMap<>();
+
+    public OnDemandResult() {}
+
+    public OnDemandResult(String sourceAgentType, CacheResult cacheResult, Map<String, Collection<String>> evictions) {
+      this.sourceAgentType = sourceAgentType;
+      this.cacheResult = cacheResult;
+      this.evictions = evictions;
+    }
+  }
+
+  OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data);
+  Collection<Map> pendingOnDemandRequests(ProviderCache providerCache);
+}

clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/caching/KubernetesCachingAgent.java

@@ -59,7 +59,7 @@ protected KubernetesCachingAgent(KubernetesNamedAccountCredentials<C> namedAccou
 
   @Override
   public String getAgentType() {
-    return String.format("%s/%s[%d/%d]", accountName, this.getClass().getSimpleName(), agentIndex, agentCount);
+    return String.format("%s/%s[%d/%d]", accountName, this.getClass().getSimpleName(), agentIndex + 1, agentCount);
   }
 
   protected void reloadNamespaces() {

clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/Keys.java

@@ -19,6 +19,7 @@
 
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesApiVersion;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesKind;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesManifest;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.extern.slf4j.Slf4j;
@@ -91,6 +92,10 @@ public static String infrastructure(KubernetesKind kind, KubernetesApiVersion ve
     return createKey(Kind.INFRASTRUCTURE, kind, version, account, namespace, name);
   }
 
+  public static String infrastructure(KubernetesManifest manifest, String account) {
+    return infrastructure(manifest.getKind(), manifest.getApiVersion(), account, manifest.getNamespace(), manifest.getName());
+  }
+
   public static Optional<CacheKey> parseKey(String key) {
     String[] parts = key.split(":", -1);
 
@@ -150,6 +155,11 @@ public ApplicationCacheKey(String[] parts) {
       name = parts[3];
     }
 
+    @Override
+    public String toString() {
+      return createKey(kind, logicalKind, name);
+    }
+
     @Override
     public String getGroup() {
       return logicalKind.toString();
@@ -173,6 +183,11 @@ public ClusterCacheKey(String[] parts) {
       name = parts[4];
     }
 
+    @Override
+    public String toString() {
+      return createKey(kind, logicalKind, account, name);
+    }
+
     @Override
     public String getGroup() {
       return logicalKind.toString();
@@ -201,6 +216,11 @@ public InfrastructureCacheKey(String[] parts) {
       name = parts[6];
     }
 
+    @Override
+    public String toString() {
+      return createKey(kind, kubernetesKind, kubernetesApiVersion, account, namespace, name);
+    }
+
     @Override
     public String getGroup() {
       return kubernetesKind.toString();

clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgent.java

@@ -60,7 +60,7 @@ public class KubernetesPodCachingAgent extends KubernetesV2CachingAgent<V1Pod> {
   );
 
   @Override
-  protected List<V1Pod> loadPrimaryResource() {
+  protected List<V1Pod> loadPrimaryResourceList() {
     return namespaces.stream()
         .map(credentials::listAllPods)
         .flatMap(Collection::stream)

clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesReplicaSetCachingAgent.java

@@ -18,10 +18,12 @@
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
 import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.cats.agent.AgentDataType;
 import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesNamedAccountCredentials;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesApiVersion;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.security.KubernetesV2Credentials;
 import io.kubernetes.client.models.V1beta1ReplicaSet;
@@ -33,13 +35,15 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
 import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE;
 
 @Slf4j
-public class KubernetesReplicaSetCachingAgent extends KubernetesV2CachingAgent<V1beta1ReplicaSet> {
+public class KubernetesReplicaSetCachingAgent extends KubernetesV2OnDemandCachingAgent<V1beta1ReplicaSet> {
   KubernetesReplicaSetCachingAgent(KubernetesNamedAccountCredentials<KubernetesV2Credentials> namedAccountCredentials,
       ObjectMapper objectMapper,
       Registry registry,
@@ -59,10 +63,44 @@ public class KubernetesReplicaSetCachingAgent extends KubernetesV2CachingAgent<V
   );
 
   @Override
-  protected List<V1beta1ReplicaSet> loadPrimaryResource() {
+  protected List<V1beta1ReplicaSet> loadPrimaryResourceList() {
     return namespaces.stream()
         .map(credentials::listAllReplicaSets)
         .flatMap(Collection::stream)
         .collect(Collectors.toList());
   }
+
+  @Override
+  protected V1beta1ReplicaSet loadPrimaryResource(String namespace, String name) {
+    return credentials.readReplicaSet(namespace, name);
+  }
+
+  @Override
+  protected OnDemandType onDemandType() {
+    return OnDemandType.ServerGroup;
+  }
+
+  @Override
+  protected KubernetesKind primaryKind() {
+    return KubernetesKind.REPLICA_SET;
+  }
+
+  @Override
+  protected KubernetesApiVersion primaryApiVersion() {
+    return KubernetesApiVersion.EXTENSIONS_V1BETA1;
+  }
+
+  @Override
+  protected Map<String, String> mapKeyToOnDemandResult(Keys.InfrastructureCacheKey key) {
+    return new ImmutableMap.Builder<String, String>()
+        .put("serverGroup", key.getName())
+        .put("account", key.getAccount())
+        .put("region", key.getNamespace())
+        .build();
+  }
+
+  @Override
+  protected Optional<String> getResourceNameFromOnDemandRequest(Map<String, ?> request) {
+    return request.containsKey("serverGroup") ? Optional.of((String) request.get("serverGroup")) : Optional.empty();
+  }
 }

clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgent.java

@@ -28,6 +28,7 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.security.KubernetesV2Credentials;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -45,8 +46,15 @@ protected KubernetesV2CachingAgent(KubernetesNamedAccountCredentials<KubernetesV
   @Override
   public CacheResult loadData(ProviderCache providerCache) {
     reloadNamespaces();
+    return buildCacheResult(loadPrimaryResourceList());
+  }
+
+  protected CacheResult buildCacheResult(T resource) {
+    return buildCacheResult(Collections.singletonList(resource));
+  }
 
-    List<CacheData> resourceData = loadPrimaryResource().stream()
+  protected CacheResult buildCacheResult(List<T> resources) {
+    List<CacheData> resourceData = resources.stream()
         .map(rs -> KubernetesCacheDataConverter.fromResource(accountName, objectMapper, rs))
         .filter(Objects::nonNull)
         .collect(Collectors.toList());
@@ -62,7 +70,8 @@ public CacheResult loadData(ProviderCache providerCache) {
     KubernetesCacheDataConverter.logStratifiedCacheData(getAgentType(), entries);
 
     return new DefaultCacheResult(entries);
+
   }
 
-  protected abstract List<T> loadPrimaryResource();
+  protected abstract List<T> loadPrimaryResourceList();
 }