feat(provider/kubernetes): search result hydrators (#2124)

spinnaker · Nov 15, 2017 · e9eab67 · e9eab67
1 parent d1acdab
commit e9eab67
Show file tree

Hide file tree

Showing 10 changed files with 143 additions and 18 deletions.
diff --git a/...ernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/Keys.java b/...ernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/Keys.java
@@ -17,6 +17,7 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.netflix.spinnaker.clouddriver.kubernetes.KubernetesCloudProvider;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
@@ -46,6 +47,7 @@ public String toString() {
       return name().toLowerCase();
     }
 
+    @JsonCreator
     public static Kind fromString(String name) {
       return Arrays.stream(values())
           .filter(k -> k.toString().equalsIgnoreCase(name))
@@ -72,6 +74,7 @@ public String singular() {
       return name.substring(0, name.length() - 1);
     }
 
+    @JsonCreator
     public static LogicalKind fromString(String name) {
       return Arrays.stream(values())
           .filter(k -> k.toString().equalsIgnoreCase(name))
@@ -159,6 +162,7 @@ public static abstract class CacheKey {
     public abstract String getName();
   }
 
+  @EqualsAndHashCode(callSuper = true)
   @Data
   public static abstract class LogicalKey extends CacheKey {
     private Kind kind = Kind.LOGICAL;

diff --git a/...y/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/KubernetesV2SearchProvider.java b/...y/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/KubernetesV2SearchProvider.java
@@ -23,6 +23,8 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys.LogicalKey;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys.LogicalKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesResourceProperties;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesResourcePropertyRegistry;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.search.SearchProvider;
@@ -51,21 +53,31 @@ public class KubernetesV2SearchProvider implements SearchProvider {
   final private KubernetesCacheUtils cacheUtils;
   final private ObjectMapper mapper;
   final private KubernetesSpinnakerKindMap kindMap;
+  final private KubernetesResourcePropertyRegistry registry;
   final private List<String> defaultTypes;
-  final private Set<LogicalKind> logicalTypes;
+  final private Set<String> logicalTypes;
+  final private Set<String> allCaches;
 
   @Autowired
-  public KubernetesV2SearchProvider(KubernetesCacheUtils cacheUtils, KubernetesSpinnakerKindMap kindMap, ObjectMapper mapper) {
+  public KubernetesV2SearchProvider(KubernetesCacheUtils cacheUtils,
+      KubernetesSpinnakerKindMap kindMap,
+      ObjectMapper mapper,
+      KubernetesResourcePropertyRegistry registry) {
     this.cacheUtils = cacheUtils;
     this.mapper = mapper;
     this.kindMap = kindMap;
+    this.registry = registry;
 
     this.defaultTypes = kindMap.allKubernetesKinds()
         .stream()
         .map(KubernetesKind::toString)
         .collect(Collectors.toList());
     this.logicalTypes = Arrays.stream(LogicalKind.values())
+        .map(LogicalKind::toString)
         .collect(Collectors.toSet());
+
+    this.allCaches = new HashSet<>(defaultTypes);
+    this.allCaches.addAll(logicalTypes);
   }
 
   @Override
@@ -110,28 +122,43 @@ private Map<String, Object> convertKeyToMap(String key) {
     }
 
     Keys.CacheKey parsedKey = optional.get();
-    Map<String, Object> result = mapper.convertValue(parsedKey, new TypeReference<Map<String, Object>>() {});
+    Map<String, Object> result;
+    String type;
 
     if (parsedKey instanceof Keys.InfrastructureCacheKey) {
       Keys.InfrastructureCacheKey infraKey = (Keys.InfrastructureCacheKey) parsedKey;
-      result.put("type", kindMap.translateKubernetesKind(infraKey.getKubernetesKind()).toString());
-      result.put("region", infraKey.getNamespace());
+      type = kindMap.translateKubernetesKind(infraKey.getKubernetesKind()).toString();
+
+      KubernetesResourceProperties properties = registry.get(infraKey.getKubernetesKind());
+      if (properties == null) {
+        log.warn("No hydrator for type {}, this is possibly a developer error", infraKey.getKubernetesKind());
+        return null;
+      }
+
+      result = properties.getHandler().hydrateSearchResult(infraKey, cacheUtils);
+    } else if (parsedKey instanceof Keys.LogicalKey) {
+      Keys.LogicalKey logicalKey = (Keys.LogicalKey) parsedKey;
+
+      result = mapper.convertValue(parsedKey, new TypeReference<Map<String, Object>>() {});
+      result.put(logicalKey.getLogicalKind().singular(), logicalKey.getName());
+      type = parsedKey.getGroup();
     } else {
-      result.put("type", parsedKey.getGroup());
+      log.warn("Unknown key type " + parsedKey + ", ignoring.");
+      return null;
     }
 
+    result.put("type", type);
     return result;
   }
 
   private Map<String, List<String>> getKeysRelatedToLogicalMatches(String matchQuery) {
     return logicalTypes.stream()
-        .map(type -> cacheUtils.getAllDataMatchingPattern(type.toString(), matchQuery)
+        .map(type -> cacheUtils.getAllDataMatchingPattern(type, matchQuery)
             .stream()
             .map(e -> e.getRelationships()
                 .values()
                 .stream()
-                .map(Collection::stream)
-                .flatMap(x -> x)
+                .flatMap(Collection::stream)
                 .filter(Objects::nonNull)
                 .map(k -> new ImmutablePair<>(k, e.getId()))
             ).flatMap(x -> x)
@@ -152,18 +179,38 @@ private Map<String, List<String>> getKeysRelatedToLogicalMatches(String matchQue
         );
   }
 
+  // TODO(lwander): use filters
   private List<Map<String, Object>> getMatches(String query, List<String> types, Map<String, String> filters) {
     String matchQuery = String.format("*%s*", query.toLowerCase());
     Set<String> typeSet = new HashSet<>(types);
 
-    List<Map<String, Object>> results = types.stream()
+    // We add k8s versions of Spinnaker types here to ensure that (for example) replica sets are returned when server groups are requested.
+    typeSet.addAll(types.stream()
+        .map(t -> {
+          try {
+            return KubernetesSpinnakerKindMap.SpinnakerKind.fromString(t);
+          } catch (IllegalArgumentException e) {
+            return null;
+          }
+        }).filter(Objects::nonNull)
+        .map(kindMap::translateSpinnakerKind)
+        .flatMap(Collection::stream)
+        .map(KubernetesKind::toString)
+        .collect(Collectors.toSet())
+    );
+
+    // Remove caches that we can't search
+    typeSet.retainAll(allCaches);
+
+    // Search caches directly
+    List<Map<String, Object>> results = typeSet.stream()
         .map(type -> cacheUtils.getAllKeysMatchingPattern(type, matchQuery))
-        .map(Collection::stream)
-        .flatMap(x -> x)
+        .flatMap(Collection::stream)
         .map(this::convertKeyToMap)
         .filter(Objects::nonNull)
         .collect(Collectors.toList());
 
+    // Search 'logical' caches (clusters, apps) for indirect matches
     Map<String, List<String>> keyToAllLogicalKeys = getKeysRelatedToLogicalMatches(matchQuery);
     results.addAll(keyToAllLogicalKeys.entrySet().stream()
         .map(kv -> {
@@ -178,18 +225,17 @@ private List<Map<String, Object>> getMatches(String query, List<String> types, M
                   .map(Optional::get)
                   .filter(LogicalKey.class::isInstance)
                   .map(k -> (LogicalKey) k)
-                  .forEach(k -> {
-                    result.put(k.getLogicalKind().singular(), k.getName());
-                  });
+                  .forEach(k -> result.put(k.getLogicalKind().singular(), k.getName()));
 
               return result;
             }
         )
-        .filter(r -> typeSet.contains(r.get("type")))
         .collect(Collectors.toList()));
 
 
-    log.info("Found {} keys matching {}", results.size(), query);
+    results = results.stream()
+        .filter(r -> typeSet.contains(r.get("type")) || typeSet.contains(r.get("group")))
+        .collect(Collectors.toList());
 
     return results;
   }

diff --git a/...lix/spinnaker/clouddriver/kubernetes/v2/caching/view/model/KubernetesV2SecurityGroup.java b/...lix/spinnaker/clouddriver/kubernetes/v2/caching/view/model/KubernetesV2SecurityGroup.java
@@ -136,11 +136,12 @@ private static class KubernetesV2SecurityGroupSummary implements SecurityGroupSu
   }
 
   @Data
-  public static class PortRule implements Rule {
+  private static class PortRule implements Rule {
     private SortedSet<PortRange> portRanges;
     private String protocol;
   }
 
+  @EqualsAndHashCode(callSuper = true)
   @Data
   public static class StringPortRange extends Rule.PortRange {
     protected String startPortName;

diff --git a/...m/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesSpinnakerKindMap.java b/...m/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesSpinnakerKindMap.java
@@ -17,9 +17,11 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.description;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import org.springframework.stereotype.Component;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -45,6 +47,14 @@ public enum SpinnakerKind {
     public String toString() {
       return id;
     }
+
+    @JsonCreator
+    public static SpinnakerKind fromString(String name) {
+      return Arrays.stream(values())
+          .filter(k -> k.toString().equalsIgnoreCase(name))
+          .findFirst()
+          .orElseThrow(() -> new IllegalArgumentException("No matching kind with name " + name + " exists"));
+    }
   }
 
   private Map<SpinnakerKind, Set<KubernetesKind>> spinnakerToKubernetes = new HashMap<>();

diff --git a/...groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesHandler.java b/...groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesHandler.java
@@ -17,10 +17,13 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.op.deployer;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.spinnaker.clouddriver.deploy.DeploymentResult;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.artifact.ArtifactReplacer;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
@@ -35,6 +38,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public abstract class KubernetesHandler {
   @Autowired
@@ -73,4 +77,11 @@ public DeploymentResult deployAugmentedManifest(KubernetesV2Credentials credenti
   protected void deploy(KubernetesV2Credentials credentials, KubernetesManifest manifest) {
     credentials.deploy(manifest);
   }
+
+  public Map<String, Object> hydrateSearchResult(Keys.InfrastructureCacheKey key, KubernetesCacheUtils cacheUtils) {
+    Map<String, Object> result = objectMapper.convertValue(key, new TypeReference<Map<String, Object>>() {});
+    result.put("region", key.getNamespace());
+    result.put("name", KubernetesManifest.getFullResourceName(key.getKubernetesKind(), key.getName()));
+    return result;
+  }
 }
diff --git a/...com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesIngressHandler.java b/...com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesIngressHandler.java
@@ -17,9 +17,11 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.op.deployer;
 
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesCacheDataConverter;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesIngressCachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
@@ -33,6 +35,7 @@
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 @Component
@@ -92,4 +95,12 @@ private static List<String> attachedServices(V1beta1Ingress ingress) {
 
     return new ArrayList<>(result);
   }
+
+  @Override
+  public Map<String, Object> hydrateSearchResult(Keys.InfrastructureCacheKey key, KubernetesCacheUtils cacheUtils) {
+    Map<String, Object> result = super.hydrateSearchResult(key, cacheUtils);
+    result.put("loadBalancer", result.get("name"));
+
+    return result;
+  }
 }
diff --git a/...ovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesPodHandler.java b/...ovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesPodHandler.java
@@ -17,14 +17,18 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.op.deployer;
 
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesPodCachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
 import com.netflix.spinnaker.clouddriver.model.Manifest.Status;
 import org.springframework.stereotype.Component;
 
+import java.util.Map;
+
 @Component
 public class KubernetesPodHandler extends KubernetesHandler implements CanDelete {
   @Override
@@ -47,6 +51,14 @@ public Status status(KubernetesManifest manifest) {
     return Status.stable();
   }
 
+  @Override
+  public Map<String, Object> hydrateSearchResult(Keys.InfrastructureCacheKey key, KubernetesCacheUtils cacheUtils) {
+    Map<String, Object> result = super.hydrateSearchResult(key, cacheUtils);
+    result.put("instanceId", result.get("name"));
+
+    return result;
+  }
+
   @Override
   public Class<? extends KubernetesV2CachingAgent> cachingAgentClass() {
     return KubernetesPodCachingAgent.class;

diff --git a/.../netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesReplicaSetHandler.java b/.../netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesReplicaSetHandler.java
@@ -19,9 +19,11 @@
 
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.artifact.ArtifactReplacer.Replacer;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.artifact.ArtifactTypes;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesCacheDataConverter;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesReplicaSetCachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
@@ -150,4 +152,12 @@ private static Map<String, String> getPodTemplateLabels(V1beta1ReplicaSet replic
   private static Map<String, String> getPodTemplateLabels(V1beta2ReplicaSet replicaSet) {
     return replicaSet.getSpec().getTemplate().getMetadata().getLabels();
   }
+
+  @Override
+  public Map<String, Object> hydrateSearchResult(Keys.InfrastructureCacheKey key, KubernetesCacheUtils cacheUtils) {
+    Map<String, Object> result = super.hydrateSearchResult(key, cacheUtils);
+    result.put("serverGroup", result.get("name"));
+
+    return result;
+  }
 }
diff --git a/...com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesServiceHandler.java b/...com/netflix/spinnaker/clouddriver/kubernetes/v2/op/deployer/KubernetesServiceHandler.java
@@ -17,9 +17,11 @@
 
 package com.netflix.spinnaker.clouddriver.kubernetes.v2.op.deployer;
 
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesCacheDataConverter;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesServiceCachingAgent;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent.KubernetesV2CachingAgent;
+import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.view.provider.KubernetesCacheUtils;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesSpinnakerKindMap.SpinnakerKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesKind;
 import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.manifest.KubernetesManifest;
@@ -65,4 +67,12 @@ public static Map<String, String> getSelector(KubernetesManifest manifest) {
         throw new IllegalArgumentException("No services with version " + manifest.getApiVersion() + " supported");
     }
   }
+
+  @Override
+  public Map<String, Object> hydrateSearchResult(Keys.InfrastructureCacheKey key, KubernetesCacheUtils cacheUtils) {
+    Map<String, Object> result = super.hydrateSearchResult(key, cacheUtils);
+    result.put("loadBalancer", result.get("name"));
+
+    return result;
+  }
 }