diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgent.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgent.java new file mode 100644 index 00000000000..785ed05e92c --- /dev/null +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgent.java @@ -0,0 +1,69 @@ +/* + * Copyright 2017 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spectator.api.Registry; +import com.netflix.spinnaker.cats.agent.AgentDataType; +import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesNamedAccountCredentials; +import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys; +import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesKind; +import com.netflix.spinnaker.clouddriver.kubernetes.v2.security.KubernetesV2Credentials; +import io.kubernetes.client.models.V1Pod; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE; +import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE; + +@Slf4j +public class KubernetesPodCachingAgent extends KubernetesV2CachingAgent { + KubernetesPodCachingAgent(KubernetesNamedAccountCredentials namedAccountCredentials, + ObjectMapper objectMapper, + Registry registry, + int agentIndex, + int agentCount) { + super(namedAccountCredentials, objectMapper, registry, agentIndex, agentCount); + } + + @Getter + final private Collection providedDataTypes = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + INFORMATIVE.forType(Keys.LogicalKind.APPLICATION.toString()), + INFORMATIVE.forType(Keys.LogicalKind.CLUSTER.toString()), + INFORMATIVE.forType(KubernetesKind.DEPLOYMENT.toString()), + INFORMATIVE.forType(KubernetesKind.REPLICA_SET.toString()), + AUTHORITATIVE.forType(KubernetesKind.POD.toString()) + )) + ); + + @Override + protected List loadPrimaryResource() { + return namespaces.stream() + .map(credentials::listAllPods) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } +} diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesReplicaSetCachingAgent.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesReplicaSetCachingAgent.java index 5d9193cdfb9..8b1f0ea6117 100644 --- a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesReplicaSetCachingAgent.java +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesReplicaSetCachingAgent.java @@ -20,11 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spectator.api.Registry; import com.netflix.spinnaker.cats.agent.AgentDataType; -import com.netflix.spinnaker.cats.agent.CacheResult; -import com.netflix.spinnaker.cats.agent.DefaultCacheResult; -import com.netflix.spinnaker.cats.cache.CacheData; -import com.netflix.spinnaker.cats.provider.ProviderCache; -import com.netflix.spinnaker.clouddriver.kubernetes.caching.KubernetesCachingAgent; import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesNamedAccountCredentials; import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys; import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesKind; @@ -38,16 +33,14 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE; import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.INFORMATIVE; @Slf4j -public class KubernetesReplicaSetCachingAgent extends KubernetesCachingAgent { - KubernetesReplicaSetCachingAgent(KubernetesNamedAccountCredentials namedAccountCredentials, +public class KubernetesReplicaSetCachingAgent extends KubernetesV2CachingAgent { + KubernetesReplicaSetCachingAgent(KubernetesNamedAccountCredentials namedAccountCredentials, ObjectMapper objectMapper, Registry registry, int agentIndex, @@ -66,28 +59,7 @@ public class KubernetesReplicaSetCachingAgent extends KubernetesCachingAgent replicaSetData = loadReplicaSets().stream() - .map(rs -> KubernetesCacheDataConverter.fromResource(accountName, objectMapper, rs)) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - List invertedRelationships = replicaSetData.stream() - .map(KubernetesCacheDataConverter::invertRelationships) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - replicaSetData.addAll(invertedRelationships); - - Map> entries = KubernetesCacheDataConverter.stratifyCacheDataByGroup(replicaSetData); - KubernetesCacheDataConverter.logStratifiedCacheData(getAgentType(), entries); - - return new DefaultCacheResult(entries); - } - - private List loadReplicaSets() { + protected List loadPrimaryResource() { return namespaces.stream() .map(credentials::listAllReplicaSets) .flatMap(Collection::stream) diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgent.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgent.java new file mode 100644 index 00000000000..57c5e1d1148 --- /dev/null +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgent.java @@ -0,0 +1,68 @@ +/* + * Copyright 2017 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spectator.api.Registry; +import com.netflix.spinnaker.cats.agent.CacheResult; +import com.netflix.spinnaker.cats.agent.DefaultCacheResult; +import com.netflix.spinnaker.cats.cache.CacheData; +import com.netflix.spinnaker.cats.provider.ProviderCache; +import com.netflix.spinnaker.clouddriver.kubernetes.caching.KubernetesCachingAgent; +import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesNamedAccountCredentials; +import com.netflix.spinnaker.clouddriver.kubernetes.v2.security.KubernetesV2Credentials; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public abstract class KubernetesV2CachingAgent extends KubernetesCachingAgent { + protected KubernetesV2CachingAgent(KubernetesNamedAccountCredentials namedAccountCredentials, + ObjectMapper objectMapper, + Registry registry, + int agentIndex, + int agentCount) { + super(namedAccountCredentials, objectMapper, registry, agentIndex, agentCount); + } + + @Override + public CacheResult loadData(ProviderCache providerCache) { + reloadNamespaces(); + + List resourceData = loadPrimaryResource().stream() + .map(rs -> KubernetesCacheDataConverter.fromResource(accountName, objectMapper, rs)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + List invertedRelationships = resourceData.stream() + .map(KubernetesCacheDataConverter::invertRelationships) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + resourceData.addAll(invertedRelationships); + + Map> entries = KubernetesCacheDataConverter.stratifyCacheDataByGroup(resourceData); + KubernetesCacheDataConverter.logStratifiedCacheData(getAgentType(), entries); + + return new DefaultCacheResult(entries); + } + + protected abstract List loadPrimaryResource(); +} diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgentDispatcher.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgentDispatcher.java index 5e768e1ebd4..7d0900dbd36 100644 --- a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgentDispatcher.java +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesV2CachingAgentDispatcher.java @@ -45,6 +45,7 @@ public List buildAllCachingAgents(KubernetesNamedAccount return IntStream.range(0, credentials.getCacheThreads()) .boxed() .map(i -> new ArrayList(Arrays.asList( + new KubernetesPodCachingAgent(credentials, objectMapper, registry, i, credentials.getCacheThreads()), new KubernetesReplicaSetCachingAgent(credentials, objectMapper, registry, i, credentials.getCacheThreads()) ))) .flatMap(Collection::stream) diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesKind.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesKind.java index 79135fafb87..7a0b5dfab6b 100644 --- a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesKind.java +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/description/KubernetesKind.java @@ -25,6 +25,7 @@ public enum KubernetesKind { DEPLOYMENT("deployment"), INGRESS("ingress"), + POD("pod"), REPLICA_SET("replicaSet"), NETWORK_POLICY("networkPolicy"), SERVICE("service"); diff --git a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/security/KubernetesV2Credentials.java b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/security/KubernetesV2Credentials.java index d20ed02134c..36c75bf35c1 100644 --- a/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/security/KubernetesV2Credentials.java +++ b/clouddriver-kubernetes/src/main/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/security/KubernetesV2Credentials.java @@ -32,6 +32,8 @@ import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.models.AppsV1beta1Deployment; +import io.kubernetes.client.models.V1Pod; +import io.kubernetes.client.models.V1PodList; import io.kubernetes.client.models.V1Service; import io.kubernetes.client.models.V1beta1Ingress; import io.kubernetes.client.models.V1beta1ReplicaSet; @@ -167,6 +169,29 @@ public void createIngress(V1beta1Ingress ingress) { }); } + public List listAllPods(String namespace) { + return listPods(namespace, new KubernetesSelectorList(), new KubernetesSelectorList()); + } + + public List listPods(String namespace, KubernetesSelectorList fieldSelectors, KubernetesSelectorList labelSelectors) { + final String methodName = "pods.list"; + final String fieldSelectorString = fieldSelectors.toString(); + final String labelSelectorString = labelSelectors.toString(); + final KubernetesApiVersion apiVersion = KubernetesApiVersion.V1; + final KubernetesKind kind = KubernetesKind.POD; + return runAndRecordMetrics(methodName, namespace, () -> { + try { + V1PodList list = coreV1Api.listNamespacedPod(namespace, PRETTY, fieldSelectorString, labelSelectorString, DEFAULT_VERSION, TIMEOUT_SECONDS, WATCH); + return annotateMissingFields(list == null ? new ArrayList<>() : list.getItems(), + V1Pod.class, + apiVersion, + kind); + } catch (ApiException e) { + throw new KubernetesApiException(methodName, e); + } + }); + } + public void createReplicaSet(V1beta1ReplicaSet replicaSet) { final String methodName = "replicaSets.create"; final String namespace = replicaSet.getMetadata().getNamespace(); diff --git a/clouddriver-kubernetes/src/test/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgentSpec.groovy b/clouddriver-kubernetes/src/test/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgentSpec.groovy new file mode 100644 index 00000000000..90a2b1b15c1 --- /dev/null +++ b/clouddriver-kubernetes/src/test/groovy/com/netflix/spinnaker/clouddriver/kubernetes/v2/caching/agent/KubernetesPodCachingAgentSpec.groovy @@ -0,0 +1,74 @@ +/* + * Copyright 2017 Google, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.agent + +import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesNamedAccountCredentials +import com.netflix.spinnaker.clouddriver.kubernetes.v2.caching.Keys +import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesApiVersion +import com.netflix.spinnaker.clouddriver.kubernetes.v2.description.KubernetesKind +import com.netflix.spinnaker.clouddriver.kubernetes.v2.security.KubernetesV2Credentials +import io.kubernetes.client.models.V1ObjectMeta +import io.kubernetes.client.models.V1Pod +import spock.lang.Specification + +class KubernetesPodCachingAgentSpec extends Specification { + def ACCOUNT = "my-account" + def CLUSTER = "my-cluster" + def APPLICATION = "my-application" + def NAME = "the-name" + def NAMESPACE = "your-namespace" + + void "invokes caching agent on output pod"() { + setup: + def pod = new V1Pod() + def annotations = [ + 'relationships.spinnaker.io/cluster': '"' + CLUSTER + '"', + 'relationships.spinnaker.io/application': '"' + APPLICATION + '"' + ] + + def metadata = new V1ObjectMeta() + metadata.setAnnotations(annotations) + metadata.setName(NAME) + metadata.setNamespace(NAMESPACE) + pod.setMetadata(metadata) + pod.setKind(KubernetesKind.REPLICA_SET.name) + pod.setApiVersion(KubernetesApiVersion.EXTENSIONS_V1BETA1.name) + + def credentials = Mock(KubernetesV2Credentials) + credentials.getDeclaredNamespaces() >> [NAMESPACE] + credentials.listAllPods(NAMESPACE) >> [pod] + + def namedAccountCredentials = Mock(KubernetesNamedAccountCredentials) + namedAccountCredentials.getCredentials() >> credentials + namedAccountCredentials.getName() >> ACCOUNT + + def cachingAgent = new KubernetesPodCachingAgent(namedAccountCredentials, new ObjectMapper(), null, 0, 1) + + when: + def result = cachingAgent.loadData(null) + + then: + result.cacheResults[KubernetesKind.REPLICA_SET.name].size() == 1 + def cacheData = result.cacheResults[KubernetesKind.REPLICA_SET.name].iterator().next() + cacheData.relationships.get(Keys.LogicalKind.CLUSTER.toString()) == [Keys.cluster(ACCOUNT, CLUSTER)] + cacheData.relationships.get(Keys.LogicalKind.APPLICATION.toString()) == [Keys.application(APPLICATION)] + cacheData.attributes.get("name") == NAME + cacheData.attributes.get("namespace") == NAMESPACE + } +}