ECS Container Instance caching classes and tests. (#2143)

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/ContainerInstanceCacheClient.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.client;
+
+import com.netflix.spinnaker.cats.cache.Cache;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.ContainerInstance;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES;
+
+public class ContainerInstanceCacheClient extends AbstractCacheClient<ContainerInstance> {
+
+  @Autowired
+  public ContainerInstanceCacheClient(Cache cacheView) {
+    super(cacheView, CONTAINER_INSTANCES.toString());
+  }
+
+  @Override
+  protected ContainerInstance convert(CacheData cacheData) {
+    ContainerInstance containerInstance = new ContainerInstance();
+    Map<String, Object> attributes = cacheData.getAttributes();
+    containerInstance.setArn((String) attributes.get("containerInstanceArn"));
+    containerInstance.setEc2InstanceId((String) attributes.get("ec2InstanceId"));
+
+    return containerInstance;
+  }
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/model/ContainerInstance.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.model;
+
+import lombok.Data;
+
+@Data
+public class ContainerInstance {
+  String arn;
+  String ec2InstanceId;
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/ContainerInstanceCachingAgent.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.ecs.AmazonECS;
+import com.amazonaws.services.ecs.model.ContainerInstance;
+import com.amazonaws.services.ecs.model.DescribeContainerInstancesRequest;
+import com.amazonaws.services.ecs.model.ListContainerInstancesRequest;
+import com.amazonaws.services.ecs.model.ListContainerInstancesResult;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.cats.agent.AgentDataType;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES;
+
+public class ContainerInstanceCachingAgent extends AbstractEcsOnDemandAgent<ContainerInstance> {
+  private static final Collection<AgentDataType> types = Collections.unmodifiableCollection(Arrays.asList(
+    AUTHORITATIVE.forType(CONTAINER_INSTANCES.toString())
+  ));
+  private final Logger log = LoggerFactory.getLogger(getClass());
+
+  public ContainerInstanceCachingAgent(String accountName, String region, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) {
+    super(accountName, region, amazonClientProvider, awsCredentialsProvider, registry);
+  }
+
+  @Override
+  public String getAgentType() {
+    return ContainerInstanceCachingAgent.class.getSimpleName();
+  }
+
+  @Override
+  public Collection<AgentDataType> getProvidedDataTypes() {
+    return types;
+  }
+
+  @Override
+  protected List<ContainerInstance> getItems(AmazonECS ecs, ProviderCache providerCache) {
+    List<ContainerInstance> containerInstanceList = new LinkedList<>();
+    Set<String> clusters = getClusters(ecs, providerCache);
+
+    for (String cluster : clusters) {
+      String nextToken = null;
+      do {
+        ListContainerInstancesRequest listContainerInstancesRequest = new ListContainerInstancesRequest().withCluster(cluster);
+        if (nextToken != null) {
+          listContainerInstancesRequest.setNextToken(nextToken);
+        }
+
+        ListContainerInstancesResult listContainerInstancesResult = ecs.listContainerInstances(listContainerInstancesRequest);
+        List<String> containerInstanceArns = listContainerInstancesResult.getContainerInstanceArns();
+        if (containerInstanceArns.size() == 0) {
+          continue;
+        }
+
+        List<ContainerInstance> containerInstances = ecs.describeContainerInstances(new DescribeContainerInstancesRequest()
+          .withCluster(cluster).withContainerInstances(containerInstanceArns)).getContainerInstances();
+        containerInstanceList.addAll(containerInstances);
+
+        nextToken = listContainerInstancesResult.getNextToken();
+      } while (nextToken != null && nextToken.length() != 0);
+    }
+    return containerInstanceList;
+  }
+
+  @Override
+  protected Map<String, Collection<CacheData>> generateFreshData(Collection<ContainerInstance> containerInstances) {
+    Collection<CacheData> dataPoints = new LinkedList<>();
+
+    for (ContainerInstance containerInstance : containerInstances) {
+      Map<String, Object> attributes = convertContainerInstanceToAttributes(containerInstance);
+
+      String key = Keys.getContainerInstanceKey(accountName, region, containerInstance.getContainerInstanceArn());
+      dataPoints.add(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+    }
+
+    log.info("Caching " + dataPoints.size() + " container instances in " + getAgentType());
+    Map<String, Collection<CacheData>> dataMap = new HashMap<>();
+    dataMap.put(CONTAINER_INSTANCES.toString(), dataPoints);
+
+    return dataMap;
+  }
+
+  public static  Map<String, Object> convertContainerInstanceToAttributes(ContainerInstance containerInstance){
+    Map<String, Object> attributes = new HashMap<>();
+    attributes.put("containerInstanceArn", containerInstance.getContainerInstanceArn());
+    attributes.put("ec2InstanceId", containerInstance.getEc2InstanceId());
+    return attributes;
+  }
+}

clouddriver-ecs/src/test/java/com/netflix/spinnaker/clouddriver/ecs/cache/ContainerInstanceCacheClientTest.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache;
+
+import com.amazonaws.services.ecs.model.ContainerInstance;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ContainerInstanceCacheClient;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.ContainerInstanceCachingAgent;
+import org.junit.Test;
+import spock.lang.Subject;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+public class ContainerInstanceCacheClientTest extends CommonCacheClient {
+  @Subject
+  private final ContainerInstanceCacheClient client = new ContainerInstanceCacheClient(cacheView);
+
+  @Test
+  public void shouldConvert() {
+    //Given
+    String containerInstanceArn = "arn:aws:ecs:" + REGION + ":012345678910:container-instance/14e8cce9-0b16-4af4-bfac-a85f7587aa98";
+    String key = Keys.getContainerInstanceKey(ACCOUNT, REGION, containerInstanceArn);
+
+    ContainerInstance containerInstance = new ContainerInstance();
+    containerInstance.setEc2InstanceId("i-deadbeef");
+    containerInstance.setContainerInstanceArn(containerInstanceArn);
+
+    Map<String, Object> attributes = ContainerInstanceCachingAgent.convertContainerInstanceToAttributes(containerInstance);
+    when(cacheView.get(CONTAINER_INSTANCES.toString(), key)).thenReturn(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+
+    //When
+    com.netflix.spinnaker.clouddriver.ecs.cache.model.ContainerInstance ecsContainerInstance = client.get(key);
+
+    //Then
+    assertTrue("Expected the EC2 instance ID to be " + containerInstance.getEc2InstanceId() + " but got " + ecsContainerInstance.getEc2InstanceId(),
+      containerInstance.getEc2InstanceId().equals(ecsContainerInstance.getEc2InstanceId()));
+
+    assertTrue("Expected the container instance ARN to be " + containerInstance.getContainerInstanceArn() + " but got " + ecsContainerInstance.getArn(),
+      containerInstance.getContainerInstanceArn().equals(ecsContainerInstance.getArn()));
+  }
+}

clouddriver-ecs/src/test/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/ContainerInstanceCacheTest.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.services.ecs.model.ContainerInstance;
+import com.amazonaws.services.ecs.model.DescribeContainerInstancesRequest;
+import com.amazonaws.services.ecs.model.DescribeContainerInstancesResult;
+import com.amazonaws.services.ecs.model.ListClustersRequest;
+import com.amazonaws.services.ecs.model.ListClustersResult;
+import com.amazonaws.services.ecs.model.ListContainerInstancesRequest;
+import com.amazonaws.services.ecs.model.ListContainerInstancesResult;
+import com.netflix.spinnaker.cats.agent.CacheResult;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ContainerInstanceCacheClient;
+import org.junit.Test;
+import spock.lang.Subject;
+
+import java.util.Collection;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES;
+import static junit.framework.TestCase.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+
+public class ContainerInstanceCacheTest extends CommonCachingAgent {
+  @Subject
+  private final ContainerInstanceCachingAgent agent = new ContainerInstanceCachingAgent(ACCOUNT, REGION, clientProvider, credentialsProvider, registry);
+  @Subject
+  private final ContainerInstanceCacheClient client = new ContainerInstanceCacheClient(providerCache);
+
+  @Test
+  public void shouldRetrieveFromWrittenCache() {
+    //Given
+    String key = Keys.getContainerInstanceKey(ACCOUNT, REGION, CONTAINER_INSTANCE_ARN_1);
+
+    ContainerInstance containerInstance = new ContainerInstance();
+    containerInstance.setContainerInstanceArn(CONTAINER_INSTANCE_ARN_1);
+    containerInstance.setEc2InstanceId(EC2_INSTANCE_ID_1);
+
+    when(ecs.listClusters(any(ListClustersRequest.class))).thenReturn(new ListClustersResult().withClusterArns(CLUSTER_ARN_1));
+    when(ecs.listContainerInstances(any(ListContainerInstancesRequest.class))).thenReturn(new ListContainerInstancesResult().withContainerInstanceArns(CONTAINER_INSTANCE_ARN_1));
+    when(ecs.describeContainerInstances(any(DescribeContainerInstancesRequest.class))).thenReturn(new DescribeContainerInstancesResult().withContainerInstances(containerInstance));
+
+    //When
+    CacheResult cacheResult = agent.loadData(providerCache);
+    when(providerCache.get(CONTAINER_INSTANCES.toString(), key)).thenReturn(cacheResult.getCacheResults().get(CONTAINER_INSTANCES.toString()).iterator().next());
+
+    //Then
+    Collection<CacheData> cacheData = cacheResult.getCacheResults().get(CONTAINER_INSTANCES.toString());
+    com.netflix.spinnaker.clouddriver.ecs.cache.model.ContainerInstance ecsContainerInstance = client.get(key);
+
+    assertTrue("Expected CacheData to be returned but null is returned", cacheData != null);
+    assertTrue("Expected 1 CacheData but returned " + cacheData.size(), cacheData.size() == 1);
+    String retrievedKey = cacheData.iterator().next().getId();
+    assertTrue("Expected CacheData with ID " + key + " but retrieved ID " + retrievedKey, retrievedKey.equals(key));
+
+    assertTrue("Expected the container instance to have EC2 instance ID of " + containerInstance.getEc2InstanceId() + " but got " + ecsContainerInstance.getEc2InstanceId(),
+      containerInstance.getEc2InstanceId().equals(ecsContainerInstance.getEc2InstanceId()));
+    assertTrue("Expected the container instance to have the ARN " + containerInstance.getContainerInstanceArn() + " but got " + ecsContainerInstance.getArn(),
+      containerInstance.getContainerInstanceArn().equals(ecsContainerInstance.getArn()));
+  }
+}