feat(provider/ecs): ECS Cluster caching classes and tests. (#2091)

* ECS Cluster caching classes and tests.

* Base abstraction.

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/EcsClusterCacheClient.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.client;
+
+import com.netflix.spinnaker.cats.cache.Cache;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsCluster;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
+
+public class EcsClusterCacheClient extends AbstractCacheClient<EcsCluster>{
+
+  @Autowired
+  public EcsClusterCacheClient(Cache cacheView) {
+    super(cacheView, ECS_CLUSTERS.toString());
+  }
+
+  @Override
+  protected EcsCluster convert(CacheData cacheData) {
+    EcsCluster ecsCluster = new EcsCluster();
+    Map<String, Object> attributes = cacheData.getAttributes();
+
+    ecsCluster.setAccount((String) attributes.get("account"));
+    ecsCluster.setRegion((String) attributes.get("region"));
+    ecsCluster.setName((String) attributes.get("clusterName"));
+    ecsCluster.setArn((String) attributes.get("clusterArn"));
+
+    return ecsCluster;
+  }
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/model/EcsCluster.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.model;
+
+import lombok.Data;
+
+@Data
+public class EcsCluster {
+  String account;
+  String region;
+  String name;
+  String arn;
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/AbstractEcsOnDemandAgent.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.ecs.AmazonECS;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.cats.agent.CacheResult;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
+import com.netflix.spinnaker.clouddriver.cache.OnDemandAgent;
+import com.netflix.spinnaker.clouddriver.cache.OnDemandMetricsSupport;
+import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider;
+import groovy.lang.Closure;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+abstract class AbstractEcsOnDemandAgent<T> extends AbstractEcsCachingAgent<T> implements OnDemandAgent {
+  final OnDemandMetricsSupport metricsSupport;
+
+  AbstractEcsOnDemandAgent(String accountName, String region, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) {
+    super(accountName, region, amazonClientProvider, awsCredentialsProvider);
+    this.metricsSupport = new OnDemandMetricsSupport(registry, this, EcsCloudProvider.ID + ":" + EcsCloudProvider.ID + ":${OnDemandAgent.OnDemandType.ServerGroup}");
+  }
+
+  @Override
+  public OnDemandMetricsSupport getMetricsSupport() {
+    return metricsSupport;
+  }
+
+  @Override
+  public Collection<Map> pendingOnDemandRequests(ProviderCache providerCache) {
+    return new LinkedList<>();
+  }
+
+  @Override
+  public String getOnDemandAgentType() {
+    return getAgentType();
+  }
+
+  @Override
+  public boolean handles(OnDemandType type, String cloudProvider) {
+    return type.equals(OnDemandType.ServerGroup) && cloudProvider.equals(EcsCloudProvider.ID);
+  }
+
+  @Override
+  public OnDemandResult handle(ProviderCache providerCache, Map<String, ?> data) {
+    if (!data.get("account").equals(accountName) || !data.get("region").equals(region)) {
+      return null;
+    }
+
+    AmazonECS ecs = amazonClientProvider.getAmazonEcs(accountName, awsCredentialsProvider, region);
+
+    List<T> items = metricsSupport.readData(new Closure<List<T>>(this, this) {
+      public List<T> doCall() {
+        return getItems(ecs, providerCache);
+      }
+    });
+
+    storeOnDemand(providerCache, data);
+
+    CacheResult cacheResult = metricsSupport.transformData(new Closure<CacheResult>(this, this) {
+      public CacheResult doCall() {
+        return buildCacheResult(getAuthoritativeKeyName(), items, providerCache);
+      }
+    });
+
+    return new OnDemandResult(getAgentType(), cacheResult, null); // TODO(Bruno Carrier) - evictions should happen properly instead of having a null here
+  }
+
+  void storeOnDemand(ProviderCache providerCache, Map<String, ?> data) {
+    // TODO: Overwrite if needed.
+  }
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/EcsClusterCachingAgent.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.ecs.AmazonECS;
+import com.amazonaws.services.ecs.model.ListClustersRequest;
+import com.amazonaws.services.ecs.model.ListClustersResult;
+import com.netflix.spinnaker.cats.agent.AgentDataType;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
+
+public class EcsClusterCachingAgent extends AbstractEcsCachingAgent<String> {
+  private static final Collection<AgentDataType> types = Collections.unmodifiableCollection(Arrays.asList(
+    AUTHORITATIVE.forType(ECS_CLUSTERS.toString())
+  ));
+  private final Logger log = LoggerFactory.getLogger(getClass());
+
+  public EcsClusterCachingAgent(String accountName, String region, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider) {
+    super(accountName, region, amazonClientProvider, awsCredentialsProvider);
+  }
+
+  @Override
+  public String getAgentType() {
+    return EcsClusterCachingAgent.class.getSimpleName();
+  }
+
+  @Override
+  public Collection<AgentDataType> getProvidedDataTypes() {
+    return types;
+  }
+
+  @Override
+  protected List<String> getItems(AmazonECS ecs, ProviderCache providerCache) {
+    List<String> allClusterArns = new LinkedList<>();
+    String nextToken = null;
+    do {
+      ListClustersRequest listClustersRequest = new ListClustersRequest();
+      if (nextToken != null) {
+        listClustersRequest.setNextToken(nextToken);
+      }
+
+      ListClustersResult listClustersResult = ecs.listClusters(listClustersRequest);
+      allClusterArns.addAll(listClustersResult.getClusterArns());
+
+      nextToken = listClustersResult.getNextToken();
+    } while (nextToken != null && nextToken.length() != 0);
+
+    return allClusterArns;
+  }
+
+  @Override
+  protected Map<String, Collection<CacheData>> generateFreshData(Collection<String> clusterArns) {
+    Collection<CacheData> dataPoints = new LinkedList<>();
+    for (String clusterArn : clusterArns) {
+      String clusterName = StringUtils.substringAfterLast(clusterArn, "/");
+
+      Map<String, Object> attributes = convertClusterArnToAttributes(accountName, region, clusterArn);
+
+      String key = Keys.getClusterKey(accountName, region, clusterName);
+      dataPoints.add(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+    }
+
+    log.info("Caching " + dataPoints.size() + " ECS clusters in " + getAgentType());
+    Map<String, Collection<CacheData>> dataMap = new HashMap<>();
+    dataMap.put(ECS_CLUSTERS.toString(), dataPoints);
+
+    return dataMap;
+  }
+  public static  Map<String, Object> convertClusterArnToAttributes(String accountName, String region, String clusterArn){
+    String clusterName = StringUtils.substringAfterLast(clusterArn, "/");
+
+    Map<String, Object> attributes = new HashMap<>();
+    attributes.put("account", accountName);
+    attributes.put("region", region);
+    attributes.put("clusterName", clusterName);
+    attributes.put("clusterArn", clusterArn);
+
+    return attributes;
+  }
+}

clouddriver-ecs/src/test/java/com/netflix/spinnaker/clouddriver/ecs/cache/CommonCacheClient.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache;
+
+import com.netflix.spinnaker.cats.cache.Cache;
+
+import static org.mockito.Mockito.mock;
+
+class CommonCacheClient {
+  static final String REGION = "us-west-2";
+  static final String ACCOUNT = "test-account";
+
+  final Cache cacheView = mock(Cache.class);
+}

clouddriver-ecs/src/test/java/com/netflix/spinnaker/clouddriver/ecs/cache/EcsClusterCacheClientTest.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache;
+
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.EcsClusterCacheClient;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsCluster;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.EcsClusterCachingAgent;
+import org.junit.Test;
+import spock.lang.Subject;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+public class EcsClusterCacheClientTest extends CommonCacheClient {
+  @Subject
+  private final EcsClusterCacheClient client = new EcsClusterCacheClient(cacheView);
+
+  @Test
+  public void shouldConvert() {
+    //Given
+    String clusterName = "test-cluster";
+    String clusterArn = "arn:aws:ecs:" + REGION + ":012345678910:cluster/" + clusterName;
+    String key = Keys.getClusterKey(ACCOUNT, REGION, clusterName);
+
+    Map<String, Object> attributes = EcsClusterCachingAgent.convertClusterArnToAttributes(ACCOUNT, REGION, clusterArn);
+
+    when(cacheView.get(ECS_CLUSTERS.toString(), key)).thenReturn(new DefaultCacheData(key, attributes, Collections.emptyMap()));
+
+    //When
+    EcsCluster ecsCluster = client.get(key);
+
+    //Then
+    assertTrue("Expected cluster name to be " + clusterName + " but got " + ecsCluster.getName(), clusterName.equals(ecsCluster.getName()));
+    assertTrue("Expected cluster ARN to be " + clusterArn + " but got " + ecsCluster.getArn(), clusterArn.equals(ecsCluster.getArn()));
+    assertTrue("Expected cluster account to be " + ACCOUNT + " but got " + ecsCluster.getAccount(), ACCOUNT.equals(ecsCluster.getAccount()));
+    assertTrue("Expected cluster region to be " + REGION + " but got " + ecsCluster.getRegion(), REGION.equals(ecsCluster.getRegion()));
+  }
+}