feat(provider/ecs): ECS Scalable Targets caching classes and tests

clouddriver-aws/src/main/groovy/com/netflix/spinnaker/clouddriver/aws/security/AmazonClientProvider.java

@@ -20,6 +20,8 @@
 import com.amazonaws.handlers.RequestHandler2;
 import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.retry.RetryPolicy;
+import com.amazonaws.services.applicationautoscaling.AWSApplicationAutoScaling;
+import com.amazonaws.services.applicationautoscaling.AWSApplicationAutoScalingClientBuilder;
 import com.amazonaws.services.autoscaling.AmazonAutoScaling;
 import com.amazonaws.services.autoscaling.AmazonAutoScalingClientBuilder;
 import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
@@ -431,4 +433,8 @@ public AWSShield getAmazonShield(NetflixAmazonCredentials amazonCredentials, Str
   public AWSShield getAmazonShield(String accountName, AWSCredentialsProvider awsCredentialsProvider, String region) {
     return awsSdkClientSupplier.getClient(AWSShieldClientBuilder.class, AWSShield.class, accountName, awsCredentialsProvider, region);
   }
+
+  public AWSApplicationAutoScaling getAmazonApplicationAutoScaling(String accountName, AWSCredentialsProvider awsCredentialsProvider, String region) {
+    return awsSdkClientSupplier.getClient(AWSApplicationAutoScalingClientBuilder.class, AWSApplicationAutoScaling.class, accountName, awsCredentialsProvider, region);
+  }
 }

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/Keys.java

@@ -33,7 +33,8 @@ public enum Namespace {
     TASKS,
     CONTAINER_INSTANCES,
     TASK_DEFINITIONS,
-    ALARMS;
+    ALARMS,
+    SCALABLE_TARGETS;
 
     public final String ns;
 
@@ -108,6 +109,10 @@ public static Map<String, String> parse(String key) {
         break;
       case IAM_ROLE:
         result.put("roleName", parts[3]);
+        break;
+      case SCALABLE_TARGETS:
+        result.put("resource", parts[4]);
+        break;
       default:
         break;
     }
@@ -148,6 +153,10 @@ public static String getAlarmKey(String account, String region, String alarmArn)
     return buildKey(Namespace.ALARMS.ns, account, region, alarmArn);
   }
 
+  public static String getScalableTargetKey(String account, String region, String resourceId) {
+    return buildKey(Namespace.SCALABLE_TARGETS.ns, account, region, resourceId);
+  }
+
   public static String getIamRoleKey(String account, String iamRoleName) {
     return ID + SEPARATOR + Namespace.IAM_ROLE + SEPARATOR + account + SEPARATOR + iamRoleName;
   }

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/ScalableTargetCacheClient.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.client;
+
+import com.amazonaws.services.applicationautoscaling.model.ScalableTarget;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.cats.cache.Cache;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SCALABLE_TARGETS;
+
+@Component
+public class ScalableTargetCacheClient extends AbstractCacheClient<ScalableTarget> {
+  private final ObjectMapper objectMapper;
+
+  @Autowired
+  public ScalableTargetCacheClient(Cache cacheView, ObjectMapper objectMapper) {
+    super(cacheView, SCALABLE_TARGETS.toString());
+    this.objectMapper = objectMapper;
+  }
+
+  @Override
+  protected ScalableTarget convert(CacheData cacheData) {
+    ScalableTarget scalableTarget;
+    scalableTarget = objectMapper.convertValue(cacheData.getAttributes(), ScalableTarget.class);
+    return scalableTarget;
+  }
+
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/EcsProvider.java

@@ -34,8 +34,10 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ALARMS;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SCALABLE_TARGETS;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS;
@@ -46,7 +48,8 @@ public class EcsProvider extends AgentSchedulerAware implements SearchableProvid
 
   private static final Set<String> defaultCaches = new HashSet<>(Arrays.asList(
     SERVICES.toString(), ECS_CLUSTERS.toString(), TASKS.toString(),
-    CONTAINER_INSTANCES.toString(), TASK_DEFINITIONS.toString()));
+    CONTAINER_INSTANCES.toString(), TASK_DEFINITIONS.toString(), ALARMS.toString(),
+    SCALABLE_TARGETS.toString()));
 
   private static final Map<String, String> urlMappingTemplates = new HashMap<>();
 

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/ScalableTargetsCachingAgent.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.applicationautoscaling.AWSApplicationAutoScaling;
+import com.amazonaws.services.applicationautoscaling.model.DescribeScalableTargetsRequest;
+import com.amazonaws.services.applicationautoscaling.model.DescribeScalableTargetsResult;
+import com.amazonaws.services.applicationautoscaling.model.ScalableTarget;
+import com.amazonaws.services.applicationautoscaling.model.ServiceNamespace;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.cats.agent.AgentDataType;
+import com.netflix.spinnaker.cats.agent.CacheResult;
+import com.netflix.spinnaker.cats.agent.CachingAgent;
+import com.netflix.spinnaker.cats.agent.DefaultCacheResult;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
+import com.netflix.spinnaker.cats.provider.ProviderCache;
+import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import com.netflix.spinnaker.clouddriver.ecs.provider.EcsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SCALABLE_TARGETS;
+
+public class ScalableTargetsCachingAgent implements CachingAgent {
+  static final Collection<AgentDataType> types = Collections.unmodifiableCollection(Arrays.asList(
+    AUTHORITATIVE.forType(SCALABLE_TARGETS.toString())
+  ));
+
+  private final Logger log = LoggerFactory.getLogger(getClass());
+  private final ObjectMapper objectMapper;
+  private AmazonClientProvider amazonClientProvider;
+  private AWSCredentialsProvider awsCredentialsProvider;
+  private String accountName;
+  private String region;
+
+  public ScalableTargetsCachingAgent(String accountName, String region,
+                                     AmazonClientProvider amazonClientProvider,
+                                     AWSCredentialsProvider awsCredentialsProvider,
+                                     ObjectMapper objectMapper) {
+    this.region = region;
+    this.accountName = accountName;
+    this.amazonClientProvider = amazonClientProvider;
+    this.awsCredentialsProvider = awsCredentialsProvider;
+    this.objectMapper = objectMapper;
+  }
+
+  public static Map<String, Object> convertMetricAlarmToAttributes(ScalableTarget scalableTarget, ObjectMapper objectMapper) {
+    return objectMapper.convertValue(scalableTarget, Map.class);
+  }
+
+  @Override
+  public Collection<AgentDataType> getProvidedDataTypes() {
+    return types;
+  }
+
+  @Override
+  public CacheResult loadData(ProviderCache providerCache) {
+    AWSApplicationAutoScaling autoScalingClient = amazonClientProvider.getAmazonApplicationAutoScaling(accountName, awsCredentialsProvider, region);
+
+    Set<ScalableTarget> scalableTargets = fetchScalableTargets(autoScalingClient);
+    Map<String, Collection<CacheData>> newDataMap = generateFreshData(scalableTargets);
+    Collection<CacheData> newData = newDataMap.get(SCALABLE_TARGETS.toString());
+
+    Set<String> oldKeys = providerCache.getAll(SCALABLE_TARGETS.toString()).stream()
+      .map(CacheData::getId).collect(Collectors.toSet());
+    Map<String, Collection<String>> evictionsByKey = computeEvictableData(newData, oldKeys);
+
+    return new DefaultCacheResult(newDataMap, evictionsByKey);
+  }
+
+  private Map<String, Collection<String>> computeEvictableData(Collection<CacheData> newData, Collection<String> oldKeys) {
+    Set<String> newKeys = newData.stream().map(CacheData::getId).collect(Collectors.toSet());
+    Set<String> evictedKeys = oldKeys.stream().filter(oldKey -> !newKeys.contains(oldKey)).collect(Collectors.toSet());
+
+    Map<String, Collection<String>> evictionsByKey = new HashMap<>();
+    evictionsByKey.put(SCALABLE_TARGETS.toString(), evictedKeys);
+    log.info("Evicting " + evictedKeys.size() + " scalable targets in " + getAgentType());
+    return evictionsByKey;
+  }
+
+  Map<String, Collection<CacheData>> generateFreshData(Set<ScalableTarget> scalableTargets) {
+    Collection<CacheData> dataPoints = new HashSet<>();
+    Map<String, Collection<CacheData>> newDataMap = new HashMap<>();
+
+    for (ScalableTarget scalableTarget : scalableTargets) {
+      String key = Keys.getScalableTargetKey(accountName, region, scalableTarget.getResourceId());
+      Map<String, Object> attributes = convertMetricAlarmToAttributes(scalableTarget, objectMapper);
+
+      CacheData data = new DefaultCacheData(key, attributes, Collections.emptyMap());
+      dataPoints.add(data);
+    }
+
+    log.info("Caching " + dataPoints.size() + " scalable targets in " + getAgentType());
+    newDataMap.put(SCALABLE_TARGETS.toString(), dataPoints);
+    return newDataMap;
+  }
+
+  Set<ScalableTarget> fetchScalableTargets(AWSApplicationAutoScaling autoScalingClient) {
+    Set<ScalableTarget> scalableTargets = new HashSet<>();
+    String nextToken = null;
+    do {
+      DescribeScalableTargetsRequest request = new DescribeScalableTargetsRequest().withServiceNamespace(ServiceNamespace.Ecs);
+      if (nextToken != null) {
+        request.setNextToken(nextToken);
+      }
+
+      DescribeScalableTargetsResult result = autoScalingClient.describeScalableTargets(request);
+      scalableTargets.addAll(result.getScalableTargets());
+
+      nextToken = result.getNextToken();
+    } while (nextToken != null && nextToken.length() != 0);
+
+    return scalableTargets;
+  }
+
+  @Override
+  public String getAgentType() {
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public String getProviderName() {
+    return EcsProvider.NAME;
+  }
+}

clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/cache/KeysSpec.groovy

@@ -21,7 +21,7 @@ import spock.lang.Specification
 import static com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider.ID
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.*
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.SEPARATOR
-import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH;
+import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH
 
 class KeysSpec extends Specification {
 
@@ -44,6 +44,8 @@ class KeysSpec extends Specification {
     'test-account-3' | 'us-west-3' | ECS_CLUSTERS.ns        | 'test-cluster-1'                                                                                  | buildParsedKey(account, region, namespace, [clusterName: identifier])
     'test-account-4' | 'us-west-4' | CONTAINER_INSTANCES.ns | 'arn:aws:ecs:' + region + ':012345678910:container-instance/14e8cce9-0b16-4af4-bfac-a85f7587aa98' | buildParsedKey(account, region, namespace, [containerInstanceArn: identifier])
     'test-account-5' | 'us-west-5' | TASK_DEFINITIONS.ns    | 'arn:aws:ecs:' + region + ':012345678910:task-definition/hello_world:10'                          | buildParsedKey(account, region, namespace, [taskDefinitionArn: identifier])
+    'test-account-6' | 'us-west-6' | ALARMS.ns    | 'arn:aws:ecs:' + region + ':012345678910:alarms/14e8cce9-0b16-4af4-bfac-a85f7587aa98'                          | buildParsedKey(account, region, namespace, [alarmArn: identifier])
+    'test-account-7' | 'us-west-7' | SCALABLE_TARGETS.ns    | 'service/test-cluster/test-service'                          | buildParsedKey(account, region, namespace, [resource: identifier])
 
   }
 
@@ -126,4 +128,14 @@ class KeysSpec extends Specification {
     'us-west-1' | 'test-account-1' | '1dc5c17a-422b-4dc4-b493-371970c6c4d6'
     'us-west-2' | 'test-account-2' | 'deadbeef-422b-4dc4-b493-371970c6c4d6'
   }
+
+  def 'should generate the proper scalable target key'() {
+    expect:
+    Keys.getScalableTargetKey(account, region, taskId) == buildKey(SCALABLE_TARGETS.ns, account, region, taskId)
+
+    where:
+    region      | account          | taskId
+    'us-west-1' | 'test-account-1' | 'service/test-cluster/test-service'
+    'us-west-2' | 'test-account-2' | 'service/mycluster/myservice'
+  }
 }

clouddriver-ecs/src/test/groovy/com/netflix/spinnaker/clouddriver/ecs/cache/ScalableTargetCacheClientSpec.groovy

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2017 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache
+
+import com.amazonaws.services.applicationautoscaling.model.ScalableTarget
+import com.amazonaws.services.applicationautoscaling.model.ServiceNamespace
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.netflix.spinnaker.cats.cache.Cache
+import com.netflix.spinnaker.cats.cache.DefaultCacheData
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ScalableTargetCacheClient
+import spock.lang.Specification
+import spock.lang.Subject
+
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SCALABLE_TARGETS
+
+class ScalableTargetCacheClientSpec extends Specification {
+  def cacheView = Mock(Cache)
+  def objectMapper = new ObjectMapper()
+    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+
+  @Subject
+  ScalableTargetCacheClient client = new ScalableTargetCacheClient(cacheView, objectMapper)
+
+  def 'should convert cache data into object'() {
+    given:
+    def resourceId = 'service/test-cluster/test-service'
+    def scalableTargetKey = Keys.getScalableTargetKey('test-account', 'us-west-1', resourceId)
+
+    def givenScalableTarget = new ScalableTarget(
+      serviceNamespace: ServiceNamespace.Ecs,
+      resourceId: resourceId,
+      scalableDimension: 'scalable-dimension',
+      minCapacity: 0,
+      maxCapacity: 9001,
+      roleARN: 'role-arn',
+      creationTime: new Date()
+    )
+
+    objectMapper
+
+    def attributes = objectMapper.convertValue(givenScalableTarget, Map)
+    cacheView.get(SCALABLE_TARGETS.ns, scalableTargetKey) >> new DefaultCacheData(scalableTargetKey, attributes, [:])
+
+    when:
+    def retrievedScalableTarget = client.get(scalableTargetKey)
+
+    then:
+    retrievedScalableTarget == givenScalableTarget
+  }
+}