refactor(provider/ecs): Caching agents enabled, code cleanup.

clouddriver-ecs/clouddriver-ecs.gradle

@@ -1,7 +1,7 @@
 dependencies {
   compile project(":clouddriver-aws")
+  compile spinnaker.dependency('lombok')
 
   spinnaker.group('amazon')
   spinnaker.group('retrofitDefault')
-
 }

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/EcsCloudProvider.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.netflix.spinnaker.clouddriver.ecs;
 
 import org.springframework.stereotype.Component;

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/EcsConfiguration.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.netflix.spinnaker.clouddriver.ecs;
 
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/EcsConfigurationProperties.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.netflix.spinnaker.clouddriver.ecs;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/EcsOperation.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 Lookout, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.netflix.spinnaker.clouddriver.ecs;
 
 import java.lang.annotation.ElementType;

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/AbstractCacheClient.java

@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 abstract class AbstractCacheClient<T> {
 
@@ -80,11 +81,9 @@ public T get(String key) {
    * @return A collection of generic typ objects.
    */
   private Collection<T> convertAll(Collection<CacheData> cacheData) {
-    Set<T> itemSet = new HashSet<>();
-    for (CacheData cacheDatum : cacheData) {
-      itemSet.add(convert(cacheDatum));
-    }
-    return itemSet;
+    return cacheData.stream()
+      .map(this::convert)
+      .collect(Collectors.toList());
   }
 
   /**

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/EcsClusterCacheClient.java

@@ -20,11 +20,13 @@
 import com.netflix.spinnaker.cats.cache.CacheData;
 import com.netflix.spinnaker.clouddriver.ecs.cache.model.EcsCluster;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.Map;
 
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
 
+@Component
 public class EcsClusterCacheClient extends AbstractCacheClient<EcsCluster>{
 
   @Autowired

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/IamRoleCacheClient.java

@@ -21,6 +21,7 @@
 import com.netflix.spinnaker.clouddriver.ecs.cache.model.IamRole;
 import com.netflix.spinnaker.clouddriver.ecs.provider.agent.IamTrustRelationship;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.Collection;
 import java.util.HashSet;
@@ -31,6 +32,7 @@
 
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.IAM_ROLE;
 
+@Component
 public class IamRoleCacheClient extends AbstractCacheClient<IamRole> {
 
   @Autowired

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/ServiceCacheClient.java

@@ -22,6 +22,7 @@
 import com.netflix.spinnaker.cats.cache.CacheData;
 import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -30,6 +31,7 @@
 
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
 
+@Component
 public class ServiceCacheClient extends AbstractCacheClient<Service> {
   private ObjectMapper objectMapper;
 

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/TaskCacheClient.java

@@ -22,6 +22,7 @@
 import com.netflix.spinnaker.cats.cache.CacheData;
 import com.netflix.spinnaker.clouddriver.ecs.cache.model.Task;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -30,6 +31,7 @@
 
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS;
 
+@Component
 public class TaskCacheClient extends AbstractCacheClient<Task> {
   private ObjectMapper objectMapper;
 

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/TaskDefinitionCacheClient.java

@@ -22,6 +22,7 @@
 import com.netflix.spinnaker.cats.cache.Cache;
 import com.netflix.spinnaker.cats.cache.CacheData;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -30,6 +31,7 @@
 
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS;
 
+@Component
 public class TaskDefinitionCacheClient extends AbstractCacheClient<TaskDefinition> {
   private ObjectMapper objectMapper;
 
@@ -45,6 +47,7 @@ protected TaskDefinition convert(CacheData cacheData) {
     Map<String, Object> attributes = cacheData.getAttributes();
 
     taskDefinition.setTaskDefinitionArn((String) attributes.get("taskDefinitionArn"));
+    taskDefinition.setTaskRoleArn((String) attributes.get("taskRoleArn"));
 
     if (attributes.containsKey("containerDefinitions")) {
       List<Map<String, Object>> containerDefinitions = (List<Map<String, Object>>) attributes.get("containerDefinitions");

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/EcsProvider.java

@@ -1,5 +1,5 @@
 /*
- * * Copyright 2017 Lookout, Inc.
+ * Copyright 2017 Lookout, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,8 +28,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -98,7 +96,8 @@ public Collection<Agent> getAgents() {
   public void synchronizeHealthAgents() {
     healthAgents = Collections.unmodifiableCollection(agents.stream()
       .filter(a -> a instanceof HealthProvidingCachingAgent)
-      .map(a -> (HealthProvidingCachingAgent) a).collect(Collectors.toList()));
+      .map(a -> (HealthProvidingCachingAgent) a)
+      .collect(Collectors.toList()));
   }
 
   public Collection<HealthProvidingCachingAgent> getHealthAgents() {

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/config/EcsProviderConfig.java

@@ -1,5 +1,5 @@
 /*
- * * Copyright 2017 Lookout, Inc.
+ * Copyright 2017 Lookout, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,16 +17,26 @@
 package com.netflix.spinnaker.clouddriver.ecs.provider.config;
 
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.cats.agent.Agent;
 import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
 import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
 import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider;
 import com.netflix.spinnaker.clouddriver.ecs.provider.EcsProvider;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.ContainerInstanceCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.EcsCloudMetricAlarmCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.EcsClusterCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.IamPolicyReader;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.IamRoleCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.ScalableTargetsCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.ServiceCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.TaskCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.TaskDefinitionCachingAgent;
+import com.netflix.spinnaker.clouddriver.ecs.provider.agent.TaskHealthCachingAgent;
 import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository;
 import com.netflix.spinnaker.clouddriver.security.ProviderUtils;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
@@ -43,30 +53,46 @@
 @Configuration
 public class EcsProviderConfig {
 
+  @Bean
+  public IamPolicyReader iamPolicyReader(ObjectMapper objectMapper) {
+    return new IamPolicyReader(objectMapper);
+  }
+
   @Bean
   @DependsOn("netflixECSCredentials")
-  public EcsProvider ecsProvider(AccountCredentialsRepository accountCredentialsRepository, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) {
+  public EcsProvider ecsProvider(AccountCredentialsRepository accountCredentialsRepository, AmazonClientProvider amazonClientProvider,
+                                 AWSCredentialsProvider awsCredentialsProvider, Registry registry, IamPolicyReader iamPolicyReader,
+                                 ObjectMapper objectMapper) {
     EcsProvider provider = new EcsProvider(accountCredentialsRepository, Collections.newSetFromMap(new ConcurrentHashMap<Agent, Boolean>()));
-    synchronizeEcsProvider(provider, accountCredentialsRepository, amazonClientProvider, awsCredentialsProvider, registry);
+    synchronizeEcsProvider(provider, accountCredentialsRepository, amazonClientProvider, awsCredentialsProvider, registry, iamPolicyReader, objectMapper);
     return provider;
   }
 
   @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
   @Bean
   public EcsProviderSynchronizer synchronizeEcsProvider(EcsProvider ecsProvider, AccountCredentialsRepository accountCredentialsRepository,
-                                                        AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) {
+                                                        AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry,
+                                                        IamPolicyReader iamPolicyReader,
+                                                        ObjectMapper objectMapper) {
 
     Set<String> scheduledAccounts = ProviderUtils.getScheduledAccounts(ecsProvider);
     Set<NetflixAmazonCredentials> allAccounts = ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository, NetflixAmazonCredentials.class);
     List<Agent> newAgents = new LinkedList<>();
 
     for (NetflixAmazonCredentials credentials : allAccounts) {
       if (credentials.getCloudProvider().equals(EcsCloudProvider.ID)) {
+        newAgents.add(new IamRoleCachingAgent(credentials.getName(), amazonClientProvider, awsCredentialsProvider, iamPolicyReader)); // IAM is region-agnostic, so one caching agent per account is enough
 
         for (AWSRegion region : credentials.getRegions()) {
           if (!scheduledAccounts.contains(credentials.getName())) {
-            //Soon to come:
-            //newAgents.add(new EcsClusterCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider));
+            newAgents.add(new EcsClusterCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider));
+            newAgents.add(new ServiceCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, registry));
+            newAgents.add(new TaskCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, registry));
+            newAgents.add(new ContainerInstanceCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, registry));
+            newAgents.add(new TaskDefinitionCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, registry, objectMapper));
+            newAgents.add(new TaskHealthCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, objectMapper));
+            newAgents.add(new EcsCloudMetricAlarmCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider));
+            newAgents.add(new ScalableTargetsCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider, objectMapper));
           }
         }
       }

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/view/EcsApplicationProvider.java

@@ -111,7 +111,7 @@ private Set<Application> transposeApplicationMapToSet(HashMap<String, Applicatio
 
   private HashMap<String, Application> inferApplicationFromServices(HashMap<String, Application> applicationHashMap, Service service, boolean expand) {
 
-    HashMap<String, String> attributes = new HashMap<>();
+    HashMap<String, String> attributes = new HashMap<>();  // After POC we'll figure exactly what info we want to put in here
     String appName = service.getApplicationName();
     String serviceName = service.getServiceName();
     attributes.put("iamRole", service.getRoleArn());
@@ -129,7 +129,9 @@ private HashMap<String, Application> inferApplicationFromServices(HashMap<String
       applicationHashMap.put(appName, application);
     } else {
       applicationHashMap.get(appName).getAttributes().putAll(application.getAttributes());
-      applicationHashMap.get(appName).getClusterNames().get(appName).add(serviceName);
+      if (expand) {
+        applicationHashMap.get(appName).getClusterNames().get(appName).add(serviceName);
+      }
     }
 
     return applicationHashMap;