diff --git a/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/Keys.java b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/Keys.java new file mode 100644 index 00000000000..3f730183b38 --- /dev/null +++ b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/Keys.java @@ -0,0 +1,146 @@ +/* + * * Copyright 2017 Lookout, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.ecs.cache; + +import com.google.common.base.CaseFormat; +import com.netflix.spinnaker.clouddriver.cache.KeyParser; + +import java.util.HashMap; +import java.util.Map; + +import static com.netflix.spinnaker.clouddriver.core.provider.agent.Namespace.HEALTH; +import static com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider.ID; + +public class Keys implements KeyParser { + public enum Namespace { + IAM_ROLE, + SERVICES, + ECS_CLUSTERS, + TASKS, + CONTAINER_INSTANCES, + TASK_DEFINITIONS; + + final String ns; + + Namespace() { + ns = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, this.name()); + } + + public String toString() { + return ns; + } + } + + public static final String SEPARATOR = ":"; + + public static Map parse(String key) { + String[] parts = key.split(SEPARATOR); + + if (parts.length < 3 || !parts[0].equals(ID)) { + return null; + } + + Map result = new HashMap<>(); + result.put("provider", parts[0]); + result.put("type", parts[1]); + result.put("account", parts[2]); + + + Namespace namespace = Namespace.valueOf(CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, parts[1])); + + if (!namespace.equals(Namespace.IAM_ROLE)) { + result.put("region", parts[3]); + } + + switch (namespace) { + case SERVICES: + result.put("serviceName", parts[4]); + break; + case ECS_CLUSTERS: + result.put("clusterName", parts[4]); + break; + case TASKS: + result.put("taskName", parts[4]); + break; + case CONTAINER_INSTANCES: + result.put("containerInstanceArn", parts[4]); + break; + case TASK_DEFINITIONS: + result.put("taskDefinitionArn", parts[4]); + break; + case IAM_ROLE: + result.put("roleName", parts[3]); + default: + break; + } + + return result; + } + + public static String getServiceKey(String account, String region, String serviceName) { + return ID + SEPARATOR + Namespace.SERVICES + SEPARATOR + account + SEPARATOR + region + SEPARATOR + serviceName; + } + + public static String getClusterKey(String account, String region, String clusterName) { + return ID + SEPARATOR + Namespace.ECS_CLUSTERS + SEPARATOR + account + SEPARATOR + region + SEPARATOR + clusterName; + } + + public static String getTaskKey(String account, String region, String taskId) { + return ID + SEPARATOR + Namespace.TASKS + SEPARATOR + account + SEPARATOR + region + SEPARATOR + taskId; + } + + public static String getTaskHealthKey(String account, String region, String taskId) { + return ID + SEPARATOR + HEALTH + SEPARATOR + account + SEPARATOR + region + SEPARATOR + taskId; + } + + public static String getContainerInstanceKey(String account, String region, String containerInstanceArn) { + return ID + SEPARATOR + Namespace.CONTAINER_INSTANCES + SEPARATOR + account + SEPARATOR + region + SEPARATOR + containerInstanceArn; + } + + public static String getTaskDefinitionKey(String account, String region, String taskDefinitionArn) { + return ID + SEPARATOR + Namespace.TASK_DEFINITIONS + SEPARATOR + account + SEPARATOR + region + SEPARATOR + taskDefinitionArn; + } + + public static String getIamRoleKey(String account, String iamRoleName) { + return ID + SEPARATOR + Namespace.IAM_ROLE + SEPARATOR + account + SEPARATOR + iamRoleName; + } + + @Override + public String getCloudProvider() { + return ID; + } + + @Override + public Map parseKey(String key) { + return parse(key); + } + + @Override + public Boolean canParseType(String type) { + for (Namespace key : Namespace.values()) { + if (key.toString().equals(type)) { + return true; + } + } + return false; + } + + @Override + public Boolean canParseField(String type) { + return false; + } +} diff --git a/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/AbstractCacheClient.java b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/AbstractCacheClient.java new file mode 100644 index 00000000000..6b004616e6e --- /dev/null +++ b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/AbstractCacheClient.java @@ -0,0 +1,96 @@ +package com.netflix.spinnaker.clouddriver.ecs.cache.client; + +import com.netflix.spinnaker.cats.cache.Cache; +import com.netflix.spinnaker.cats.cache.CacheData; +import com.netflix.spinnaker.clouddriver.ecs.cache.Keys; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +abstract class AbstractCacheClient { + + private final String keyNamespace; + private final Cache cacheView; + + /** + * @param cacheView The Cache that the client will query. + * @param keyNamespace The key namespace that the client is responsible for. + */ + AbstractCacheClient(Cache cacheView, String keyNamespace) { + this.cacheView = cacheView; + this.keyNamespace = keyNamespace; + } + + /** + * @param cacheData CacheData that will be converted into an object. + * @return An object of the generic type. + */ + protected abstract T convert(CacheData cacheData); + + /** + * @return A list of all generic type objects belonging to the key namespace. + */ + public Collection getAll() { + Collection allData = cacheView.getAll(keyNamespace); + return convertAll(allData); + } + + /** + * @param account name of the AWS account, as defined in clouddriver.yml + * @param region region of the AWS account, as defined in clouddriver.yml + * @return A list of all generic type objects belonging to the account and region in the key namespace. + */ + public Collection getAll(String account, String region) { + Collection data = fetchFromCache(account, region); + return convertAll(data); + } + + /** + * @param key A key within the key namespace that will be used to retrieve the object. + * @return An object of the generic type that is associated to the key. + */ + public T get(String key) { + CacheData cacheData = cacheView.get(keyNamespace, key); + if (cacheData != null) { + return convert(cacheData); + } + return null; + } + + /** + * @param cacheData A collection of CacheData that will be converted into a collection of generic typ objects. + * @return A collection of generic typ objects. + */ + private Collection convertAll(Collection cacheData) { + Set itemSet = new HashSet<>(); + for (CacheData cacheDatum : cacheData) { + itemSet.add(convert(cacheDatum)); + } + return itemSet; + } + + /** + * @param account name of the AWS account, as defined in clouddriver.yml + * @param region region of the AWS account, as defined in clouddriver.yml + * @return + */ + private Collection fetchFromCache(String account, String region) { + String accountFilter = account != null ? account + Keys.SEPARATOR : "*" + Keys.SEPARATOR; + String regionFilter = region != null ? region + Keys.SEPARATOR : "*" + Keys.SEPARATOR; + Set keys = new HashSet<>(); + String pattern = "ecs" + Keys.SEPARATOR + keyNamespace + Keys.SEPARATOR + accountFilter + regionFilter + "*"; + Collection nameMatches = cacheView.filterIdentifiers(keyNamespace, pattern); + + keys.addAll(nameMatches); + + Collection allData = cacheView.getAll(keyNamespace, keys); + + if (allData == null) { + return Collections.emptyList(); + } + + return allData; + } +} diff --git a/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/EcsProvider.java b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/EcsProvider.java new file mode 100644 index 00000000000..e2d9ac2d924 --- /dev/null +++ b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/EcsProvider.java @@ -0,0 +1,105 @@ +/* + * * Copyright 2017 Lookout, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.ecs.provider; + +import com.netflix.spinnaker.cats.agent.Agent; +import com.netflix.spinnaker.cats.agent.AgentSchedulerAware; +import com.netflix.spinnaker.clouddriver.cache.SearchableProvider; +import com.netflix.spinnaker.clouddriver.core.provider.agent.HealthProvidingCachingAgent; +import com.netflix.spinnaker.clouddriver.ecs.cache.Keys; +import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.CONTAINER_INSTANCES; +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS; +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES; +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASKS; +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.TASK_DEFINITIONS; + + +public class EcsProvider extends AgentSchedulerAware implements SearchableProvider { + public static final String NAME = EcsProvider.class.getName(); + + private static final Set defaultCaches = new HashSet<>(Arrays.asList( + SERVICES.toString(), ECS_CLUSTERS.toString(), TASKS.toString(), + CONTAINER_INSTANCES.toString(), TASK_DEFINITIONS.toString())); + + private static final Map urlMappingTemplates = new HashMap<>(); + + private final Collection agents; + private final AccountCredentialsRepository accountCredentialsRepository; + private final Keys keys = new Keys(); + private Collection healthAgents; + + public EcsProvider(AccountCredentialsRepository accountCredentialsRepository, Collection agents) { + this.agents = agents; + this.accountCredentialsRepository = accountCredentialsRepository; + } + + @Override + public Set getDefaultCaches() { + return defaultCaches; + } + + @Override + public Map getUrlMappingTemplates() { + return urlMappingTemplates; + } + + @Override + public Map getSearchResultHydrators() { + //TODO: Implement if needed - see InstanceSearchResultHydrator as an example. + return Collections.emptyMap(); + } + + @Override + public Map parseKey(String key) { + return keys.parseKey(key); + } + + @Override + public String getProviderName() { + return NAME; + } + + @Override + public Collection getAgents() { + return agents; + } + + + public void synchronizeHealthAgents() { + healthAgents = Collections.unmodifiableCollection(agents.stream() + .filter(a -> a instanceof HealthProvidingCachingAgent) + .map(a -> (HealthProvidingCachingAgent) a).collect(Collectors.toList())); + } + + public Collection getHealthAgents() { + return Collections.unmodifiableCollection(healthAgents); + } + +} diff --git a/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/AbstractEcsCachingAgent.java b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/AbstractEcsCachingAgent.java new file mode 100644 index 00000000000..eb4bff01115 --- /dev/null +++ b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/AbstractEcsCachingAgent.java @@ -0,0 +1,168 @@ +/* + * * Copyright 2017 Lookout, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.ecs.provider.agent; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.ecs.AmazonECS; +import com.amazonaws.services.ecs.model.ListClustersRequest; +import com.amazonaws.services.ecs.model.ListClustersResult; +import com.google.common.base.CaseFormat; +import com.netflix.spinnaker.cats.agent.AgentDataType; +import com.netflix.spinnaker.cats.agent.CacheResult; +import com.netflix.spinnaker.cats.agent.CachingAgent; +import com.netflix.spinnaker.cats.agent.DefaultCacheResult; +import com.netflix.spinnaker.cats.cache.CacheData; +import com.netflix.spinnaker.cats.provider.ProviderCache; +import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider; +import com.netflix.spinnaker.clouddriver.ecs.provider.EcsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE; +import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS; + +abstract class AbstractEcsCachingAgent implements CachingAgent { + private final Logger log = LoggerFactory.getLogger(getClass()); + + final AmazonClientProvider amazonClientProvider; + final AWSCredentialsProvider awsCredentialsProvider; + final String region; + final String accountName; + + AbstractEcsCachingAgent(String accountName, String region, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider) { + this.accountName = accountName; + this.region = region; + this.amazonClientProvider = amazonClientProvider; + this.awsCredentialsProvider = awsCredentialsProvider; + } + + /** + * Fetches items from the ECS service. + * @param ecs The AmazonECS client that will be used to make the queries. + * @param providerCache A ProviderCache that is used to access already existing cache. + * @return A list of generic type objects. + */ + protected abstract List getItems(AmazonECS ecs, ProviderCache providerCache); + + /** + * Generates a map of CacheData collections associated to a key namespace from a given collection of generic type objects. + * @param cacheableItems A collection of generic type objects. + * @return A map of CacheData collections belonging to a key namespace. + */ + protected abstract Map> generateFreshData(Collection cacheableItems); + + @Override + public String getProviderName() { + return EcsProvider.NAME; + } + + @Override + public CacheResult loadData(ProviderCache providerCache) { + String authoritativeKeyName = getAuthoritativeKeyName(); + + AmazonECS ecs = amazonClientProvider.getAmazonEcs(accountName, awsCredentialsProvider, region); + List items = getItems(ecs, providerCache); + return buildCacheResult(authoritativeKeyName, items, providerCache); + } + + /** + * Provides a set of ECS cluster ARNs. + * Either uses the cache, or queries the ECS service. + * @param ecs The AmazonECS client to use for querying. + * @param providerCache The ProviderCache to retrieve clusters from. + * @return A set of ECS cluster ARNs. + */ + Set getClusters(AmazonECS ecs, ProviderCache providerCache) { + Set clusters = providerCache.getAll(ECS_CLUSTERS.toString()).stream() + .map(cacheData -> (String) cacheData.getAttributes().get("clusterArn")) + .collect(Collectors.toSet()); + + if (clusters.isEmpty()) { + clusters = new HashSet<>(); + String nextToken = null; + do { + ListClustersRequest listClustersRequest = new ListClustersRequest(); + if (nextToken != null) { + listClustersRequest.setNextToken(nextToken); + } + ListClustersResult listClustersResult = ecs.listClusters(listClustersRequest); + clusters.addAll(listClustersResult.getClusterArns()); + + nextToken = listClustersResult.getNextToken(); + } while (nextToken != null && nextToken.length() != 0); + } + + return clusters; + } + + /** + * Provides the key namespace that the caching agent is authoritative of. + * Currently only supports the caching agent being authoritative over one key namespace. + * @return Key namespace. + */ + String getAuthoritativeKeyName() { + Collection authoritativeNamespaces = getProvidedDataTypes().stream() + .filter(agentDataType -> agentDataType.getAuthority().equals(AUTHORITATIVE)) + .collect(Collectors.toSet()); + + if (authoritativeNamespaces.size() != 1) { + throw new RuntimeException("AbstractEcsCachingAgent supports only one authoritative key namespace. " + + authoritativeNamespaces.size() + " authoritative key namespace were given."); + } + + return authoritativeNamespaces.iterator().next().getTypeName(); + } + + CacheResult buildCacheResult(String authoritativeKeyName, List items, ProviderCache providerCache) { + String prettyKeyName = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, authoritativeKeyName); + + Map> dataMap = generateFreshData(items); + + Set oldKeys = providerCache.getAll(authoritativeKeyName).stream() + .map(cache -> cache.getId()) + .collect(Collectors.toSet()); + + Map> evictions = computeEvictableData(dataMap.get(authoritativeKeyName), oldKeys); + log.info("Evicting " + evictions.size() + " " + prettyKeyName + (evictions.size() > 1 ? "s" : "") + " in " + getAgentType()); + + return new DefaultCacheResult(dataMap, evictions); + } + + /** + * Evicts cache that does not belong to an entity on the ECS service. + * This is done by evicting old keys that are no longer found in the new keys provided by the new data. + * @param newData New data that contains new keys. + * @param oldKeys Old keys. + * @return Key collection associated to the key namespace the the caching agent is authoritative of. + */ + private Map> computeEvictableData(Collection newData, Collection oldKeys) { + Set newKeys = newData.stream().map(CacheData::getId).collect(Collectors.toSet()); + Set evictedKeys = oldKeys.stream().filter(oldKey -> !newKeys.contains(oldKey)).collect(Collectors.toSet()); + + Map> evictionsByKey = new HashMap<>(); + evictionsByKey.put(getAuthoritativeKeyName(), evictedKeys); + return evictionsByKey; + } +} diff --git a/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/config/EcsProviderConfig.java b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/config/EcsProviderConfig.java new file mode 100644 index 00000000000..293d221e5ae --- /dev/null +++ b/clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/config/EcsProviderConfig.java @@ -0,0 +1,82 @@ +/* + * * Copyright 2017 Lookout, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.ecs.provider.config; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.netflix.spectator.api.Registry; +import com.netflix.spinnaker.cats.agent.Agent; +import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider; +import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials; +import com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider; +import com.netflix.spinnaker.clouddriver.ecs.provider.EcsProvider; +import com.netflix.spinnaker.clouddriver.security.AccountCredentialsRepository; +import com.netflix.spinnaker.clouddriver.security.ProviderUtils; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Scope; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static com.netflix.spinnaker.clouddriver.aws.security.AmazonCredentials.AWSRegion; + +@Configuration +public class EcsProviderConfig { + + @Bean + @DependsOn("netflixECSCredentials") + public EcsProvider ecsProvider(AccountCredentialsRepository accountCredentialsRepository, AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) { + EcsProvider provider = new EcsProvider(accountCredentialsRepository, Collections.newSetFromMap(new ConcurrentHashMap())); + synchronizeEcsProvider(provider, accountCredentialsRepository, amazonClientProvider, awsCredentialsProvider, registry); + return provider; + } + + @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) + @Bean + public EcsProviderSynchronizer synchronizeEcsProvider(EcsProvider ecsProvider, AccountCredentialsRepository accountCredentialsRepository, + AmazonClientProvider amazonClientProvider, AWSCredentialsProvider awsCredentialsProvider, Registry registry) { + + Set scheduledAccounts = ProviderUtils.getScheduledAccounts(ecsProvider); + Set allAccounts = ProviderUtils.buildThreadSafeSetOfAccounts(accountCredentialsRepository, NetflixAmazonCredentials.class); + List newAgents = new LinkedList<>(); + + for (NetflixAmazonCredentials credentials : allAccounts) { + if (credentials.getCloudProvider().equals(EcsCloudProvider.ID)) { + + for (AWSRegion region : credentials.getRegions()) { + if (!scheduledAccounts.contains(credentials.getName())) { + //Soon to come: + //newAgents.add(new EcsClusterCachingAgent(credentials.getName(), region.getName(), amazonClientProvider, awsCredentialsProvider)); + } + } + } + } + + ecsProvider.getAgents().addAll(newAgents); + ecsProvider.synchronizeHealthAgents(); + return new EcsProviderSynchronizer(); + } + + class EcsProviderSynchronizer { + } +}