From b5fb6ccf646c0b05ec0bcc589890ad6355fd29e8 Mon Sep 17 00:00:00 2001 From: tpetr Date: Thu, 8 Jan 2015 12:10:22 -0500 Subject: [PATCH 1/4] remove getActiveTasks(host) + add getActiveTasksOnSlave(slaveId) to SingularityClient --- .../com/hubspot/singularity/client/SingularityClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 90aee7b91f..0652d83507 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -71,7 +71,7 @@ public class SingularityClient { private static final String TASKS_FORMAT = "http://%s/%s/tasks"; private static final String TASKS_KILL_TASK_FORMAT = TASKS_FORMAT + "/task/%s"; private static final String TASKS_GET_ACTIVE_FORMAT = TASKS_FORMAT + "/active"; - private static final String TASKS_GET_ACTIVE_PER_HOST_FORMAT = TASKS_FORMAT + "/active/%s"; + private static final String TASKS_GET_ACTIVE_ON_SLAVE_FORMAT = TASKS_FORMAT + "/active/slave/%s"; private static final String TASKS_GET_SCHEDULED_FORMAT = TASKS_FORMAT + "/scheduled"; private static final String HISTORY_FORMAT = "http://%s/%s/history"; @@ -506,10 +506,10 @@ public Collection getActiveTasks() { return getCollection(requestUri, "active tasks", TASKS_COLLECTION); } - public Collection getActiveTasks(final String host) { - final String requestUri = String.format(TASKS_GET_ACTIVE_PER_HOST_FORMAT, getHost(), contextPath, host); + public Collection getActiveTasksOnSlave(final String slaveId) { + final String requestUri = String.format(TASKS_GET_ACTIVE_ON_SLAVE_FORMAT, getHost(), contextPath, slaveId); - return getCollection(requestUri, String.format("active tasks on %s", host), TASKS_COLLECTION); + return getCollection(requestUri, String.format("active tasks on slave %s", slaveId), TASKS_COLLECTION); } public Optional killTask(String taskId, Optional user) { From 92b39b53518fb9a5ed1bea2c749cd464eeb0dc6a Mon Sep 17 00:00:00 2001 From: tpetr Date: Thu, 8 Jan 2015 12:11:01 -0500 Subject: [PATCH 2/4] look up running tasks on slave via slave ID, not hostname --- .../cleanup/SingularityExecutorCleanup.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java index d74dbec18e..073e06f60e 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java @@ -11,15 +11,20 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; -import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.google.inject.Inject; +import com.google.inject.name.Named; +import com.hubspot.horizon.HttpClient; +import com.hubspot.horizon.HttpRequest; +import com.hubspot.horizon.HttpResponse; import com.hubspot.mesos.JavaUtils; +import com.hubspot.mesos.json.MesosSlaveStateObject; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskHistory; import com.hubspot.singularity.SingularityTaskHistoryUpdate; import com.hubspot.singularity.client.SingularityClient; import com.hubspot.singularity.client.SingularityClientException; +import com.hubspot.singularity.client.SingularityClientModule; import com.hubspot.singularity.executor.SingularityExecutorCleanupStatistics; import com.hubspot.singularity.executor.SingularityExecutorCleanupStatistics.SingularityExecutorCleanupStatisticsBuilder; import com.hubspot.singularity.executor.TemplateManager; @@ -35,19 +40,23 @@ public class SingularityExecutorCleanup { private static final Logger LOG = LoggerFactory.getLogger(SingularityExecutorCleanup.class); + public static final String LOCAL_SLAVE_STATE_URL_FORMAT = "http://%s:5051/slave(1)/state.json"; + private final JsonObjectFileHelper jsonObjectFileHelper; private final SingularityExecutorConfiguration executorConfiguration; private final SingularityClient singularityClient; private final TemplateManager templateManager; private final SingularityExecutorCleanupConfiguration cleanupConfiguration; + private final HttpClient httpClient; @Inject - public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager) { + public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager, @Named(SingularityClientModule.HTTP_CLIENT_NAME) HttpClient httpClient) { this.jsonObjectFileHelper = jsonObjectFileHelper; this.executorConfiguration = executorConfiguration; this.cleanupConfiguration = cleanupConfiguration; this.singularityClient = singularityClient; this.templateManager = templateManager; + this.httpClient = httpClient; } public SingularityExecutorCleanupStatistics clean() { @@ -58,9 +67,9 @@ public SingularityExecutorCleanupStatistics clean() { try { runningTaskIds = getRunningTaskIds(); - } catch (SingularityClientException sce) { - LOG.error("While fetching running tasks from singularity", sce); - statisticsBldr.setErrorMessage(sce.getMessage()); + } catch (Exception e) { + LOG.error("While fetching running tasks from singularity", e); + statisticsBldr.setErrorMessage(e.getMessage()); return statisticsBldr.build(); } @@ -129,16 +138,23 @@ public SingularityExecutorCleanupStatistics clean() { return statisticsBldr.build(); } - private Collection getActiveTasksOnSlave() { + private String getLocalSlaveID() { try { - return singularityClient.getActiveTasks(JavaUtils.getHostAddress()); - } catch (SocketException e) { - throw Throwables.propagate(e); + final HttpRequest request = HttpRequest.newBuilder().setUrl(String.format(LOCAL_SLAVE_STATE_URL_FORMAT, JavaUtils.getHostAddress())).build(); + final HttpResponse response = httpClient.execute(request); + + if (response.isSuccess()) { + return response.getAs(MesosSlaveStateObject.class).getId(); + } else { + throw new RuntimeException(String.format("Failed to get local Slave ID -- HTTP %d: %s", response.getStatusCode(), response.getAsString())); + } + } catch (SocketException se) { + throw new RuntimeException("Failed to get host address", se); } } private Set getRunningTaskIds() { - final Collection activeTasks = getActiveTasksOnSlave(); + final Collection activeTasks = singularityClient.getActiveTasksOnSlave(getLocalSlaveID()); final Set runningTaskIds = Sets.newHashSet(); From 46dbe67d603e9db1f3eb01b2cee7c38a466fbdfa Mon Sep 17 00:00:00 2001 From: tpetr Date: Thu, 8 Jan 2015 14:01:52 -0500 Subject: [PATCH 3/4] need to add HorizonCore dep --- SingularityExecutorCleanup/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SingularityExecutorCleanup/pom.xml b/SingularityExecutorCleanup/pom.xml index 0c0e4cbf78..fcf279546b 100644 --- a/SingularityExecutorCleanup/pom.xml +++ b/SingularityExecutorCleanup/pom.xml @@ -40,6 +40,11 @@ SingularityClient + + com.hubspot + HorizonCore + + com.google.inject guice From c96e6243da2b6ceed34f02ac6b849231936a0957 Mon Sep 17 00:00:00 2001 From: tpetr Date: Fri, 9 Jan 2015 09:41:45 -0500 Subject: [PATCH 4/4] use MesosClient instead --- SingularityExecutorCleanup/pom.xml | 5 +++ .../cleanup/SingularityExecutorCleanup.java | 39 +++++++------------ .../SingularityExecutorCleanupRunner.java | 3 +- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/SingularityExecutorCleanup/pom.xml b/SingularityExecutorCleanup/pom.xml index fcf279546b..4f25213d0d 100644 --- a/SingularityExecutorCleanup/pom.xml +++ b/SingularityExecutorCleanup/pom.xml @@ -40,6 +40,11 @@ SingularityClient + + com.hubspot + SingularityMesosClient + + com.hubspot HorizonCore diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java index 073e06f60e..9bee218421 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -18,6 +19,7 @@ import com.hubspot.horizon.HttpRequest; import com.hubspot.horizon.HttpResponse; import com.hubspot.mesos.JavaUtils; +import com.hubspot.mesos.client.MesosClient; import com.hubspot.mesos.json.MesosSlaveStateObject; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskHistory; @@ -40,23 +42,21 @@ public class SingularityExecutorCleanup { private static final Logger LOG = LoggerFactory.getLogger(SingularityExecutorCleanup.class); - public static final String LOCAL_SLAVE_STATE_URL_FORMAT = "http://%s:5051/slave(1)/state.json"; - private final JsonObjectFileHelper jsonObjectFileHelper; private final SingularityExecutorConfiguration executorConfiguration; private final SingularityClient singularityClient; private final TemplateManager templateManager; private final SingularityExecutorCleanupConfiguration cleanupConfiguration; - private final HttpClient httpClient; + private final MesosClient mesosClient; @Inject - public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager, @Named(SingularityClientModule.HTTP_CLIENT_NAME) HttpClient httpClient) { + public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager, MesosClient mesosClient) { this.jsonObjectFileHelper = jsonObjectFileHelper; this.executorConfiguration = executorConfiguration; this.cleanupConfiguration = cleanupConfiguration; this.singularityClient = singularityClient; this.templateManager = templateManager; - this.httpClient = httpClient; + this.mesosClient = mesosClient; } public SingularityExecutorCleanupStatistics clean() { @@ -138,31 +138,22 @@ public SingularityExecutorCleanupStatistics clean() { return statisticsBldr.build(); } - private String getLocalSlaveID() { + private Set getRunningTaskIds() { try { - final HttpRequest request = HttpRequest.newBuilder().setUrl(String.format(LOCAL_SLAVE_STATE_URL_FORMAT, JavaUtils.getHostAddress())).build(); - final HttpResponse response = httpClient.execute(request); + final String slaveId = mesosClient.getSlaveState(mesosClient.getSlaveUri(JavaUtils.getHostAddress())).getId(); - if (response.isSuccess()) { - return response.getAs(MesosSlaveStateObject.class).getId(); - } else { - throw new RuntimeException(String.format("Failed to get local Slave ID -- HTTP %d: %s", response.getStatusCode(), response.getAsString())); - } - } catch (SocketException se) { - throw new RuntimeException("Failed to get host address", se); - } - } + final Collection activeTasks = singularityClient.getActiveTasksOnSlave(slaveId); - private Set getRunningTaskIds() { - final Collection activeTasks = singularityClient.getActiveTasksOnSlave(getLocalSlaveID()); + final Set runningTaskIds = Sets.newHashSet(); - final Set runningTaskIds = Sets.newHashSet(); + for (SingularityTask task : activeTasks) { + runningTaskIds.add(task.getTaskId().getId()); + } - for (SingularityTask task : activeTasks) { - runningTaskIds.add(task.getTaskId().getId()); + return runningTaskIds; + } catch (SocketException se) { + throw Throwables.propagate(se); } - - return runningTaskIds; } private boolean cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional taskHistory) { diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanupRunner.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanupRunner.java index c9a5dc0641..e8793502db 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanupRunner.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanupRunner.java @@ -8,6 +8,7 @@ import com.google.inject.Injector; import com.google.inject.Stage; import com.hubspot.mesos.JavaUtils; +import com.hubspot.mesos.client.SingularityMesosClientModule; import com.hubspot.singularity.client.SingularityClientModule; import com.hubspot.singularity.executor.SingularityExecutorCleanupStatistics; import com.hubspot.singularity.executor.cleanup.config.SingularityExecutorCleanupConfiguration; @@ -27,7 +28,7 @@ public static void main(String... args) { final long start = System.currentTimeMillis(); try { - final Injector injector = Guice.createInjector(Stage.PRODUCTION, new SingularityRunnerBaseModule(new SingularityS3ConfigurationLoader(), new SingularityExecutorConfigurationLoader(), new SingularityExecutorCleanupConfigurationLoader()), new SingularityExecutorModule(), new SingularityClientModule()); + final Injector injector = Guice.createInjector(Stage.PRODUCTION, new SingularityRunnerBaseModule(new SingularityS3ConfigurationLoader(), new SingularityExecutorConfigurationLoader(), new SingularityExecutorCleanupConfigurationLoader()), new SingularityExecutorModule(), new SingularityClientModule(), new SingularityMesosClientModule()); final SingularityExecutorCleanupRunner runner = injector.getInstance(SingularityExecutorCleanupRunner.class); LOG.info("Starting cleanup");