From b547c87aed443c2b4752a8aa42c280b916869039 Mon Sep 17 00:00:00 2001 From: jaredstehler Date: Tue, 9 May 2017 15:39:37 -0400 Subject: [PATCH 1/3] upgrade docker-client to 8.4.0 --- .../SingularityExecutorThreadChecker.java | 2 +- .../SingularityExecutorDockerAuthConfig.java | 10 ++++++++ .../config/SingularityExecutorModule.java | 25 +++++++++++++------ .../task/SingularityExecutorTaskCleanup.java | 4 +-- ...SingularityExecutorTaskProcessBuilder.java | 2 +- .../executor/utils/DockerUtils.java | 5 ++-- pom.xml | 6 ++++- 7 files changed, 40 insertions(+), 14 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutorThreadChecker.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutorThreadChecker.java index 80432cc5b0..5cd74ae57f 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutorThreadChecker.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/SingularityExecutorThreadChecker.java @@ -31,7 +31,7 @@ import com.hubspot.singularity.executor.utils.DockerUtils; import com.hubspot.singularity.runner.base.shared.ProcessFailedException; import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; -import com.spotify.docker.client.DockerException; +import com.spotify.docker.client.exceptions.DockerException; @Singleton public class SingularityExecutorThreadChecker { diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorDockerAuthConfig.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorDockerAuthConfig.java index 165e4eb3f5..4671b6697a 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorDockerAuthConfig.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorDockerAuthConfig.java @@ -2,11 +2,21 @@ public class SingularityExecutorDockerAuthConfig { + private boolean fromDockerConfig; + private String username; private String password; private String email; private String serverAddress; + public boolean isFromDockerConfig() { + return fromDockerConfig; + } + + public void setFromDockerConfig(boolean fromDockerConfig) { + this.fromDockerConfig = fromDockerConfig; + } + public String getUsername() { return username; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java index 1ab2bbaf84..89b79a6907 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java @@ -6,6 +6,7 @@ import com.github.jknack.handlebars.Handlebars; import com.github.jknack.handlebars.Template; +import com.google.common.base.Throwables; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; @@ -21,7 +22,7 @@ import com.spotify.docker.client.DefaultDockerClient; import com.spotify.docker.client.DefaultDockerClient.Builder; import com.spotify.docker.client.DockerClient; -import com.spotify.docker.client.messages.AuthConfig; +import com.spotify.docker.client.messages.RegistryAuth; public class SingularityExecutorModule extends AbstractModule { @@ -108,12 +109,22 @@ public DockerClient providesDockerClient(SingularityExecutorConfiguration config if(configuration.getDockerAuthConfig().isPresent()) { SingularityExecutorDockerAuthConfig authConfig = configuration.getDockerAuthConfig().get(); - dockerClientBuilder.authConfig(AuthConfig.builder() - .email(authConfig.getEmail()) - .username(authConfig.getUsername()) - .password(authConfig.getPassword()) - .serverAddress(authConfig.getServerAddress()) - .build()); + if(authConfig.isFromDockerConfig()) { + try { + dockerClientBuilder.registryAuth(RegistryAuth.fromDockerConfig().build()); + + } catch(IOException e) { + throw Throwables.propagate(e); + } + } + else { + dockerClientBuilder.registryAuth(RegistryAuth.builder() + .email(authConfig.getEmail()) + .username(authConfig.getUsername()) + .password(authConfig.getPassword()) + .serverAddress(authConfig.getServerAddress()) + .build()); + } } return dockerClientBuilder.build(); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java index fe33c94bb9..a3482aac0c 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java @@ -13,8 +13,8 @@ import com.hubspot.singularity.executor.utils.DockerUtils; import com.hubspot.singularity.runner.base.shared.ExceptionChainParser; import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; -import com.spotify.docker.client.ContainerNotFoundException; -import com.spotify.docker.client.DockerException; +import com.spotify.docker.client.exceptions.ContainerNotFoundException; +import com.spotify.docker.client.exceptions.DockerException; import com.spotify.docker.client.messages.ContainerInfo; public class SingularityExecutorTaskCleanup { diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 2734213115..32d942cece 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -30,7 +30,7 @@ import com.hubspot.singularity.executor.utils.DockerUtils; import com.hubspot.singularity.executor.utils.ExecutorUtils; import com.hubspot.singularity.runner.base.shared.ProcessFailedException; -import com.spotify.docker.client.DockerException; +import com.spotify.docker.client.exceptions.DockerException; public class SingularityExecutorTaskProcessBuilder implements Callable { diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/utils/DockerUtils.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/utils/DockerUtils.java index 7e88bcaf5d..1be538f2ee 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/utils/DockerUtils.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/utils/DockerUtils.java @@ -13,7 +13,8 @@ import com.google.inject.Inject; import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.spotify.docker.client.DockerClient; -import com.spotify.docker.client.DockerException; +import com.spotify.docker.client.DockerClient.RemoveContainerParam; +import com.spotify.docker.client.exceptions.DockerException; import com.spotify.docker.client.messages.Container; import com.spotify.docker.client.messages.ContainerInfo; @@ -97,7 +98,7 @@ public void stopContainer(final String containerId, final int timeout) throws Do public void removeContainer(final String containerId, final boolean removeRunning) throws DockerException { Callable callable = new Callable() { @Override public Void call() throws Exception { - dockerClient.removeContainer(containerId, removeRunning); + dockerClient.removeContainer(containerId, RemoveContainerParam.removeVolumes(removeRunning)); return null; } }; diff --git a/pom.xml b/pom.xml index fbf69ea896..cedffc8118 100644 --- a/pom.xml +++ b/pom.xml @@ -174,7 +174,7 @@ com.spotify docker-client - 3.1.5 + 8.4.0 commons-logging @@ -192,6 +192,10 @@ org.glassfish.hk2.external aopalliance-repackaged + + org.apache.httpcomponents + httpcore + From 9318fa18e4e157c8f0726142607dc185f027a15c Mon Sep 17 00:00:00 2001 From: jaredstehler Date: Tue, 9 May 2017 15:48:57 -0400 Subject: [PATCH 2/3] add test --- .../config/SingularityExecutorConfigurationTest.java | 11 ++++++++++- .../config/executor-conf-dockerauth-fromconfig.yaml | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 SingularityExecutor/src/test/resources/config/executor-conf-dockerauth-fromconfig.yaml diff --git a/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java b/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java index 7d30654673..ea865fc2f6 100644 --- a/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java +++ b/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java @@ -21,13 +21,22 @@ public class SingularityExecutorConfigurationTest { @Test public void itLoadsDockerAuthConfig() { SingularityExecutorConfiguration config = loadConfig("config/executor-conf-dockerauth.yaml"); - + assertThat(config.getDockerAuthConfig().isPresent()).isTrue(); + assertThat(config.getDockerAuthConfig().get().isFromDockerConfig()).isFalse(); assertThat(config.getDockerAuthConfig().get().getUsername()).isEqualTo("dockeruser"); assertThat(config.getDockerAuthConfig().get().getPassword()).isEqualTo("dockerpassword"); assertThat(config.getDockerAuthConfig().get().getServerAddress()).isEqualTo("https://private.docker.registry/path"); } + @Test + public void itLoadsDockerAuthFromConfigFileIfSpecified() { + SingularityExecutorConfiguration config = loadConfig("config/executor-conf-dockerauth-fromconfig.yaml"); + + assertThat(config.getDockerAuthConfig().isPresent()).isTrue(); + assertThat(config.getDockerAuthConfig().get().isFromDockerConfig()).isTrue(); + } + private SingularityExecutorConfiguration loadConfig(String file) { try { ObjectMapper mapper = new SingularityRunnerBaseModule(null).providesYamlMapper(); diff --git a/SingularityExecutor/src/test/resources/config/executor-conf-dockerauth-fromconfig.yaml b/SingularityExecutor/src/test/resources/config/executor-conf-dockerauth-fromconfig.yaml new file mode 100644 index 0000000000..f230d159a7 --- /dev/null +++ b/SingularityExecutor/src/test/resources/config/executor-conf-dockerauth-fromconfig.yaml @@ -0,0 +1,4 @@ +executor: + + dockerAuthConfig: + fromDockerConfig: true From ef15b1a2c94eab8952e4c8d9a7677b54f2e5583a Mon Sep 17 00:00:00 2001 From: jaredstehler Date: Tue, 9 May 2017 18:16:32 -0400 Subject: [PATCH 3/3] need to add httpcore dep --- SingularityExecutor/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/SingularityExecutor/pom.xml b/SingularityExecutor/pom.xml index 59c09cdc87..b4a9966822 100644 --- a/SingularityExecutor/pom.xml +++ b/SingularityExecutor/pom.xml @@ -90,6 +90,12 @@ docker-client + + org.apache.httpcomponents + httpcore + runtime + + com.ning async-http-client