diff --git a/.gitignore b/.gitignore index 8e548131d5d..5ae25f9d5ac 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ node_modules/ .gradle/ build/ +out/ +*.class diff --git a/CHANGELOG.md b/CHANGELOG.md index f7a0287426b..be092a82f92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,12 @@ All notable changes to this project will be documented in this file. - Fix for setting `ryuk.container.timeout` causes a `ClassCastException` ([\#684](https://github.com/testcontainers/testcontainers-java/issues/684)) ### Changed +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) ## [1.7.2] - 2018-04-30 ### Fixed +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662)) - Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664)) @@ -24,6 +26,7 @@ All notable changes to this project will be documented in this file. - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640)) - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643)) - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648)) +- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) ### Changed - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630)) diff --git a/circle.yml b/circle.yml index a793bf8cb6a..61a4c8041a4 100644 --- a/circle.yml +++ b/circle.yml @@ -14,6 +14,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit modules-no-jdbc-test-no-selenium: steps: - checkout @@ -30,6 +32,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit modules-jdbc-test: steps: - checkout @@ -46,6 +50,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit selenium: steps: - checkout @@ -59,6 +65,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit workflows: version: 2 diff --git a/core/build.gradle b/core/build.gradle index cdc9172517c..fcaa62f132e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -120,7 +120,7 @@ dependencies { } shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1' shaded 'org.rnorth:tcp-unix-socket-proxy:1.0.2' - shaded 'org.zeroturnaround:zt-exec:1.8' + shaded 'org.zeroturnaround:zt-exec:1.10' shaded 'commons-lang:commons-lang:2.6' shaded 'commons-io:commons-io:2.5' shaded 'commons-codec:commons-codec:1.11' diff --git a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java index 9554793c004..bb0d0c969ce 100644 --- a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java +++ b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java @@ -3,6 +3,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.exception.DockerClientException; +import com.github.dockerjava.api.model.AuthConfig; import com.github.dockerjava.api.model.Image; import com.github.dockerjava.core.command.PullImageResultCallback; import lombok.NonNull; @@ -14,6 +15,7 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; import org.testcontainers.utility.LazyFuture; +import org.testcontainers.utility.RegistryAuthLocator; import java.util.HashSet; import java.util.List; @@ -93,10 +95,14 @@ protected final String resolve() { // The image is not available locally - pull it try { + final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig()); + final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName); + final PullImageResultCallback callback = new PullImageResultCallback(); dockerClient .pullImageCmd(imageName.getUnversionedPart()) .withTag(imageName.getVersionPart()) + .withAuthConfig(effectiveAuthConfig) .exec(callback); callback.awaitCompletion(); AVAILABLE_IMAGE_NAME_CACHE.add(imageName); diff --git a/core/src/main/java/org/testcontainers/utility/DockerImageName.java b/core/src/main/java/org/testcontainers/utility/DockerImageName.java index 212b81a20eb..8795e14b2cc 100644 --- a/core/src/main/java/org/testcontainers/utility/DockerImageName.java +++ b/core/src/main/java/org/testcontainers/utility/DockerImageName.java @@ -94,7 +94,11 @@ public String getVersionPart() { @Override public String toString() { - return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + if (versioning == null) { + return getUnversionedPart(); + } else { + return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + } } /** @@ -116,6 +120,10 @@ public void assertValid() { } } + public String getRegistry() { + return registry; + } + private interface Versioning { boolean isValid(); String getSeparator(); diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java new file mode 100644 index 00000000000..d211e2dd3b1 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -0,0 +1,127 @@ +package org.testcontainers.utility; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.zeroturnaround.exec.ProcessExecutor; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.util.concurrent.TimeUnit; + +import static org.slf4j.LoggerFactory.getLogger; + +/** + * Utility to look up registry authentication information for an image. + */ +public class RegistryAuthLocator { + + private static final Logger log = getLogger(RegistryAuthLocator.class); + + private final AuthConfig defaultAuthConfig; + private final File configFile; + private final String commandPathPrefix; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @VisibleForTesting + RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) { + this.defaultAuthConfig = defaultAuthConfig; + this.configFile = configFile; + this.commandPathPrefix = commandPathPrefix; + } + + /** + * @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication + * available for images that are looked up + */ + public RegistryAuthLocator(AuthConfig defaultAuthConfig) { + this.defaultAuthConfig = defaultAuthConfig; + final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG", + System.getProperty("user.home") + "/.docker"); + this.configFile = new File(dockerConfigLocation + "/config.json"); + this.commandPathPrefix = ""; + } + + /** + * Looks up an AuthConfig for a given image name. + * + * @param dockerImageName image name to be looked up (potentially including a registry URL part) + * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for + * this {@link RegistryAuthLocator}. + */ + public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { + log.debug("Looking up auth config for image: {}", dockerImageName); + + log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}", + configFile, + configFile.exists() ? "exists" : "does not exist", + commandPathPrefix); + + try { + final JsonNode config = OBJECT_MAPPER.readTree(configFile); + + final String reposName = dockerImageName.getRegistry(); + final JsonNode auths = config.at("/auths/" + reposName); + + if (!auths.isMissingNode() && auths.size() == 0) { + // auths/ is an empty dict - use a credential helper + return authConfigUsingCredentialsStoreOrHelper(reposName, config); + } + // otherwise, defaultAuthConfig should already contain any credentials available + } catch (Exception e) { + log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " + + "Falling back to docker-java default behaviour", + dockerImageName, + configFile, + e); + } + return defaultAuthConfig; + } + + private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, JsonNode config) throws Exception { + + final JsonNode credsStoreName = config.at("/credsStore"); + final JsonNode credHelper = config.at("/credHelpers/" + hostName); + + if (!credHelper.isMissingNode()) { + return runCredentialProvider(hostName, credHelper.asText()); + } else if (!credsStoreName.isMissingNode()) { + return runCredentialProvider(hostName, credsStoreName.asText()); + } else { + throw new IllegalStateException("Unsupported Docker config auths settings!"); + } + } + + private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception { + final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper; + String data; + + log.debug("Executing docker credential helper: {} to locate auth config for: {}", + credentialHelperName, hostName); + + try { + data = new ProcessExecutor() + .command(credentialHelperName, "get") + .redirectInput(new ByteArrayInputStream(hostName.getBytes())) + .readOutput(true) + .exitValueNormal() + .timeout(30, TimeUnit.SECONDS) + .execute() + .outputUTF8() + .trim(); + } catch (Exception e) { + log.error("Failure running docker credential helper ({})", credentialHelperName); + throw e; + } + + final JsonNode helperResponse = OBJECT_MAPPER.readTree(data); + log.debug("Credential helper provided auth config for: {}", hostName); + + return new AuthConfig() + .withRegistryAddress(helperResponse.at("/ServerURL").asText()) + .withUsername(helperResponse.at("/Username").asText()) + .withPassword(helperResponse.at("/Secret").asText()); + } +} diff --git a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java index f06ff40fc40..1dabc02ac8d 100644 --- a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java +++ b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java @@ -22,7 +22,7 @@ public static String[] parameters() { "gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085", "quay.io/testcontainers/ryuk:latest", "quay.io/testcontainers/ryuk:0.2.2", - "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da" + "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da", }; } diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java new file mode 100644 index 00000000000..ec5dfaa2e88 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -0,0 +1,63 @@ +package org.testcontainers.utility; + +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.io.Resources; +import org.apache.commons.lang.SystemUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.net.URISyntaxException; + +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.rnorth.visibleassertions.VisibleAssertions.assertNull; + +public class RegistryAuthLocatorTest { + + @BeforeClass + public static void nonWindowsTest() throws Exception { + Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); + } + + @Test + public void lookupAuthConfigWithoutCredentials() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("unauthenticated.registry.org/org/repo")); + + assertEquals("Default docker registry URL is set on auth config", "https://index.docker.io/v1/", authConfig.getRegistryAddress()); + assertNull("No username is set", authConfig.getUsername()); + assertNull("No password is set", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingStore() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingHelper() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @NotNull + private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException { + final File configFile = new File(Resources.getResource("auth-config/" + configName).toURI()); + return new RegistryAuthLocator(new AuthConfig(), configFile, configFile.getParentFile().getAbsolutePath() + "/"); + } + +} diff --git a/core/src/test/resources/auth-config/config-with-helper-and-store.json b/core/src/test/resources/auth-config/config-with-helper-and-store.json new file mode 100644 index 00000000000..cd623369d78 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper-and-store.json @@ -0,0 +1,12 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake", + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-helper.json b/core/src/test/resources/auth-config/config-with-helper.json new file mode 100644 index 00000000000..eaa670e4296 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper.json @@ -0,0 +1,11 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-store.json b/core/src/test/resources/auth-config/config-with-store.json new file mode 100644 index 00000000000..d165997e52f --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-store.json @@ -0,0 +1,9 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake" +} diff --git a/core/src/test/resources/auth-config/docker-credential-fake b/core/src/test/resources/auth-config/docker-credential-fake new file mode 100755 index 00000000000..0cfa0dfb72d --- /dev/null +++ b/core/src/test/resources/auth-config/docker-credential-fake @@ -0,0 +1,13 @@ +#!/bin/bash + +if [[ $1 != "get" ]]; then + exit 1 +fi + +read > /dev/null + +echo '{' \ + ' "ServerURL": "url",' \ + ' "Username": "username",' \ + ' "Secret": "secret"' \ + '}'