diff --git a/.gitignore b/.gitignore index 4c3f24f5bb4..c95a6ece0cd 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ node_modules/ .gradle/ build/ +out/ +*.class # Eclipse IDE files **/.project diff --git a/CHANGELOG.md b/CHANGELOG.md index f49c8b5b753..9cb73ecbed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,10 @@ All notable changes to this project will be documented in this file. ## [1.7.2] - 2018-04-30 +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) + ### Fixed +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662)) - Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664)) @@ -57,6 +60,7 @@ All notable changes to this project will be documented in this file. - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640)) - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643)) - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648)) +- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) ### Changed - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630)) diff --git a/circle.yml b/circle.yml index b216656417f..625c5026db0 100644 --- a/circle.yml +++ b/circle.yml @@ -14,6 +14,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit okhttp: steps: - checkout @@ -45,6 +47,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit modules-jdbc-test: steps: - checkout @@ -61,6 +65,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit selenium: steps: - checkout @@ -74,6 +80,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit workflows: version: 2 diff --git a/core/build.gradle b/core/build.gradle index 15a0300738f..3781c90002f 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -104,7 +104,7 @@ dependencies { shaded 'com.squareup.okhttp3:okhttp:3.10.0' shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1' - shaded 'org.zeroturnaround:zt-exec:1.8' + shaded 'org.zeroturnaround:zt-exec:1.10' shaded 'commons-lang:commons-lang:2.6' shaded 'commons-io:commons-io:2.5' shaded 'commons-codec:commons-codec:1.11' diff --git a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java index 9554793c004..bb0d0c969ce 100644 --- a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java +++ b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java @@ -3,6 +3,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.exception.DockerClientException; +import com.github.dockerjava.api.model.AuthConfig; import com.github.dockerjava.api.model.Image; import com.github.dockerjava.core.command.PullImageResultCallback; import lombok.NonNull; @@ -14,6 +15,7 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; import org.testcontainers.utility.LazyFuture; +import org.testcontainers.utility.RegistryAuthLocator; import java.util.HashSet; import java.util.List; @@ -93,10 +95,14 @@ protected final String resolve() { // The image is not available locally - pull it try { + final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig()); + final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName); + final PullImageResultCallback callback = new PullImageResultCallback(); dockerClient .pullImageCmd(imageName.getUnversionedPart()) .withTag(imageName.getVersionPart()) + .withAuthConfig(effectiveAuthConfig) .exec(callback); callback.awaitCompletion(); AVAILABLE_IMAGE_NAME_CACHE.add(imageName); diff --git a/core/src/main/java/org/testcontainers/utility/DockerImageName.java b/core/src/main/java/org/testcontainers/utility/DockerImageName.java index 50b2d57b743..3f6d7124891 100644 --- a/core/src/main/java/org/testcontainers/utility/DockerImageName.java +++ b/core/src/main/java/org/testcontainers/utility/DockerImageName.java @@ -94,7 +94,11 @@ public String getVersionPart() { @Override public String toString() { - return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + if (versioning == null) { + return getUnversionedPart(); + } else { + return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + } } /** @@ -116,6 +120,10 @@ public void assertValid() { } } + public String getRegistry() { + return registry; + } + private interface Versioning { boolean isValid(); String getSeparator(); diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java new file mode 100644 index 00000000000..eb290cae115 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -0,0 +1,184 @@ +package org.testcontainers.utility; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.SystemUtils; +import org.slf4j.Logger; +import org.zeroturnaround.exec.ProcessExecutor; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.slf4j.LoggerFactory.getLogger; + +/** + * Utility to look up registry authentication information for an image. + */ +public class RegistryAuthLocator { + + private static final Logger log = getLogger(RegistryAuthLocator.class); + + private final AuthConfig defaultAuthConfig; + private final File configFile; + private final String commandPathPrefix; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @VisibleForTesting + RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) { + this.defaultAuthConfig = defaultAuthConfig; + this.configFile = configFile; + this.commandPathPrefix = commandPathPrefix; + } + + /** + * @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication + * available for images that are looked up + */ + public RegistryAuthLocator(AuthConfig defaultAuthConfig) { + this.defaultAuthConfig = defaultAuthConfig; + final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG", + System.getProperty("user.home") + "/.docker"); + this.configFile = new File(dockerConfigLocation + "/config.json"); + this.commandPathPrefix = ""; + } + + /** + * Looks up an AuthConfig for a given image name. + *

+ * Lookup is performed in following order: + *

    + *
  1. {@code auths} is checked for existing credentials for the specified registry.
    2. + *
  2. if no existing auth is found, {@code credHelpers} are checked for helper for the specified registry.
    3. + *
  3. if no suitable {@code credHelpers} found, {@code credsStore} is used.
    4. + *
  4. if no {@code credsStore} is found then the default configuration is returned.
    5. + *
+ * + * @param dockerImageName image name to be looked up (potentially including a registry URL part) + * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for + * this {@link RegistryAuthLocator}. + */ + public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { + + if (SystemUtils.IS_OS_WINDOWS) { + log.debug("RegistryAuthLocator is not supported on Windows. Please help test or improve it and update " + + "https://github.com/testcontainers/testcontainers-java/issues/756"); + return defaultAuthConfig; + } + + log.debug("Looking up auth config for image: {}", dockerImageName); + + log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}", + configFile, + configFile.exists() ? "exists" : "does not exist", + commandPathPrefix); + + try { + final JsonNode config = OBJECT_MAPPER.readTree(configFile); + final String reposName = dockerImageName.getRegistry(); + + final AuthConfig existingAuthConfig = findExistingAuthConfig(config, reposName); + if (existingAuthConfig != null) { + return existingAuthConfig; + } + // auths is empty, using helper: + final AuthConfig helperAuthConfig = authConfigUsingHelper(config, reposName); + if (helperAuthConfig != null) { + return helperAuthConfig; + } + // no credsHelper to use, using credsStore: + final AuthConfig storeAuthConfig = authConfigUsingStore(config, reposName); + if (storeAuthConfig != null) { + return storeAuthConfig; + } + // otherwise, defaultAuthConfig should already contain any credentials available + } catch (Exception e) { + log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " + + "Falling back to docker-java default behaviour", + dockerImageName, + configFile, + e); + } + return defaultAuthConfig; + } + + private AuthConfig findExistingAuthConfig(final JsonNode config, final String reposName) throws Exception { + final Map.Entry entry = findAuthNode(config, reposName); + if (entry != null && entry.getValue() != null && entry.getValue().size() > 0) { + return OBJECT_MAPPER + .treeToValue(entry.getValue(), AuthConfig.class) + .withRegistryAddress(entry.getKey()); + } + return null; + } + + private AuthConfig authConfigUsingHelper(final JsonNode config, final String reposName) throws Exception { + final JsonNode credHelpers = config.get("credHelpers"); + if (credHelpers != null && credHelpers.size() > 0) { + final JsonNode helperNode = credHelpers.get(reposName); + if (helperNode != null && helperNode.isTextual()) { + final String helper = helperNode.asText(); + return runCredentialProvider(reposName, helper); + } + } + return null; + } + + private AuthConfig authConfigUsingStore(final JsonNode config, final String reposName) throws Exception { + final JsonNode credsStoreNode = config.get("credsStore"); + if (credsStoreNode != null && !credsStoreNode.isMissingNode() && credsStoreNode.isTextual()) { + final String credsStore = credsStoreNode.asText(); + return runCredentialProvider(reposName, credsStore); + } + return null; + } + + private Map.Entry findAuthNode(final JsonNode config, final String reposName) throws Exception { + final JsonNode auths = config.get("auths"); + if (auths != null && auths.size() > 0) { + final Iterator> fields = auths.fields(); + while (fields.hasNext()) { + final Map.Entry entry = fields.next(); + if (entry.getKey().endsWith("://" + reposName)) { + return entry; + } + } + } + return null; + } + + private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception { + final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper; + String data; + + log.debug("Executing docker credential helper: {} to locate auth config for: {}", + credentialHelperName, hostName); + + try { + data = new ProcessExecutor() + .command(credentialHelperName, "get") + .redirectInput(new ByteArrayInputStream(hostName.getBytes())) + .readOutput(true) + .exitValueNormal() + .timeout(30, TimeUnit.SECONDS) + .execute() + .outputUTF8() + .trim(); + } catch (Exception e) { + log.error("Failure running docker credential helper ({})", credentialHelperName); + throw e; + } + + final JsonNode helperResponse = OBJECT_MAPPER.readTree(data); + log.debug("Credential helper provided auth config for: {}", hostName); + + return new AuthConfig() + .withRegistryAddress(helperResponse.at("/ServerURL").asText()) + .withUsername(helperResponse.at("/Username").asText()) + .withPassword(helperResponse.at("/Secret").asText()); + } +} diff --git a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java index bfd03d2c9be..cb87caf16d2 100644 --- a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java +++ b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java @@ -23,7 +23,7 @@ public static String[] parameters() { "gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085", "quay.io/testcontainers/ryuk:latest", "quay.io/testcontainers/ryuk:0.2.2", - "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da" + "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da", }; } diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java new file mode 100644 index 00000000000..7a79f6bd40c --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -0,0 +1,86 @@ +package org.testcontainers.utility; + +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.io.Resources; +import org.apache.commons.lang.SystemUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.net.URISyntaxException; + +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.rnorth.visibleassertions.VisibleAssertions.assertNull; + +public class RegistryAuthLocatorTest { + + @BeforeClass + public static void nonWindowsTest() throws Exception { + Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); + } + + @Test + public void lookupAuthConfigWithoutCredentials() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-empty.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("unauthenticated.registry.org/org/repo")); + + assertEquals("Default docker registry URL is set on auth config", "https://index.docker.io/v1/", authConfig.getRegistryAddress()); + assertNull("No username is set", authConfig.getUsername()); + assertNull("No password is set", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingStore() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingHelper() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupUsingHelperEmptyAuth() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-empty-auth-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupNonEmptyAuthWithHelper() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-existing-auth-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "https://registry.example.com", authConfig.getRegistryAddress()); + assertNull("No username is set", authConfig.getUsername()); + assertEquals("Correct email is obtained from a credential store", "not@val.id", authConfig.getEmail()); + assertEquals("Correct auth is obtained from a credential store", "encoded auth token", authConfig.getAuth()); + } + + @NotNull + private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException { + final File configFile = new File(Resources.getResource("auth-config/" + configName).toURI()); + return new RegistryAuthLocator(new AuthConfig(), configFile, configFile.getParentFile().getAbsolutePath() + "/"); + } + +} diff --git a/core/src/test/resources/auth-config/config-empty-auth-with-helper.json b/core/src/test/resources/auth-config/config-empty-auth-with-helper.json new file mode 100644 index 00000000000..8d8864815e3 --- /dev/null +++ b/core/src/test/resources/auth-config/config-empty-auth-with-helper.json @@ -0,0 +1,10 @@ +{ + "auths": { + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-empty.json b/core/src/test/resources/auth-config/config-empty.json new file mode 100644 index 00000000000..38a103a2459 --- /dev/null +++ b/core/src/test/resources/auth-config/config-empty.json @@ -0,0 +1,5 @@ +{ + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + } +} diff --git a/core/src/test/resources/auth-config/config-existing-auth-with-helper.json b/core/src/test/resources/auth-config/config-existing-auth-with-helper.json new file mode 100644 index 00000000000..2c877b88f90 --- /dev/null +++ b/core/src/test/resources/auth-config/config-existing-auth-with-helper.json @@ -0,0 +1,14 @@ +{ + "auths": { + "https://registry.example.com": { + "email": "not@val.id", + "auth": "encoded auth token" + } + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-helper-and-store.json b/core/src/test/resources/auth-config/config-with-helper-and-store.json new file mode 100644 index 00000000000..cd623369d78 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper-and-store.json @@ -0,0 +1,12 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake", + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-helper.json b/core/src/test/resources/auth-config/config-with-helper.json new file mode 100644 index 00000000000..eaa670e4296 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper.json @@ -0,0 +1,11 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-store.json b/core/src/test/resources/auth-config/config-with-store.json new file mode 100644 index 00000000000..d165997e52f --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-store.json @@ -0,0 +1,9 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake" +} diff --git a/core/src/test/resources/auth-config/docker-credential-fake b/core/src/test/resources/auth-config/docker-credential-fake new file mode 100755 index 00000000000..0cfa0dfb72d --- /dev/null +++ b/core/src/test/resources/auth-config/docker-credential-fake @@ -0,0 +1,13 @@ +#!/bin/bash + +if [[ $1 != "get" ]]; then + exit 1 +fi + +read > /dev/null + +echo '{' \ + ' "ServerURL": "url",' \ + ' "Username": "username",' \ + ' "Secret": "secret"' \ + '}'