Skip to content

Commit

Permalink
Add extra debugging logs to help diagnose any future docker daemon co…
Browse files Browse the repository at this point in the history 
…mpatibility issues

Refactor Docker compose support, to allow scaling, better output logs, and eventually docker-compose v2 format. Refs #146, #147

Start support for docker-compose v2 environments, putting ambassador containers onto the right network to be able to access the compose-launched containers.

Fix Netty conflicts by using Jedis for testing (simpler dependencies)
  • Loading branch information
@rnorth
rnorth committed Jul 3, 2016
1 parent 68fb35f commit 3478853
Show file tree
Hide file tree
Showing 17 changed files with 429 additions and 116 deletions.
12 changes: 6 additions & 6 deletions core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@
</dependency>

<!-- Test dependencies -->
<!--<dependency>-->
<!--<groupId>org.redisson</groupId>-->
<!--<artifactId>redisson</artifactId>-->
<!--<version>1.3.0</version>-->
<!--<scope>test</scope>-->
<!--</dependency>-->
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/java/org/testcontainers/DockerClientFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.Image;
import com.github.dockerjava.api.model.Info;
import com.github.dockerjava.core.DockerClientBuilder;
import com.github.dockerjava.core.DockerClientConfig;
import com.github.dockerjava.core.command.LogContainerResultCallback;
Expand Down Expand Up @@ -85,6 +86,15 @@ public DockerClient client(boolean failFast) {
DockerClient client = DockerClientBuilder.getInstance(config).withDockerCmdExecFactory(nettyExecFactory).build();

if (!preconditionsChecked) {
Info dockerInfo = client.infoCmd().exec();
LOGGER.info("Connected to docker: \n" +
" Server Version: " + dockerInfo.getServerVersion() + "\n" +
" Operating System: " + dockerInfo.getOperatingSystem() + "\n" +
" Total Memory: " + dockerInfo.getMemTotal() + "\n" +
" HTTP Proxy: " + dockerInfo.getHttpProxy() + "\n" +
" HTTPS Proxy: " + dockerInfo.getHttpsProxy()
);

String version = client.versionCmd().exec().getVersion();
checkVersion(version);
checkDiskSpaceAndHandleExceptions(client);
Expand Down
4 changes: 0 additions & 4 deletions core/src/main/java/org/testcontainers/containers/Container.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,6 @@ ExecResult execInContainer(Charset outputCharset, String... command)

Map<String, LinkableContainer> getLinkedContainers();

Duration getMinimumRunningDuration();

DockerClient getDockerClient();

Info getDockerDaemonInfo();
Expand All @@ -339,8 +337,6 @@ ExecResult execInContainer(Charset outputCharset, String... command)

void setLinkedContainers(Map<String, LinkableContainer> linkedContainers);

void setMinimumRunningDuration(Duration minimumRunningDuration);

void setDockerClient(DockerClient dockerClient);

void setDockerDaemonInfo(Info dockerDaemonInfo);
Expand Down
161 changes: 117 additions & 44 deletions core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,21 @@
package org.testcontainers.containers;

import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.exception.DockerException;
import com.github.dockerjava.api.model.Container;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.runner.Description;
import org.rnorth.ducttape.ratelimits.RateLimiter;
import org.rnorth.ducttape.ratelimits.RateLimiterBuilder;
import org.rnorth.ducttape.unreliables.Unreliables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.profiler.Profiler;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.output.OutputFrame;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.traits.LinkableContainer;
import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.ContainerReaper;

Expand All @@ -25,62 +33,78 @@
/**
* Container which launches Docker Compose, for the purposes of launching a defined set of containers.
*/
public class DockerComposeContainer<SELF extends DockerComposeContainer<SELF>> extends GenericContainer<SELF> implements LinkableContainer {
public class DockerComposeContainer<SELF extends DockerComposeContainer<SELF>> extends FailureDetectingExternalResource {

/**
* Random identifier which will become part of spawned containers names, so we can shut them down
*/
private final String identifier;
private final Map<String, AmbassadorContainer> ambassadorContainers = new HashMap<>();
private final File composeFile;
private Set<String> spawnedContainerIds;
private Map<String, Integer> scalingPreferences = new HashMap<>();
private DockerClient dockerClient;

private static final RateLimiter AMBASSADOR_CREATION_RATE_LIMITER = RateLimiterBuilder
.newBuilder()
.withRate(6, TimeUnit.MINUTES)
.withConstantThroughput()
.build();

public DockerComposeContainer(File composeFile) {
this(composeFile, "up -d");
}

@SuppressWarnings("WeakerAccess")
public DockerComposeContainer(File composeFile, String command) {
super("dduportal/docker-compose:1.6.0");

// Create a unique identifier and tell compose
identifier = Base58.randomString(6).toLowerCase();
addEnv("COMPOSE_PROJECT_NAME", identifier);
this(composeFile, command, Base58.randomString(6).toLowerCase());
}

// Map the docker compose file into the container
addEnv("COMPOSE_FILE", "/compose/" + composeFile.getAbsoluteFile().getName());
addFileSystemBind(composeFile.getAbsoluteFile().getParentFile().getAbsolutePath(), "/compose", READ_ONLY);
@SuppressWarnings("WeakerAccess")
public DockerComposeContainer(File composeFile, String command, String identifier) {
this.composeFile = composeFile;

// Ensure that compose can access docker. Since the container is assumed to be running on the same machine
// as the docker daemon, just mapping the docker control socket is OK.
// As there seems to be a problem with mapping to the /var/run directory in certain environments (e.g. CircleCI)
// we map the socket file outside of /var/run, as just /docker.sock
addFileSystemBind("/var/run/docker.sock", "/docker.sock", READ_WRITE);
addEnv("DOCKER_HOST", "unix:///docker.sock");
// Use a unique identifier so that containers created for this compose environment can be identified
this.identifier = identifier;

if (command != null) {
setCommand(command);
}
this.dockerClient = DockerClientFactory.instance().client();
}

@Override
public void start() {

protected void starting(Description description) {
final Profiler profiler = new Profiler("Docker compose container rule");
profiler.setLogger(logger());
profiler.start("Docker compose container startup");

applyScaling(); // scale before up, so that all scaled instances are available first for linking
createServices();
registerContainersForShutdown();
startAmbassadorContainers(profiler);

}


private void createServices() {
// Start the docker-compose container, which starts up the services
super.start();
followOutput(new Slf4jLogConsumer(logger()), OutputFrame.OutputType.STDERR);
new DockerCompose(composeFile, identifier)
.withCommand("up -d")
.start();
}

// wait for the compose container to stop, which should only happen after it has spawned all the service containers
logger().info("Docker compose container is running - service creation will start now");
while (isRunning()) {
logger().trace("Compose container is still running");
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
private void applyScaling() {
// Apply scaling
if (!scalingPreferences.isEmpty()) {
StringBuffer sb = new StringBuffer("scale");
for (Map.Entry<String, Integer> scale : scalingPreferences.entrySet()) {
sb.append(" ").append(scale.getKey()).append("=").append(scale.getValue());
}

new DockerCompose(composeFile, identifier)
.withCommand(sb.toString())
.start();
}
logger().info("Docker compose has finished running");
}

private void registerContainersForShutdown() {
// Ensure that all service containers that were launched by compose will be killed at shutdown
try {
List<Container> containers = dockerClient.listContainersCmd()
Expand All @@ -101,7 +125,9 @@ public void start() {
} catch (DockerException e) {
logger().debug("Failed to stop a service container with exception", e);
}
}

private void startAmbassadorContainers(Profiler profiler) {
for (final Map.Entry<String, AmbassadorContainer> address : ambassadorContainers.entrySet()) {

try {
Expand All @@ -110,11 +136,14 @@ public void start() {

final AmbassadorContainer ambassadorContainer = address.getValue();
Unreliables.retryUntilSuccess(120, TimeUnit.SECONDS, () -> {
Profiler localProfiler = profiler.startNested("Ambassador container: " + ambassadorContainer.getContainerName());

localProfiler.start("Start ambassador container");
AMBASSADOR_CREATION_RATE_LIMITER.doWhenReady(() -> {
Profiler localProfiler = profiler.startNested("Ambassador container: " + ambassadorContainer.getContainerName());

ambassadorContainer.start();
localProfiler.start("Start ambassador container");

ambassadorContainer.start();
});

return null;
});
Expand All @@ -126,10 +155,21 @@ public void start() {
}
}

private Logger logger() {
return LoggerFactory.getLogger(DockerComposeContainer.class);
}

@Override
public void stop() {
// this, the compose container, should not be running, but just in case something has gone wrong
super.stop();
protected void finished(Description description) {


// Kill the services using docker-compose
new DockerCompose(composeFile, identifier)
.withCommand("kill")
.start();
new DockerCompose(composeFile, identifier)
.withCommand("rm -f -v")
.start();

// shut down all the ambassador containers
ambassadorContainers.forEach((String address, AmbassadorContainer container) -> container.stop());
Expand All @@ -139,12 +179,6 @@ public void stop() {
spawnedContainerIds.clear();
}

@Override
@Deprecated
public SELF withExposedPorts(Integer... ports) {
throw new UnsupportedOperationException("Use withExposedService instead");
}

public SELF withExposedService(String serviceName, int servicePort) {

/**
Expand All @@ -157,12 +191,12 @@ public SELF withExposedService(String serviceName, int servicePort) {
* This avoids the need for the docker compose file to explicitly expose ports on all the
* services.
*/
AmbassadorContainer ambassadorContainer = new AmbassadorContainer(new FutureContainer(this.identifier + "_" + serviceName), serviceName, servicePort);
AmbassadorContainer ambassadorContainer = new AmbassadorContainer<>(new FutureContainer(this.identifier + "_" + serviceName), serviceName, servicePort);

// Ambassador containers will all be started together after docker compose has started
ambassadorContainers.put(serviceName + ":" + servicePort, ambassadorContainer);

return self();
return (SELF) this;
}

/**
Expand Down Expand Up @@ -192,4 +226,43 @@ public String getServiceHost(String serviceName, Integer servicePort) {
public Integer getServicePort(String serviceName, Integer servicePort) {
return ambassadorContainers.get(serviceName + ":" + servicePort).getMappedPort(servicePort);
}

public SELF withScaledService(String serviceBaseName, int numInstances) {
scalingPreferences.put(serviceBaseName, numInstances);

return (SELF) this;
}
}

class DockerCompose extends GenericContainer<DockerCompose> {
public DockerCompose(File composeFile, String identifier) {

super("dduportal/docker-compose:1.6.0");
addEnv("COMPOSE_PROJECT_NAME", identifier);
// Map the docker compose file into the container
addEnv("COMPOSE_FILE", "/compose/" + composeFile.getAbsoluteFile().getName());
addFileSystemBind(composeFile.getAbsoluteFile().getParentFile().getAbsolutePath(), "/compose", READ_ONLY);
// Ensure that compose can access docker. Since the container is assumed to be running on the same machine
// as the docker daemon, just mapping the docker control socket is OK.
// As there seems to be a problem with mapping to the /var/run directory in certain environments (e.g. CircleCI)
// we map the socket file outside of /var/run, as just /docker.sock
addFileSystemBind("/var/run/docker.sock", "/docker.sock", READ_WRITE);
addEnv("DOCKER_HOST", "unix:///docker.sock");
setStartupCheckStrategy(new OneShotStartupCheckStrategy());
}

@Override
public void start() {
super.start();

this.followOutput(new Slf4jLogConsumer(logger()), OutputFrame.OutputType.STDERR);

// wait for the compose container to stop, which should only happen after it has spawned all the service containers
logger().info("Docker compose container is running for command: {}", Joiner.on(" ").join(this.getCommandParts()));
while (this.isRunning()) {
logger().trace("Compose container is still running");
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
}
logger().info("Docker compose has finished running");
}
}

0 comments on commit 3478853

Please sign in to comment.