Skip to content

Commit

Permalink
Run correct subset of docker compose containers when withServices/`…
Browse files Browse the repository at this point in the history
…withScaledService` used (#2922)

Co-authored-by: Afshin <mnafshin@gmail.com>
  • Loading branch information
rnorth and mnafshin committed Jun 24, 2020
1 parent 3f64d0c commit 53e02a5
Show file tree
Hide file tree
Showing 4 changed files with 184 additions and 50 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.model.Container;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Uninterruptibles;
Expand Down Expand Up @@ -56,6 +56,7 @@

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static org.testcontainers.containers.BindMode.READ_ONLY;
Expand Down Expand Up @@ -196,23 +197,36 @@ public SELF withServices(@NonNull String... services) {
}

private void createServices() {
// Apply scaling
final String servicesWithScalingSettings = Stream.concat(services.stream(), scalingPreferences.keySet().stream())
.map(service -> "--scale " + service + "=" + scalingPreferences.getOrDefault(service, 1))
// services that have been explicitly requested to be started. If empty, all services should be started.
final String serviceNameArgs = Stream.concat(
services.stream(), // services that have been specified with `withServices`
scalingPreferences.keySet().stream() // services that are implicitly needed via `withScaledService`
)
.distinct()
.collect(joining(" "));

String flags = "-d";
// Apply scaling for the services specified using `withScaledService`
final String scalingOptions = scalingPreferences.entrySet().stream()
.map(entry -> "--scale " + entry.getKey() + "=" + entry.getValue())
.distinct()
.collect(joining(" "));

String command = "up -d";

if (build) {
flags += " --build";
command += " --build";
}

// Run the docker-compose container, which starts up the services
if(Strings.isNullOrEmpty(servicesWithScalingSettings)) {
runWithCompose("up " + flags);
} else {
runWithCompose("up " + flags + " " + servicesWithScalingSettings);
if (!isNullOrEmpty(scalingOptions)) {
command += " " + scalingOptions;
}

if (!isNullOrEmpty(serviceNameArgs)) {
command += " " + serviceNameArgs;
}

// Run the docker-compose container, which starts up the services
runWithCompose(command);
}

private void waitUntilServiceStarted() {
Expand Down Expand Up @@ -250,7 +264,7 @@ private void createServiceInstance(Container container) {

private void waitUntilServiceStarted(String serviceName, ComposeServiceWaitStrategyTarget serviceInstance) {
final WaitAllStrategy waitAllStrategy = waitStrategyMap.get(serviceName);
if(waitAllStrategy != null) {
if (waitAllStrategy != null) {
waitAllStrategy.waitUntilReady(serviceInstance);
}
}
Expand All @@ -273,24 +287,25 @@ private void runWithCompose(String cmd) {
}

dockerCompose
.withCommand(cmd)
.withEnv(env)
.invoke();
.withCommand(cmd)
.withEnv(env)
.invoke();
}

private void registerContainersForShutdown() {
ResourceReaper.instance().registerFilterForCleanup(Arrays.asList(
new SimpleEntry<>("label", "com.docker.compose.project=" + project)
new SimpleEntry<>("label", "com.docker.compose.project=" + project)
));
}

private List<Container> listChildContainers() {
@VisibleForTesting
List<Container> listChildContainers() {
return dockerClient.listContainersCmd()
.withShowAll(true)
.exec().stream()
.filter(container -> Arrays.stream(container.getNames()).anyMatch(name ->
name.startsWith("/" + project)))
.collect(toList());
.withShowAll(true)
.exec().stream()
.filter(container -> Arrays.stream(container.getNames()).anyMatch(name ->
name.startsWith("/" + project)))
.collect(toList());
}

private void startAmbassadorContainers() {
Expand Down Expand Up @@ -378,12 +393,12 @@ private void addWaitStrategy(String serviceInstanceName, @NonNull WaitStrategy w
}

/**
Specify the {@link WaitStrategy} to use to determine if the container is ready.
* Specify the {@link WaitStrategy} to use to determine if the container is ready.
*
* @see org.testcontainers.containers.wait.strategy.Wait#defaultWaitStrategy()
* @param serviceName the name of the service to wait for
* @param serviceName the name of the service to wait for
* @param waitStrategy the WaitStrategy to use
* @return this
* @see org.testcontainers.containers.wait.strategy.Wait#defaultWaitStrategy()
*/
public SELF waitingFor(String serviceName, @NonNull WaitStrategy waitStrategy) {
String serviceInstanceName = getServiceInstanceName(serviceName);
Expand Down Expand Up @@ -420,8 +435,8 @@ public Integer getServicePort(String serviceName, Integer servicePort) {

if (portMap == null) {
throw new IllegalArgumentException("Could not get a port for '" + serviceName + "'. " +
"Testcontainers does not have an exposed port configured for '" + serviceName + "'. "+
"To fix, please ensure that the service '" + serviceName + "' has ports exposed using .withExposedService(...)");
"Testcontainers does not have an exposed port configured for '" + serviceName + "'. " +
"To fix, please ensure that the service '" + serviceName + "' has ports exposed using .withExposedService(...)");
} else {
return ambassadorContainer.getMappedPort(portMap.get(servicePort));
}
Expand Down Expand Up @@ -479,7 +494,7 @@ public SELF withTailChildContainers(boolean tailChildContainers) {
* More than one consumer may be registered.
*
* @param serviceName the name of the service as set in the docker-compose.yml file
* @param consumer consumer that output frames should be sent to
* @param consumer consumer that output frames should be sent to
* @return this instance, for chaining
*/
public SELF withLogConsumer(String serviceName, Consumer<OutputFrame> consumer) {
Expand Down Expand Up @@ -579,10 +594,10 @@ public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {
final String containerPwd = MountableFile.forHostPath(pwd).getFilesystemPath();

final List<String> absoluteDockerComposeFiles = composeFiles.stream()
.map(File::getAbsolutePath)
.map(MountableFile::forHostPath)
.map(MountableFile::getFilesystemPath)
.collect(toList());
.map(File::getAbsolutePath)
.map(MountableFile::forHostPath)
.map(MountableFile::getFilesystemPath)
.collect(toList());
final String composeFileEnvVariableValue = Joiner.on(UNIX_PATH_SEPERATOR).join(absoluteDockerComposeFiles); // we always need the UNIX path separator
logger().debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue);
addEnv(ENV_COMPOSE_FILE, composeFileEnvVariableValue);
Expand All @@ -600,8 +615,8 @@ public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {

private String getDockerSocketHostPath() {
return SystemUtils.IS_OS_WINDOWS
? "/" + DOCKER_SOCKET_PATH
: DOCKER_SOCKET_PATH;
? "/" + DOCKER_SOCKET_PATH
: DOCKER_SOCKET_PATH;
}

@Override
Expand All @@ -621,16 +636,16 @@ public void invoke() {
AuditLogger.doComposeLog(this.getCommandParts(), this.getEnv());

final Integer exitCode = this.dockerClient.inspectContainerCmd(getContainerId())
.exec()
.getState()
.getExitCode();
.exec()
.getState()
.getExitCode();

if (exitCode == null || exitCode != 0) {
throw new ContainerLaunchException(
"Containerised Docker Compose exited abnormally with code " +
exitCode +
" whilst running command: " +
StringUtils.join(this.getCommandParts(), ' '));
"Containerised Docker Compose exited abnormally with code " +
exitCode +
" whilst running command: " +
StringUtils.join(this.getCommandParts(), ' '));
}
}
}
Expand Down Expand Up @@ -691,23 +706,23 @@ public void invoke() {
logger().info("Local Docker Compose is running command: {}", cmd);

final List<String> command = Splitter.onPattern(" ")
.omitEmptyStrings()
.splitToList(COMPOSE_EXECUTABLE + " " + cmd);
.omitEmptyStrings()
.splitToList(COMPOSE_EXECUTABLE + " " + cmd);

try {
new ProcessExecutor().command(command)
.redirectOutput(Slf4jStream.of(logger()).asInfo())
.redirectError(Slf4jStream.of(logger()).asInfo()) // docker-compose will log pull information to stderr
.environment(environment)
.directory(pwd)
.exitValueNormal()
.executeNoTimeout();
.redirectOutput(Slf4jStream.of(logger()).asInfo())
.redirectError(Slf4jStream.of(logger()).asInfo()) // docker-compose will log pull information to stderr
.environment(environment)
.directory(pwd)
.exitValueNormal()
.executeNoTimeout();

logger().info("Docker Compose has finished running");

} catch (InvalidExitValueException e) {
throw new ContainerLaunchException("Local Docker Compose exited abnormally with code " +
e.getExitValue() + " whilst running command: " + cmd);
e.getExitValue() + " whilst running command: " + cmd);

} catch (Exception e) {
throw new ContainerLaunchException("Error running local Docker Compose command: " + cmd, e);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package org.testcontainers.containers;

import org.junit.Test;

import java.io.File;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;


public class DockerComposeContainerWithServicesTest {

public static final File SIMPLE_COMPOSE_FILE = new File("src/test/resources/compose-scaling-multiple-containers.yml");
public static final File COMPOSE_FILE_WITH_INLINE_SCALE = new File("src/test/resources/compose-with-inline-scale-test.yml");

@Test
public void testDesiredSubsetOfServicesAreStarted() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(SIMPLE_COMPOSE_FILE)
.withServices("redis")
) {
compose.start();

verifyStartedContainers(compose, "redis_1");
}
}

@Test
public void testDesiredSubsetOfScaledServicesAreStarted() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(SIMPLE_COMPOSE_FILE)
.withScaledService("redis", 2)
) {
compose.start();

verifyStartedContainers(compose, "redis_1", "redis_2");
}
}

@Test
public void testDesiredSubsetOfSpecifiedAndScaledServicesAreStarted() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(SIMPLE_COMPOSE_FILE)
.withServices("redis")
.withScaledService("redis", 2)
) {
compose.start();

verifyStartedContainers(compose, "redis_1", "redis_2");
}
}

@Test
public void testDesiredSubsetOfSpecifiedOrScaledServicesAreStarted() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(SIMPLE_COMPOSE_FILE)
.withServices("other")
.withScaledService("redis", 2)
) {
compose.start();

verifyStartedContainers(compose, "redis_1", "redis_2", "other_1");
}
}

@Test
public void testAllServicesAreStartedIfNotSpecified() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(SIMPLE_COMPOSE_FILE)
) {
compose.start();

verifyStartedContainers(compose, "redis_1", "other_1");
}
}

@Test
public void testScaleInComposeFileIsRespected() {
try (
DockerComposeContainer<?> compose = new DockerComposeContainer<>(COMPOSE_FILE_WITH_INLINE_SCALE)
) {
compose.start();

// the compose file includes `scale: 3` for the redis container
verifyStartedContainers(compose, "redis_1", "redis_2", "redis_3");
}
}

private void verifyStartedContainers(final DockerComposeContainer<?> compose, final String... names) {
final List<String> containerNames = compose.listChildContainers().stream()
.flatMap(container -> Stream.of(container.getNames()))
.collect(Collectors.toList());

assertEquals("number of running services of docker-compose is the same as length of listOfServices",
names.length, containerNames.size());

for (final String expectedName : names) {
final long matches = containerNames.stream()
.filter(foundName -> foundName.endsWith(expectedName))
.count();

assertEquals("container with name starting '" + expectedName + "' should be running", 1L, matches);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
version: '2.4'
services:
redis:
image: redis
other:
image: alpine:3.5
command: sleep 10000
5 changes: 5 additions & 0 deletions core/src/test/resources/compose-with-inline-scale-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
version: '2.4'
services:
redis:
image: redis
scale: 3 # legacy mechanism to specify scale

0 comments on commit 53e02a5

Please sign in to comment.