Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Resolve excessive JDBC container creation when using pools and driverClassName #195

Merged
merged 2 commits into from
Aug 16, 2016
Merged

Resolve excessive JDBC container creation when using pools and driverClassName #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
32d252c
Add a basic fix for #159
rnorth Aug 2, 2016
f1bc5c6
Add convenience methods to clean up database containers. Refs #159.
rnorth Aug 7, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
206 changes: 121 additions & 85 deletions modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,16 @@

/**
* Test Containers JDBC proxy driver. This driver will handle JDBC URLs of the form:
*
* <p>
* <code>jdbc:tc:<i>type</i>://<i>host</i>:<i>port</i>/<i>database</i>?<i>querystring</i></code>
*
* <p>
* where <i>type</i> is a supported database type (e.g. mysql, postgresql, oracle). Behind the scenes a new
* docker container will be launched running the required database engine. New JDBC connections will be created
* using the database's standard driver implementation, connected to the container.
*
* <p>
* If <code>TC_INITSCRIPT</code> is set in <i>querystring</i>, it will be used as the path for an init script that
* should be run to initialize the database after the container is created. This should be a classpath resource.
*
* <p>
* Similarly <code>TC_INITFUNCTION</code> may be a method reference for a function that can initialize the database.
* Such a function must accept a javax.sql.Connection as its only parameter.
* An example of a valid method reference would be <code>com.myapp.SomeClass::initFunction</code>
Expand All @@ -48,9 +48,9 @@ public class ContainerDatabaseDriver implements Driver {
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(ContainerDatabaseDriver.class);

private Driver delegate;
private final Map<JdbcDatabaseContainer, Set<Connection>> containerConnections = new HashMap<>();
private final Map<String, JdbcDatabaseContainer> jdbcUrlContainerCache = new HashMap<>();
private final Set<JdbcDatabaseContainer> initializedContainers = new HashSet<>();
private static final Map<String, Set<Connection>> containerConnections = new HashMap<>();
private static final Map<String, JdbcDatabaseContainer> jdbcUrlContainerCache = new HashMap<>();
private static final Set<String> initializedContainers = new HashSet<>();

static {
load();
Expand All @@ -72,90 +72,93 @@ public boolean acceptsURL(String url) throws SQLException {
@Override
public synchronized Connection connect(String url, final Properties info) throws SQLException {

/*
The driver should return "null" if it realizes it is the wrong kind of driver to connect to the given URL.
*/
if(!acceptsURL(url)) {
return null;
}

String queryString = "";
/*
If we already have a running container for this exact connection string, we want to connect
to that rather than create a new container
The driver should return "null" if it realizes it is the wrong kind of driver to connect to the given URL.
*/
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(url);
if (container == null) {
/*
Extract from the JDBC connection URL:
* The database type (e.g. mysql, postgresql, ...)
* The docker tag, if provided.
* The URL query string, if provided
*/
Matcher urlMatcher = URL_MATCHING_PATTERN.matcher(url);
if (!urlMatcher.matches()) {
throw new IllegalArgumentException("JDBC URL matches jdbc:tc: prefix but the database or tag name could not be identified");
}
String databaseType = urlMatcher.group(1);
String tag = urlMatcher.group(3);
if (tag == null) {
tag = "latest";
}

queryString = urlMatcher.group(4);
if (queryString == null) {
queryString = "";
}
if (!acceptsURL(url)) {
return null;
}

Map<String, String> parameters = getContainerParameters(url);
synchronized (jdbcUrlContainerCache) {

String queryString = "";
/*
Find a matching container type using ServiceLoader.
If we already have a running container for this exact connection string, we want to connect
to that rather than create a new container
*/
ServiceLoader<JdbcDatabaseContainerProvider> databaseContainers = ServiceLoader.load(JdbcDatabaseContainerProvider.class);
for (JdbcDatabaseContainerProvider candidateContainerType : databaseContainers) {
if (candidateContainerType.supports(databaseType)) {
container = candidateContainerType.newInstance(tag);
delegate = container.getJdbcDriverInstance();
}
}
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(url);
if (container == null) {
throw new UnsupportedOperationException("Database name " + databaseType + " not supported");
}
/*
Extract from the JDBC connection URL:
* The database type (e.g. mysql, postgresql, ...)
* The docker tag, if provided.
* The URL query string, if provided
*/
Matcher urlMatcher = URL_MATCHING_PATTERN.matcher(url);
if (!urlMatcher.matches()) {
throw new IllegalArgumentException("JDBC URL matches jdbc:tc: prefix but the database or tag name could not be identified");
}
String databaseType = urlMatcher.group(1);
String tag = urlMatcher.group(3);
if (tag == null) {
tag = "latest";
}

/*
Cache the container before starting to prevent race conditions when a connection
pool is started up
*/
jdbcUrlContainerCache.put(url, container);
queryString = urlMatcher.group(4);
if (queryString == null) {
queryString = "";
}

Map<String, String> parameters = getContainerParameters(url);

/*
Find a matching container type using ServiceLoader.
*/
ServiceLoader<JdbcDatabaseContainerProvider> databaseContainers = ServiceLoader.load(JdbcDatabaseContainerProvider.class);
for (JdbcDatabaseContainerProvider candidateContainerType : databaseContainers) {
if (candidateContainerType.supports(databaseType)) {
container = candidateContainerType.newInstance(tag);
delegate = container.getJdbcDriverInstance();
}
}
if (container == null) {
throw new UnsupportedOperationException("Database name " + databaseType + " not supported");
}

/*
Cache the container before starting to prevent race conditions when a connection
pool is started up
*/
jdbcUrlContainerCache.put(url, container);

/*
Pass possible container-specific parameters
*/
container.setParameters(parameters);

/*
Start the container
*/
container.start();
}

/*
Pass possible container-specific parameters
Create a connection using the delegated driver. The container must be ready to accept connections.
*/
container.setParameters(parameters);
Connection connection = container.createConnection(queryString);

/*
Start the container
If this container has not been initialized, AND
an init script or function has been specified, use it
*/
container.start();
}

/*
Create a connection using the delegated driver. The container must be ready to accept connections.
*/
Connection connection = container.createConnection(queryString);
if (!initializedContainers.contains(container.getContainerId())) {
runInitScriptIfRequired(url, connection);
runInitFunctionIfRequired(url, connection);
initializedContainers.add(container.getContainerId());
}

/*
If this container has not been initialized, AND
an init script or function has been specified, use it
*/
if (!initializedContainers.contains(container)) {
runInitScriptIfRequired(url, connection);
runInitFunctionIfRequired(url, connection);
initializedContainers.add(container);
return wrapConnection(connection, container, url);
}

return wrapConnection(connection, container, url);
}

private Map<String, String> getContainerParameters(String url) {
Expand All @@ -174,20 +177,20 @@ private Map<String, String> getContainerParameters(String url) {

/**
* Wrap the connection, setting up a callback to be called when the connection is closed.
*
* <p>
* When there are no more open connections, the container itself will be stopped.
*
* @param connection the new connection to be wrapped
* @param container the container which the connection is associated with
* @param url the testcontainers JDBC URL for this connection
* @return the connection, wrapped
* @param connection the new connection to be wrapped
* @param container the container which the connection is associated with
* @param url the testcontainers JDBC URL for this connection
* @return the connection, wrapped
*/
private Connection wrapConnection(final Connection connection, final JdbcDatabaseContainer container, final String url) {
Set<Connection> connections = containerConnections.get(container);
Set<Connection> connections = containerConnections.get(container.getContainerId());

if(connections == null) {
if (connections == null) {
connections = new HashSet<>();
containerConnections.put(container, connections);
containerConnections.put(container.getContainerId(), connections);
}

connections.add(connection);
Expand All @@ -206,7 +209,7 @@ private Connection wrapConnection(final Connection connection, final JdbcDatabas
/**
* Run an init script from the classpath.
*
* @param url the JDBC URL to check for init script declarations.
* @param url the JDBC URL to check for init script declarations.
* @param connection JDBC connection to apply init scripts to.
* @throws SQLException on script or DB error
*/
Expand All @@ -231,7 +234,7 @@ private void runInitScriptIfRequired(String url, Connection connection) throws S
/**
* Run an init function (must be a public static method on an accessible class).
*
* @param url the JDBC URL to check for init function declarations.
* @param url the JDBC URL to check for init function declarations.
* @param connection JDBC connection to apply init functions to.
* @throws SQLException on script or DB error
*/
Expand Down Expand Up @@ -277,4 +280,37 @@ public boolean jdbcCompliant() {
public Logger getParentLogger() throws SQLFeatureNotSupportedException {
return delegate.getParentLogger();
}

/**
* Utility method to kill ALL database containers directly from test support code. It shouldn't be necessary to use this,
* but it is provided for convenience - e.g. for situations where many different database containers are being
* tested and cleanup is needed to limit resource usage.
*/
public static void killContainers() {
synchronized (jdbcUrlContainerCache) {
jdbcUrlContainerCache.values().forEach(JdbcDatabaseContainer::stop);
jdbcUrlContainerCache.clear();
containerConnections.clear();
initializedContainers.clear();
}

}

/**
* Utility method to kill a database container directly from test support code. It shouldn't be necessary to use this,
* but it is provided for convenience - e.g. for situations where many different database containers are being
* tested and cleanup is needed to limit resource usage.
* @param jdbcUrl the JDBC URL of the container which should be killed
*/
public static void killContainer(String jdbcUrl) {
synchronized (jdbcUrlContainerCache) {
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(jdbcUrl);
if (container != null) {
container.stop();
jdbcUrlContainerCache.remove(jdbcUrl);
containerConnections.remove(container.getContainerId());
initializedContainers.remove(container.getContainerId());
}
}
}
}
10 changes: 5 additions & 5 deletions modules/jdbc/src/test/java/org/testcontainers/jdbc/ContainerDatabaseDriverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
package org.testcontainers.jdbc;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;

import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;

public class ContainerDatabaseDriverTest {

private static final String PLAIN_POSTGRESQL_JDBC_URL = "jdbc:postgresql://localhost:5432/test";
Expand Down
10 changes: 10 additions & 0 deletions modules/mysql/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,15 @@
<version>1.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jdbc</artifactId>
<version>8.5.4</version>
</dependency>
<dependency>
<groupId>org.vibur</groupId>
<artifactId>vibur-dbcp</artifactId>
<version>9.0</version>
</dependency>
</dependencies>
</project>
43 changes: 8 additions & 35 deletions modules/mysql/src/test/java/org/testcontainers/jdbc/JDBCDriverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,27 @@
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
import org.apache.commons.lang.SystemUtils;
import org.junit.After;
import org.junit.Test;

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;
import static org.junit.Assume.assumeFalse;
import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;

/**
*
*/
public class JDBCDriverTest {

@After
public void testCleanup() {
ContainerDatabaseDriver.killContainers();
}

@Test
public void testMySQLWithVersion() throws SQLException {
performSimpleTest("jdbc:tc:mysql:5.5.43://hostname/databasename");
Expand All @@ -32,7 +38,7 @@ public void testMySQLWithNoSpecifiedVersion() throws SQLException {

@Test
public void testMySQLWithCustomIniFile() throws SQLException {
assumeFalse(SystemUtils.IS_OS_WINDOWS);
assumeFalse(SystemUtils.IS_OS_WINDOWS);
HikariDataSource ds = getDataSource("jdbc:tc:mysql:5.6://hostname/databasename?TC_MY_CNF=somepath/mysql_conf_override", 1);
Statement statement = ds.getConnection().createStatement();
statement.execute("SELECT @@GLOBAL.innodb_file_format");
Expand All @@ -58,39 +64,6 @@ public void testMySQLWithClasspathInitFunction() throws SQLException {
performTestForScriptedSchema("jdbc:tc:mysql://hostname/databasename?TC_INITFUNCTION=org.testcontainers.jdbc.JDBCDriverTest::sampleInitFunction");
}

@Test
public void testMySQLWithConnectionPoolUsingSameContainer() throws SQLException {
HikariDataSource dataSource = getDataSource("jdbc:tc:mysql://hostname/databasename?TC_INITFUNCTION=org.testcontainers.jdbc.JDBCDriverTest::sampleInitFunction", 10);
for (int i = 0; i < 100; i++) {
new QueryRunner(dataSource).insert("INSERT INTO my_counter (n) VALUES (5)", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
return true;
}
});
}

new QueryRunner(dataSource).query("SELECT COUNT(1) FROM my_counter", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
rs.next();
int resultSetInt = rs.getInt(1);
assertEquals("Reuse of a datasource points to the same DB container", 100, resultSetInt);
return true;
}
});

new QueryRunner(dataSource).query("SELECT SUM(n) FROM my_counter", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
rs.next();
int resultSetInt = rs.getInt(1);
assertEquals("Reuse of a datasource points to the same DB container", 500, resultSetInt);
return true;
}
});
}

@Test
public void testMySQLWithQueryParams() throws SQLException {
performSimpleTestWithCharacterSet("jdbc:tc:mysql://hostname/databasename?useUnicode=yes&characterEncoding=utf8");
Expand Down