Skip to content

Commit

Permalink
Merge pull request #195 from testcontainers/fix-jdbc-driver-initializ…
Browse files Browse the repository at this point in the history 
…ation

Resolve excessive JDBC container creation when using pools and driverClassName
  • Loading branch information
@rnorth
rnorth authored Aug 16, 2016
2 parents ade0d9b + f1bc5c6 commit 0134e9b
Show file tree
Hide file tree
Showing 5 changed files with 309 additions and 125 deletions.
206 changes: 121 additions & 85 deletions modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,16 @@

/**
* Test Containers JDBC proxy driver. This driver will handle JDBC URLs of the form:
*
* <p>
* <code>jdbc:tc:<i>type</i>://<i>host</i>:<i>port</i>/<i>database</i>?<i>querystring</i></code>
*
* <p>
* where <i>type</i> is a supported database type (e.g. mysql, postgresql, oracle). Behind the scenes a new
* docker container will be launched running the required database engine. New JDBC connections will be created
* using the database's standard driver implementation, connected to the container.
*
* <p>
* If <code>TC_INITSCRIPT</code> is set in <i>querystring</i>, it will be used as the path for an init script that
* should be run to initialize the database after the container is created. This should be a classpath resource.
*
* <p>
* Similarly <code>TC_INITFUNCTION</code> may be a method reference for a function that can initialize the database.
* Such a function must accept a javax.sql.Connection as its only parameter.
* An example of a valid method reference would be <code>com.myapp.SomeClass::initFunction</code>
Expand All @@ -48,9 +48,9 @@ public class ContainerDatabaseDriver implements Driver {
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(ContainerDatabaseDriver.class);

private Driver delegate;
private final Map<JdbcDatabaseContainer, Set<Connection>> containerConnections = new HashMap<>();
private final Map<String, JdbcDatabaseContainer> jdbcUrlContainerCache = new HashMap<>();
private final Set<JdbcDatabaseContainer> initializedContainers = new HashSet<>();
private static final Map<String, Set<Connection>> containerConnections = new HashMap<>();
private static final Map<String, JdbcDatabaseContainer> jdbcUrlContainerCache = new HashMap<>();
private static final Set<String> initializedContainers = new HashSet<>();

static {
load();
Expand All @@ -72,90 +72,93 @@ public boolean acceptsURL(String url) throws SQLException {
@Override
public synchronized Connection connect(String url, final Properties info) throws SQLException {

/*
The driver should return "null" if it realizes it is the wrong kind of driver to connect to the given URL.
*/
if(!acceptsURL(url)) {
return null;
}

String queryString = "";
/*
If we already have a running container for this exact connection string, we want to connect
to that rather than create a new container
The driver should return "null" if it realizes it is the wrong kind of driver to connect to the given URL.
*/
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(url);
if (container == null) {
/*
Extract from the JDBC connection URL:
* The database type (e.g. mysql, postgresql, ...)
* The docker tag, if provided.
* The URL query string, if provided
*/
Matcher urlMatcher = URL_MATCHING_PATTERN.matcher(url);
if (!urlMatcher.matches()) {
throw new IllegalArgumentException("JDBC URL matches jdbc:tc: prefix but the database or tag name could not be identified");
}
String databaseType = urlMatcher.group(1);
String tag = urlMatcher.group(3);
if (tag == null) {
tag = "latest";
}

queryString = urlMatcher.group(4);
if (queryString == null) {
queryString = "";
}
if (!acceptsURL(url)) {
return null;
}

Map<String, String> parameters = getContainerParameters(url);
synchronized (jdbcUrlContainerCache) {

String queryString = "";
/*
Find a matching container type using ServiceLoader.
If we already have a running container for this exact connection string, we want to connect
to that rather than create a new container
*/
ServiceLoader<JdbcDatabaseContainerProvider> databaseContainers = ServiceLoader.load(JdbcDatabaseContainerProvider.class);
for (JdbcDatabaseContainerProvider candidateContainerType : databaseContainers) {
if (candidateContainerType.supports(databaseType)) {
container = candidateContainerType.newInstance(tag);
delegate = container.getJdbcDriverInstance();
}
}
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(url);
if (container == null) {
throw new UnsupportedOperationException("Database name " + databaseType + " not supported");
}
/*
Extract from the JDBC connection URL:
* The database type (e.g. mysql, postgresql, ...)
* The docker tag, if provided.
* The URL query string, if provided
*/
Matcher urlMatcher = URL_MATCHING_PATTERN.matcher(url);
if (!urlMatcher.matches()) {
throw new IllegalArgumentException("JDBC URL matches jdbc:tc: prefix but the database or tag name could not be identified");
}
String databaseType = urlMatcher.group(1);
String tag = urlMatcher.group(3);
if (tag == null) {
tag = "latest";
}

/*
Cache the container before starting to prevent race conditions when a connection
pool is started up
*/
jdbcUrlContainerCache.put(url, container);
queryString = urlMatcher.group(4);
if (queryString == null) {
queryString = "";
}

Map<String, String> parameters = getContainerParameters(url);

/*
Find a matching container type using ServiceLoader.
*/
ServiceLoader<JdbcDatabaseContainerProvider> databaseContainers = ServiceLoader.load(JdbcDatabaseContainerProvider.class);
for (JdbcDatabaseContainerProvider candidateContainerType : databaseContainers) {
if (candidateContainerType.supports(databaseType)) {
container = candidateContainerType.newInstance(tag);
delegate = container.getJdbcDriverInstance();
}
}
if (container == null) {
throw new UnsupportedOperationException("Database name " + databaseType + " not supported");
}

/*
Cache the container before starting to prevent race conditions when a connection
pool is started up
*/
jdbcUrlContainerCache.put(url, container);

/*
Pass possible container-specific parameters
*/
container.setParameters(parameters);

/*
Start the container
*/
container.start();
}

/*
Pass possible container-specific parameters
Create a connection using the delegated driver. The container must be ready to accept connections.
*/
container.setParameters(parameters);
Connection connection = container.createConnection(queryString);

/*
Start the container
If this container has not been initialized, AND
an init script or function has been specified, use it
*/
container.start();
}

/*
Create a connection using the delegated driver. The container must be ready to accept connections.
*/
Connection connection = container.createConnection(queryString);
if (!initializedContainers.contains(container.getContainerId())) {
runInitScriptIfRequired(url, connection);
runInitFunctionIfRequired(url, connection);
initializedContainers.add(container.getContainerId());
}

/*
If this container has not been initialized, AND
an init script or function has been specified, use it
*/
if (!initializedContainers.contains(container)) {
runInitScriptIfRequired(url, connection);
runInitFunctionIfRequired(url, connection);
initializedContainers.add(container);
return wrapConnection(connection, container, url);
}

return wrapConnection(connection, container, url);
}

private Map<String, String> getContainerParameters(String url) {
Expand All @@ -174,20 +177,20 @@ private Map<String, String> getContainerParameters(String url) {

/**
* Wrap the connection, setting up a callback to be called when the connection is closed.
*
* <p>
* When there are no more open connections, the container itself will be stopped.
*
* @param connection the new connection to be wrapped
* @param container the container which the connection is associated with
* @param url the testcontainers JDBC URL for this connection
* @return the connection, wrapped
* @param connection the new connection to be wrapped
* @param container the container which the connection is associated with
* @param url the testcontainers JDBC URL for this connection
* @return the connection, wrapped
*/
private Connection wrapConnection(final Connection connection, final JdbcDatabaseContainer container, final String url) {
Set<Connection> connections = containerConnections.get(container);
Set<Connection> connections = containerConnections.get(container.getContainerId());

if(connections == null) {
if (connections == null) {
connections = new HashSet<>();
containerConnections.put(container, connections);
containerConnections.put(container.getContainerId(), connections);
}

connections.add(connection);
Expand All @@ -206,7 +209,7 @@ private Connection wrapConnection(final Connection connection, final JdbcDatabas
/**
* Run an init script from the classpath.
*
* @param url the JDBC URL to check for init script declarations.
* @param url the JDBC URL to check for init script declarations.
* @param connection JDBC connection to apply init scripts to.
* @throws SQLException on script or DB error
*/
Expand All @@ -231,7 +234,7 @@ private void runInitScriptIfRequired(String url, Connection connection) throws S
/**
* Run an init function (must be a public static method on an accessible class).
*
* @param url the JDBC URL to check for init function declarations.
* @param url the JDBC URL to check for init function declarations.
* @param connection JDBC connection to apply init functions to.
* @throws SQLException on script or DB error
*/
Expand Down Expand Up @@ -277,4 +280,37 @@ public boolean jdbcCompliant() {
public Logger getParentLogger() throws SQLFeatureNotSupportedException {
return delegate.getParentLogger();
}

/**
* Utility method to kill ALL database containers directly from test support code. It shouldn't be necessary to use this,
* but it is provided for convenience - e.g. for situations where many different database containers are being
* tested and cleanup is needed to limit resource usage.
*/
public static void killContainers() {
synchronized (jdbcUrlContainerCache) {
jdbcUrlContainerCache.values().forEach(JdbcDatabaseContainer::stop);
jdbcUrlContainerCache.clear();
containerConnections.clear();
initializedContainers.clear();
}

}

/**
* Utility method to kill a database container directly from test support code. It shouldn't be necessary to use this,
* but it is provided for convenience - e.g. for situations where many different database containers are being
* tested and cleanup is needed to limit resource usage.
* @param jdbcUrl the JDBC URL of the container which should be killed
*/
public static void killContainer(String jdbcUrl) {
synchronized (jdbcUrlContainerCache) {
JdbcDatabaseContainer container = jdbcUrlContainerCache.get(jdbcUrl);
if (container != null) {
container.stop();
jdbcUrlContainerCache.remove(jdbcUrl);
containerConnections.remove(container.getContainerId());
initializedContainers.remove(container.getContainerId());
}
}
}
}
10 changes: 5 additions & 5 deletions modules/jdbc/src/test/java/org/testcontainers/jdbc/ContainerDatabaseDriverTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
package org.testcontainers.jdbc;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;

import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;

public class ContainerDatabaseDriverTest {

private static final String PLAIN_POSTGRESQL_JDBC_URL = "jdbc:postgresql://localhost:5432/test";
Expand Down
10 changes: 10 additions & 0 deletions modules/mysql/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,15 @@
<version>1.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jdbc</artifactId>
<version>8.5.4</version>
</dependency>
<dependency>
<groupId>org.vibur</groupId>
<artifactId>vibur-dbcp</artifactId>
<version>9.0</version>
</dependency>
</dependencies>
</project>
43 changes: 8 additions & 35 deletions modules/mysql/src/test/java/org/testcontainers/jdbc/JDBCDriverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,27 @@
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
import org.apache.commons.lang.SystemUtils;
import org.junit.After;
import org.junit.Test;

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;
import static org.junit.Assume.assumeFalse;
import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;

/**
*
*/
public class JDBCDriverTest {

@After
public void testCleanup() {
ContainerDatabaseDriver.killContainers();
}

@Test
public void testMySQLWithVersion() throws SQLException {
performSimpleTest("jdbc:tc:mysql:5.5.43://hostname/databasename");
Expand All @@ -32,7 +38,7 @@ public void testMySQLWithNoSpecifiedVersion() throws SQLException {

@Test
public void testMySQLWithCustomIniFile() throws SQLException {
assumeFalse(SystemUtils.IS_OS_WINDOWS);
assumeFalse(SystemUtils.IS_OS_WINDOWS);
HikariDataSource ds = getDataSource("jdbc:tc:mysql:5.6://hostname/databasename?TC_MY_CNF=somepath/mysql_conf_override", 1);
Statement statement = ds.getConnection().createStatement();
statement.execute("SELECT @@GLOBAL.innodb_file_format");
Expand All @@ -58,39 +64,6 @@ public void testMySQLWithClasspathInitFunction() throws SQLException {
performTestForScriptedSchema("jdbc:tc:mysql://hostname/databasename?TC_INITFUNCTION=org.testcontainers.jdbc.JDBCDriverTest::sampleInitFunction");
}

@Test
public void testMySQLWithConnectionPoolUsingSameContainer() throws SQLException {
HikariDataSource dataSource = getDataSource("jdbc:tc:mysql://hostname/databasename?TC_INITFUNCTION=org.testcontainers.jdbc.JDBCDriverTest::sampleInitFunction", 10);
for (int i = 0; i < 100; i++) {
new QueryRunner(dataSource).insert("INSERT INTO my_counter (n) VALUES (5)", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
return true;
}
});
}

new QueryRunner(dataSource).query("SELECT COUNT(1) FROM my_counter", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
rs.next();
int resultSetInt = rs.getInt(1);
assertEquals("Reuse of a datasource points to the same DB container", 100, resultSetInt);
return true;
}
});

new QueryRunner(dataSource).query("SELECT SUM(n) FROM my_counter", new ResultSetHandler<Object>() {
@Override
public Object handle(ResultSet rs) throws SQLException {
rs.next();
int resultSetInt = rs.getInt(1);
assertEquals("Reuse of a datasource points to the same DB container", 500, resultSetInt);
return true;
}
});
}

@Test
public void testMySQLWithQueryParams() throws SQLException {
performSimpleTestWithCharacterSet("jdbc:tc:mysql://hostname/databasename?useUnicode=yes&characterEncoding=utf8");
Expand Down
Loading

0 comments on commit 0134e9b

Please sign in to comment.