adds time stamp to all logs and creates thread dump of neo4j if container fails to start

pom.xml

@@ -12,6 +12,7 @@
 	<properties>
         <java.version>17</java.version>
         <junit.version>5.10.0</junit.version>
+		<log4j.version>1.7.32</log4j.version>
         <neo4j.driver.version>4.4.11</neo4j.driver.version>
         <surefire.version>3.1.2</surefire.version>
         <testcontainers.version>1.18.0</testcontainers.version>
@@ -35,6 +36,12 @@
                 <profile.include>BundleTest</profile.include>
             </properties>
         </profile>
+        <profile>
+            <id>all-tests</id>
+            <properties>
+                <profile.include></profile.include>
+            </properties>
+        </profile>
     </profiles>
 
 	<build>
@@ -65,12 +72,12 @@
 			<!-- logging library -->
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
-			<version>1.7.32</version>
+			<version>${log4j.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-log4j12</artifactId>
-			<version>1.7.32</version>
+			<version>${log4j.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.junit.jupiter</groupId>

src/main/resources/log4j.properties

@@ -1,3 +1,4 @@
 log4j.rootLogger=INFO, console 
 log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSSZ} %-5p [%c{1}] %m%n

src/test/java/com/neo4j/docker/coredb/TestBasic.java

@@ -4,6 +4,7 @@
 import com.github.dockerjava.api.command.StopContainerCmd;
 import com.neo4j.docker.utils.DatabaseIO;
 import com.neo4j.docker.utils.Neo4jVersion;
+import com.neo4j.docker.utils.Network;
 import com.neo4j.docker.utils.TemporaryFolderManager;
 import com.neo4j.docker.utils.TestSettings;
 import com.neo4j.docker.utils.WaitStrategies;
@@ -26,10 +27,6 @@
 import java.util.List;
 import java.util.stream.Stream;
 
-import static com.neo4j.docker.utils.Network.getUniqueHostPort;
-import static com.neo4j.docker.utils.WaitStrategies.waitForBoltReady;
-import static com.neo4j.docker.utils.WaitStrategies.waitForNeo4jReady;
-
 public class TestBasic
 {
     private static Logger log = LoggerFactory.getLogger( TestBasic.class );
@@ -50,7 +47,7 @@ void testListensOn7687()
     {
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor(WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
             Assertions.assertTrue( container.isRunning() );
             String stdout = container.getLogs();
@@ -66,7 +63,7 @@ void testNoUnexpectedErrors()
                                  "UBI8 based images are expected to have a warning in stderr" );
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
             Assertions.assertTrue( container.isRunning() );
 
@@ -111,7 +108,7 @@ void testLicenseAcceptanceAvoidsWarning()
                                 "No unified license acceptance method before 5.0.0" );
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
             Assertions.assertTrue( container.isRunning() );
 
@@ -132,7 +129,7 @@ void testLicenseAcceptanceAvoidsWarning_evaluation()
         try ( GenericContainer container = createBasicContainer() )
         {
             container.withEnv( "NEO4J_ACCEPT_LICENSE_AGREEMENT", "eval" )
-                     .waitingFor( waitForNeo4jReady( "neo4j" ) );
+                     .waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
             Assertions.assertTrue( container.isRunning() );
 
@@ -149,7 +146,7 @@ void testCypherShellOnPath() throws Exception
         String expectedCypherShellPath = "/var/lib/neo4j/bin/cypher-shell";
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
 
             Container.ExecResult whichResult = container.execInContainer( "which", "cypher-shell" );
@@ -163,7 +160,7 @@ void testCanChangeWorkDir()
     {
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.setWorkingDirectory( "/tmp" );
             Assertions.assertDoesNotThrow( container::start,
                                            "Could not start neo4j from workdir other than NEO4J_HOME" );
@@ -177,7 +174,7 @@ void testPackagingInfoContainsDocker() throws Exception
                 "No packaging_info file before 5.0.0" );
         try ( GenericContainer container = createBasicContainer() )
         {
-            container.waitingFor( waitForNeo4jReady( "neo4j" ) );
+            container.waitingFor( WaitStrategies.waitForNeo4jReady( "neo4j" ) );
             container.start();
             String packagingInfo = container.execInContainer("cat", "/var/lib/neo4j/packaging_info").getStdout();
             List<String> actualPackageType = Stream.of(packagingInfo.split( "\n" ))
@@ -197,7 +194,7 @@ void testShutsDownCleanly( String signal )
         try ( GenericContainer container = createBasicContainer() )
         {
             container.withEnv( "NEO4J_AUTH", "none" )
-                     .waitingFor( waitForNeo4jReady( "none" ) );
+                     .waitingFor( WaitStrategies.waitForNeo4jReady( "none" ) );
             container.start();
             DatabaseIO dbio = new DatabaseIO( container );
             dbio.putInitialDataIntoContainer( "neo4j", "none" );
@@ -238,10 +235,10 @@ void testContainerCanBeRestartedAfterUnexpectedTermination() throws IOException
     {
         try ( GenericContainer container = createBasicContainer() )
         {
-            int boltHostPort = getUniqueHostPort();
-            int browserHostPort = getUniqueHostPort();
+            int boltHostPort = Network.getUniqueHostPort();
+            int browserHostPort = Network.getUniqueHostPort();
 
-            container.waitingFor( waitForBoltReady( Duration.ofSeconds( 90 ) ) );
+            container.waitingFor( WaitStrategies.waitForBoltReady( Duration.ofSeconds( 90 ) ) );
             container.withEnv( "NEO4J_AUTH", "none" );
 
             // Ensuring host ports are constant with container restarts
@@ -260,7 +257,7 @@ void testContainerCanBeRestartedAfterUnexpectedTermination() throws IOException
             container.getDockerClient().startContainerCmd( container.getContainerId() ).exec();
 
             // Applying the Waiting strategy to ensure container is correctly running, because DockerClient does not check
-            waitForBoltReady( Duration.ofSeconds( 90 ) ).waitUntilReady( container );
+            WaitStrategies.waitForBoltReady( Duration.ofSeconds( 90 ) ).waitUntilReady( container );
         }
     }
 }

src/test/java/com/neo4j/docker/coredb/plugins/TestBundledPluginInstallation.java

@@ -208,7 +208,7 @@ public void testBundledPlugin_downloadsIfNotAvailableLocally() throws Exception
                 if(plugins.isEmpty())
                 {
                     // no plugins were downloaded, which is correct if we are testing an unreleased neo4j
-                    String expectedError = String.format(".*No compatible \"%s\" plugin found for Neo4j %s community\\.",
+                    String expectedError = String.format(".*No compatible \"%s\" plugin found for Neo4j %s.*",
                             plugin.name, TestSettings.NEO4J_VERSION);
                     Assertions.assertTrue(
                         Stream.of(errlogs.split( "\n" ))

src/test/java/com/neo4j/docker/neo4jadmin/TestBackupRestore.java

@@ -105,11 +105,13 @@ private void testCanBackupAndRestore(boolean asDefaultUser, String password) thr
     {
         final String dbUser = "neo4j";
         Path backupDir;
+        Path adminLogDir;
 
         // BACKUP
         // start a database and populate data
         try(GenericContainer neo4j = createDBContainer( asDefaultUser, password ))
         {
+            temporaryFolderManager.createFolderAndMountAsVolume(neo4j, "/logs");
             Path dataDir = temporaryFolderManager.createFolderAndMountAsVolume(neo4j, "/data");
             neo4j.start();
             DatabaseIO dbio = new DatabaseIO( neo4j );
@@ -129,7 +131,7 @@ private void testCanBackupAndRestore(boolean asDefaultUser, String password) thr
                                          "--include-metadata=all",
                                          "--from=" + neoDBAddress,
                                          "neo4j" );
-
+                adminLogDir = temporaryFolderManager.createNamedFolderAndMountAsVolume(adminBackup, "admin-logs", "/logs");
                 backupDir = temporaryFolderManager.createFolderAndMountAsVolume(adminBackup, "/backups");
                 adminBackup.start();
 
@@ -164,6 +166,7 @@ private void testCanBackupAndRestore(boolean asDefaultUser, String password) thr
                                           "--overwrite-destination=true",
                                           "--from-path=/backups/" + backupFile.getName(),
                                           "neo4j" );
+                temporaryFolderManager.mountHostFolderAsVolume( adminRestore, adminLogDir, "/logs" );
                 temporaryFolderManager.mountHostFolderAsVolume( adminRestore, backupDir, "/backups" );
                 temporaryFolderManager.mountHostFolderAsVolume( adminRestore, dataDir, "/data" );
                 adminRestore.start();

src/test/java/com/neo4j/docker/neo4jadmin/TestDumpLoad.java

@@ -100,10 +100,13 @@ private void shouldCreateDumpAndLoadDump( boolean asDefaultUser, String password
         Path firstDataDir;
         Path secondDataDir;
         Path backupDir;
+        Path logs;
+        Path adminLogs;
 
         // start a database and populate it
         try(GenericContainer container = createDBContainer( asDefaultUser, password ))
         {
+            logs = temporaryFolderManager.createFolderAndMountAsVolume(container, "/logs");
             firstDataDir = temporaryFolderManager.createNamedFolderAndMountAsVolume(container,"data1", "/data");
             container.start();
             DatabaseIO dbio = new DatabaseIO( container );
@@ -114,9 +117,10 @@ private void shouldCreateDumpAndLoadDump( boolean asDefaultUser, String password
         // use admin container to create dump
         try(GenericContainer admin = createAdminContainer( asDefaultUser ))
         {
-            temporaryFolderManager.mountHostFolderAsVolume( admin, firstDataDir, "/data" );
+            adminLogs = temporaryFolderManager.createNamedFolderAndMountAsVolume(admin,"admin-logs", "/logs");
+            temporaryFolderManager.mountHostFolderAsVolume(admin, firstDataDir, "/data");
             backupDir = temporaryFolderManager.createFolderAndMountAsVolume(admin, "/backups");
-            admin.withCommand( "neo4j-admin", "database", "dump", "neo4j", "--to-path=/backups" );
+            admin.withCommand("neo4j-admin", "database", "dump", "neo4j", "--to-path=/backups");
             admin.start();
         }
         Assertions.assertTrue( backupDir.resolve( "neo4j.dump" ).toFile().exists(), "dump file not created");
@@ -126,6 +130,7 @@ private void shouldCreateDumpAndLoadDump( boolean asDefaultUser, String password
         try(GenericContainer admin = createAdminContainer( asDefaultUser ))
         {
             secondDataDir = temporaryFolderManager.createNamedFolderAndMountAsVolume(admin, "data2", "/data");
+            temporaryFolderManager.mountHostFolderAsVolume(admin, adminLogs, "/logs");
             temporaryFolderManager.mountHostFolderAsVolume( admin, backupDir, "/backups" );
             admin.withCommand( "neo4j-admin", "database", "load", "neo4j", "--from-path=/backups" );
             admin.start();
@@ -134,6 +139,7 @@ private void shouldCreateDumpAndLoadDump( boolean asDefaultUser, String password
         // verify data in 2nd data directory by starting a database and verifying data we populated earlier
         try(GenericContainer container = createDBContainer( asDefaultUser, password ))
         {
+            temporaryFolderManager.mountHostFolderAsVolume(container, logs, "/logs");
             temporaryFolderManager.mountHostFolderAsVolume( container, secondDataDir, "/data" );
             container.start();
             DatabaseIO dbio = new DatabaseIO( container );

src/test/java/com/neo4j/docker/utils/Neo4jWaitStrategy.java

@@ -0,0 +1,77 @@
+package com.neo4j.docker.utils;
+
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.ContainerLaunchException;
+import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
+import org.testcontainers.containers.wait.strategy.WaitStrategy;
+
+import java.io.IOException;
+import java.time.Duration;
+
+public class Neo4jWaitStrategy extends HttpWaitStrategy
+{
+    private static Logger log = LoggerFactory.getLogger(Neo4jWaitStrategy.class);
+
+    public static WaitStrategy waitForNeo4jReady(String username, String password, String database, Duration timeout )
+    {
+        if (TestSettings.EDITION == TestSettings.Edition.ENTERPRISE &&
+                TestSettings.NEO4J_VERSION.isAtLeastVersion(Neo4jVersion.NEO4J_VERSION_500)) {
+            return new Neo4jWaitStrategy()
+                    .forPath("/db/" + database + "/cluster/available")
+                    .withBasicCredentials(username, password)
+                    .forPort(7474)
+                    .forStatusCode(200)
+                    .withStartupTimeout(timeout);
+        } else
+        {
+            return waitForBoltReady( timeout );
+        }
+    }
+
+    public static WaitStrategy waitForBoltReady( Duration timeout )
+    {
+        return new Neo4jWaitStrategy()
+                .forPath("/")
+                .forPort(7687)
+                .forStatusCode(200)
+                .withStartupTimeout(timeout);
+    }
+
+    @Override
+    protected void waitUntilReady()
+    {
+        try {
+            super.waitUntilReady();
+        }
+        catch (ContainerLaunchException ex)
+        {
+            InspectContainerResponse containerInfo = waitStrategyTarget.getCurrentContainerInfo();
+            log.error("Failed to start container. State:\n"+containerInfo.toString());
+            String doThreadDumpCmd = "jcmd $(cat /var/lib/neo4j/run/neo4j.pid) Thread.print > /var/lib/neo4j/logs/threaddump";
+            // if running as default user then we need to be `neo4j` to query the neo4j process.
+            if(containerInfo.getConfig().getUser().isEmpty())
+            {
+                doThreadDumpCmd = "su-exec neo4j " + doThreadDumpCmd;
+            }
+            try
+            {
+                String fullCommand =
+                        "if [ -f /var/lib/neo4j/run/neo4j.pid ]; then\n" +
+                                doThreadDumpCmd + "\n" +
+                            "else\n" +
+                                "echo >&2 \"could not dump threads, Neo4j is not running.\"\n" +
+                            "fi";
+                Container.ExecResult threadDumpResponse = waitStrategyTarget.execInContainer("sh", "-c", fullCommand);
+                log.warn(threadDumpResponse.getStderr());
+            }
+            catch (IOException | InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+            throw ex;
+        }
+    }
+}

src/test/java/com/neo4j/docker/utils/WaitStrategies.java

@@ -19,17 +19,7 @@ private WaitStrategies() {}
 
     public static WaitStrategy waitForNeo4jReady( String username, String password, String database, Duration timeout )
     {
-        if (TestSettings.EDITION == TestSettings.Edition.ENTERPRISE &&
-            TestSettings.NEO4J_VERSION.isAtLeastVersion(Neo4jVersion.NEO4J_VERSION_500)) {
-            return Wait.forHttp("/db/" + database + "/cluster/available")
-                       .withBasicCredentials(username, password)
-                       .forPort(7474)
-                       .forStatusCode(200)
-                       .withStartupTimeout(timeout);
-        } else
-        {
-            return waitForBoltReady( timeout );
-        }
+        return Neo4jWaitStrategy.waitForNeo4jReady(username, password, database, timeout);
     }
 
     public static WaitStrategy waitForNeo4jReady( String password ) {
@@ -46,10 +36,7 @@ public static WaitStrategy waitForNeo4jReady( String user, String password, Dura
 
     public static WaitStrategy waitForBoltReady( Duration timeout )
     {
-        return Wait.forHttp("/")
-                   .forPort(7687)
-                   .forStatusCode(200)
-                   .withStartupTimeout(timeout);
+        return Neo4jWaitStrategy.waitForBoltReady(timeout);
     }
 
     /**For containers that will just run a command and exit automatically.