[PAN-2009] - Fix cluster clean start after stop in Acceptance tests

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/ClusterAcceptanceTest.java

@@ -35,4 +35,14 @@ public void shouldConnectToOtherPeer() {
     minerNode.verify(net.awaitPeerCount(1));
     fullNode.verify(net.awaitPeerCount(1));
   }
+
+  @Test
+  public void shouldRestartAfterStop() {
+    minerNode.verify(net.awaitPeerCount(1));
+    fullNode.verify(net.awaitPeerCount(1));
+    cluster.stop();
+    cluster.start(minerNode, fullNode);
+    minerNode.verify(net.awaitPeerCount(1));
+    fullNode.verify(net.awaitPeerCount(1));
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/PantheonNode.java

@@ -478,7 +478,7 @@ public boolean isBootnodeEligible() {
   }
 
   @Override
-  public void getBootnodes(final List<URI> bootnodes) {
+  public void setBootnodes(final List<URI> bootnodes) {
     this.bootnodes.clear();
     this.bootnodes.addAll(bootnodes);
   }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/ThreadPantheonNodeRunner.java

@@ -62,6 +62,19 @@ public class ThreadPantheonNodeRunner implements PantheonNodeRunner {
 
   private final Map<Node, PantheonPluginContextImpl> pantheonPluginContextMap = new HashMap<>();
 
+  private PantheonPluginContextImpl buildPluginContext(final PantheonNode node) {
+    PantheonPluginContextImpl pantheonPluginContext = new PantheonPluginContextImpl();
+    final Path pluginsPath = node.homeDirectory().resolve("plugins");
+    final File pluginsDirFile = pluginsPath.toFile();
+    if (!pluginsDirFile.isDirectory()) {
+      pluginsDirFile.mkdirs();
+      pluginsDirFile.deleteOnExit();
+    }
+    System.setProperty("pantheon.plugins.dir", pluginsPath.toString());
+    pantheonPluginContext.registerPlugins(pluginsPath);
+    return pantheonPluginContext;
+  }
+
   @Override
   @SuppressWarnings("UnstableApiUsage")
   public void startNode(final PantheonNode node) {
@@ -71,16 +84,9 @@ public void startNode(final PantheonNode node) {
 
     final CommandLine commandLine = new CommandLine(CommandSpec.create());
     final PantheonPluginContextImpl pantheonPluginContext =
-        pantheonPluginContextMap.computeIfAbsent(node, n -> new PantheonPluginContextImpl());
+        pantheonPluginContextMap.computeIfAbsent(node, n -> buildPluginContext(node));
     pantheonPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine));
-    final Path pluginsPath = node.homeDirectory().resolve("plugins");
-    final File pluginsDirFile = pluginsPath.toFile();
-    if (!pluginsDirFile.isDirectory()) {
-      pluginsDirFile.mkdirs();
-      pluginsDirFile.deleteOnExit();
-    }
-    System.setProperty("pantheon.plugins.dir", pluginsPath.toString());
-    pantheonPluginContext.registerPlugins(pluginsPath);
+
     commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0]));
 
     final MetricsSystem noOpMetricsSystem = new NoOpMetricsSystem();
@@ -150,13 +156,20 @@ public void startNode(final PantheonNode node) {
 
   @Override
   public void stopNode(final PantheonNode node) {
-    pantheonPluginContextMap.get(node).stopPlugins();
+    PantheonPluginContextImpl pluginContext = pantheonPluginContextMap.remove(node);
+    if (pluginContext != null) {
+      pluginContext.stopPlugins();
+    }
     node.stop();
     killRunner(node.getName());
   }
 
   @Override
   public void shutdown() {
+    // stop all plugins from pluginContext
+    pantheonPluginContextMap.values().forEach(PantheonPluginContextImpl::stopPlugins);
+    pantheonPluginContextMap.clear();
+
     // iterate over a copy of the set so that pantheonRunner can be updated when a runner is killed
     new HashSet<>(pantheonRunners.keySet()).forEach(this::killRunner);
     try {

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/cluster/Cluster.java

@@ -13,7 +13,6 @@
 package tech.pegasys.pantheon.tests.acceptance.dsl.node.cluster;
 
 import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -25,6 +24,7 @@
 import tech.pegasys.pantheon.tests.acceptance.dsl.node.RunnableNode;
 
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -42,7 +42,7 @@ public class Cluster implements AutoCloseable {
   private final NetConditions net;
   private final ClusterConfiguration clusterConfiguration;
   private List<? extends RunnableNode> originalNodes = emptyList();
-  private List<URI> bootnodes = emptyList();
+  private List<URI> bootnodes = new ArrayList<>();
 
   public Cluster(final NetConditions net) {
     this(new ClusterConfigurationBuilder().build(), net, PantheonNodeRunner.instance());
@@ -78,6 +78,7 @@ public void start(final List<? extends RunnableNode> nodes) {
     }
     this.originalNodes = nodes;
     this.nodes.clear();
+    this.bootnodes.clear();
     nodes.forEach(node -> this.nodes.put(node.getName(), node));
 
     final Optional<? extends RunnableNode> bootnode = selectAndStartBootnode(nodes);
@@ -103,9 +104,13 @@ private Optional<? extends RunnableNode> selectAndStartBootnode(
             .filter(node -> node.getConfiguration().isDiscoveryEnabled())
             .findFirst();
 
-    bootnode.ifPresent((b) -> LOG.info("Selected node {} as bootnode", b.getName()));
-    bootnode.ifPresent(this::startNode);
-    bootnodes = bootnode.map(node -> singletonList(node.enodeUrl())).orElse(emptyList());
+    bootnode.ifPresent(
+        b -> {
+          LOG.info("Selected node {} as bootnode", b.getName());
+          startNode(b, true);
+          bootnodes.add(b.enodeUrl());
+        });
+
     return bootnode;
   }
 
@@ -122,9 +127,12 @@ public void addNode(final Node node) {
   }
 
   private void startNode(final RunnableNode node) {
-    if (node.getConfiguration().getBootnodes().isEmpty()) {
-      node.getConfiguration().getBootnodes(bootnodes);
-    }
+    this.startNode(node, false);
+  }
+
+  private void startNode(final RunnableNode node, final boolean isBootNode) {
+    node.getConfiguration().setBootnodes(isBootNode ? emptyList() : bootnodes);
+
     node.getConfiguration()
         .getGenesisConfigProvider()
         .create(originalNodes)
@@ -138,10 +146,15 @@ private void startNode(final RunnableNode node) {
   }
 
   public void stop() {
+    // stops nodes but do not shutdown pantheonNodeRunner
     for (final RunnableNode node : nodes.values()) {
-      node.stop();
+      if (node instanceof PantheonNode) {
+        pantheonNodeRunner.stopNode(
+            (PantheonNode) node); // pantheonNodeRunner.stopNode also calls node.stop
+      } else {
+        node.stop();
+      }
     }
-    pantheonNodeRunner.shutdown();
   }
 
   public void stopNode(final PantheonNode node) {
@@ -150,6 +163,7 @@ public void stopNode(final PantheonNode node) {
 
   @Override
   public void close() {
+    stop();
     for (final RunnableNode node : nodes.values()) {
       node.close();
     }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/configuration/NodeConfiguration.java

@@ -20,7 +20,7 @@
 
 public interface NodeConfiguration {
 
-  void getBootnodes(List<URI> bootnodes);
+  void setBootnodes(List<URI> bootnodes);
 
   List<URI> getBootnodes();