Add support to run sanity test on remote cluster (#394)

Signed-off-by: Ankit Kala <ankikala@amazon.com>

build.gradle

@@ -613,7 +613,7 @@ def securityPluginOld = new Callable<RegularFile>() {
 }
 
 // We maintain 2 set of clusters here. One for full cluster restart and one for rolling restart + mixed cluster.
-List<String> clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1","singleCluster"]
+List<String> clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1"]
 // TODO: Make BWC test work with security plugin
 clusters.each { name ->
     testClusters {
@@ -860,24 +860,12 @@ task "bwcTestSuite"(type: RestIntegTestTask) {
     dependsOn tasks.named("fullRestartClusterTask")
 }
 
-task integTestSingleCluster(type: RestIntegTestTask) {
-    useCluster testClusters.singleCluster
+task integTestRemote(type: RestIntegTestTask) {
     doFirst {
-        getClusters().forEach { cluster ->
-            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllTransportPortURI().stream()
-            }.collect(Collectors.joining(","))
-            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
-                node.getAllHttpSocketURI().stream()
-            }.collect(Collectors.joining(","))
-
-            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
-            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
-            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
-            configureCluster(cluster, securityEnabled)
-        }
+        systemProperty "tests.cluster.follower.http_hosts", System.getProperty("follower.http_host")
+        systemProperty "tests.cluster.follower.transport_hosts", System.getProperty("follower.transport_host")
+        systemProperty "tests.cluster.follower.security_enabled", System.getProperty("security_enabled")
     }
-
     filter {
         setIncludePatterns("org.opensearch.replication.singleCluster.SingleClusterSanityIT")
     }

scripts/integtest.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+function usage() {
+    echo ""
+    echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster."
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo "None"
+    echo ""
+    echo "Optional arguments:"
+    echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location."
+    echo -e "-p BIND_PORT\t, defaults to 9200, can be changed to any port for the cluster location."
+    echo -e "-t TRANSPORT_PORT\t, defaults to 9300, can be changed to any port for the cluster location."
+    echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not."
+    echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true."
+    echo -e "-h\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":h:b:p:s:c:v:n:t:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        b)
+            BIND_ADDRESS=$OPTARG
+            ;;
+        p)
+            BIND_PORT=$OPTARG
+            ;;
+        t)
+            TRANSPORT_PORT=$OPTARG
+            ;;
+        s)
+            SECURITY_ENABLED=$OPTARG
+            ;;
+        c)
+            CREDENTIAL=$OPTARG
+            ;;
+        :)
+            echo "-${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${OPTARG}"
+            exit 1
+            ;;
+    esac
+done
+
+
+if [ -z "$BIND_ADDRESS" ]
+then
+  BIND_ADDRESS="localhost"
+fi
+
+if [ -z "$BIND_PORT" ]
+then
+  BIND_PORT="9200"
+fi
+
+if [ -z "$TRANSPORT_PORT" ]
+then
+  TRANSPORT_PORT="9300"
+fi
+
+if [ -z "$SECURITY_ENABLED" ]
+then
+  SECURITY_ENABLED="true"
+fi
+
+if [ -z "$CREDENTIAL" ]
+then
+  CREDENTIAL="admin:admin"
+fi
+
+USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'`
+PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'`
+
+./gradlew integTestRemote -Dfollower.http_host="$BIND_ADDRESS:$BIND_PORT" -Dfollower.transport_host="$BIND_ADDRESS:$TRANSPORT_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain

src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt

@@ -454,6 +454,10 @@ abstract class MultiClusterRestTestCase : OpenSearchTestCase() {
         return OpenSearchRestTestCase.entityAsMap(client.performRequest(Request("GET", endpoint)))
     }
 
+    fun getAsList(client: RestClient, endpoint: String): List<Any> {
+        return OpenSearchRestTestCase.entityAsList(client.performRequest(Request("GET", endpoint)))
+    }
+
     protected fun createConnectionBetweenClusters(fromClusterName: String, toClusterName: String, connectionName: String="source") {
         val toCluster = getNamedCluster(toClusterName)
         val fromCluster = getNamedCluster(fromClusterName)

src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt

@@ -9,6 +9,7 @@ import org.opensearch.replication.StartReplicationRequest
 import org.opensearch.replication.startReplication
 import org.assertj.core.api.Assertions.assertThatThrownBy
 import org.junit.Assert
+import org.opensearch.client.Request
 import org.opensearch.replication.stopReplication
 import java.util.stream.Collectors
 
@@ -19,16 +20,15 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() {
 
     companion object {
         private val log = LogManager.getLogger(SingleClusterSanityIT::class.java)
-        private const val standaloneClusterName = "singleCluster"
+        private const val followerClusterName = "follower"
         private const val REPLICATION_PLUGIN_NAME = "opensearch-cross-cluster-replication"
-        private const val NUM_NODES = 3
         private const val SAMPLE_INDEX = "sample_test_index"
 
         @BeforeClass
         @JvmStatic
         fun setupTestClusters() {
             val clusters = HashMap<String, TestCluster>()
-            clusters.put(standaloneClusterName, createTestCluster(standaloneClusterName, true, true, true, false))
+            clusters.put(followerClusterName, createTestCluster(followerClusterName, true, true, true, false))
             testClusters = clusters
         }
 
@@ -49,15 +49,17 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() {
     }
 
     fun basicReplicationSanityWithSingleCluster() {
-        verifyReplicationPluginInstallationOnAllNodes(standaloneClusterName)
-        VerifyReplicationApis(standaloneClusterName)
+        verifyReplicationPluginInstallationOnAllNodes(followerClusterName)
+        VerifyReplicationApis(followerClusterName)
     }
 
     @Throws(java.lang.Exception::class)
     private fun verifyReplicationPluginInstallationOnAllNodes(clusterName: String) {
         val restClient = getClientForCluster(clusterName)
-        for (i in 0 until NUM_NODES) {
-            val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$clusterName-$i/plugins")["nodes"]
+        val nodes = getAsList(restClient.lowLevelClient, "_cat/nodes?format=json") as List<Map<String, String>>
+        nodes.forEach { node ->
+            val nodeName = node["name"]
+            val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$nodeName/plugins")["nodes"]
                     as Map<String, Map<String, Any>>?
             Assert.assertTrue(responseMap!!.values.isNotEmpty())
             for (response in responseMap!!.values) {
@@ -72,17 +74,17 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() {
 
     @Throws(java.lang.Exception::class)
     private fun VerifyReplicationApis(clusterName: String) {
-        val follower = getClientForCluster(standaloneClusterName)
+        val follower = getClientForCluster(followerClusterName)
         assertThatThrownBy {
             follower.startReplication(
                 StartReplicationRequest("sample_connection", SAMPLE_INDEX, SAMPLE_INDEX),
                 waitForRestore = true
             )
         }.isInstanceOf(ResponseException::class.java).hasMessageContaining("no such remote cluster")
         assertThatThrownBy {
-            follower.stopReplication(standaloneClusterName)
+            follower.stopReplication(followerClusterName)
         }.isInstanceOf(ResponseException::class.java)
-            .hasMessageContaining("No replication in progress for index:"+standaloneClusterName)
+            .hasMessageContaining("No replication in progress for index:"+followerClusterName)
     }
 
 }