diff --git a/.github/workflows/reports-scheduler-test-and-build-workflow.yml b/.github/workflows/reports-scheduler-test-and-build-workflow.yml index 1e389afa..be4282e8 100644 --- a/.github/workflows/reports-scheduler-test-and-build-workflow.yml +++ b/.github/workflows/reports-scheduler-test-and-build-workflow.yml @@ -19,6 +19,13 @@ jobs: - name: Checkout Reports Scheduler uses: actions/checkout@v2 + - name: RunBackwards Compatibility Tests + run: | + cd reports-scheduler + echo "Running backwards compatibility tests ..." + ./gradlew bwcTestSuite + + - name: Build with Gradle run: | cd reports-scheduler diff --git a/reports-scheduler/build.gradle b/reports-scheduler/build.gradle index 395af841..b75794c4 100644 --- a/reports-scheduler/build.gradle +++ b/reports-scheduler/build.gradle @@ -5,6 +5,7 @@ import org.opensearch.gradle.test.RestIntegTestTask import java.util.concurrent.Callable +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask buildscript { ext { @@ -21,6 +22,7 @@ buildscript { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } jcenter() } @@ -115,6 +117,9 @@ allprojects { repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + jcenter() } dependencies { @@ -222,6 +227,12 @@ integTest { if (System.getProperty("tests.clustername") != null) { exclude 'org/opensearch/reportsscheduler/ReportsSchedulerPluginIT.class' } + + if (System.getProperty("tests.rest.bwcsuite") == null) { + filter { + excludeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + } } Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); @@ -242,6 +253,7 @@ testClusters.integTest { } } })) + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore @@ -257,6 +269,169 @@ testClusters.integTest { setting 'path.repo', repo.absolutePath } +// For job-scheduler and reports-scheduler, the latest opendistro releases appear to be 1.13.0.0. +String bwcVersion = "1.13.0.0" +String baseName = "reportsSchedulerBwcCluster" +String bwcFilePath = "src/test/resources/bwc" + +2.times {i -> + testClusters { + "${baseName}$i" { + testDistribution = "ARCHIVE" + versions = ["7.10.2","1.3.0-SNAPSHOT"] + numberOfNodes = 3 + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + "/job-scheduler/" + bwcVersion).getSingleFile() + } + } + } + })) + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + "/reports-scheduler/" + bwcVersion).getSingleFile() + } + } + } + })) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' + } + } +} + +List> plugins = [] + +// Ensure the artifact for the current project version is available to be used for the bwc tests +task prepareBwcTests { + dependsOn bundle + doLast { + plugins = [ + provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + "/job-scheduler/" + project.version).getSingleFile() + } + } + } + }), + project.getObjects().fileProperty().value(bundle.getArchiveFile()) + ] + } +} + +// Create two test clusters with 3 nodes of the old version +2.times {i -> + task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { + dependsOn 'prepareBwcTests' + useCluster testClusters."${baseName}$i" + filter { + includeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'old_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'old' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") + } +} + +// Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version. +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +// This is also used as a one third upgraded cluster for a rolling upgrade. +task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}0" + dependsOn "${baseName}#oldVersionClusterTask0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'first' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. +// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +// This is used for rolling upgrade. +task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#mixedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'second' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. +// This results in a fully upgraded cluster. +// This is used for rolling upgrade. +task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdsUpgradedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + mustRunAfter "${baseName}#mixedClusterTask" + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'third' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +// at the same time resulting in a fully upgraded cluster. +task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldVersionClusterTask1" + useCluster testClusters."${baseName}1" + doFirst { + testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.reportsscheduler.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") +} + +// A bwc test suite which runs all the bwc tasks combined +task bwcTestSuite(type: StandaloneRestIntegTestTask) { + exclude '**/*Test*' + exclude '**/*IT*' + dependsOn tasks.named("${baseName}#mixedClusterTask") + dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") + dependsOn tasks.named("${baseName}#fullRestartClusterTask") +} + + run { doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/PluginRestTestCase.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/PluginRestTestCase.kt index 1a1b575d..a14d7d73 100644 --- a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/PluginRestTestCase.kt +++ b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/PluginRestTestCase.kt @@ -64,9 +64,12 @@ abstract class PluginRestTestCase : OpenSearchRestTestCase() { return true } + open fun preserveODFEIndicesAfterTest(): Boolean = false + @Throws(IOException::class) @After open fun wipeAllODFEIndices() { + if (preserveODFEIndicesAfterTest()) return val response = client().performRequest(Request("GET", "/_cat/indices?format=json&expand_wildcards=all")) val xContentType = XContentType.fromMediaTypeOrFormat(response.entity.contentType.value) xContentType.xContent().createParser( diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/bwc/ReportsSchedulerBackwardsCompatibilityIT.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/bwc/ReportsSchedulerBackwardsCompatibilityIT.kt new file mode 100644 index 00000000..571b4d45 --- /dev/null +++ b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/bwc/ReportsSchedulerBackwardsCompatibilityIT.kt @@ -0,0 +1,161 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.reportsscheduler.bwc + +import org.junit.Assert +import org.opensearch.common.settings.Settings +import org.opensearch.reportsscheduler.PluginRestTestCase +import org.opensearch.reportsscheduler.ReportsSchedulerPlugin.Companion.BASE_REPORTS_URI +import org.opensearch.reportsscheduler.ReportsSchedulerPlugin.Companion.LEGACY_BASE_REPORTS_URI +import org.opensearch.reportsscheduler.constructReportDefinitionRequest +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestStatus +import java.time.Instant + +class ReportsSchedulerBackwardsCompatibilityIT : PluginRestTestCase() { + + companion object { + private val CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite")) + private val CLUSTER_NAME = System.getProperty("tests.clustername") + } + + override fun preserveIndicesUponCompletion(): Boolean = true + + override fun preserveReposUponCompletion(): Boolean = true + + override fun preserveTemplatesUponCompletion(): Boolean = true + + override fun preserveODFEIndicesAfterTest(): Boolean = true + + override fun restClientSettings(): Settings { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(CLIENT_SOCKET_TIMEOUT, "90s") + .build() + } + + @Throws(Exception::class) + @Suppress("UNCHECKED_CAST") + fun `test backwards compatibility`() { + val uri = getPluginUri() + val responseMap = getAsMap(uri)["nodes"] as Map> + for (response in responseMap.values) { + val plugins = response["plugins"] as List> + val pluginNames = plugins.map { plugin -> plugin["name"] }.toSet() + when (CLUSTER_TYPE) { + ClusterType.OLD -> { + assertTrue(pluginNames.contains("opendistro-reports-scheduler")) + assertTrue(pluginNames.contains("opendistro-job-scheduler")) + createBasicReportDefinition() + } + ClusterType.MIXED -> { + assertTrue(pluginNames.contains("opensearch-reports-scheduler")) + assertTrue(pluginNames.contains("opensearch-job-scheduler")) + verifyReportDefinitionExists(LEGACY_BASE_REPORTS_URI) + verifyReportInstanceExists(LEGACY_BASE_REPORTS_URI) + } + ClusterType.UPGRADED -> { + assertTrue(pluginNames.contains("opensearch-reports-scheduler")) + assertTrue(pluginNames.contains("opensearch-job-scheduler")) + verifyReportDefinitionExists(BASE_REPORTS_URI) + verifyReportInstanceExists(BASE_REPORTS_URI) + } + } + break + } + } + + private enum class ClusterType { + OLD, + MIXED, + UPGRADED; + + companion object { + fun parse(value: String): ClusterType { + return when (value) { + "old_cluster" -> OLD + "mixed_cluster" -> MIXED + "upgraded_cluster" -> UPGRADED + else -> throw AssertionError("Unknown cluster type: $value") + } + } + } + } + + private fun getPluginUri(): String { + return when (CLUSTER_TYPE) { + ClusterType.OLD -> "_nodes/$CLUSTER_NAME-0/plugins" + ClusterType.MIXED -> { + when (System.getProperty("tests.rest.bwcsuite_round")) { + "second" -> "_nodes/$CLUSTER_NAME-1/plugins" + "third" -> "_nodes/$CLUSTER_NAME-2/plugins" + else -> "_nodes/$CLUSTER_NAME-0/plugins" + } + } + ClusterType.UPGRADED -> "_nodes/plugins" + } + } + + @Throws(Exception::class) + private fun createBasicReportDefinition() { + val timeStampMillis = Instant.now().toEpochMilli() + val trigger = """ + "trigger":{ + "triggerType":"IntervalSchedule", + "schedule":{ + "interval":{ + "start_time":$timeStampMillis, + "period":"1", + "unit":"Minutes" + } + } + }, + """.trimIndent() + val reportDefinitionRequest = constructReportDefinitionRequest(trigger) + + // legacy test + val legacyReportDefinitionResponse = executeRequest( + RestRequest.Method.POST.name, + "$LEGACY_BASE_REPORTS_URI/definition", + reportDefinitionRequest, + RestStatus.OK.status + ) + val legacyReportDefinitionId = legacyReportDefinitionResponse.get("reportDefinitionId").asString + Assert.assertNotNull("reportDefinitionId should be generated", legacyReportDefinitionId) + // adding this wait time for the scheduler to create a report instance in the report-instance index, + // based on this report definition + Thread.sleep(610000) + } + + @Throws(Exception::class) + @Suppress("UNCHECKED_CAST") + private fun verifyReportDefinitionExists(uri: String) { + val listReportDefinitions = executeRequest( + RestRequest.Method.GET.name, + "$uri/definitions", + "", + RestStatus.OK.status + ) + val totalHits = listReportDefinitions.get("totalHits").asInt + Assert.assertEquals(totalHits, 1) + } + + @Throws(Exception::class) + @Suppress("UNCHECKED_CAST") + private fun verifyReportInstanceExists(uri: String) { + val listReportInstances = executeRequest( + RestRequest.Method.GET.name, + "$uri/instances", + "", + RestStatus.OK.status + ) + val totalHits = listReportInstances.get("totalHits").asInt + assertTrue("Actual report instances counts ($totalHits) should be greater than or equal to (1)", totalHits >= 1) + } +} diff --git a/reports-scheduler/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip b/reports-scheduler/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip new file mode 100644 index 00000000..daf3b8f7 Binary files /dev/null and b/reports-scheduler/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip differ diff --git a/reports-scheduler/src/test/resources/bwc/job-scheduler/1.3.0.0-SNAPSHOT/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip b/reports-scheduler/src/test/resources/bwc/job-scheduler/1.3.0.0-SNAPSHOT/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip new file mode 100644 index 00000000..49f516a1 Binary files /dev/null and b/reports-scheduler/src/test/resources/bwc/job-scheduler/1.3.0.0-SNAPSHOT/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip differ diff --git a/reports-scheduler/src/test/resources/bwc/reports-scheduler/1.13.0.0/opendistro-reports-scheduler-1.13.0.0.zip b/reports-scheduler/src/test/resources/bwc/reports-scheduler/1.13.0.0/opendistro-reports-scheduler-1.13.0.0.zip new file mode 100644 index 00000000..79abef18 Binary files /dev/null and b/reports-scheduler/src/test/resources/bwc/reports-scheduler/1.13.0.0/opendistro-reports-scheduler-1.13.0.0.zip differ