Action Validation framework and Explain API integration

README.md

@@ -2,7 +2,7 @@
 [![codecov](https://codecov.io/gh/opensearch-project/index-management/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/index-management)
 [![Roadmap](https://img.shields.io/badge/roadmap-checkout-ff69b4)](https://github.com/opensearch-project/index-management/projects/1)
 [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://opensearch.org/docs/im-plugin/index/)
-[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/index-management/)
+[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/plugins/index-management/47)
 ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)
 
 <img src="https://opensearch.org/assets/brand/SVG/Logo/opensearch_logo_default.svg" height="64px"/>
@@ -59,7 +59,7 @@ See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project
 
 If you find a bug, or have a feature request, please don't hesitate to open an issue in this repository.
 
-For more information, see [project website](https://opensearch.org/) and [documentation](https://opensearch.org/docs/). If you need help and are unsure where to open an issue, try [forums](https://discuss.opendistrocommunity.dev/).
+For more information, see [project website](https://opensearch.org/) and [documentation](https://opensearch.org/docs/latest/im-plugin/index/). If you need help and are unsure where to open an issue, try [Forum](https://forum.opensearch.org/c/plugins/index-management/47).
 
 ## Code of Conduct
 

build.gradle

@@ -14,7 +14,7 @@ import java.util.function.Predicate
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         // 2.2.0-SNAPSHOT -> 2.2.0.0-SNAPSHOT
         version_tokens = opensearch_version.tokenize('-')
@@ -59,7 +59,7 @@ buildscript {
         classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
         classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
-        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.17.1"
+        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.21.0"
         classpath "org.jacoco:org.jacoco.agent:0.8.7"
     }
 }
@@ -95,7 +95,7 @@ configurations.all {
         force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
         force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
         force 'com.fasterxml.jackson.core:jackson-databind:2.10.4'
-        force 'org.yaml:snakeyaml:1.26'
+        force 'org.yaml:snakeyaml:1.32'
         force 'org.codehaus.plexus:plexus-utils:3.0.24'
     }
 }
@@ -180,12 +180,12 @@ dependencies {
     implementation "com.github.seancfoley:ipaddress:5.3.3"
     implementation "commons-codec:commons-codec:${versions.commonscodec}"
     implementation "org.apache.httpcomponents:httpclient:4.5.13"
-    implementation "org.apache.httpcomponents:httpcore:4.4.12"
+    implementation "org.apache.httpcomponents:httpcore:4.4.15"
 
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
     testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
     testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
-    testImplementation "org.mockito:mockito-core:4.6.1"
+    testImplementation "org.mockito:mockito-core:4.7.0"
 
     add("ktlint", "com.pinterest:ktlint:0.45.1") {
         attributes {

release-notes/opensearch-index-management.release-notes-2.2.0.0.md

@@ -0,0 +1,13 @@
+## Version 2.2.0.0 2022-08-05
+
+Compatible with OpenSearch 2.2.0
+
+### Features
+* Ability to count the number of documents from source index ([#439](https://github.com/opensearch-project/index-management/pull/439)) and ([#3985](https://github.com/opensearch-project/OpenSearch/pull/3985))
+
+### Enhancements
+* Support mustache scripting of rollup.target_index field ([#435](https://github.com/opensearch-project/index-management/pull/435))
+* Support searching multiple rollup indices with same mapping ([#440](https://github.com/opensearch-project/index-management/pull/440))
+
+### Maintenance
+* version upgrade to 2.2.0 ([#446](https://github.com/opensearch-project/index-management/pull/446))

release-notes/opensearch-index-management.release-notes-2.3.0.0.md

@@ -0,0 +1,23 @@
+## Version 2.3.0.0 2022-09-09
+
+Compatible with OpenSearch 2.3.0
+
+### Enhancements
+* Replica Count Validation when awareness replica balance is enabled ([#429](https://github.com/opensearch-project/index-management/pull/429))
+* Updated detekt plugin, snakeyaml dependency and code to reduce the number of issues after static analysis ([#483](https://github.com/opensearch-project/index-management/pull/483))
+* Transform max_clauses optimization: limit amount of modified buckets being processed at a time and added capping of pageSize to avoid maxClause exception ([#477](https://github.com/opensearch-project/index-management/pull/477))
+* Remove HOST_DENY_LIST usage as Notification plugin will own it ([#488](https://github.com/opensearch-project/index-management/pull/488))
+* Deprecate Master nonmenclature ([#502](https://github.com/opensearch-project/index-management/pull/502))
+
+### Bug Fixes
+* Failed concurrent creates of ISM policies should return http 409 ([#464](https://github.com/opensearch-project/index-management/pull/464))
+* Disable detekt to avoid the CVE issues ([#500](https://github.com/opensearch-project/index-management/pull/500))
+
+### Infrastructure
+* Staging for version increment automation ([#409](https://github.com/opensearch-project/index-management/pull/409))
+
+### Maintenance
+* Version upgrade to 2.3.0 ([#484](https://github.com/opensearch-project/index-management/pull/484))
+
+### Documentation
+* Added 2.3 release note ([#507](https://github.com/opensearch-project/index-management/pull/507))

spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Validate.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.indexmanagement.spi.indexstatemanagement
+
+import org.opensearch.cluster.service.ClusterService
+import org.opensearch.common.io.stream.StreamInput
+import org.opensearch.common.io.stream.StreamOutput
+import org.opensearch.common.io.stream.Writeable
+import org.opensearch.common.settings.Settings
+import org.opensearch.monitor.jvm.JvmService
+import java.util.Locale
+
+abstract class Validate(
+    val settings: Settings,
+    val clusterService: ClusterService,
+    val jvmService: JvmService
+) {
+
+    var validationStatus = ValidationStatus.PASSED
+    var validationMessage: String? = "Starting Validation"
+
+    abstract fun execute(indexName: String): Validate
+
+    enum class ValidationStatus(val status: String) : Writeable {
+        PASSED("passed"),
+        RE_VALIDATING("re_validating"),
+        FAILED("failed");
+
+        override fun toString(): String {
+            return status
+        }
+
+        override fun writeTo(out: StreamOutput) {
+            out.writeString(status)
+        }
+
+        companion object {
+            fun read(streamInput: StreamInput): ValidationStatus {
+                return valueOf(streamInput.readString().uppercase(Locale.ROOT))
+            }
+        }
+    }
+}

spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ManagedIndexMetaData.kt

@@ -241,7 +241,6 @@ data class ManagedIndexMetaData(
             var action: ActionMetaData? = null
             var step: StepMetaData? = null
             var retryInfo: PolicyRetryInfoMetaData? = null
-
             var info: Map<String, Any>? = null
 
             XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)

spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ValidationResult.kt

@@ -0,0 +1,94 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.indexmanagement.spi.indexstatemanagement.model
+
+import org.opensearch.common.Strings
+import org.opensearch.common.io.stream.StreamInput
+import org.opensearch.common.io.stream.StreamOutput
+import org.opensearch.common.io.stream.Writeable
+import org.opensearch.common.xcontent.LoggingDeprecationHandler
+import org.opensearch.common.xcontent.NamedXContentRegistry
+import org.opensearch.common.xcontent.ToXContent
+import org.opensearch.common.xcontent.ToXContentFragment
+import org.opensearch.common.xcontent.XContentBuilder
+import org.opensearch.common.xcontent.XContentParser
+import org.opensearch.common.xcontent.XContentParserUtils
+import org.opensearch.common.xcontent.XContentType
+import org.opensearch.indexmanagement.spi.indexstatemanagement.Validate
+import java.io.ByteArrayInputStream
+import java.nio.charset.StandardCharsets
+import java.util.Locale
+
+data class ValidationResult(
+    val validationMessage: String,
+    val validationStatus: Validate.ValidationStatus
+) : Writeable, ToXContentFragment {
+
+    override fun writeTo(out: StreamOutput) {
+        out.writeString(validationMessage)
+        validationStatus.writeTo(out)
+    }
+
+    override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+        builder
+            .field(VALIDATION_MESSAGE, validationMessage)
+            .field(VALIDATION_STATUS, validationStatus.toString())
+        return builder
+    }
+
+    fun getMapValueString(): String {
+        return Strings.toString(this, false, false)
+    }
+
+    companion object {
+        const val VALIDATE = "validate"
+        const val VALIDATION_MESSAGE = "validation_message"
+        const val VALIDATION_STATUS = "validation_status"
+
+        fun fromStreamInput(si: StreamInput): ValidationResult {
+            val validationMessage: String? = si.readString()
+            val validationStatus: Validate.ValidationStatus? = Validate.ValidationStatus.read(si)
+
+            return ValidationResult(
+                requireNotNull(validationMessage) { "$VALIDATION_MESSAGE is null" },
+                requireNotNull(validationStatus) { "$VALIDATION_STATUS is null" }
+            )
+        }
+
+        fun fromManagedIndexMetaDataMap(map: Map<String, String?>): ValidationResult? {
+            val stepJsonString = map[VALIDATE]
+            return if (stepJsonString != null) {
+                val inputStream = ByteArrayInputStream(stepJsonString.toByteArray(StandardCharsets.UTF_8))
+                val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, inputStream)
+                parser.nextToken()
+                parse(parser)
+            } else {
+                null
+            }
+        }
+
+        fun parse(xcp: XContentParser): ValidationResult {
+            var validationMessage: String? = null
+            var validationStatus: Validate.ValidationStatus? = null
+
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)
+            while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
+                val fieldName = xcp.currentName()
+                xcp.nextToken()
+
+                when (fieldName) {
+                    VALIDATION_MESSAGE -> validationMessage = xcp.text()
+                    VALIDATION_STATUS -> validationStatus = Validate.ValidationStatus.valueOf(xcp.text().uppercase(Locale.ROOT))
+                }
+            }
+
+            return ValidationResult(
+                requireNotNull(validationMessage) { "$VALIDATION_MESSAGE is null" },
+                requireNotNull(validationStatus) { "$VALIDATION_STATUS is null" }
+            )
+        }
+    }
+}

src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt

@@ -37,6 +37,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementH
 import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator
 import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner
 import org.opensearch.indexmanagement.indexstatemanagement.MetadataService
+import org.opensearch.indexmanagement.indexstatemanagement.PluginVersionSweepCoordinator
 import org.opensearch.indexmanagement.indexstatemanagement.SkipExecution
 import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig
 import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
@@ -73,7 +74,8 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retr
 import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction
 import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction
 import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
-import org.opensearch.indexmanagement.migration.ISMTemplateService
+import org.opensearch.indexmanagement.indexstatemanagement.validation.ValidationService
+import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
 import org.opensearch.indexmanagement.refreshanalyzer.RefreshSearchAnalyzerAction
 import org.opensearch.indexmanagement.refreshanalyzer.RestRefreshSearchAnalyzerAction
 import org.opensearch.indexmanagement.refreshanalyzer.TransportRefreshSearchAnalyzerAction
@@ -186,6 +188,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
 
     private val logger = LogManager.getLogger(javaClass)
     lateinit var indexManagementIndices: IndexManagementIndices
+    lateinit var validationService: ValidationService
     lateinit var clusterService: ClusterService
     lateinit var indexNameExpressionResolver: IndexNameExpressionResolver
     lateinit var rollupInterceptor: RollupInterceptor
@@ -280,15 +283,15 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
         indexManagementExtensions.forEach { extension ->
             val extensionName = extension.getExtensionName()
             if (extensionName in extensions) {
-                throw IllegalStateException("Multiple extensions of IndexManagement have same name $extensionName - not supported")
+                error("Multiple extensions of IndexManagement have same name $extensionName - not supported")
             }
             extension.getISMActionParsers().forEach { parser ->
                 ISMActionsParser.instance.addParser(parser, extensionName)
             }
             indexMetadataServices.add(extension.getIndexMetadataService())
             extension.overrideClusterStateIndexUuidSetting()?.let {
                 if (customIndexUUIDSetting != null) {
-                    throw IllegalStateException(
+                    error(
                         "Multiple extensions of IndexManagement plugin overriding ClusterStateIndexUUIDSetting - not supported"
                     )
                 }
@@ -364,13 +367,13 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
             xContentRegistry,
             settings,
             indexNameExpressionResolver,
-            jvmService,
+            jvmService, // put jvm
             threadPool
         )
         fieldCapsFilter = FieldCapsFilter(clusterService, settings, indexNameExpressionResolver)
         this.indexNameExpressionResolver = indexNameExpressionResolver
 
-        val skipFlag = SkipExecution(client, clusterService)
+        val skipFlag = SkipExecution(client)
         RollupFieldValueExpressionResolver.registerScriptService(scriptService)
         val rollupRunner = RollupRunner
             .registerClient(client)
@@ -386,6 +389,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
             .registerConsumers()
             .registerClusterConfigurationProvider(skipFlag)
         indexManagementIndices = IndexManagementIndices(settings, client.admin().indices(), clusterService)
+        validationService = ValidationService(settings, clusterService, jvmService)
         val indexStateManagementHistory =
             IndexStateManagementHistory(
                 settings,
@@ -407,6 +411,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
         val managedIndexRunner = ManagedIndexRunner
             .registerClient(client)
             .registerClusterService(clusterService)
+            .registerValidationService(validationService)
             .registerNamedXContentRegistry(xContentRegistry)
             .registerScriptService(scriptService)
             .registerSettings(settings)
@@ -428,15 +433,19 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
 
         val smRunner = SMRunner.init(client, threadPool, settings, indexManagementIndices, clusterService)
 
+        val pluginVersionSweepCoordinator = PluginVersionSweepCoordinator(skipFlag, settings, threadPool, clusterService)
+
         return listOf(
             managedIndexRunner,
             rollupRunner,
             transformRunner,
             indexManagementIndices,
+            validationService,
             managedIndexCoordinator,
             indexStateManagementHistory,
             indexMetadataProvider,
-            smRunner
+            smRunner,
+            pluginVersionSweepCoordinator
         )
     }
 
@@ -454,13 +463,15 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
             ManagedIndexSettings.ROLLOVER_ALIAS,
             ManagedIndexSettings.ROLLOVER_SKIP,
             ManagedIndexSettings.INDEX_STATE_MANAGEMENT_ENABLED,
+            ManagedIndexSettings.VALIDATION_SERVICE_ENABLED,
             ManagedIndexSettings.METADATA_SERVICE_ENABLED,
             ManagedIndexSettings.AUTO_MANAGE,
             ManagedIndexSettings.METADATA_SERVICE_STATUS,
             ManagedIndexSettings.TEMPLATE_MIGRATION_CONTROL,
             ManagedIndexSettings.JITTER,
             ManagedIndexSettings.JOB_INTERVAL,
             ManagedIndexSettings.SWEEP_PERIOD,
+            ManagedIndexSettings.SWEEP_SKIP_PERIOD,
             ManagedIndexSettings.COORDINATOR_BACKOFF_COUNT,
             ManagedIndexSettings.COORDINATOR_BACKOFF_MILLIS,
             ManagedIndexSettings.ALLOW_LIST,

src/main/kotlin/org/opensearch/indexmanagement/common/model/notification/Channel.kt

@@ -61,7 +61,7 @@ data class Channel(val id: String) : ToXContent, Writeable {
                 when (fieldName) {
                     ID -> id = xcp.text()
                     else -> {
-                        throw IllegalStateException("Unexpected field: $fieldName, while parsing Channel destination")
+                        error("Unexpected field: $fieldName, while parsing Channel destination")
                     }
                 }
             }

src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt

@@ -73,7 +73,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed
 import org.opensearch.indexmanagement.indexstatemanagement.util.isPolicyCompleted
 import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest
 import org.opensearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest
-import org.opensearch.indexmanagement.migration.ISMTemplateService
+import org.opensearch.indexmanagement.indexstatemanagement.migration.ISMTemplateService
 import org.opensearch.indexmanagement.opensearchapi.IndexManagementSecurityContext
 import org.opensearch.indexmanagement.opensearchapi.contentParser
 import org.opensearch.indexmanagement.opensearchapi.parseFromSearchResponse