fix bug Observability plugin calling createIndex from onNodeStarted m…

.github/workflows/opensearch-observability-test-and-build-workflow.yml

@@ -11,27 +11,29 @@ jobs:
   build-linux:
     needs: Get-CI-Image-Tag
     strategy:
+      # Run all jobs
+      fail-fast: false
       matrix:
         java: [11, 17, 21]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
-
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-      - uses: actions/checkout@v1
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
 
+      - uses: actions/checkout@v4
+
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
-
+          distribution: 'temurin'
+
       - name: Run Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests ..."
@@ -56,7 +58,7 @@ jobs:
           cp -r ./build/distributions/*.zip opensearch-observability-builds/
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: opensearch-observability-ubuntu-latest
           path: opensearch-observability-builds
@@ -71,11 +73,12 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
       - name: Build with Gradle
@@ -88,7 +91,7 @@ jobs:
           cp -r ./build/distributions/*.zip opensearch-observability-builds/
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: opensearch-observability-${{ matrix.os }}
           path: opensearch-observability-builds

src/main/kotlin/org/opensearch/observability/index/ObservabilityIndex.kt

@@ -96,32 +96,41 @@ internal object ObservabilityIndex : LifecycleListener() {
      */
     @Suppress("TooGenericExceptionCaught")
     private fun createIndex() {
-        if (!isIndexExists(INDEX_NAME)) {
-            val classLoader = ObservabilityIndex::class.java.classLoader
-            val indexMappingSource = classLoader.getResource(OBSERVABILITY_MAPPING_FILE_NAME)?.readText()!!
-            val indexSettingsSource = classLoader.getResource(OBSERVABILITY_SETTINGS_FILE_NAME)?.readText()!!
-            val request = CreateIndexRequest(INDEX_NAME)
-                .mapping(indexMappingSource, XContentType.YAML)
-                .settings(indexSettingsSource, XContentType.YAML)
-            try {
-                val actionFuture = client.admin().indices().create(request)
-                val response = actionFuture.actionGet(PluginSettings.operationTimeoutMs)
-                if (response.isAcknowledged) {
-                    log.info("$LOG_PREFIX:Index $INDEX_NAME creation Acknowledged")
-                    reindexNotebooks()
-                } else {
-                    error("$LOG_PREFIX:Index $INDEX_NAME creation not Acknowledged")
-                }
-            } catch (exception: ResourceAlreadyExistsException) {
-                log.warn("message: ${exception.message}")
-            } catch (exception: Exception) {
-                if (exception.cause !is ResourceAlreadyExistsException) {
-                    throw exception
+        try {
+            // wait for the cluster here until it will finish managed node election
+            while (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) {
+                log.info("Wait for cluster to be available ...")
+                TimeUnit.SECONDS.sleep(1)
+            }
+            if (!isIndexExists(INDEX_NAME)) {
+                val classLoader = ObservabilityIndex::class.java.classLoader
+                val indexMappingSource = classLoader.getResource(OBSERVABILITY_MAPPING_FILE_NAME)?.readText()!!
+                val indexSettingsSource = classLoader.getResource(OBSERVABILITY_SETTINGS_FILE_NAME)?.readText()!!
+                val request = CreateIndexRequest(INDEX_NAME)
+                    .mapping(indexMappingSource, XContentType.YAML)
+                    .settings(indexSettingsSource, XContentType.YAML)
+                try {
+                    val actionFuture = client.admin().indices().create(request)
+                    val response = actionFuture.actionGet(PluginSettings.operationTimeoutMs)
+                    if (response.isAcknowledged) {
+                        log.info("$LOG_PREFIX:Index $INDEX_NAME creation Acknowledged")
+                        reindexNotebooks()
+                    } else {
+                        error("$LOG_PREFIX:Index $INDEX_NAME creation not Acknowledged")
+                    }
+                } catch (exception: ResourceAlreadyExistsException) {
+                    log.warn("message: ${exception.message}")
+                } catch (exception: Exception) {
+                    if (exception.cause !is ResourceAlreadyExistsException) {
+                        throw exception
+                    }
                 }
+                this.mappingsUpdated = true
+            } else if (!this.mappingsUpdated) {
+                updateMappings()
             }
-            this.mappingsUpdated = true
-        } else if (!this.mappingsUpdated) {
-            updateMappings()
+        } catch (exception: Exception) {
+            log.error("Unexpected exception while initializing node $exception", exception)
         }
     }