Merge branch 'main' into add-primary-term

Signed-off-by: Craig Perkins <cwperx@amazon.com>

.github/workflows/gradle-check.yml

@@ -12,13 +12,28 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
+  check-files:
+    runs-on: ubuntu-latest
+    outputs: 
+      RUN_GRADLE_CHECK: ${{ steps.changed-files-specific.outputs.any_changed }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get changed files
+      id: changed-files-specific
+      uses: tj-actions/changed-files@v44
+      with:
+        files_ignore: |
+          release-notes/*.md
+          .github/**
+          *.md
+    
   gradle-check:
-    if: github.repository == 'opensearch-project/OpenSearch'
+    needs: check-files
+    if: github.repository == 'opensearch-project/OpenSearch' && needs.check-files.outputs.RUN_GRADLE_CHECK == 'true'
     permissions:
       contents: read # to fetch code (actions/checkout)
       pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
       issues: write # To create an issue if check fails on push.
-
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -151,3 +166,12 @@ jobs:
         with:
           assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
           filename: .github/ISSUE_TEMPLATE/failed_check.md
+
+  check-result:
+    needs: [check-files, gradle-check]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if gradle-check fails
+        if: ${{ needs.check-files.outputs.RUN_GRADLE_CHECK && needs.gradle-check.result == 'failure' }}
+        run: exit 1

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
 - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
 - Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
+- [Remote State] Add async remote state deletion task running on an interval, configurable by a setting ([#13131](https://github.com/opensearch-project/OpenSearch/pull/13131))
 - Remove deprecated constant META_FIELDS_BEFORE_7DOT8 ([#13860](https://github.com/opensearch-project/OpenSearch/pull/13860))
 
 ### Dependencies

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0-snapshot-fb97840
+lucene            = 9.11.0-snapshot-4be6531
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9

libs/core/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+3c2361bd633374ae3814b175cc25ccf773f67026

modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+8752daf173a642ae02e081cc0398f2ce59278200

plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+12630ff9c56e2a372ba57f519c579ff9e728208a

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+752bfc61c7829be6c27d9c1764250196e2c6b06b

plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+5ca56d42b24498a226cf91f48b94e010b6af5fe2

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+8eb59a89aa8984457798ccffb8e97e5351bebc1f

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+851c1bd99eaef368e84335853dd448e4f56cdbc8

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+63647085d41ae231733580c20a498ce7c9134ce5

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+a3ba7dd03b1df9efed08eb544689d51d2be22aa5

server/licenses/lucene-analysis-common-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+9cc4e600289bf1171b47de74536bd34c476f85a8

server/licenses/lucene-backward-codecs-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+8babfe85be7e36c893741e08072c11e71db09715

server/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+3c2361bd633374ae3814b175cc25ccf773f67026

server/licenses/lucene-grouping-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+d9f29b49cd1e0a061ff7fa4a53e8605bd49bd3d0

server/licenses/lucene-highlighter-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+33bc26d46d62bb1cf3bf725db637226a43db7625

server/licenses/lucene-join-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+82966698abdb8f0367a162f642560566a6085dc8

server/licenses/lucene-memory-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+746f392e7ec27a7cd6ca2add7dd8441d2a6085da

server/licenses/lucene-misc-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+0eb06ecc39c0ec0db380a6e5aad1b16907e0bfd9

server/licenses/lucene-queries-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+0e56eb18cceffcd5ce2e47b679e873420254df74

server/licenses/lucene-queryparser-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+dee3997a72eeae905e92930f53e724b6bef279da

server/licenses/lucene-sandbox-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+946bc45b87b3d770ab6828b0d0a5f8684f2c3624

server/licenses/lucene-spatial-extras-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+d73667f61fb5e7fde4cec52fcfbbfd9847068aec

server/licenses/lucene-spatial3d-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+a8e8ab80bfb6abd70932e50fe31e13ecf2e00987

server/licenses/lucene-suggest-9.11.0-snapshot-4be6531.jar.sha1

@@ -0,0 +1 @@
+45d6f0facd45d4e49585f0dabfa62ed5a1883033

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java

@@ -0,0 +1,145 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.Before;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT;
+import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING;
+import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS;
+import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteClusterStateCleanupManagerIT extends RemoteStoreBaseIntegTestCase {
+
+    private static final String INDEX_NAME = "test-index";
+
+    @Before
+    public void setup() {
+        asyncUploadMockFsRepo = false;
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build();
+    }
+
+    private Map<String, Long> initialTestSetup(int shardCount, int replicaCount, int dataNodeCount, int clusterManagerNodeCount) {
+        prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount);
+        Map<String, Long> indexStats = indexData(1, false, INDEX_NAME);
+        assertEquals(shardCount * (replicaCount + 1), getNumShards(INDEX_NAME).totalNumShards);
+        ensureGreen(INDEX_NAME);
+        return indexStats;
+    }
+
+    public void testRemoteCleanupTaskUpdated() {
+        int shardCount = randomIntBetween(1, 2);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 1;
+
+        initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
+        RemoteClusterStateCleanupManager remoteClusterStateCleanupManager = internalCluster().getClusterManagerNodeInstance(
+            RemoteClusterStateCleanupManager.class
+        );
+
+        assertEquals(CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval());
+        assertTrue(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isScheduled());
+
+        // now disable
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1))
+            .get();
+
+        assertEquals(-1, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval().getMillis());
+        assertFalse(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isScheduled());
+
+        // now set Clean up interval to 1 min
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m"))
+            .get();
+        assertEquals(1, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval().getMinutes());
+    }
+
+    public void testRemoteCleanupDeleteStale() throws Exception {
+        int shardCount = randomIntBetween(1, 2);
+        int replicaCount = 1;
+        int dataNodeCount = shardCount * (replicaCount + 1);
+        int clusterManagerNodeCount = 1;
+
+        initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount);
+
+        // set cleanup interval to 100 ms to make the test faster
+        ClusterUpdateSettingsResponse response = client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "100ms"))
+            .get();
+
+        assertTrue(response.isAcknowledged());
+
+        // update cluster state 21 times to ensure that clean up has run after this will upload 42 manifest files
+        // to repository, if manifest files are less than that it means clean up has run
+        updateClusterStateNTimes(RETAINED_MANIFESTS + SKIP_CLEANUP_STATE_CHANGES + 1);
+
+        RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class);
+        BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME);
+        BlobPath baseMetadataPath = repository.basePath()
+            .add(
+                Base64.getUrlEncoder()
+                    .withoutPadding()
+                    .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8))
+            )
+            .add("cluster-state")
+            .add(getClusterState().metadata().clusterUUID());
+        BlobPath manifestContainerPath = baseMetadataPath.add("manifest");
+
+        assertBusy(() -> {
+            int manifestFiles = repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size();
+            logger.info("number of current manifest file: {}", manifestFiles);
+            // we can't guarantee that we have same number of manifest as Retained manifest in our repo as there can be other queued task
+            // other than replica count change which can upload new manifest files, that's why we check that number of manifests is between
+            // Retained manifests and Retained manifests + 2 * Skip cleanup state changes (each cluster state update uploads 2 manifests)
+            assertTrue(
+                "Current number of manifest files: " + manifestFiles,
+                manifestFiles >= RETAINED_MANIFESTS && manifestFiles < RETAINED_MANIFESTS + 2 * SKIP_CLEANUP_STATE_CHANGES
+            );
+        }, 500, TimeUnit.MILLISECONDS);
+    }
+
+    private void updateClusterStateNTimes(int n) {
+        int newReplicaCount = randomIntBetween(0, 3);
+        for (int i = n; i > 0; i--) {
+            ClusterUpdateSettingsResponse response = client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), i, TimeUnit.SECONDS))
+                .get();
+            assertTrue(response.isAcknowledged());
+        }
+    }
+}