From b45ad8c07ad369e93fc750b22fcbde3e7c673209 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 27 Aug 2024 12:39:09 +0530 Subject: [PATCH] Address comments Signed-off-by: Arpit Bandejiya --- .../RemoteClusterStateCleanupManagerIT.java | 5 ++--- .../PublicationTransportHandler.java | 2 ++ .../RemoteClusterStateCleanupManager.java | 2 +- .../remote/RemoteClusterStateService.java | 12 ++++++++++++ .../gateway/remote/RemoteDownloadStats.java | 17 +++++++++++++++++ .../gateway/remote/RemotePersistenceStats.java | 8 ++++++++ .../RemoteClusterStateCleanupManagerTests.java | 2 +- .../remote/RemoteClusterStateServiceTests.java | 12 ++++++------ 8 files changed, 49 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index 7d2e24c777da3..a4b7b7114fdac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -254,9 +254,8 @@ private void verifyIndexRoutingFilesDeletion( assertNotNull(discoveryStats.getClusterStateStats()); for (PersistedStateStats persistedStateStats : discoveryStats.getClusterStateStats().getPersistenceStats()) { Map extendedFields = persistedStateStats.getExtendedFields(); - assertTrue(extendedFields.containsKey(RemotePersistenceStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT)); - long cleanupAttemptFailedCount = extendedFields.get(RemotePersistenceStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT) - .get(); + assertTrue(extendedFields.containsKey(RemoteUploadStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT)); + long cleanupAttemptFailedCount = extendedFields.get(RemoteUploadStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT).get(); assertEquals(0, cleanupAttemptFailedCount); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index b24a65f78e3a1..620f5f862f649 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -262,6 +262,7 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest } if (applyFullState == true) { + remoteClusterStateService.fullDownloadState(); logger.debug( () -> new ParameterizedMessage( "Downloading full cluster state for term {}, version {}, stateUUID {}", @@ -281,6 +282,7 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest lastSeenClusterState.set(clusterState); return response; } else { + remoteClusterStateService.diffDownloadState(); logger.debug( () -> new ParameterizedMessage( "Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 02db15477ff95..d9bd9669f138c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -81,7 +81,7 @@ public RemoteClusterStateCleanupManager( RemoteRoutingTableService remoteRoutingTableService ) { this.remoteClusterStateService = remoteClusterStateService; - this.remoteStateStats = remoteClusterStateService.getStats(); + this.remoteStateStats = remoteClusterStateService.getRemoteStateStats(); ClusterSettings clusterSettings = clusterService.getClusterSettings(); this.clusterApplierService = clusterService.getClusterApplierService(); this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2dd9d22dd932c..1d3899e71cdc2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1662,6 +1662,18 @@ public void readMetadataFailed() { remoteStateStats.stateDownloadFailed(); } + public void fullDownloadState() { + remoteStateStats.fullDownloadState(); + } + + public void diffDownloadState() { + remoteStateStats.diffDownloadState(); + } + + public RemotePersistenceStats getRemoteStateStats() { + return remoteStateStats; + } + public PersistedStateStats getUploadStats() { return remoteStateStats.getUploadStats(); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java index 37c47257838d7..8c1505ab5ed89 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java @@ -10,10 +10,27 @@ import org.opensearch.cluster.coordination.PersistedStateStats; +import java.util.concurrent.atomic.AtomicLong; + public class RemoteDownloadStats extends PersistedStateStats { static final String REMOTE_DOWNLOAD = "remote_download"; + static final String DIFF_DOWNLOAD = "diff_download"; + private AtomicLong diffDownloadCount = new AtomicLong(0); + static final String FULL_DOWNLOAD = "full_download"; + private AtomicLong fullDownloadCount = new AtomicLong(0); public RemoteDownloadStats() { super(REMOTE_DOWNLOAD); + addToExtendedFields(DIFF_DOWNLOAD, diffDownloadCount); + addToExtendedFields(FULL_DOWNLOAD, fullDownloadCount); + } + + public void diffDownloadState() { + diffDownloadCount.incrementAndGet(); } + + public void fullDownloadState() { + fullDownloadCount.incrementAndGet(); + } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 915c718947cc1..e9fe9a94c0377 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -81,4 +81,12 @@ public PersistedStateStats getDownloadStats() { return remoteDownloadStats; } + public void diffDownloadState() { + remoteDownloadStats.diffDownloadState(); + } + + public void fullDownloadState() { + remoteDownloadStats.fullDownloadState(); + } + } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 1b15973c6dc16..8e114c9a26534 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -144,7 +144,7 @@ public void setup() { remoteManifestManager = mock(RemoteManifestManager.class); remoteClusterStateService = mock(RemoteClusterStateService.class); when(remoteClusterStateService.getRemoteManifestManager()).thenReturn(remoteManifestManager); - when(remoteClusterStateService.getStats()).thenReturn(new RemotePersistenceStats()); + when(remoteClusterStateService.getRemoteStateStats()).thenReturn(new RemotePersistenceStats()); when(remoteClusterStateService.getThreadpool()).thenReturn(threadPool); when(remoteClusterStateService.getBlobStore()).thenReturn(blobStore); when(remoteClusterStateService.getBlobStoreRepository()).thenReturn(blobStoreRepository); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 168c74c7cb584..cd03d948f76ef 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -575,7 +575,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx RemoteStateTransferException.class, () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) ); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOException { @@ -587,7 +587,7 @@ public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOExc null ); Assert.assertThat(manifestDetails, nullValue()); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } public void testFailWriteIncrementalMetadataWhenTermChanged() { @@ -2600,10 +2600,10 @@ public void testRemoteStateStats() throws IOException { MANIFEST_CURRENT_CODEC_VERSION ).getClusterMetadataManifest(); - assertTrue(remoteClusterStateService.getStats() != null); - assertEquals(1, remoteClusterStateService.getStats().getSuccessCount()); - assertEquals(0, remoteClusterStateService.getStats().getCleanupAttemptFailedCount()); - assertEquals(0, remoteClusterStateService.getStats().getFailedCount()); + assertTrue(remoteClusterStateService.getUploadStats() != null); + assertEquals(1, remoteClusterStateService.getUploadStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getCleanupAttemptFailedCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getFailedCount()); } public void testRemoteRoutingTableNotInitializedWhenDisabled() {