Create Remote Object managers and use them in orchestration from Remo…

…teClusterStateService Signed-off-by: Shivansh Arora <hishiv@amazon.com>
opensearch-project · Jun 6, 2024 · 9b53c30 · 9b53c30
1 parent 3de5d3c
commit 9b53c30
Show file tree

Hide file tree

Showing 12 changed files with 1,368 additions and 1,025 deletions.
diff --git a/...c/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/...c/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
@@ -26,13 +26,13 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA;
-import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX;
+import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA;
+import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA;
+import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA;
+import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase {

diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java
@@ -8,16 +8,99 @@
 
 package org.opensearch.gateway.remote;
 
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 
 /**
  * Utility class for Remote Cluster State
  */
 public class RemoteClusterStateUtils {
     public static final String PATH_DELIMITER = "/";
+    public static final String METADATA_NAME_FORMAT = "%s.dat";
+    public static final String METADATA_FILE_PREFIX = "metadata";
+    public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state";
+    public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata";
+    public static final String DELIMITER = "__";
+
+    // ToXContent Params with gateway mode.
+    // We are using gateway context mode to persist all custom metadata.
+    public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(
+        Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)
+    );
+
+    public static BlobPath getCusterMetadataBasePath(BlobStoreRepository blobStoreRepository, String clusterName, String clusterUUID) {
+        return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID);
+    }
 
     public static String encodeString(String content) {
         return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8));
     }
+
+    public static String getFormattedFileName(String fileName, int codecVersion) {
+        return String.format(Locale.ROOT, METADATA_NAME_FORMAT, fileName);
+    }
+
+    static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepository, String clusterName) {
+        return blobStoreRepository.blobStore()
+            .blobContainer(
+                blobStoreRepository.basePath()
+                    .add(Base64.getUrlEncoder().withoutPadding().encodeToString(clusterName.getBytes(StandardCharsets.UTF_8)))
+                    .add(CLUSTER_STATE_PATH_TOKEN)
+            );
+    }
+
+    /**
+     * Exception for Remote state transfer.
+     */
+    public static class RemoteStateTransferException extends RuntimeException {
+
+        public RemoteStateTransferException(String errorDesc) {
+            super(errorDesc);
+        }
+
+        public RemoteStateTransferException(String errorDesc, Throwable cause) {
+            super(errorDesc, cause);
+        }
+    }
+
+    public static class UploadedMetadataResults {
+        List<ClusterMetadataManifest.UploadedIndexMetadata> uploadedIndexMetadata;
+        Map<String, ClusterMetadataManifest.UploadedMetadataAttribute> uploadedCustomMetadataMap;
+        ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata;
+        ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata;
+        ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata;
+
+        public UploadedMetadataResults(
+            List<ClusterMetadataManifest.UploadedIndexMetadata> uploadedIndexMetadata,
+            Map<String, ClusterMetadataManifest.UploadedMetadataAttribute> uploadedCustomMetadataMap,
+            ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata,
+            ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata,
+            ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata
+        ) {
+            this.uploadedIndexMetadata = uploadedIndexMetadata;
+            this.uploadedCustomMetadataMap = uploadedCustomMetadataMap;
+            this.uploadedCoordinationMetadata = uploadedCoordinationMetadata;
+            this.uploadedSettingsMetadata = uploadedSettingsMetadata;
+            this.uploadedTemplatesMetadata = uploadedTemplatesMetadata;
+        }
+
+        public UploadedMetadataResults() {
+            this.uploadedIndexMetadata = new ArrayList<>();
+            this.uploadedCustomMetadataMap = new HashMap<>();
+            this.uploadedCoordinationMetadata = null;
+            this.uploadedSettingsMetadata = null;
+            this.uploadedTemplatesMetadata = null;
+        }
+    }
 }