Skip to content

Commit

Permalink
Add versioning for UploadedIndexMetadata (#14677)
Browse files Browse the repository at this point in the history 
* Add versioning for UploadedIndexMetadata
* Handle componentPrefix for backward compatibility

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
  • Loading branch information
@soosinha
soosinha committed Jul 11, 2024
1 parent 125b773 commit 2d8c68c
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 21 deletions.
65 changes: 52 additions & 13 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.gateway.remote.ClusterMetadataManifest.Builder;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -243,7 +244,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareBoolean(ConstructingObjectParser.constructorArg(), COMMITTED_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_FIELD
);
parser.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
Expand Down Expand Up @@ -277,7 +278,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD);
parser.declareObjectArray(
ConstructingObjectParser.constructorArg(),
(p, c) -> UploadedIndexMetadata.fromXContent(p),
(p, c) -> UploadedIndexMetadata.fromXContent(p, codec_version),
INDICES_ROUTING_FIELD
);
parser.declareNamedObject(
Expand Down Expand Up @@ -1112,16 +1113,30 @@ private static String componentPrefix(Object[] fields) {
return (String) fields[3];
}

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER = new ConstructingObjectParser<>(
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
"uploaded_index_metadata",
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
);

private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;

static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
declareParser(PARSER_V0, CODEC_V0);
declareParser(PARSER_V2, CODEC_V2);
}

private static void declareParser(ConstructingObjectParser<UploadedIndexMetadata, Void> parser, long codec_version) {
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD);
parser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD);
if (codec_version >= CODEC_V2) {
parser.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD);
}
}

static final String COMPONENT_PREFIX = "index--";
Expand All @@ -1130,15 +1145,32 @@ private static String componentPrefix(Object[] fields) {
private final String indexUUID;
private final String uploadedFilename;

private long codecVersion = CODEC_V2;

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX);
this(indexName, indexUUID, uploadedFileName, CODEC_V2);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, long codecVersion) {
this(indexName, indexUUID, uploadedFileName, COMPONENT_PREFIX, codecVersion);
}

public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) {
this(indexName, indexUUID, uploadedFileName, componentPrefix, CODEC_V2);
}

public UploadedIndexMetadata(
String indexName,
String indexUUID,
String uploadedFileName,
String componentPrefix,
long codecVersion
) {
this.componentPrefix = componentPrefix;
this.indexName = indexName;
this.indexUUID = indexUUID;
this.uploadedFilename = uploadedFileName;
this.codecVersion = codecVersion;
}

public UploadedIndexMetadata(StreamInput in) throws IOException {
Expand Down Expand Up @@ -1175,10 +1207,13 @@ public String getComponentPrefix() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())
.field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID())
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath())
.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
.field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath());
if (codecVersion >= CODEC_V2) {
builder.field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix());
}
return builder;
}

@Override
Expand Down Expand Up @@ -1214,9 +1249,13 @@ public String toString() {
return Strings.toString(MediaTypeRegistry.JSON, this);
}

public static UploadedIndexMetadata fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
public static UploadedIndexMetadata fromXContent(XContentParser parser, long codecVersion) throws IOException {
if (codecVersion >= CODEC_V2) {
return CURRENT_PARSER.parse(parser, null);
}
return PARSER_V0.parse(parser, null);
}

}

/**
Expand Down
23 changes: 21 additions & 2 deletions server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
public class ClusterMetadataManifestTests extends OpenSearchTestCase {

public void testClusterMetadataManifestXContentV0() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand All @@ -74,7 +74,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException {
}

public void testClusterMetadataManifestXContentV1() throws IOException {
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path");
UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V1);
ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder()
.clusterTerm(1L)
.stateVersion(1L)
Expand Down Expand Up @@ -619,6 +619,24 @@ public void testUploadedIndexMetadataSerializationEqualsHashCode() {
);
}

public void testUploadedIndexMetadataWithoutComponentPrefix() throws IOException {
final UploadedIndexMetadata originalUploadedIndexMetadata = new UploadedIndexMetadata(
"test-index",
"test-index-uuid",
"test_file_name",
CODEC_V1
);
final XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
originalUploadedIndexMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();

try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
final UploadedIndexMetadata fromXContentUploadedIndexMetadata = UploadedIndexMetadata.fromXContent(parser, 1L);
assertEquals(originalUploadedIndexMetadata, fromXContentUploadedIndexMetadata);
}
}

private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedIndexMetadata uploadedIndexMetadata) {
switch (randomInt(2)) {
case 0:
Expand All @@ -642,4 +660,5 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde
}
return uploadedIndexMetadata;
}

}
14 changes: 8 additions & 6 deletions server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2254,13 +2254,14 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
.stateVersion(1L)
.stateUUID("state-uuid")
.clusterUUID("cluster-uuid")
.codecVersion(CODEC_V2)
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.previousClusterUUID("prev-cluster-uuid")
.build();

BlobContainer blobContainer = mockBlobStoreObjects();
mockBlobContainer(blobContainer, expectedManifest, Map.of());
mockBlobContainer(blobContainer, expectedManifest, Map.of(), CODEC_V2);
when(blobContainer.readBlob(uploadedIndexMetadata.getUploadedFilename())).thenThrow(FileNotFoundException.class);

remoteClusterStateService.start();
Expand Down Expand Up @@ -2288,11 +2289,11 @@ public void testReadLatestMetadataManifestSuccess() throws IOException {
.clusterUUID("cluster-uuid")
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.codecVersion(ClusterMetadataManifest.CODEC_V0)
.codecVersion(CODEC_V2)
.previousClusterUUID("prev-cluster-uuid")
.build();

mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>());
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2);
remoteClusterStateService.start();
final ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
clusterState.getClusterName().value(),
Expand Down Expand Up @@ -2416,10 +2417,10 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
.nodeId("nodeA")
.opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
.previousClusterUUID("prev-cluster-uuid")
.codecVersion(ClusterMetadataManifest.CODEC_V0)
.codecVersion(CODEC_V2)
.build();

mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata), CODEC_V2);

Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestClusterState(
clusterState.getClusterName().value(),
Expand Down Expand Up @@ -2664,6 +2665,7 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx
.clusterUUID("cluster-uuid")
.previousClusterUUID("prev-cluster-uuid")
.routingTableVersion(1)
.codecVersion(CODEC_V2)
.indicesRouting(List.of(uploadedIndiceRoutingMetadata))
.build();

Expand Down Expand Up @@ -3081,7 +3083,7 @@ private void mockBlobContainerForGlobalMetadata(
FORMAT_PARAMS
);
when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes()));
if (codecVersion >= ClusterMetadataManifest.CODEC_V2) {
if (codecVersion >= CODEC_V2) {
String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename());
when(blobContainer.readBlob(COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer((invocationOnMock) -> {
BytesReference bytesReference = COORDINATION_METADATA_FORMAT.serialize(
Expand Down

0 comments on commit 2d8c68c

Please sign in to comment.