From d844a7fe45bcdc5a2f2f01dd0a308a4b57849d1b Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Sun, 26 May 2024 21:37:06 +0530 Subject: [PATCH 1/3] Custom toXContent implementations Signed-off-by: Shivansh Arora --- .../cluster/RepositoryCleanupInProgress.java | 36 +++ .../opensearch/cluster/RestoreInProgress.java | 211 ++++++++++++--- .../cluster/SnapshotDeletionsInProgress.java | 94 ++++++- .../cluster/SnapshotsInProgress.java | 243 ++++++++++++++++++ .../org/opensearch/repositories/IndexId.java | 24 ++ .../org/opensearch/snapshots/SnapshotId.java | 21 ++ 6 files changed, 596 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java index 72a3519aca6f8..636db24d1dbad 100644 --- a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java @@ -33,12 +33,15 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.repositories.RepositoryOperation; import java.io.IOException; @@ -101,6 +104,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field("repository", entry.repository); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field("repository_state_id", entry.repositoryStateId); + } // else we don't serialize it } builder.endObject(); } @@ -108,6 +114,36 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static RepositoryCleanupInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + XContentParserUtils.ensureFieldName(parser, parser.currentToken(), TYPE); + parser.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List entries = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + String repository = null; + long repositoryStateId = -1L; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + parser.nextToken(); + if ("repository".equals(currentFieldName)) { + repository = parser.text(); + } else if ("repository_state_id".equals(currentFieldName)) { + // only XContent parsed with {@link Metadata.CONTEXT_MODE_GATEWAY} will have the repository state id and can be deserialized + repositoryStateId = parser.longValue(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + entries.add(new Entry(repository, repositoryStateId)); + } + return new RepositoryCleanupInProgress(entries); + } + @Override public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this); diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java index 769f97373f7b7..627a04bd8277b 100644 --- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java @@ -32,18 +32,25 @@ package org.opensearch.cluster; +import org.elasticsearch.snapshots.SnapshotId; import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -52,6 +59,8 @@ import java.util.Objects; import java.util.UUID; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * Meta data about restore processes that are currently executing * @@ -144,7 +153,7 @@ public RestoreInProgress build() { * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class Entry { + public static class Entry implements ToXContentFragment { private final String uuid; private final State state; private final Snapshot snapshot; @@ -236,6 +245,135 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(uuid, snapshot, state, indices, shards); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean isGatewayXContent = params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API) + .equals(Metadata.CONTEXT_MODE_GATEWAY); + builder.startObject(); + builder.field("snapshot", snapshot().getSnapshotId().getName()); + builder.field("repository", snapshot().getRepository()); + builder.field("state", state()); + if (isGatewayXContent) { + builder.field("snapshot_uuid", snapshot().getSnapshotId().getUUID()); + builder.field("uuid", uuid()); + } + builder.startArray("indices"); + { + for (String index : indices()) { + builder.value(index); + } + } + builder.endArray(); + builder.startArray("shards"); + { + for (final Map.Entry shardEntry : shards.entrySet()) { + ShardId shardId = shardEntry.getKey(); + ShardRestoreStatus status = shardEntry.getValue(); + builder.startObject(); + { + builder.field("index", shardId.getIndex()); + builder.field("shard", shardId.getId()); + builder.field("state", status.state()); + if (isGatewayXContent) { + builder.field("index_uuid", shardId.getIndex().getUUID()); + if (status.nodeId() != null) builder.field("node_id", status.nodeId()); + if (status.reason() != null) builder.field("reason", status.reason()); + } + } + builder.endObject(); + } + } + + builder.endArray(); + builder.endObject(); + return builder; + } + + public static Entry fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String snapshotName = null; + String snapshotRepository = null; + String snapshotUUID = null; + int state = -1; + String uuid = null; + List indices = new ArrayList<>(); + Map shards = new HashMap<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case "snapshot": + snapshotName = parser.text(); + break; + case "repository": + snapshotRepository = parser.text(); + break; + case "state": + state = parser.intValue(); + break; + case "snapshot_uuid": + snapshotUUID = parser.text(); + break; + case "uuid": + uuid = parser.text(); + break; + case "indices": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + indices.add(parser.text()); + } + break; + case "shards": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String indexName = null; + String indexUUID = null; + int shardId = -1; + int restoreState = -1; + String nodeId = null; + String reason = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentShardFieldName = parser.currentName(); + parser.nextToken(); + switch (currentShardFieldName) { + case "index": + indexName = parser.text(); + break; + case "shard": + shardId = parser.intValue(); + break; + case "state": + restoreState = parser.intValue(); + break; + case "index_uuid": + indexUUID = parser.text(); + break; + case "node_id": + nodeId = parser.text(); + break; + case "reason": + reason = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + shards.put(new ShardId(indexName, indexUUID, shardId), new ShardRestoreStatus(nodeId, State.fromValue((byte) restoreState), reason)); + } + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new Entry(uuid, new Snapshot(snapshotRepository, new SnapshotId(snapshotName, snapshotUUID)), State.fromValue((byte) state), indices, shards); + } } /** @@ -495,46 +633,57 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startArray("snapshots"); for (final Entry entry : entries.values()) { - toXContent(entry, builder); + toXContent(entry, builder, ToXContent.EMPTY_PARAMS); } builder.endArray(); return builder; } + public static RestoreInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + final Map entries = new HashMap<>(); + XContentParserUtils.ensureFieldName(parser, parser.currentToken(), "snapshots"); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + final Entry entry = Entry.fromXContent(parser); + entries.put(entry.uuid, entry); + } + return new RestoreInProgress(entries); + } + /** * Serializes single restore operation * * @param entry restore operation metadata * @param builder XContent builder + * @param params */ - public void toXContent(Entry entry, XContentBuilder builder) throws IOException { - builder.startObject(); - builder.field("snapshot", entry.snapshot().getSnapshotId().getName()); - builder.field("repository", entry.snapshot().getRepository()); - builder.field("state", entry.state()); - builder.startArray("indices"); - { - for (String index : entry.indices()) { - builder.value(index); - } - } - builder.endArray(); - builder.startArray("shards"); - { - for (final Map.Entry shardEntry : entry.shards.entrySet()) { - ShardId shardId = shardEntry.getKey(); - ShardRestoreStatus status = shardEntry.getValue(); - builder.startObject(); - { - builder.field("index", shardId.getIndex()); - builder.field("shard", shardId.getId()); - builder.field("state", status.state()); - } - builder.endObject(); - } - } - - builder.endArray(); - builder.endObject(); + public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + entry.toXContent(builder, params); } } + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index e33245e02f75c..fed4332e99361 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -34,12 +34,14 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.UUIDs; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.snapshots.SnapshotId; @@ -52,6 +54,9 @@ import java.util.Objects; import java.util.Set; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; + /** * A class that represents the snapshot deletions that are in progress in the cluster. * @@ -72,7 +77,7 @@ private SnapshotDeletionsInProgress(List entries) { assert assertNoConcurrentDeletionsForSameRepository(entries); } - public static SnapshotDeletionsInProgress of(List entries) { + public static SnapshotDeletionsInProgress of(List entries) { if (entries.isEmpty()) { return EMPTY; } @@ -190,11 +195,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository", entry.repository()); builder.startArray("snapshots"); for (SnapshotId snapshot : entry.snapshots) { - builder.value(snapshot.getName()); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.startObject(); + builder.field("name", snapshot.getName()); + builder.field("uuid", snapshot.getUUID()); + builder.endObject(); + } else { + builder.value(snapshot.getName()); + } } builder.endArray(); builder.humanReadableField("start_time_millis", "start_time", new TimeValue(entry.startTime)); builder.field("repository_state_id", entry.repositoryStateId); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field("state", entry.state().value); + } // else we don't serialize it } builder.endObject(); } @@ -202,6 +217,70 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { + ensureFieldName(parser, parser.currentToken(), TYPE); + parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List entries = new ArrayList<>(); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String repository = null; + long repositoryStateId = -1; + byte stateValue = -1; + List snapshotIds = new ArrayList<>(); + TimeValue startTime = null; + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case "repository": + repository = parser.text(); + break; + case "snapshots": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + String name = null; + String uuid = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String currentFieldName = parser.currentName(); + parser.nextToken(); + if ("name".equals(currentFieldName)) { + name = parser.text(); + } else if ("uuid".equals(currentFieldName)) { + uuid = parser.text(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + snapshotIds.add(new SnapshotId(name, uuid)); + } + break; + case "start_time_millis": + startTime = TimeValue.timeValueMillis(parser.longValue()); + break; + case "start_time": + startTime = TimeValue.parseTimeValue(parser.text(), "start_time"); + break; + case "repository_state_id": + repositoryStateId = parser.longValue(); + break; + case "state": + stateValue = (byte) parser.intValue(); + break; + default: + throw new IllegalArgumentException("unknown field [" + fieldName + "]"); + } + } + assert startTime != null; + entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue))); + } + return SnapshotDeletionsInProgress.of(entries); + } + + @Override public String toString() { StringBuilder builder = new StringBuilder("SnapshotDeletionsInProgress["); @@ -389,6 +468,17 @@ public static State readFrom(StreamInput in) throws IOException { } } + public static State fromValue(byte value) { + switch (value) { + case 0: + return WAITING; + case 1: + return STARTED; + default: + throw new IllegalArgumentException("No snapshot delete state for value [" + value + "]"); + } + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeByte(value); diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index d658f38430dd9..700b25d3d1e54 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.unit.TimeValue; @@ -45,6 +46,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.repositories.RepositoryShardId; @@ -53,6 +55,7 @@ import org.opensearch.snapshots.SnapshotId; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -63,6 +66,10 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; + /** * Meta data about snapshots that are currently executing * @@ -732,16 +739,232 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SHARD, shardId.getId()); builder.field(STATE, status.state()); builder.field(NODE, status.nodeId()); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field(INDEX_UUID, shardId.getIndex().getUUID()); + if (status.generation() != null) builder.field(GENERATION, status.generation()); + if (status.reason() != null) builder.field(REASON, status.reason()); + } } builder.endObject(); } } builder.endArray(); builder.array(DATA_STREAMS, dataStreams.toArray(new String[0])); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field(VERSION, version); + if (failure != null) builder.field(FAILURE, failure); + if (source != null) { + builder.field(SOURCE, source); + } + builder.field(USER_METADATA, userMetadata); + builder.startArray(CLONES); + for (RepositoryShardId shardId : clones.keySet()) { + builder.startObject(); + builder.field(INDEX, shardId.index().getName()); + builder.field(INDEX_ID, shardId.index().getId()); + builder.field(SHARD, shardId.shardId()); + ShardSnapshotStatus status = clones.get(shardId); + if (status.nodeId() != null) builder.field(NODE, status.nodeId()); + builder.field(STATE, status.state()); + if (status.generation() != null) builder.field(GENERATION, status.generation()); + if (status.reason() != null) builder.field(REASON, status.reason()); + builder.endObject(); + } + builder.field(REMOTE_STORE_INDEX_SHALLOW_COPY, remoteStoreIndexShallowCopy); + } builder.endObject(); return builder; } + public static Entry fromXContent(XContentParser parser) throws IOException { + String repository = null; + String snapshotName = null; + String snapshotUUID = null; + boolean includeGlobalState = false; + boolean partial = false; + String failure = null; + Version version = null; + SnapshotId source = null; + Map metadata = null; + byte state = -1; + List indices = new ArrayList<>(); + long startTime = 0; + long repositoryStateId = -1L; + Map shards = new HashMap<>(); + List dataStreams = new ArrayList<>(); + Map clones = new HashMap<>(); + boolean remoteStoreIndexShallowCopy = false; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String currentFieldName = parser.currentName(); + parser.nextToken(); + + switch (currentFieldName) { + case REPOSITORY: + repository = parser.text(); + break; + case SNAPSHOT: + snapshotName = parser.text(); + break; + case UUID: + snapshotUUID = parser.text(); + break; + case INCLUDE_GLOBAL_STATE: + includeGlobalState = parser.booleanValue(); + break; + case PARTIAL: + partial = parser.booleanValue(); + break; + case STATE: + state = (byte) parser.intValue(); + break; + case INDICES: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + indices.add(IndexId.fromXContent(parser)); + } + break; + case START_TIME_MILLIS: + startTime = parser.longValue(); + break; + case START_TIME: + startTime = TimeValue.parseTimeValue(parser.text(), null, currentFieldName).millis(); + break; + case REPOSITORY_STATE_ID: + repositoryStateId = parser.longValue(); + break; + case SHARDS: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String index = null; + String indexUUID = null; + int shardId = -1; + String nodeId = null; + ShardState shardState = null; + String reason = null; + String generation = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + final String currentShardField = parser.currentName(); + parser.nextToken(); + switch (currentShardField) { + case INDEX: + index = parser.text(); + break; + case SHARD: + shardId = parser.intValue(); + break; + case INDEX_UUID: + indexUUID = parser.text(); + break; + case NODE: + nodeId = parser.text(); + break; + case STATE: + shardState = ShardState.fromValue((byte) parser.intValue()); + break; + case REASON: + reason = parser.text(); + break; + case GENERATION: + generation = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); + } + } + shards.put(new ShardId(index, indexUUID, shardId), + reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : + new ShardSnapshotStatus(nodeId, shardState, generation)); + } + break; + case DATA_STREAMS: + dataStreams = parseStringList(parser); + break; + case FAILURE: + failure = parser.text(); + break; + case SOURCE: + source = SnapshotId.fromXContent(parser); + break; + case USER_METADATA: + metadata = parser.map(); + break; + case CLONES: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String index = null; + String indexId = null; + int shardId = -1; + byte snapshotShardStatus = -1; + String nodeId = null; + String reason = null; + String generation = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case INDEX: + index = parser.text(); + break; + case INDEX_ID: + indexId = parser.text(); + break; + case SHARD: + shardId = parser.intValue(); + break; + case STATE: + snapshotShardStatus = (byte) parser.intValue(); + break; + case NODE: + nodeId = parser.text(); + break; + case REASON: + reason = parser.text(); + break; + case GENERATION: + generation = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + clones.put(new RepositoryShardId(new IndexId(index, indexId), shardId), + reason != null ? new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), reason, generation) : + new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), generation)); + } + break; + case VERSION: + version = Version.fromString(parser.text()); + break; + case REMOTE_STORE_INDEX_SHALLOW_COPY: + remoteStoreIndexShallowCopy = parser.booleanValue(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + Snapshot snapshot = new Snapshot(repository, new SnapshotId(snapshotName, snapshotUUID)); + return new Entry( + snapshot, + includeGlobalState, + partial, + State.fromValue(state), + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + failure, + metadata, + version, + source, + clones, + remoteStoreIndexShallowCopy + ); + } + @Override public void writeTo(StreamOutput out) throws IOException { snapshot.writeTo(out); @@ -1057,6 +1280,12 @@ public void writeTo(StreamOutput out) throws IOException { private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; private static final String PARTIAL = "partial"; private static final String STATE = "state"; + private static final String VERSION = "version"; + private static final String FAILURE = "failure"; + private static final String SOURCE = "source"; + private static final String USER_METADATA = "user_metadata"; + private static final String CLONES = "clones"; + private static final String REMOTE_STORE_INDEX_SHALLOW_COPY = "remote_store_index_shallow_copy"; private static final String INDICES = "indices"; private static final String DATA_STREAMS = "data_streams"; private static final String START_TIME_MILLIS = "start_time_millis"; @@ -1064,6 +1293,10 @@ public void writeTo(StreamOutput out) throws IOException { private static final String REPOSITORY_STATE_ID = "repository_state_id"; private static final String SHARDS = "shards"; private static final String INDEX = "index"; + private static final String INDEX_ID = "index_id"; + private static final String INDEX_UUID = "index_uuid"; + private static final String GENERATION = "generation"; + private static final String REASON = "reason"; private static final String SHARD = "shard"; private static final String NODE = "node"; @@ -1077,6 +1310,16 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par return builder; } + public static SnapshotsInProgress fromXContent(XContentParser parser) throws IOException { + ensureFieldName(parser, parser.currentToken(), SNAPSHOTS); + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); + List entries = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + entries.add(Entry.fromXContent(parser)); + } + return SnapshotsInProgress.of(entries); + } + /** * The shard state. * diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 87a0063e8c21b..1a2e797c62847 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -36,6 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; @@ -44,6 +45,8 @@ import java.io.IOException; import java.util.Objects; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * Represents a single snapshotted index in the repository. * @@ -133,4 +136,25 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.endObject(); return builder; } + + public static IndexId fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + String name = null; + String id = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case NAME: + name = parser.text(); + break; + case ID: + id = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new IndexId(name, id); + } } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java index 4eeb956a0cb19..31b18f14c854b 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java @@ -38,6 +38,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.Objects; @@ -145,4 +147,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + public static SnapshotId fromXContent(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String name = null; + String uuid = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String currentFieldName = parser.currentName(); + parser.nextToken(); + if (NAME.equals(currentFieldName)) { + name = parser.text(); + } else if (UUID.equals(currentFieldName)) { + uuid = parser.text(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new SnapshotId(name, uuid); + } } From 5a0cf23dcc27b4da3111473018393c908015abfa Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 4 Jun 2024 14:25:53 +0530 Subject: [PATCH 2/3] Fix snapshot de/ser Signed-off-by: Shivansh Arora --- .../cluster/SnapshotDeletionsInProgress.java | 13 +++- .../cluster/SnapshotsInProgress.java | 75 +++++++++++++++---- .../org/opensearch/repositories/IndexId.java | 2 +- ...SnapshotsInProgressSerializationTests.java | 52 ++++++++++++- 4 files changed, 122 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index fed4332e99361..b59e0d8214932 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -209,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository_state_id", entry.repositoryStateId); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { builder.field("state", entry.state().value); + builder.field("uuid", entry.uuid()); } // else we don't serialize it } builder.endObject(); @@ -218,6 +219,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), TYPE); parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -229,6 +236,7 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th byte stateValue = -1; List snapshotIds = new ArrayList<>(); TimeValue startTime = null; + String entryUUID = null; while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); final String fieldName = parser.currentName(); @@ -270,12 +278,15 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th case "state": stateValue = (byte) parser.intValue(); break; + case "uuid": + entryUUID = parser.text(); + break; default: throw new IllegalArgumentException("unknown field [" + fieldName + "]"); } } assert startTime != null; - entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue))); + entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue), entryUUID)); } return SnapshotDeletionsInProgress.of(entries); } diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 700b25d3d1e54..9a3f582995814 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -66,6 +67,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.common.xcontent.XContentUtils.readValue; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; @@ -740,7 +742,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATE, status.state()); builder.field(NODE, status.nodeId()); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { - builder.field(INDEX_UUID, shardId.getIndex().getUUID()); if (status.generation() != null) builder.field(GENERATION, status.generation()); if (status.reason() != null) builder.field(REASON, status.reason()); } @@ -770,6 +771,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (status.reason() != null) builder.field(REASON, status.reason()); builder.endObject(); } + builder.endArray(); builder.field(REMOTE_STORE_INDEX_SHALLOW_COPY, remoteStoreIndexShallowCopy); } builder.endObject(); @@ -786,7 +788,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { Version version = null; SnapshotId source = null; Map metadata = null; - byte state = -1; + State state = null; List indices = new ArrayList<>(); long startTime = 0; long repositoryStateId = -1L; @@ -794,11 +796,10 @@ public static Entry fromXContent(XContentParser parser) throws IOException { List dataStreams = new ArrayList<>(); Map clones = new HashMap<>(); boolean remoteStoreIndexShallowCopy = false; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String currentFieldName = parser.currentName(); parser.nextToken(); - switch (currentFieldName) { case REPOSITORY: repository = parser.text(); @@ -816,7 +817,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { partial = parser.booleanValue(); break; case STATE: - state = (byte) parser.intValue(); + state = State.fromString(parser.text()); break; case INDICES: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -836,32 +837,28 @@ public static Entry fromXContent(XContentParser parser) throws IOException { case SHARDS: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - String index = null; - String indexUUID = null; + Index index = null; int shardId = -1; String nodeId = null; ShardState shardState = null; String reason = null; String generation = null; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { final String currentShardField = parser.currentName(); parser.nextToken(); switch (currentShardField) { case INDEX: - index = parser.text(); + index = Index.fromXContent(parser); break; case SHARD: shardId = parser.intValue(); break; - case INDEX_UUID: - indexUUID = parser.text(); - break; case NODE: - nodeId = parser.text(); + nodeId = (String) readValue(parser, parser.currentToken()); break; case STATE: - shardState = ShardState.fromValue((byte) parser.intValue()); + shardState = ShardState.fromString(parser.text()); break; case REASON: reason = parser.text(); @@ -873,7 +870,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); } } - shards.put(new ShardId(index, indexUUID, shardId), + shards.put(new ShardId(index, shardId), reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : new ShardSnapshotStatus(nodeId, shardState, generation)); } @@ -950,7 +947,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { snapshot, includeGlobalState, partial, - State.fromValue(state), + state, indices, dataStreams, startTime, @@ -1203,6 +1200,25 @@ public static State fromValue(byte value) { throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); } } + + public static State fromString(String value) { + switch(value) { + case "INIT": + return INIT; + case "STARTED": + return STARTED; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "PARTIAL": + return PARTIAL; + default: + throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); + } + } } private final List entries; @@ -1311,6 +1327,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static SnapshotsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), SNAPSHOTS); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); List entries = new ArrayList<>(); @@ -1380,5 +1402,26 @@ public static ShardState fromValue(byte value) { throw new IllegalArgumentException("No shard snapshot state for value [" + value + "]"); } } + + public static ShardState fromString(String state) { + switch (state) { + case "INIT": + return INIT; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "MISSING": + return MISSING; + case "WAITING": + return WAITING; + case "QUEUED": + return QUEUED; + default: + throw new IllegalArgumentException("No shard snapshot state for value [" + state + "]"); + } + } } } diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 1a2e797c62847..2f97368d42038 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -138,7 +138,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static IndexId fromXContent(XContentParser parser) throws IOException { - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); String name = null; String id = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index 8fd1f44286094..b19d84815ded0 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -36,17 +36,27 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.SnapshotsInProgress.Entry; import org.opensearch.cluster.SnapshotsInProgress.ShardState; import org.opensearch.cluster.SnapshotsInProgress.State; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.IndexId; import org.opensearch.test.AbstractDiffableWireSerializationTestCase; import org.opensearch.test.VersionUtils; @@ -60,6 +70,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static java.lang.Math.abs; +import static java.util.Collections.singletonMap; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.opensearch.test.VersionUtils.randomVersion; public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase { @@ -84,7 +97,7 @@ private Entry randomSnapshot() { for (int i = 0; i < numberOfIndices; i++) { indices.add(new IndexId(randomAlphaOfLength(10), randomAlphaOfLength(10))); } - long startTime = randomLong(); + long startTime = abs(randomLong()); long repositoryStateId = randomLong(); Map builder = new HashMap<>(); final List esIndices = indices.stream() @@ -183,6 +196,41 @@ protected Custom mutateInstance(Custom instance) { return SnapshotsInProgress.of(entries); } + public void testToXContent() throws IOException { + SnapshotsInProgress sip = SnapshotsInProgress.of(List.of(randomSnapshot(), randomSnapshot())); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotsInProgress parsed = SnapshotsInProgress.fromXContent(parser); + assertEquals(sip, parsed); + } + } + + public void testToXContent_deletion() throws IOException { + SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(List.of(new SnapshotId("name1", "uuid1")), "repo", 10000000L, 10000L, SnapshotDeletionsInProgress.State.WAITING); + SnapshotDeletionsInProgress sdip = SnapshotDeletionsInProgress.of(List.of(entry)); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sdip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotDeletionsInProgress parsed = SnapshotDeletionsInProgress.fromXContent(parser); + assertEquals(sdip, parsed); + } + } + public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry( new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), @@ -191,7 +239,7 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgressSerializationTests.randomState(Map.of()), Collections.emptyList(), Collections.emptyList(), - Math.abs(randomLong()), + abs(randomLong()), randomIntBetween(0, 1000), Map.of(), null, From 5a2cc2098f4546ac37e3a8b3e722ea4a64fb8453 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 5 Jun 2024 19:12:52 +0530 Subject: [PATCH 3/3] Add UT Signed-off-by: Shivansh Arora --- .../core/xcontent/XContentParserUtils.java | 12 +++ .../cluster/RepositoryCleanupInProgress.java | 3 +- .../opensearch/cluster/RestoreInProgress.java | 38 +++------- .../cluster/SnapshotDeletionsInProgress.java | 1 - .../cluster/SnapshotsInProgress.java | 20 +++-- .../org/opensearch/repositories/IndexId.java | 2 +- .../org/opensearch/snapshots/SnapshotId.java | 3 + .../opensearch/snapshots/SnapshotIdTests.java | 74 +++++++++++++++++++ ...SnapshotsInProgressSerializationTests.java | 9 ++- 9 files changed, 123 insertions(+), 39 deletions(-) create mode 100644 server/src/test/java/org/opensearch/snapshots/SnapshotIdTests.java diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java index b10be393f9adb..a543556baa262 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java @@ -38,6 +38,8 @@ import org.opensearch.core.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.function.Consumer; @@ -178,4 +180,14 @@ public static void parseTypedKeysObject(XContentParser parser, String delimi throw new ParsingException(parser.getTokenLocation(), "Failed to parse object: empty key"); } } + + public static List parseStringList(XContentParser parser) throws IOException { + List valueList = new ArrayList<>(); + ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != Token.END_ARRAY) { + ensureExpectedToken(Token.VALUE_STRING, parser.currentToken(), parser); + valueList.add(parser.text()); + } + return valueList; + } } diff --git a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java index 636db24d1dbad..cffe85cc37a02 100644 --- a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java @@ -133,7 +133,8 @@ public static RepositoryCleanupInProgress fromXContent(XContentParser parser) th if ("repository".equals(currentFieldName)) { repository = parser.text(); } else if ("repository_state_id".equals(currentFieldName)) { - // only XContent parsed with {@link Metadata.CONTEXT_MODE_GATEWAY} will have the repository state id and can be deserialized + // only XContent parsed with {@link Metadata.CONTEXT_MODE_GATEWAY} will have the repository state id and can be + // deserialized repositoryStateId = parser.longValue(); } else { throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java index 627a04bd8277b..dd3f7e5227bfa 100644 --- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java @@ -32,7 +32,6 @@ package org.opensearch.cluster; -import org.elasticsearch.snapshots.SnapshotId; import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.metadata.Metadata; @@ -365,14 +364,23 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); } } - shards.put(new ShardId(indexName, indexUUID, shardId), new ShardRestoreStatus(nodeId, State.fromValue((byte) restoreState), reason)); + shards.put( + new ShardId(indexName, indexUUID, shardId), + new ShardRestoreStatus(nodeId, State.fromValue((byte) restoreState), reason) + ); } break; default: throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); } } - return new Entry(uuid, new Snapshot(snapshotRepository, new SnapshotId(snapshotName, snapshotUUID)), State.fromValue((byte) state), indices, shards); + return new Entry( + uuid, + new Snapshot(snapshotRepository, new SnapshotId(snapshotName, snapshotUUID)), + State.fromValue((byte) state), + indices, + shards + ); } } @@ -663,27 +671,3 @@ public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params p entry.toXContent(builder, params); } } - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index b59e0d8214932..d33f652a5d6f9 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -291,7 +291,6 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th return SnapshotDeletionsInProgress.of(entries); } - @Override public String toString() { StringBuilder builder = new StringBuilder("SnapshotDeletionsInProgress["); diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 9a3f582995814..9e84278391c86 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -870,9 +870,12 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); } } - shards.put(new ShardId(index, shardId), - reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : - new ShardSnapshotStatus(nodeId, shardState, generation)); + shards.put( + new ShardId(index, shardId), + reason != null + ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) + : new ShardSnapshotStatus(nodeId, shardState, generation) + ); } break; case DATA_STREAMS: @@ -927,9 +930,12 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); } } - clones.put(new RepositoryShardId(new IndexId(index, indexId), shardId), - reason != null ? new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), reason, generation) : - new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), generation)); + clones.put( + new RepositoryShardId(new IndexId(index, indexId), shardId), + reason != null + ? new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), reason, generation) + : new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), generation) + ); } break; case VERSION: @@ -1202,7 +1208,7 @@ public static State fromValue(byte value) { } public static State fromString(String value) { - switch(value) { + switch (value) { case "INIT": return INIT; case "STARTED": diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 2f97368d42038..d083cadbb9738 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -36,11 +36,11 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java index 31b18f14c854b..d74895de4caf7 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java @@ -149,6 +149,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static SnapshotId fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); String name = null; String uuid = null; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotIdTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotIdTests.java new file mode 100644 index 0000000000000..56ba35863e420 --- /dev/null +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotIdTests.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.XContentTestUtils; + +import java.io.IOException; +import java.util.Collections; + +public class SnapshotIdTests extends OpenSearchTestCase { + public void testToXContent() throws IOException { + SnapshotId snapshotId = new SnapshotId("repo", "snapshot"); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + snapshotId.toXContent(builder, null); + assertEquals("{\n" + " \"name\" : \"repo\",\n" + " \"uuid\" : \"snapshot\"\n" + "}", builder.toString()); + } + + public void testFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + public void testFromXContentWithRandomField() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + SnapshotId snapshotId = new SnapshotId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10)); + boolean humanReadable = randomBoolean(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + snapshotId, + mediaType, + new ToXContent.MapParams(Collections.emptyMap()), + humanReadable + ); + + if (addRandomFields) { + String unsupportedField = "unsupported_field"; + BytesReference mutated = BytesReference.bytes( + XContentTestUtils.insertIntoXContent( + mediaType.xContent(), + originalBytes, + Collections.singletonList(""), + () -> unsupportedField, + () -> randomAlphaOfLengthBetween(3, 10) + ) + ); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> SnapshotId.fromXContent(createParser(mediaType.xContent(), mutated)) + ); + assertEquals("unknown field [" + unsupportedField + "]", iae.getMessage()); + } else { + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotId parsedSnapshotId = SnapshotId.fromXContent(parser); + assertEquals(snapshotId, parsedSnapshotId); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index b19d84815ded0..e4cd8b6b11ab6 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -42,7 +42,6 @@ import org.opensearch.cluster.SnapshotsInProgress.ShardState; import org.opensearch.cluster.SnapshotsInProgress.State; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.json.JsonXContent; @@ -214,7 +213,13 @@ public void testToXContent() throws IOException { } public void testToXContent_deletion() throws IOException { - SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(List.of(new SnapshotId("name1", "uuid1")), "repo", 10000000L, 10000L, SnapshotDeletionsInProgress.State.WAITING); + SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry( + List.of(new SnapshotId("name1", "uuid1")), + "repo", + 10000000L, + 10000L, + SnapshotDeletionsInProgress.State.WAITING + ); SnapshotDeletionsInProgress sdip = SnapshotDeletionsInProgress.of(List.of(entry)); boolean humanReadable = false; XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();