Skip to content

Commit

Permalink
Add UTs
Browse files Browse the repository at this point in the history 
Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
@ashking94
ashking94 committed Aug 28, 2024
1 parent e21d937 commit c04eb67
Show file tree
Hide file tree
Showing 8 changed files with 383 additions and 133 deletions.
4 changes: 4 additions & 0 deletions server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ BlobPath hashPath() {
return BlobPath.cleanPath().add(shardId).add(indexUUID());
}

public String shardId() {
return shardId;
}

/**
* Returns a new builder for {@link SnapshotShardPathInput}.
*/
Expand Down
17 changes: 12 additions & 5 deletions server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2076,17 +2076,24 @@ private Set<String> writeNewIndexShardPaths(
private String writeIndexShardPaths(IndexId indexId, SnapshotId snapshotId, int shardCount) {
try {
List<String> paths = getShardPaths(indexId, shardCount);
String pathType = String.valueOf(indexId.getShardPathType());
String pathHashAlgorithm = String.valueOf(FNV_1A_COMPOSITE_1.getCode());
int pathType = indexId.getShardPathType();
int pathHashAlgorithm = FNV_1A_COMPOSITE_1.getCode();
String blobName = String.join(
SnapshotShardPaths.DELIMITER,
indexId.getId(),
indexId.getName(),
String.valueOf(shardCount),
pathType,
pathHashAlgorithm
String.valueOf(pathType),
String.valueOf(pathHashAlgorithm)
);
SnapshotShardPaths shardPaths = new SnapshotShardPaths(
paths,
indexId.getId(),
indexId.getName(),
shardCount,
PathType.fromCode(pathType),
PathHashAlgorithm.fromCode(pathHashAlgorithm)
);
SnapshotShardPaths shardPaths = new SnapshotShardPaths(paths);
SNAPSHOT_SHARD_PATHS_FORMAT.writeAsyncWithUrgentPriority(
shardPaths,
snapshotShardPathBlobContainer(),
Expand Down
89 changes: 41 additions & 48 deletions server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,13 @@

package org.opensearch.snapshots;

import com.fasterxml.jackson.core.JsonParseException;

import org.opensearch.OpenSearchParseException;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
Expand All @@ -34,19 +31,49 @@ public class SnapshotShardPaths implements ToXContent {
public static final String FILE_NAME_FORMAT = "%s";

private static final String PATHS_FIELD = "paths";
private static final String INDEX_ID_FIELD = "indexId";
private static final String INDEX_NAME_FIELD = "indexName";
private static final String NUMBER_OF_SHARDS_FIELD = "number_of_shards";
private static final String SHARD_PATH_TYPE_FIELD = "shard_path_type";
private static final String SHARD_PATH_HASH_ALGORITHM_FIELD = "shard_path_hash_algorithm";

private final List<String> paths;

public SnapshotShardPaths(List<String> paths) {
this.paths = Collections.unmodifiableList(paths);
}

public List<String> getPaths() {
return paths;
private final String indexId;
private final String indexName;
private final int numberOfShards;
private final PathType shardPathType;
private final PathHashAlgorithm shardPathHashAlgorithm;

public SnapshotShardPaths(
List<String> paths,
String indexId,
String indexName,
int numberOfShards,
PathType shardPathType,
PathHashAlgorithm shardPathHashAlgorithm
) {
assert !paths.isEmpty() : "paths must not be empty";
assert indexId != null && !indexId.isEmpty() : "indexId must not be empty";
assert indexName != null && !indexName.isEmpty() : "indexName must not be empty";
assert numberOfShards > 0 : "numberOfShards must be > 0";
assert shardPathType != null : "shardPathType must not be null";
assert shardPathHashAlgorithm != null : "shardPathHashAlgorithm must not be null";

this.paths = paths;
this.indexId = indexId;
this.indexName = indexName;
this.numberOfShards = numberOfShards;
this.shardPathType = shardPathType;
this.shardPathHashAlgorithm = shardPathHashAlgorithm;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(INDEX_ID_FIELD, indexId);
builder.field(INDEX_NAME_FIELD, indexName);
builder.field(NUMBER_OF_SHARDS_FIELD, numberOfShards);
builder.field(SHARD_PATH_TYPE_FIELD, shardPathType.getCode());
builder.field(SHARD_PATH_HASH_ALGORITHM_FIELD, shardPathHashAlgorithm.getCode());
builder.startArray(PATHS_FIELD);
for (String path : paths) {
builder.value(path);
Expand All @@ -55,41 +82,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder;
}

public static SnapshotShardPaths fromXContent(XContentParser parser) throws IOException {
List<String> paths = new ArrayList<>();

try {
XContentParser.Token token = parser.currentToken();
if (token == null) {
token = parser.nextToken();
}

if (token != XContentParser.Token.START_OBJECT) {
throw new OpenSearchParseException("Expected a start object");
}

token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
throw new OpenSearchParseException("Missing [" + PATHS_FIELD + "] field");
}

while (token != XContentParser.Token.END_OBJECT) {
String fieldName = parser.currentName();
if (PATHS_FIELD.equals(fieldName)) {
if (parser.nextToken() != XContentParser.Token.START_ARRAY) {
throw new OpenSearchParseException("Expected an array for field [" + PATHS_FIELD + "]");
}
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
paths.add(parser.text());
}
} else {
throw new OpenSearchParseException("Unexpected field [" + fieldName + "]");
}
token = parser.nextToken();
}
} catch (JsonParseException e) {
throw new OpenSearchParseException("Failed to parse SnapshotIndexIdPaths", e);
}
return new SnapshotShardPaths(paths);
public static SnapshotShardPaths fromXContent(XContentParser ignored) {
throw new UnsupportedOperationException("SnapshotShardPaths.fromXContent() is not supported");
}
}
42 changes: 42 additions & 0 deletions server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.opensearch.index.remote.RemoteStoreEnums.DataType;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput;
import org.opensearch.index.remote.RemoteStorePathStrategy.SnapshotShardPathInput;
import org.opensearch.test.OpenSearchTestCase;

import java.util.ArrayList;
Expand Down Expand Up @@ -597,6 +598,47 @@ public void testGeneratePathForHashedInfixType() {
assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
}

public void testGeneratePathForSnapshotShardPathInput() {
BlobPath blobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij");
String indexUUID = "dsdkjsu8832njn";
String shardId = "10";
SnapshotShardPathInput pathInput = SnapshotShardPathInput.builder()
.basePath(blobPath)
.indexUUID(indexUUID)
.shardId(shardId)
.build();

// FIXED PATH
BlobPath result = FIXED.path(pathInput, null);
String expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
String actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_PREFIX - FNV_1A_COMPOSITE_1
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
expected = "_11001000010110/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_PREFIX - FNV_1A_BASE64
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
expected = "_yFiSl_VGGM/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_INFIX - FNV_1A_COMPOSITE_1
result = HASHED_INFIX.path(pathInput, FNV_1A_COMPOSITE_1);
expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_11001000010110/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);

// HASHED_INFIX - FNV_1A_BASE64
result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_yFiSl_VGGM/indices/dsdkjsu8832njn/10/";
actual = result.buildAsString();
assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected);
}

private String derivePath(String basePath, ShardDataPathInput pathInput) {
return "".equals(basePath)
? String.join(
Expand Down
40 changes: 40 additions & 0 deletions server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,46 @@ public void testFixedSubPath() {
.dataType(DATA)
.build();
assertEquals(BlobPath.cleanPath().add(INDEX_UUID).add(SHARD_ID).add(TRANSLOG.getName()).add(DATA.getName()), input2.fixedSubPath());
}

public void testSnapshotShardPathInput() {
assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().build());
assertThrows(
NullPointerException.class,
() -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().basePath(BASE_PATH).build()
);
assertThrows(
NullPointerException.class,
() -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().indexUUID(INDEX_UUID).build()
);
assertThrows(NullPointerException.class, () -> RemoteStorePathStrategy.SnapshotShardPathInput.builder().shardId(SHARD_ID).build());

RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BASE_PATH, input.basePath());
assertEquals(INDEX_UUID, input.indexUUID());
assertEquals(SHARD_ID, input.shardId());
}

public void testSnapshotShardPathInputFixedSubPath() {
RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BlobPath.cleanPath().add("indices").add(INDEX_UUID).add(SHARD_ID), input.fixedSubPath());
}

public void testSnapshotShardPathInputHashPath() {
RemoteStorePathStrategy.SnapshotShardPathInput input = RemoteStorePathStrategy.SnapshotShardPathInput.builder()
.basePath(BASE_PATH)
.indexUUID(INDEX_UUID)
.shardId(SHARD_ID)
.build();
assertEquals(BlobPath.cleanPath().add(SHARD_ID).add(INDEX_UUID), input.hashPath());
}

}
71 changes: 53 additions & 18 deletions server/src/test/java/org/opensearch/repositories/IndexIdTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,15 @@

package org.opensearch.repositories;

import org.opensearch.Version;
import org.opensearch.common.UUIDs;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.remote.RemoteStoreEnums;
import org.opensearch.test.OpenSearchTestCase;

import java.io.IOException;
Expand All @@ -52,51 +54,84 @@ public void testEqualsAndHashCode() {
// assert equals and hashcode
String name = randomAlphaOfLength(8);
String id = UUIDs.randomBase64UUID();
IndexId indexId1 = new IndexId(name, id);
IndexId indexId2 = new IndexId(name, id);
int shardPathType = randomIntBetween(0, 2);
IndexId indexId1 = new IndexId(name, id, shardPathType);
IndexId indexId2 = new IndexId(name, id, shardPathType);
assertEquals(indexId1, indexId2);
assertEquals(indexId1.hashCode(), indexId2.hashCode());
// assert equals when using index name for id
id = name;
indexId1 = new IndexId(name, id);
indexId2 = new IndexId(name, id);
indexId1 = new IndexId(name, id, shardPathType);
indexId2 = new IndexId(name, id, shardPathType);
assertEquals(indexId1, indexId2);
assertEquals(indexId1.hashCode(), indexId2.hashCode());
// assert not equals when name or id differ
indexId2 = new IndexId(randomAlphaOfLength(8), id);
// assert not equals when name, id, or shardPathType differ
indexId2 = new IndexId(randomAlphaOfLength(8), id, shardPathType);
assertNotEquals(indexId1, indexId2);
assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
indexId2 = new IndexId(name, UUIDs.randomBase64UUID());
indexId2 = new IndexId(name, UUIDs.randomBase64UUID(), shardPathType);
assertNotEquals(indexId1, indexId2);
assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
int newShardPathType = randomIntBetween(0, 2);
indexId2 = new IndexId(name, id, newShardPathType);
if (shardPathType == newShardPathType) {
assertEquals(indexId1, indexId2);
assertEquals(indexId1.hashCode(), indexId2.hashCode());
} else {
assertNotEquals(indexId1, indexId2);
assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
}
}

public void testSerialization() throws IOException {
IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID(), randomIntBetween(0, 2));
BytesStreamOutput out = new BytesStreamOutput();
out.setVersion(Version.CURRENT);
indexId.writeTo(out);
assertEquals(indexId, new IndexId(out.bytes().streamInput()));
}

public void testXContent() throws IOException {
IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
String name = randomAlphaOfLength(8);
String id = UUIDs.randomBase64UUID();
int shardPathType = randomIntBetween(0, 2);
IndexId indexId = new IndexId(name, id, shardPathType);
XContentBuilder builder = JsonXContent.contentBuilder();
indexId.toXContent(builder, ToXContent.EMPTY_PARAMS);
XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder));
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
String name = null;
String id = null;
String parsedName = null;
String parsedId = null;
int parsedShardPathType = -1;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
final String currentFieldName = parser.currentName();
parser.nextToken();
if (currentFieldName.equals(IndexId.NAME)) {
name = parser.text();
} else if (currentFieldName.equals(IndexId.ID)) {
id = parser.text();
switch (currentFieldName) {
case IndexId.NAME:
parsedName = parser.text();
break;
case IndexId.ID:
parsedId = parser.text();
break;
case IndexId.SHARD_PATH_TYPE:
parsedShardPathType = parser.intValue();
break;
}
}
assertNotNull(name);
assertNotNull(id);
assertEquals(indexId, new IndexId(name, id));
parser.close();
assertNotNull(parsedName);
assertNotNull(parsedId);
assertNotEquals(-1, parsedShardPathType);
assertEquals(name, parsedName);
assertEquals(id, parsedId);
assertEquals(shardPathType, parsedShardPathType);
}

public void testDefaultShardPathType() {
String name = randomAlphaOfLength(8);
String id = UUIDs.randomBase64UUID();
IndexId indexId = new IndexId(name, id);
assertEquals(IndexId.DEFAULT_SHARD_PATH_TYPE, indexId.getShardPathType());
assertEquals(RemoteStoreEnums.PathType.FIXED.getCode(), IndexId.DEFAULT_SHARD_PATH_TYPE);
}
}
Loading

0 comments on commit c04eb67

Please sign in to comment.