Skip to content

Commit

Permalink
refactoring for bwc
Browse files Browse the repository at this point in the history 
Signed-off-by: Himshikha Gupta <himshikh@amazon.com>
  • Loading branch information
Himshikha Gupta committed Jun 11, 2024
1 parent 8b36f07 commit fc4770c
Show file tree
Hide file tree
Showing 11 changed files with 106 additions and 80 deletions.
2 changes: 1 addition & 1 deletion ...rc/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ public CheckedRunnable<IOException> getIndexRoutingAsyncAction(

BlobPath indexRoutingPath = clusterBasePath.add(INDEX_ROUTING_PATH_TOKEN);
BlobPath path = pathType.path(
RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(),
RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID(indexRouting.getIndex().getUUID()).build(),
pathHashAlgo
);
final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path);
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
import org.opensearch.index.remote.RemoteStoreEnums.DataType;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.index.remote.RemoteStorePathStrategy.BasePathInput;
import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput;
import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput;

import java.io.IOException;
import java.util.Collections;
Expand Down Expand Up @@ -142,7 +142,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
DataCategory dataCategory = entry.getKey();
for (DataType type : entry.getValue()) {
for (int shardNo = 0; shardNo < shardCount; shardNo++) {
PathInput pathInput = ShardDataPathInput.builder()
BasePathInput pathInput = PathInput.builder()
.basePath(new BlobPath().add(basePath))
.indexUUID(indexUUID)
.shardId(Integer.toString(shardNo))
Expand Down
24 changes: 15 additions & 9 deletions server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.hash.FNV1a;
import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput;
import org.opensearch.index.remote.RemoteStorePathStrategy.BasePathInput;

import java.util.HashMap;
import java.util.Locale;
Expand Down Expand Up @@ -92,7 +92,7 @@ public String getName() {
public enum PathType {
FIXED(0) {
@Override
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
public BlobPath generatePath(BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type";
// Hash algorithm is not used in FIXED path type
return pathInput.basePath().add(pathInput.fixedSubPath());
Expand All @@ -105,7 +105,7 @@ boolean requiresHashAlgorithm() {
},
HASHED_PREFIX(1) {
@Override
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
public BlobPath generatePath(BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
return BlobPath.cleanPath().add(hashAlgorithm.hash(pathInput)).add(pathInput.basePath()).add(pathInput.fixedSubPath());
}
Expand All @@ -117,7 +117,7 @@ boolean requiresHashAlgorithm() {
},
HASHED_INFIX(2) {
@Override
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
public BlobPath generatePath(BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
return pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.fixedSubPath());
}
Expand Down Expand Up @@ -170,12 +170,18 @@ public static PathType fromCode(int code) {
* @param hashAlgorithm hashing algorithm.
* @return the blob path for the path input.
*/
public BlobPath path(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
public BlobPath path(BasePathInput pathInput, PathHashAlgorithm hashAlgorithm) {
pathInput.assertIsValid();
return generatePath(pathInput, hashAlgorithm);
}

protected abstract BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm);
// Added for BWC
public BlobPath path(RemoteStorePathStrategy.PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
pathInput.assertIsValid();
return generatePath(pathInput, hashAlgorithm);
}

protected abstract BlobPath generatePath(BasePathInput pathInput, PathHashAlgorithm hashAlgorithm);

abstract boolean requiresHashAlgorithm();

Expand Down Expand Up @@ -205,7 +211,7 @@ public enum PathHashAlgorithm {

FNV_1A_BASE64(0) {
@Override
String hash(PathInput pathInput) {
String hash(BasePathInput pathInput) {
StringBuilder input = new StringBuilder();
for (String path : pathInput.fixedSubPath().toArray()) {
input.append(path);
Expand All @@ -220,7 +226,7 @@ String hash(PathInput pathInput) {
*/
FNV_1A_COMPOSITE_1(1) {
@Override
String hash(PathInput pathInput) {
String hash(BasePathInput pathInput) {
StringBuilder input = new StringBuilder();
for (String path : pathInput.fixedSubPath().toArray()) {
input.append(path);
Expand Down Expand Up @@ -264,7 +270,7 @@ public static PathHashAlgorithm fromCode(int code) {
return CODE_TO_ENUM.get(code);
}

abstract String hash(PathInput pathInput);
abstract String hash(BasePathInput pathInput);

public static PathHashAlgorithm parseString(String pathHashAlgorithm) {
try {
Expand Down
46 changes: 33 additions & 13 deletions server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,16 @@ public String toString() {
return "RemoteStorePathStrategy{" + "type=" + type + ", hashAlgorithm=" + hashAlgorithm + '}';
}

public BlobPath generatePath(BasePathInput pathInput) {
return type.path(pathInput, hashAlgorithm);
}

// Added for BWC
public BlobPath generatePath(PathInput pathInput) {
return type.path(pathInput, hashAlgorithm);
}


/**
* Wrapper class for the path input required to generate path for remote store uploads. This input is composed of
* basePath and indexUUID.
Expand All @@ -79,11 +85,17 @@ public BlobPath generatePath(PathInput pathInput) {
*/
@PublicApi(since = "2.14.0")
@ExperimentalApi
public static class PathInput {
public static class BasePathInput {
private final BlobPath basePath;
private final String indexUUID;

public PathInput(Builder<?> builder) {
//Adding for BWC
public BasePathInput(BlobPath basePath, String indexUUID) {
this.basePath = basePath;
this.indexUUID = indexUUID;
}

public BasePathInput(Builder<?> builder) {
this.basePath = Objects.requireNonNull(builder.basePath);
this.indexUUID = Objects.requireNonNull(builder.indexUUID);
}
Expand All @@ -101,7 +113,7 @@ BlobPath fixedSubPath() {
}

/**
* Returns a new builder for {@link PathInput}.
* Returns a new builder for {@link BasePathInput}.
*/
public static Builder<?> builder() {
return new Builder<>();
Expand All @@ -112,7 +124,7 @@ public void assertIsValid() {
}

/**
* Builder for {@link PathInput}.
* Builder for {@link BasePathInput}.
*
* @opensearch.internal
*/
Expand All @@ -136,8 +148,8 @@ protected T self() {
return (T) this;
}

public PathInput build() {
return new PathInput(this);
public BasePathInput build() {
return new BasePathInput(this);
}
}
}
Expand All @@ -150,12 +162,20 @@ public PathInput build() {
*/
@PublicApi(since = "2.14.0")
@ExperimentalApi
public static class ShardDataPathInput extends PathInput {
public static class PathInput extends BasePathInput {
private final String shardId;
private final DataCategory dataCategory;
private final DataType dataType;

public ShardDataPathInput(Builder builder) {
//Adding for BWC
public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) {
super(basePath, indexUUID);
this.shardId = shardId;
this.dataCategory = dataCategory;
this.dataType = dataType;
}

public PathInput(Builder builder) {
super(builder);
this.shardId = Objects.requireNonNull(builder.shardId);
this.dataCategory = Objects.requireNonNull(builder.dataCategory);
Expand Down Expand Up @@ -186,20 +206,20 @@ BlobPath fixedSubPath() {
}

/**
* Returns a new builder for {@link ShardDataPathInput}.
* Returns a new builder for {@link PathInput}.
*/
public static Builder builder() {
return new Builder();
}

/**
* Builder for {@link ShardDataPathInput}.
* Builder for {@link PathInput}.
*
* @opensearch.internal
*/
@PublicApi(since = "2.14.0")
@ExperimentalApi
public static class Builder extends PathInput.Builder<Builder> {
public static class Builder extends BasePathInput.Builder<Builder> {
private String shardId;
private DataCategory dataCategory;
private DataType dataType;
Expand Down Expand Up @@ -234,8 +254,8 @@ protected Builder self() {
return this;
}

public ShardDataPathInput build() {
return new ShardDataPathInput(this);
public PathInput build() {
return new PathInput(this);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
BlobPath repositoryBasePath = blobStoreRepository.basePath();
String shardIdStr = String.valueOf(shardId.id());

RemoteStorePathStrategy.ShardDataPathInput dataPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder()
RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder()
.basePath(repositoryBasePath)
.indexUUID(indexUUID)
.shardId(shardIdStr)
Expand All @@ -80,7 +80,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
blobStoreRepository::maybeRateLimitRemoteDownloadTransfers
);

RemoteStorePathStrategy.ShardDataPathInput mdPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder()
RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder()
.basePath(repositoryBasePath)
.indexUUID(indexUUID)
.shardId(shardIdStr)
Expand Down
2 changes: 1 addition & 1 deletion ...r/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public static RemoteStoreMetadataLockManager newLockManager(
assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath();

RemoteStorePathStrategy.ShardDataPathInput lockFilesPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder()
RemoteStorePathStrategy.PathInput lockFilesPathInput = RemoteStorePathStrategy.PathInput.builder()
.basePath(repositoryBasePath)
.indexUUID(indexUUID)
.shardId(shardId)
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,15 +304,15 @@ public static TranslogTransferManager buildTranslogTransferManager(
assert Objects.nonNull(pathStrategy);
String indexUUID = shardId.getIndex().getUUID();
String shardIdStr = String.valueOf(shardId.id());
RemoteStorePathStrategy.ShardDataPathInput dataPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder()
RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder()
.basePath(blobStoreRepository.basePath())
.indexUUID(indexUUID)
.shardId(shardIdStr)
.dataCategory(TRANSLOG)
.dataType(DATA)
.build();
BlobPath dataPath = pathStrategy.generatePath(dataPathInput);
RemoteStorePathStrategy.ShardDataPathInput mdPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder()
RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder()
.basePath(blobStoreRepository.basePath())
.indexUUID(indexUUID)
.shardId(shardIdStr)
Expand Down
2 changes: 1 addition & 1 deletion ...r/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ private ClusterState createClusterState(String indexName) {
private BlobPath getPath() {
BlobPath indexRoutingPath = basePath.add(INDEX_ROUTING_PATH_TOKEN);
return RemoteStoreEnums.PathType.HASHED_PREFIX.path(
RemoteStorePathStrategy.PathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(),
RemoteStorePathStrategy.BasePathInput.builder().basePath(indexRoutingPath).indexUUID("uuid").build(),
RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64
);
}
Expand Down
Loading

0 comments on commit fc4770c

Please sign in to comment.