Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store Disk Threshold Ignore Setting in IndexMetadata #78672

Merged
merged 1 commit into from
Oct 5, 2021
Merged

Store Disk Threshold Ignore Setting in IndexMetadata #78672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
import org.elasticsearch.cluster.routing.IndexRouting;
import org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater;
import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
import org.elasticsearch.common.collect.ImmutableOpenIntMap;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.MapBuilder;
Expand Down Expand Up @@ -397,6 +398,8 @@ public static APIBlock readFrom(StreamInput input) throws IOException {

private final long creationDate;

private final boolean ignoreDiskWatermarks;

private IndexMetadata(
final Index index,
final long version,
Expand Down Expand Up @@ -425,7 +428,9 @@ private IndexMetadata(
final boolean isHidden,
final IndexLongFieldRange timestampRange,
final int priority,
final long creationDate) {
final long creationDate,
final boolean ignoreDiskWatermarks
) {

this.index = index;
this.version = version;
Expand Down Expand Up @@ -462,6 +467,7 @@ private IndexMetadata(
this.timestampRange = timestampRange;
this.priority = priority;
this.creationDate = creationDate;
this.ignoreDiskWatermarks = ignoreDiskWatermarks;
assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
}

Expand Down Expand Up @@ -556,6 +562,10 @@ public ActiveShardCount getWaitForActiveShards() {
return waitForActiveShards;
}

public boolean ignoreDiskWatermarks() {
return ignoreDiskWatermarks;
}

public Settings getSettings() {
return settings;
}
Expand Down Expand Up @@ -1329,7 +1339,8 @@ public IndexMetadata build() {
INDEX_HIDDEN_SETTING.get(settings),
timestampRange,
IndexMetadata.INDEX_PRIORITY_SETTING.get(settings),
settings.getAsLong(SETTING_CREATION_DATE, -1L)
settings.getAsLong(SETTING_CREATION_DATE, -1L),
DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS.get(settings)
);
}

Expand Down
12 changes: 6 additions & 6 deletions .../main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.snapshots.SnapshotShardSizeInfo;

import java.util.List;
import java.util.Set;

import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING;
Expand Down Expand Up @@ -114,15 +113,16 @@ public static long sizeOfRelocatingShards(RoutingNode node, boolean subtractShar
// no longer initializing because their recovery failed or was cancelled.

// Where reserved space is unavailable (e.g. stats are out-of-sync) compute a conservative estimate for initialising shards
final List<ShardRouting> initializingShards = node.shardsWithState(ShardRoutingState.INITIALIZING);
initializingShards.removeIf(shardRouting -> reservedSpace.containsShardId(shardRouting.shardId()));
for (ShardRouting routing : initializingShards) {
for (ShardRouting routing : node.shardsWithState(ShardRoutingState.INITIALIZING)) {
if (routing.relocatingNodeId() == null) {
// in practice the only initializing-but-not-relocating shards with a nonzero expected shard size will be ones created
// by a resize (shrink/split/clone) operation which we expect to happen using hard links, so they shouldn't be taking
// any additional space and can be ignored here
continue;
}
if (reservedSpace.containsShardId(routing.shardId())) {
continue;
}

final String actualPath = clusterInfo.getDataPath(routing);
// if we don't yet know the actual path of the incoming shard then conservatively assume it's going to the path with the least
Expand Down Expand Up @@ -162,7 +162,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
return decision;
}

if (SETTING_IGNORE_DISK_WATERMARKS.get(allocation.metadata().index(shardRouting.index()).getSettings())) {
if (allocation.metadata().index(shardRouting.index()).ignoreDiskWatermarks()) {
return YES_DISK_WATERMARKS_IGNORED;
}

Expand Down Expand Up @@ -333,7 +333,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
return decision;
}

if (SETTING_IGNORE_DISK_WATERMARKS.get(allocation.metadata().index(shardRouting.index()).getSettings())) {
if (allocation.metadata().index(shardRouting.index()).ignoreDiskWatermarks()) {
return YES_DISK_WATERMARKS_IGNORED;
}

Expand Down