Skip to content

Commit

Permalink
First steps of integrating converted settings into the cluster/index …
Browse files Browse the repository at this point in the history 
…settings infrastructure

The old infa has been removed in this commit such that nothing uses `DynamicSettings` anymore
and all index-scoped settings require to be registered before the node has fully started up.
  • Loading branch information
@s1monw
s1monw committed Jan 18, 2016
1 parent 211f80f commit 04045a0
Show file tree
Hide file tree
Showing 44 changed files with 342 additions and 713 deletions.
46 changes: 2 additions & 44 deletions core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,9 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.cluster.ClusterState.builder;

/**
Expand Down Expand Up @@ -57,11 +52,11 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
boolean changed = false;
Settings.Builder transientSettings = Settings.settingsBuilder();
transientSettings.put(currentState.metaData().transientSettings());
changed |= apply(transientToApply, transientSettings, transientUpdates, "transient");
changed |= clusterSettings.applyDynamicSettings(transientToApply, transientSettings, transientUpdates, "transient");

Settings.Builder persistentSettings = Settings.settingsBuilder();
persistentSettings.put(currentState.metaData().persistentSettings());
changed |= apply(persistentToApply, persistentSettings, persistentUpdates, "persistent");
changed |= clusterSettings.applyDynamicSettings(persistentToApply, persistentSettings, persistentUpdates, "persistent");

if (!changed) {
return currentState;
Expand All @@ -86,42 +81,5 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
return build;
}

private boolean apply(Settings toApply, Settings.Builder target, Settings.Builder updates, String type) {
boolean changed = false;
final Set<String> toRemove = new HashSet<>();
Settings.Builder settingsBuilder = Settings.settingsBuilder();
for (Map.Entry<String, String> entry : toApply.getAsMap().entrySet()) {
if (entry.getValue() == null) {
toRemove.add(entry.getKey());
} else if (clusterSettings.isLoggerSetting(entry.getKey()) || clusterSettings.hasDynamicSetting(entry.getKey())) {
settingsBuilder.put(entry.getKey(), entry.getValue());
updates.put(entry.getKey(), entry.getValue());
changed = true;
} else {
throw new IllegalArgumentException(type + " setting [" + entry.getKey() + "], not dynamically updateable");
}

}
changed |= applyDeletes(toRemove, target);
target.put(settingsBuilder.build());
return changed;
}

private final boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
boolean changed = false;
for (String entry : deletes) {
Set<String> keysToRemove = new HashSet<>();
Set<String> keySet = builder.internalMap().keySet();
for (String key : keySet) {
if (Regex.simpleMatch(entry, key)) {
keysToRemove.add(key);
}
}
for (String key : keysToRemove) {
builder.remove(key);
changed = true;
}
}
return changed;
}
}
15 changes: 0 additions & 15 deletions core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateFilter;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
Expand Down Expand Up @@ -55,19 +54,12 @@
import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
import org.elasticsearch.cluster.service.InternalClusterService;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.gateway.GatewayAllocator;
import org.elasticsearch.gateway.PrimaryShardAllocator;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.settings.IndexDynamicSettings;
import org.elasticsearch.index.MergePolicyConfig;

import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -98,7 +90,6 @@ public class ClusterModule extends AbstractModule {
SnapshotInProgressAllocationDecider.class));

private final Settings settings;
private final DynamicSettings.Builder indexDynamicSettings = new DynamicSettings.Builder();
private final ExtensionPoint.SelectedType<ShardsAllocator> shardsAllocators = new ExtensionPoint.SelectedType<>("shards_allocator", ShardsAllocator.class);
private final ExtensionPoint.ClassSet<AllocationDecider> allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class);
private final ExtensionPoint.ClassSet<IndexTemplateFilter> indexTemplateFilters = new ExtensionPoint.ClassSet<>("index_template_filter", IndexTemplateFilter.class);
Expand All @@ -115,10 +106,6 @@ public ClusterModule(Settings settings) {
registerShardsAllocator(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR, BalancedShardsAllocator.class);
}

public void registerIndexDynamicSetting(String setting, Validator validator) {
indexDynamicSettings.addSetting(setting, validator);
}

public void registerAllocationDecider(Class<? extends AllocationDecider> allocationDecider) {
allocationDeciders.registerExtension(allocationDecider);
}
Expand All @@ -133,8 +120,6 @@ public void registerIndexTemplateFilter(Class<? extends IndexTemplateFilter> ind

@Override
protected void configure() {
bind(DynamicSettings.class).annotatedWith(IndexDynamicSettings.class).toInstance(indexDynamicSettings.build());

// bind ShardsAllocator
String shardsAllocatorType = shardsAllocators.bindType(binder(), settings, ClusterModule.SHARDS_ALLOCATOR_TYPE_KEY, ClusterModule.BALANCED_ALLOCATOR);
if (shardsAllocatorType.equals(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR)) {
Expand Down
23 changes: 1 addition & 22 deletions core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ public static State fromString(String state) {
public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
public static final String SETTING_VERSION_MINIMUM_COMPATIBLE = "index.version.minimum_compatible";
public static final String SETTING_CREATION_DATE = "index.creation_date";
public static final Setting<Long> INDEX_CREATION_DATE_SETTING = Setting.longSetting(SETTING_CREATION_DATE, -1, -1, false, Setting.Scope.INDEX);
public static final String SETTING_PRIORITY = "index.priority";
public static final Setting<Integer> INDEX_PRIORITY_SETTING = Setting.intSetting("index.priority", 1, 0, true, Setting.Scope.INDEX);
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
Expand Down Expand Up @@ -653,10 +654,6 @@ public Builder creationDate(long creationDate) {
return this;
}

public long creationDate() {
return settings.getAsLong(SETTING_CREATION_DATE, -1l);
}

public Builder settings(Settings.Builder settings) {
this.settings = settings.build();
return this;
Expand All @@ -671,11 +668,6 @@ public MappingMetaData mapping(String type) {
return mappings.get(type);
}

public Builder removeMapping(String mappingType) {
mappings.remove(mappingType);
return this;
}

public Builder putMapping(String type, String source) throws IOException {
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
putMapping(new MappingMetaData(type, parser.mapOrdered()));
Expand Down Expand Up @@ -718,24 +710,11 @@ public Builder putCustom(String type, Custom customIndexMetaData) {
return this;
}

public Builder removeCustom(String type) {
this.customs.remove(type);
return this;
}

public Custom getCustom(String type) {
return this.customs.get(type);
}

public Builder putActiveAllocationIds(int shardId, Set<String> allocationIds) {
activeAllocationIds.put(shardId, new HashSet(allocationIds));
return this;
}

public Set<String> getActiveAllocationIds(int shardId) {
return activeAllocationIds.get(shardId);
}

public long version() {
return this.version;
}
Expand Down
5 changes: 4 additions & 1 deletion core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ public void validateIndexName(String index, ClusterState state) {
public void createIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
Settings.Builder updatedSettingsBuilder = Settings.settingsBuilder();
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
for (Map.Entry<String, String> entry : updatedSettingsBuilder.internalMap().entrySet()) {
indexScopeSettings.validate(entry.getKey(), entry.getValue());
}
request.settings(updatedSettingsBuilder.build());

clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
Expand Down Expand Up @@ -473,7 +476,7 @@ List<String> getIndexSettingsValidationErrors(Settings settings) {
validationErrors.add("custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]");
}
}
//nocommit - this can be removed?
//norelease - this can be removed?
Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null);
if (number_of_primaries != null && number_of_primaries <= 0) {
Expand Down
74 changes: 35 additions & 39 deletions core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,14 @@
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.IndexScopeSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.settings.IndexDynamicSettings;

import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -63,18 +62,17 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements

private final AllocationService allocationService;

private final DynamicSettings dynamicSettings;

private final IndexNameExpressionResolver indexNameExpressionResolver;
private final IndexScopeSettings indexScopeSettings;

@Inject
public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService, @IndexDynamicSettings DynamicSettings dynamicSettings, IndexNameExpressionResolver indexNameExpressionResolver) {
public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService, IndexScopeSettings indexScopeSettings, IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings);
this.clusterService = clusterService;
this.indexNameExpressionResolver = indexNameExpressionResolver;
this.clusterService.add(this);
this.allocationService = allocationService;
this.dynamicSettings = dynamicSettings;
this.indexScopeSettings = indexScopeSettings;
}

@Override
Expand Down Expand Up @@ -147,40 +145,32 @@ public void onFailure(Throwable t) {
public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
Settings.Builder updatedSettingsBuilder = Settings.settingsBuilder();
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
Settings.Builder settingsForClosedIndices = Settings.builder();
Settings.Builder settingsForOpenIndices = Settings.builder();
Settings.Builder skipppedSettings = Settings.builder();


// never allow to change the number of shards
for (String key : updatedSettingsBuilder.internalMap().keySet()) {
if (key.equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
for (Map.Entry<String, String> entry : updatedSettingsBuilder.internalMap().entrySet()) {
if (entry.getKey().equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
listener.onFailure(new IllegalArgumentException("can't change the number of shards for an index"));
return;
}
}

final Settings closeSettings = updatedSettingsBuilder.build();

final Set<String> removedSettings = new HashSet<>();
final Set<String> errors = new HashSet<>();
for (Map.Entry<String, String> setting : updatedSettingsBuilder.internalMap().entrySet()) {
if (!dynamicSettings.hasDynamicSetting(setting.getKey())) {
removedSettings.add(setting.getKey());
} else {
String error = dynamicSettings.validateDynamicSetting(setting.getKey(), setting.getValue(), clusterService.state());
if (error != null) {
errors.add("[" + setting.getKey() + "] - " + error);
}
Setting setting = indexScopeSettings.get(entry.getKey());
if (setting == null) {
throw new IllegalArgumentException("setting [" + entry.getKey() + "] is unknown");
}
}

if (!errors.isEmpty()) {
listener.onFailure(new IllegalArgumentException("can't process the settings: " + errors.toString()));
return;
}

if (!removedSettings.isEmpty()) {
for (String removedSetting : removedSettings) {
updatedSettingsBuilder.remove(removedSetting);
indexScopeSettings.validate(entry.getKey(), entry.getValue());
settingsForClosedIndices.put(entry.getKey(), entry.getValue());
if (setting.isDynamic()) {
settingsForOpenIndices.put(entry.getKey(), entry.getValue());
} else {
skipppedSettings.put(entry.getKey(), entry.getValue());
}
}
final Settings openSettings = updatedSettingsBuilder.build();
final Settings skippedSettigns = skipppedSettings.build();
final Settings closedSettings = settingsForClosedIndices.build();
final Settings openSettings = settingsForOpenIndices.build();

clusterService.submitStateUpdateTask("update-settings",
new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
Expand Down Expand Up @@ -208,16 +198,16 @@ public ClusterState execute(ClusterState currentState) {
}
}

if (closeIndices.size() > 0 && closeSettings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) != null) {
if (closeIndices.size() > 0 && closedSettings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) != null) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
"Can't update [%s] on closed indices [%s] - can leave index in an unopenable state", IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
closeIndices
));
}
if (!removedSettings.isEmpty() && !openIndices.isEmpty()) {
if (!skippedSettigns.getAsMap().isEmpty() && !openIndices.isEmpty()) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
"Can't update non dynamic settings[%s] for open indices [%s]",
removedSettings,
skippedSettigns.getAsMap().keySet(),
openIndices
));
}
Expand Down Expand Up @@ -272,7 +262,7 @@ public ClusterState execute(ClusterState currentState) {

if (!closeIndices.isEmpty()) {
String[] indices = closeIndices.toArray(new String[closeIndices.size()]);
metaDataBuilder.updateSettings(closeSettings, indices);
metaDataBuilder.updateSettings(closedSettings, indices);
}


Expand All @@ -281,12 +271,18 @@ public ClusterState execute(ClusterState currentState) {
// now, reroute in case things change that require it (like number of replicas)
RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update");
updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();

for (String index : openIndices) {
indexScopeSettings.dryRun(updatedState.metaData().index(index).getSettings());
}
for (String index : closeIndices) {
indexScopeSettings.dryRun(updatedState.metaData().index(index).getSettings());
}
return updatedState;
}
});
}


public void upgradeIndexSettings(final UpgradeSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {


Expand Down
Loading

0 comments on commit 04045a0

Please sign in to comment.