From a2b2d54826b24ed839e37c74c3bcd972a1079468 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 19 Dec 2014 23:14:44 -0500 Subject: [PATCH 01/20] Internal: Add support for sending cluster state using diffs [WIP] Closes #6295 --- .../cluster/AbstractClusterStatePart.java | 108 +++++ .../elasticsearch/cluster/ClusterState.java | 449 +++++++----------- .../cluster/ClusterStatePart.java | 82 ++++ .../cluster/ClusterStateParts.java | 71 +++ .../cluster/CompositeClusterStatePart.java | 224 +++++++++ ...ompatibleClusterStateVersionException.java | 34 ++ .../cluster/block/ClusterBlocks.java | 64 ++- .../cluster/metadata/MetaData.java | 130 ++++- .../cluster/node/DiscoveryNodes.java | 58 ++- .../cluster/routing/RoutingTable.java | 59 ++- .../service/InternalClusterService.java | 4 +- .../discovery/local/LocalDiscovery.java | 24 +- .../publish/PublishClusterStateAction.java | 195 +++++++- .../cluster/ClusterStateDiffTests.java | 67 +++ 14 files changed, 1254 insertions(+), 315 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java create mode 100644 src/main/java/org/elasticsearch/cluster/ClusterStatePart.java create mode 100644 src/main/java/org/elasticsearch/cluster/ClusterStateParts.java create mode 100644 src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java create mode 100644 src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java create mode 100644 src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java new file mode 100644 index 0000000000000..944997c35c7b5 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.EnumSet; + +/** + */ +public abstract class AbstractClusterStatePart implements ClusterStatePart { + + @Override + public EnumSet context() { + return API_ONLY; + } + + public static class CompleteDiff implements Diff { + private T part; + + public CompleteDiff(T part) { + this.part = part; + } + + @Override + public T apply(T state) { + return part; + } + + public void writeTo(StreamOutput out) throws IOException{ + out.writeBoolean(true); + if (part != null) { + out.writeBoolean(true); + part.writeTo(out); + } else { + out.writeBoolean(false); + } + } + } + + protected static class NoDiff implements Diff { + + public NoDiff() { + } + + @Override + public T apply(T part) { + return part; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(false); + } + } + + protected static abstract class AbstractFactory implements ClusterStatePart.Factory { + + @Override + public Diff diff(T before, T after) { + if (before.equals(after)) { + return new NoDiff(); + } else { + return new CompleteDiff(after); + } + } + + @Override + public Diff readDiffFrom(StreamInput in) throws IOException { + if(in.readBoolean()) { + if (in.readBoolean()) { + T part = readFrom(in); + return new CompleteDiff(part); + } else { + return new CompleteDiff(null); + } + } else { + return new NoDiff(); + } + } + + @Override + public Version addedIn() { + return null; + } + } + +} diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index fae22af1c260b..78d9c3567fc52 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -19,42 +19,30 @@ package org.elasticsearch.cluster; -import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticsearchIllegalArgumentException; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.*; -import org.elasticsearch.cluster.routing.allocation.AllocationExplanation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; +import java.util.*; + +import static com.google.common.collect.Sets.newHashSet; /** * @@ -78,79 +66,31 @@ public byte id() { } } - public interface Custom { - - interface Factory { - - String type(); - - T readFrom(StreamInput in) throws IOException; - - void writeTo(T customState, StreamOutput out) throws IOException; - - void toXContent(T customState, XContentBuilder builder, ToXContent.Params params); - } - } - - private final static Map customFactories = new HashMap<>(); - - /** - * Register a custom index meta data factory. Make sure to call it from a static block. - */ - public static void registerFactory(String type, Custom.Factory factory) { - customFactories.put(type, factory); - } - - @Nullable - public static Custom.Factory lookupFactory(String type) { - return customFactories.get(type); - } - - public static Custom.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { - Custom.Factory factory = customFactories.get(type); - if (factory == null) { - throw new ElasticsearchIllegalArgumentException("No custom state factory registered for type [" + type + "]"); - } - return factory; - } - - public static final long UNKNOWN_VERSION = -1; private final long version; - private final RoutingTable routingTable; - - private final DiscoveryNodes nodes; - - private final MetaData metaData; - - private final ClusterBlocks blocks; - - private final ImmutableOpenMap customs; + private final String uuid; private final ClusterName clusterName; + + private final ClusterStateParts parts; // built on demand private volatile RoutingNodes routingNodes; - private SettingsFilter settingsFilter; - private volatile ClusterStateStatus status; - public ClusterState(long version, ClusterState state) { - this(state.clusterName, version, state.metaData(), state.routingTable(), state.nodes(), state.blocks(), state.customs()); - } - - public ClusterState(ClusterName clusterName, long version, MetaData metaData, RoutingTable routingTable, DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap customs) { + public ClusterState(ClusterName clusterName, long version, String uuid, ClusterStateParts parts) { this.version = version; this.clusterName = clusterName; - this.metaData = metaData; - this.routingTable = routingTable; - this.nodes = nodes; - this.blocks = blocks; - this.customs = customs; + this.parts = parts; this.status = ClusterStateStatus.UNKNOWN; + this.uuid = uuid; + } + + public ClusterState(long version, String uuid, ClusterState state) { + this(state.clusterName, version, uuid, state.parts); } public ClusterStateStatus status() { @@ -171,7 +111,7 @@ public long getVersion() { } public DiscoveryNodes nodes() { - return this.nodes; + return (DiscoveryNodes)parts.get(DiscoveryNodes.TYPE); } public DiscoveryNodes getNodes() { @@ -179,7 +119,7 @@ public DiscoveryNodes getNodes() { } public MetaData metaData() { - return this.metaData; + return (MetaData)parts.get(MetaData.TYPE); } public MetaData getMetaData() { @@ -187,7 +127,7 @@ public MetaData getMetaData() { } public RoutingTable routingTable() { - return routingTable; + return (RoutingTable)parts.get(RoutingTable.TYPE); } public RoutingTable getRoutingTable() { @@ -195,7 +135,7 @@ public RoutingTable getRoutingTable() { } public RoutingNodes routingNodes() { - return routingTable.routingNodes(this); + return routingTable().routingNodes(this); } public RoutingNodes getRoutingNodes() { @@ -203,19 +143,11 @@ public RoutingNodes getRoutingNodes() { } public ClusterBlocks blocks() { - return this.blocks; + return (ClusterBlocks)parts.get(ClusterBlocks.TYPE); } public ClusterBlocks getBlocks() { - return blocks; - } - - public ImmutableOpenMap customs() { - return this.customs; - } - - public ImmutableOpenMap getCustoms() { - return this.customs; + return blocks(); } public ClusterName getClusterName() { @@ -230,19 +162,19 @@ public RoutingNodes readOnlyRoutingNodes() { if (routingNodes != null) { return routingNodes; } - routingNodes = routingTable.routingNodes(this); + routingNodes = routingTable().routingNodes(this); return routingNodes; } public ClusterState settingsFilter(SettingsFilter settingsFilter) { - this.settingsFilter = settingsFilter; + metaData().settingsFilter(settingsFilter); return this; } public String prettyPrint() { StringBuilder sb = new StringBuilder(); sb.append("version: ").append(version).append("\n"); - sb.append("meta data version: ").append(metaData.version()).append("\n"); + sb.append("meta data version: ").append(metaData().version()).append("\n"); sb.append(nodes().prettyPrint()); sb.append(routingTable().prettyPrint()); sb.append(readOnlyRoutingNodes().prettyPrint()); @@ -317,6 +249,10 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { EnumSet metrics = Metric.parseString(params.param("metric", "_all"), true); + Set metricStrings = newHashSet(); + for (Metric metric : metrics) { + metricStrings.add(metric.value); + } if (metrics.contains(Metric.VERSION)) { builder.field("version", version); @@ -326,156 +262,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("master_node", nodes().masterNodeId()); } - if (metrics.contains(Metric.BLOCKS)) { - builder.startObject("blocks"); - - if (!blocks().global().isEmpty()) { - builder.startObject("global"); - for (ClusterBlock block : blocks().global()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - - if (!blocks().indices().isEmpty()) { - builder.startObject("indices"); - for (Map.Entry> entry : blocks().indices().entrySet()) { - builder.startObject(entry.getKey()); - for (ClusterBlock block : entry.getValue()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - builder.endObject(); - } - - builder.endObject(); - } - - // nodes - if (metrics.contains(Metric.NODES)) { - builder.startObject("nodes"); - for (DiscoveryNode node : nodes()) { - builder.startObject(node.id(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("name", node.name()); - builder.field("transport_address", node.address().toString()); - - builder.startObject("attributes"); - for (Map.Entry attr : node.attributes().entrySet()) { - builder.field(attr.getKey(), attr.getValue()); - } - builder.endObject(); - + for(ObjectObjectCursor partIter : parts.parts()) { + if (metricStrings.contains(partIter.key)) { + builder.startObject(partIter.key); + partIter.value.toXContent(builder, params); builder.endObject(); } - builder.endObject(); - } - - // meta data - if (metrics.contains(Metric.METADATA)) { - builder.startObject("metadata"); - - builder.startObject("templates"); - for (ObjectCursor cursor : metaData().templates().values()) { - IndexTemplateMetaData templateMetaData = cursor.value; - builder.startObject(templateMetaData.name(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("template", templateMetaData.template()); - builder.field("order", templateMetaData.order()); - - builder.startObject("settings"); - Settings settings = templateMetaData.settings(); - if (settingsFilter != null) { - settings = settingsFilter.filterSettings(settings); - } - settings.toXContent(builder, params); - builder.endObject(); - - builder.startObject("mappings"); - for (ObjectObjectCursor cursor1 : templateMetaData.mappings()) { - byte[] mappingSource = cursor1.value.uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor1.key); - } - builder.field(cursor1.key); - builder.map(mapping); - } - builder.endObject(); - - - builder.endObject(); - } - builder.endObject(); - - builder.startObject("indices"); - for (IndexMetaData indexMetaData : metaData()) { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); - - builder.startObject("settings"); - Settings settings = indexMetaData.settings(); - if (settingsFilter != null) { - settings = settingsFilter.filterSettings(settings); - } - settings.toXContent(builder, params); - builder.endObject(); - - builder.startObject("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - byte[] mappingSource = cursor.value.source().uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor.key); - } - builder.field(cursor.key); - builder.map(mapping); - } - builder.endObject(); - - builder.startArray("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().keys()) { - builder.value(cursor.value); - } - builder.endArray(); - - builder.endObject(); - } - builder.endObject(); - - for (ObjectObjectCursor cursor : metaData.customs()) { - builder.startObject(cursor.key); - MetaData.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - builder.endObject(); - } - - builder.endObject(); - } - - // routing table - if (metrics.contains(Metric.ROUTING_TABLE)) { - builder.startObject("routing_table"); - builder.startObject("indices"); - for (IndexRoutingTable indexRoutingTable : routingTable()) { - builder.startObject(indexRoutingTable.index(), XContentBuilder.FieldCaseConversion.NONE); - builder.startObject("shards"); - for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { - builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); - for (ShardRouting shardRouting : indexShardRoutingTable) { - shardRouting.toXContent(builder, params); - } - builder.endArray(); - } - builder.endObject(); - builder.endObject(); - } - builder.endObject(); - builder.endObject(); } // routing nodes @@ -499,13 +291,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } - if (metrics.contains(Metric.CUSTOMS)) { - for (ObjectObjectCursor cursor : customs) { - builder.startObject(cursor.key); - lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - builder.endObject(); - } - } return builder; } @@ -522,34 +307,36 @@ public static class Builder { private final ClusterName clusterName; private long version = 0; - private MetaData metaData = MetaData.EMPTY_META_DATA; - private RoutingTable routingTable = RoutingTable.EMPTY_ROUTING_TABLE; - private DiscoveryNodes nodes = DiscoveryNodes.EMPTY_NODES; - private ClusterBlocks blocks = ClusterBlocks.EMPTY_CLUSTER_BLOCK; - private final ImmutableOpenMap.Builder customs; + private String uuid = null; + private final ImmutableOpenMap.Builder parts; public Builder(ClusterState state) { this.clusterName = state.clusterName; this.version = state.version(); - this.nodes = state.nodes(); - this.routingTable = state.routingTable(); - this.metaData = state.metaData(); - this.blocks = state.blocks(); - this.customs = ImmutableOpenMap.builder(state.customs()); + this.parts = ImmutableOpenMap.builder(state.parts.parts); + putPart(MetaData.TYPE, state.metaData()); + putPart(RoutingTable.TYPE, state.routingTable()); + putPart(DiscoveryNodes.TYPE, state.nodes()); + putPart(ClusterBlocks.TYPE, state.blocks()); } public Builder(ClusterName clusterName) { - customs = ImmutableOpenMap.builder(); this.clusterName = clusterName; + parts = ImmutableOpenMap.builder(); + putPart(MetaData.TYPE, MetaData.EMPTY_META_DATA); + putPart(RoutingTable.TYPE, RoutingTable.EMPTY_ROUTING_TABLE); + putPart(DiscoveryNodes.TYPE, DiscoveryNodes.EMPTY_NODES); + putPart(ClusterBlocks.TYPE, ClusterBlocks.EMPTY_CLUSTER_BLOCK); } + public Builder nodes(DiscoveryNodes.Builder nodesBuilder) { return nodes(nodesBuilder.build()); } public Builder nodes(DiscoveryNodes nodes) { - this.nodes = nodes; + putPart(DiscoveryNodes.TYPE, nodes); return this; } @@ -558,12 +345,12 @@ public Builder routingTable(RoutingTable.Builder routingTable) { } public Builder routingResult(RoutingAllocation.Result routingResult) { - this.routingTable = routingResult.routingTable(); + putPart(RoutingTable.TYPE, routingResult.routingTable()); return this; } public Builder routingTable(RoutingTable routingTable) { - this.routingTable = routingTable; + putPart(RoutingTable.TYPE, routingTable); return this; } @@ -572,7 +359,7 @@ public Builder metaData(MetaData.Builder metaDataBuilder) { } public Builder metaData(MetaData metaData) { - this.metaData = metaData; + putPart(MetaData.TYPE, metaData); return this; } @@ -581,7 +368,7 @@ public Builder blocks(ClusterBlocks.Builder blocksBuilder) { } public Builder blocks(ClusterBlocks blocks) { - this.blocks = blocks; + putPart(ClusterBlocks.TYPE, blocks); return this; } @@ -590,22 +377,36 @@ public Builder version(long version) { return this; } - public Custom getCustom(String type) { - return customs.get(type); + public Builder incrementVersion() { + version = version + 1; + uuid = Strings.randomBase64UUID(); + return this; } - public Builder putCustom(String type, Custom custom) { - customs.put(type, custom); + public Builder uuid(String uuid) { + this.uuid = uuid; return this; } - public Builder removeCustom(String type) { - customs.remove(type); + public T getPart(String type) { + return (T)parts.get(type); + } + + public Builder putPart(String type, ClusterStatePart custom) { + parts.put(type, custom); + return this; + } + + public Builder removePart(String type) { + parts.remove(type); return this; } public ClusterState build() { - return new ClusterState(clusterName, version, metaData, routingTable, nodes, blocks, customs.build()); + if (uuid == null) { + uuid = Strings.randomBase64UUID(); + } + return new ClusterState(clusterName, version, uuid, new ClusterStateParts(parts.build())); } public static byte[] toBytes(ClusterState state) throws IOException { @@ -629,16 +430,9 @@ public static void writeTo(ClusterState state, StreamOutput out) throws IOExcept if (state.clusterName != null) { state.clusterName.writeTo(out); } - out.writeLong(state.version()); - MetaData.Builder.writeTo(state.metaData(), out); - RoutingTable.Builder.writeTo(state.routingTable(), out); - DiscoveryNodes.Builder.writeTo(state.nodes(), out); - ClusterBlocks.Builder.writeClusterBlocks(state.blocks(), out); - out.writeVInt(state.customs().size()); - for (ObjectObjectCursor cursor : state.customs()) { - out.writeString(cursor.key); - lookupFactorySafe(cursor.key).writeTo(cursor.value, out); - } + out.writeLong(state.version); + out.writeString(state.uuid); + state.parts.writeTo(out); } /** @@ -654,17 +448,98 @@ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode loca } Builder builder = new Builder(clusterName); builder.version = in.readLong(); - builder.metaData = MetaData.Builder.readFrom(in); - builder.routingTable = RoutingTable.Builder.readFrom(in); - builder.nodes = DiscoveryNodes.Builder.readFrom(in, localNode); - builder.blocks = ClusterBlocks.Builder.readClusterBlocks(in); - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in); - builder.putCustom(type, customIndexMetaData); + builder.uuid = in.readString(); + builder.parts.putAll(ClusterStateParts.FACTORY.readFrom(in).parts()); + //TODO: Hack!!!! Need to find a better way to handle localNode + DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); + if (localNode != null) { + builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); } return builder.build(); } + + public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode, @Nullable ClusterName defaultClusterName) throws IOException { + ClusterName clusterName = defaultClusterName; + if (in.readBoolean()) { + clusterName = ClusterName.readClusterName(in); + } + long version = in.readLong(); + String previousUuid = in.readString(); + String newUuid = in.readString(); + ClusterStatePart.Diff diff = ClusterStateParts.FACTORY.readDiffFrom(in); + return new ClusterStateDiff(clusterName, version, previousUuid, newUuid, localNode, diff); + + } + + public static ClusterStateDiff diff(ClusterState before, ClusterState after) { + return new ClusterStateDiff(after.clusterName, after.version, before.uuid, after.uuid, after.nodes().localNode(), ClusterStateParts.FACTORY.diff(before.parts, after.parts) ); + } + + public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws IOException { + BytesStreamOutput os = new BytesStreamOutput(); + diff(before, after).writeTo(os); + return os.bytes().toBytes(); + } + + public static ClusterState fromDiffBytes(ClusterState before, byte[] data, DiscoveryNode localNode, ClusterName defaultClusterName) throws IOException { + ClusterStateDiff diff = readDiffFrom(new BytesStreamInput(data, false), localNode, defaultClusterName); + return diff.apply(before); + } + + + } + + public static class ClusterStateDiff { + private ClusterName clusterName; + private long version; + private String previousUuid; + private String newUuid; + private DiscoveryNode localNode; + private ClusterStatePart.Diff diff; + + public ClusterStateDiff(ClusterName clusterName, long version, String previousUuid, String newUuid, @Nullable DiscoveryNode localNode, ClusterStatePart.Diff diff) { + this.clusterName = clusterName; + this.version = version; + this.previousUuid = previousUuid; + this.newUuid = newUuid; + this.localNode = localNode; + this.diff = diff; + } + + public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { + if (!previousUuid.equals(previous.uuid)) { + throw new IncompatibleClusterStateVersionException("Expected version " + (previous.version + 1) +"/" + previous.uuid + " got version " + version + "/" + previousUuid); + } + Builder builder = new Builder(clusterName); + builder.version = version; + builder.uuid = newUuid; + builder.parts.putAll(diff.apply(previous.parts).parts); + //TODO: Is there a better way to handle it? + DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); + if (localNode != null) { + builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); + } + return builder.build(); + } + + public void writeTo(StreamOutput out) throws IOException { + if (clusterName != null) { + out.writeBoolean(true); + clusterName.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeLong(version); + out.writeString(previousUuid); + out.writeString(newUuid); + diff.writeTo(out); + + } + + public long version() { + return version; + } + } + } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java new file mode 100644 index 0000000000000..cf9e9d69751d7 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.EnumSet; + +/** + */ +public interface ClusterStatePart extends ToXContent { + + public static final String ALL = "_all"; + + public enum XContentContext { + /* Custom metadata should be returns as part of API call */ + API, + + /* Custom metadata should be stored as part of the persistent cluster state */ + GATEWAY, + + /* Custom metadata should be stored as part of a snapshot */ + SNAPSHOT; + } + + public static EnumSet API_ONLY = EnumSet.of(XContentContext.API); + public static EnumSet GATEWAY_ONLY = EnumSet.of(XContentContext.GATEWAY); + public static EnumSet API_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY); + public static EnumSet API_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT); + public static EnumSet API_GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.GATEWAY, XContentContext.SNAPSHOT); + + String type(); + + void writeTo(StreamOutput out) throws IOException; + + EnumSet context(); + + interface Factory { + + String type(); + + Diff diff(T before, T after); + + Diff readDiffFrom(StreamInput in) throws IOException; + + T readFrom(StreamInput in) throws IOException; + + T fromXContent(XContentParser parser) throws IOException; + + Version addedIn(); + + } + + interface Diff { + + T apply(T part); + + void writeTo(StreamOutput out) throws IOException; + } +} diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java b/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java new file mode 100644 index 0000000000000..63076df64b21d --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.collect.ImmutableOpenMap; + +/** + */ +public class ClusterStateParts extends CompositeClusterStatePart { + public static final String TYPE = "cluster"; + + public static final Factory FACTORY = new Factory(); + + public static class Factory extends AbstractCompositeClusterStatePartFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public ClusterStateParts fromParts(ImmutableOpenMap.Builder parts) { + return new ClusterStateParts(parts.build()); + } + } + + static { + registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); + registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); + registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); + registerFactory(MetaData.TYPE, MetaData.FACTORY); + } + + @Override + public String type() { + return TYPE; + } + + static { + registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); + registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); + registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); + registerFactory(MetaData.TYPE, MetaData.FACTORY); + } + + public ClusterStateParts(ImmutableOpenMap parts) { + super(parts); + } + +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java new file mode 100644 index 0000000000000..91ca827ca8f1d --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -0,0 +1,224 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import com.google.common.collect.ImmutableList; +import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; + +/** + */ +public abstract class CompositeClusterStatePart extends AbstractClusterStatePart { + + protected final ImmutableOpenMap parts; + + private final static Map partFactories = new HashMap<>(); + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CompositeClusterStatePart that = (CompositeClusterStatePart) o; + + if (parts != null ? !parts.equals(that.parts) : that.parts != null) return false; + + return true; + } + + @Override + public int hashCode() { + return parts != null ? parts.hashCode() : 0; + } + + public static abstract class AbstractCompositeClusterStatePartFactory extends AbstractClusterStatePart.AbstractFactory{ + + @Override + public T readFrom(StreamInput in) throws IOException { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + int partsSize = in.readVInt(); + for (int i = 0; i < partsSize; i++) { + String type = in.readString(); + ClusterStatePart part = lookupFactorySafe(type).readFrom(in); + builder.put(type, part); + } + return fromParts(builder); + } + + @Override + public T fromXContent(XContentParser parser) throws IOException { + return null; + } + + public abstract T fromParts(ImmutableOpenMap.Builder parts); + + @Override + public Diff diff(T before, T after) { + ImmutableOpenMap beforeParts = before.parts(); + ImmutableOpenMap afterParts = after.parts(); + if (before.equals(after)) { + return new NoDiff(); + } else { + Map> diffs = newHashMap(); + List deletes = newArrayList(); + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); + } + } + for (ObjectObjectCursor partIter : afterParts) { + ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); + diffs.put(partIter.key, factory.diff(beforeParts.get(partIter.key), partIter.value)); + } + return new CompositeDiff<>(this, deletes, diffs); + } + } + + @Override + public Diff readDiffFrom(StreamInput in) throws IOException { + if (in.readBoolean()) { + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i=0; i> diffs = newHashMap(); + for (int i=0; i(this, deletes, diffs); + + } else { + return new NoDiff(); + } + } + } + + private static class CompositeDiff implements Diff { + + Map> diffs; + List deletes; + AbstractCompositeClusterStatePartFactory factory; + + private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, List deletes, Map> diffs) { + this.diffs = diffs; + this.deletes = deletes; + this.factory = factory; + } + + @Override + public T apply(T part) { + ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); + parts.putAll(part.parts); + for (String delete : deletes) { + parts.remove(delete); + } + + for (Map.Entry> entry : diffs.entrySet()) { + parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); + } + return factory.fromParts(parts); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); // We have diffs + out.writeVInt(deletes.size()); + for(String delete : deletes) { + out.writeString(delete); + } + + out.writeVInt(diffs.size()); + for (Map.Entry> entry : diffs.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + } + + protected CompositeClusterStatePart(ImmutableOpenMap parts) { + this.parts = parts; + } + + public ImmutableOpenMap parts() { + return parts; + } + + /** + * Register a custom index meta data factory. Make sure to call it from a static block. + */ + public static void registerFactory(String type, ClusterStatePart.Factory factory) { + partFactories.put(type, factory); + } + + @Nullable + public static ClusterStatePart.Factory lookupFactory(String type) { + return partFactories.get(type); + } + + public static ClusterStatePart.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { + ClusterStatePart.Factory factory = partFactories.get(type); + if (factory == null) { + throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); + } + return factory; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(parts().size()); + for (ObjectObjectCursor cursor : parts()) { + out.writeString(cursor.key); + cursor.value.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (ObjectObjectCursor partIter : parts) { + builder.startObject(partIter.key); + partIter.value.toXContent(builder, params); + } + return builder; + } + + public T get(String type) { + return (T) parts.get(type); + } + +} diff --git a/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java b/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java new file mode 100644 index 0000000000000..6a602e0102ff5 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.ElasticsearchException; + +/** + */ +public class IncompatibleClusterStateVersionException extends ElasticsearchException { + public IncompatibleClusterStateVersionException(String msg) { + super(msg); + } + + public IncompatibleClusterStateVersionException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index bb7d332de4f96..c2494ef9ce16c 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -23,10 +23,15 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -36,7 +41,11 @@ /** * Represents current cluster level blocks to block dirty operations done against the cluster. */ -public class ClusterBlocks { +public class ClusterBlocks extends AbstractClusterStatePart { + + public static final String TYPE = "blocks"; + + public static final Factory FACTORY = new Factory(); public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(ImmutableSet.of(), ImmutableMap.>of()); @@ -76,6 +85,24 @@ public class ClusterBlocks { } } + public static class Factory extends AbstractFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public ClusterBlocks readFrom(StreamInput in) throws IOException { + return Builder.readClusterBlocks(in); + } + + @Override + public ClusterBlocks fromXContent(XContentParser parser) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } + } + public ImmutableSet global() { return global; } @@ -203,6 +230,41 @@ public ClusterBlockException indicesBlockedException(ClusterBlockLevel level, St return new ClusterBlockException(builder.build()); } + @Override + public String type() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeClusterBlocks(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (!global().isEmpty()) { + builder.startObject("global"); + for (ClusterBlock block : global()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + + if (!indices().isEmpty()) { + builder.startObject("indices"); + for (Map.Entry> entry : indices().entrySet()) { + builder.startObject(entry.getKey()); + for (ClusterBlock block : entry.getValue()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + builder.endObject(); + } + + return builder; + } + static class ImmutableLevelHolder { static final ImmutableLevelHolder EMPTY = new ImmutableLevelHolder(ImmutableSet.of(), ImmutableMap.>of()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 6fd591fd0bb84..9668ceec094dc 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -27,17 +27,21 @@ import com.google.common.collect.*; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.HppcMaps; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.Index; @@ -56,10 +60,126 @@ /** * */ -public class MetaData implements Iterable { +public class MetaData extends AbstractClusterStatePart implements Iterable { + + public static final String TYPE = "metadata"; + + public static final Factory FACTORY = new Factory(); public static final String ALL = "_all"; + + public static class Factory extends AbstractFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public MetaData readFrom(StreamInput in) throws IOException { + return Builder.readFrom(in); + } + + @Override + public MetaData fromXContent(XContentParser parser) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } + } + + @Override + public String type() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("templates"); + for (ObjectCursor cursor : templates().values()) { + IndexTemplateMetaData templateMetaData = cursor.value; + builder.startObject(templateMetaData.name(), XContentBuilder.FieldCaseConversion.NONE); + + builder.field("template", templateMetaData.template()); + builder.field("order", templateMetaData.order()); + + builder.startObject("settings"); + Settings settings = templateMetaData.settings(); + if (settingsFilter != null) { + settings = settingsFilter.filterSettings(settings); + } + settings.toXContent(builder, params); + builder.endObject(); + + builder.startObject("mappings"); + for (ObjectObjectCursor cursor1 : templateMetaData.mappings()) { + byte[] mappingSource = cursor1.value.uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(cursor1.key); + } + builder.field(cursor1.key); + builder.map(mapping); + } + builder.endObject(); + + + builder.endObject(); + } + builder.endObject(); + + builder.startObject("indices"); + for (IndexMetaData indexMetaData : this) { + builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + + builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); + + builder.startObject("settings"); + Settings settings = indexMetaData.settings(); + if (settingsFilter != null) { + settings = settingsFilter.filterSettings(settings); + } + settings.toXContent(builder, params); + builder.endObject(); + + builder.startObject("mappings"); + for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + byte[] mappingSource = cursor.value.source().uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(cursor.key); + } + builder.field(cursor.key); + builder.map(mapping); + } + builder.endObject(); + + builder.startArray("aliases"); + for (ObjectCursor cursor : indexMetaData.aliases().keys()) { + builder.value(cursor.value); + } + builder.endArray(); + + builder.endObject(); + } + builder.endObject(); + + for (ObjectObjectCursor cursor : customs()) { + builder.startObject(cursor.key); + MetaData.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); + builder.endObject(); + } + return builder; + } + public enum XContentContext { /* Custom metadata should be returns as part of API call */ API, @@ -159,6 +279,9 @@ public static Custom.Factory lookupFactorySafe(String type private final ImmutableOpenMap> aliases; private final ImmutableOpenMap aliasAndIndexToIndexMap; + //TODO: This is a hack - needed for plugins. We should refactor it using params + private SettingsFilter settingsFilter; + @SuppressWarnings("unchecked") MetaData(String uuid, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs) { this.uuid = uuid; @@ -254,6 +377,11 @@ public static Custom.Factory lookupFactorySafe(String type this.aliasAndIndexToIndexMap = aliasAndIndexToIndexMap.cast().build(); } + public MetaData settingsFilter(SettingsFilter settingsFilter) { + this.settingsFilter = settingsFilter; + return this; + } + public long version() { return this.version; } diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 0a4986476e5c1..e130f4f73201d 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -26,6 +26,8 @@ import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -33,8 +35,12 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,7 +51,11 @@ * This class holds all {@link DiscoveryNode} in the cluster and provides convenience methods to * access, modify merge / diff discovery nodes. */ -public class DiscoveryNodes implements Iterable { +public class DiscoveryNodes extends AbstractClusterStatePart implements Iterable { + + public static final String TYPE = "nodes"; + + public static final Factory FACTORY = new Factory(); public static final DiscoveryNodes EMPTY_NODES = builder().build(); @@ -58,6 +68,24 @@ public class DiscoveryNodes implements Iterable { private final Version minNodeVersion; private final Version minNonClientNodeVersion; + public static class Factory extends AbstractFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public DiscoveryNodes readFrom(StreamInput in) throws IOException { + return Builder.readFrom(in, null); + } + + @Override + public DiscoveryNodes fromXContent(XContentParser parser) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } + } + private DiscoveryNodes(ImmutableOpenMap nodes, ImmutableOpenMap dataNodes, ImmutableOpenMap masterNodes, String masterNodeId, String localNodeId, Version minNodeVersion, Version minNonClientNodeVersion) { this.nodes = nodes; this.dataNodes = dataNodes; @@ -467,6 +495,34 @@ public Delta emptyDelta() { return new Delta(null, null, localNodeId, DiscoveryNode.EMPTY_LIST, DiscoveryNode.EMPTY_LIST); } + @Override + public String type() { + return DiscoveryNodes.TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + DiscoveryNodes.Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (DiscoveryNode node : this) { + builder.startObject(node.id(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("name", node.name()); + builder.field("transport_address", node.address().toString()); + + builder.startObject("attributes"); + for (Map.Entry attr : node.attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); + + builder.endObject(); + } + return builder; + } + public static class Delta { private final String localNodeId; diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 6f44a1d11fc04..d46067a13356a 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -21,11 +21,16 @@ import com.carrotsearch.hppc.IntSet; import com.google.common.collect.*; +import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexMissingException; @@ -44,7 +49,11 @@ * * @see IndexRoutingTable */ -public class RoutingTable implements Iterable { +public class RoutingTable extends AbstractClusterStatePart implements Iterable { + + public static final String TYPE = "routing_table"; + + public static final Factory FACTORY = new Factory(); public static final RoutingTable EMPTY_ROUTING_TABLE = builder().build(); @@ -58,6 +67,24 @@ public class RoutingTable implements Iterable { this.indicesRouting = ImmutableMap.copyOf(indicesRouting); } + + public static class Factory extends AbstractFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public RoutingTable readFrom(StreamInput in) throws IOException { + return Builder.readFrom(in); + } + + @Override + public RoutingTable fromXContent(XContentParser parser) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } + } /** * Returns the version of the {@link RoutingTable}. * @@ -295,6 +322,36 @@ public static Builder builder(RoutingTable routingTable) { return new Builder(routingTable); } + @Override + public String type() { + return TYPE; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("indices"); + for (IndexRoutingTable indexRoutingTable : this) { + builder.startObject(indexRoutingTable.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject("shards"); + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); + for (ShardRouting shardRouting : indexShardRoutingTable) { + shardRouting.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); + } + builder.endObject(); + return builder; + } + public static class Builder { private long version; diff --git a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index 82f3116b32af6..8f2dd44c4d761 100644 --- a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; @@ -367,7 +368,8 @@ public void run() { Discovery.AckListener ackListener = new NoOpAckListener(); if (newClusterState.nodes().localNodeMaster()) { // only the master controls the version numbers - Builder builder = ClusterState.builder(newClusterState).version(newClusterState.version() + 1); + Builder builder = ClusterState.builder(newClusterState).incrementVersion(); + builder.uuid(Strings.randomBase64UUID()); if (previousClusterState.routingTable() != newClusterState.routingTable()) { builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1)); } diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 6a49fe99a0245..1accdf2d5d3e8 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -40,6 +40,8 @@ import org.elasticsearch.node.service.NodeService; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.Arrays; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -75,6 +77,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem private static final ConcurrentMap clusterGroups = ConcurrentCollections.newConcurrentMap(); + private volatile ClusterState lastProcessedClusterState; + @Inject public LocalDiscovery(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService, DiscoveryNodeService discoveryNodeService, Version version, DiscoverySettings discoverySettings) { @@ -294,14 +298,26 @@ private LocalDiscovery[] members() { private void publish(LocalDiscovery[] members, ClusterState clusterState, final ClusterStatePublishResponseHandler publishResponseHandler) { try { - // we do the marshaling intentionally, to check it works well... - final byte[] clusterStateBytes = Builder.toBytes(clusterState); + byte[] clusterStateBytes = null; + byte[] clusterStateDiffBytes = null; for (final LocalDiscovery discovery : members) { if (discovery.master) { continue; } - final ClusterState nodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode, clusterName); + final ClusterState nodeSpecificClusterState; + // we do the marshaling intentionally, to check it works well... + if (lastProcessedClusterState != null && lastProcessedClusterState.nodes().nodeExists(discovery.localNode.id())) { + if (clusterStateDiffBytes == null) { + clusterStateDiffBytes = Builder.toDiffBytes(lastProcessedClusterState, clusterState); + } + nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode, clusterName ); + } else { + if (clusterStateBytes == null) { + clusterStateBytes = Builder.toBytes(clusterState); + } + nodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode, clusterName); + } nodeSpecificClusterState.status(ClusterState.ClusterStateStatus.RECEIVED); // ignore cluster state messages that do not include "me", not in the game yet... if (nodeSpecificClusterState.nodes().localNode() != null) { @@ -362,7 +378,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS Thread.currentThread().interrupt(); } } - + lastProcessedClusterState = clusterState; } catch (Exception e) { // failure to marshal or un-marshal diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 1e46bbb0171ab..7038731b51f21 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -23,8 +23,10 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; @@ -39,8 +41,15 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.*; +import java.io.IOException; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static com.google.common.collect.Maps.newHashMap; /** * @@ -66,6 +75,9 @@ static interface NewStateProcessed { private final NewClusterStateListener listener; private final DiscoverySettings discoverySettings; private final ClusterName clusterName; + private final Map lastFullVersionSent = newHashMap(); + private final Lock versionMapLock = new ReentrantLock(); + private volatile ClusterState lastProcessedClusterState; public PublishClusterStateAction(Settings settings, TransportService transportService, DiscoveryNodesProvider nodesProvider, NewClusterStateListener listener, DiscoverySettings discoverySettings, ClusterName clusterName) { @@ -76,6 +88,7 @@ public PublishClusterStateAction(Settings settings, TransportService transportSe this.discoverySettings = discoverySettings; this.clusterName = clusterName; transportService.registerHandler(ACTION_NAME, new PublishClusterStateRequestHandler()); + lastProcessedClusterState = null; } public void close() { @@ -90,10 +103,25 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR DiscoveryNode localNode = nodesProvider.nodes().localNode(); - Map serializedStates = Maps.newHashMap(); + Map serializedStates = newHashMap(); + Map serializedDiffs = newHashMap(); final AtomicBoolean timedOutWaitingForNodes = new AtomicBoolean(false); final TimeValue publishTimeout = discoverySettings.getPublishTimeout(); + final boolean sendFullVersion = lastProcessedClusterState == null || clusterState.version() - lastProcessedClusterState.version() != 1; + + // Remove nodes that are no longer part of the cluster + versionMapLock.lock(); + try { + Iterator iterator = lastFullVersionSent.keySet().iterator(); + while (iterator.hasNext()) { + if (!clusterState.nodes().nodeExists(iterator.next())) { + iterator.remove(); + } + } + } finally { + versionMapLock.unlock(); + } for (final DiscoveryNode node : clusterState.nodes()) { if (node.equals(localNode)) { @@ -101,20 +129,37 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR } // try and serialize the cluster state once (or per version), so we don't serialize it // per node when we send it over the wire, compress it while we are at it... - BytesReference bytes = serializedStates.get(node.version()); - if (bytes == null) { + BytesReference bytes; + boolean newlyAddedNode = !lastProcessedClusterState.nodes().nodeExists(node.id()); + if (sendFullVersion || newlyAddedNode) { + versionMapLock.lock(); try { - BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput stream = new HandlesStreamOutput(CompressorFactory.defaultCompressor().streamOutput(bStream)); - stream.setVersion(node.version()); - ClusterState.Builder.writeTo(clusterState, stream); - stream.close(); - bytes = bStream.bytes(); - serializedStates.put(node.version(), bytes); - } catch (Throwable e) { - logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); - publishResponseHandler.onFailure(node, e); - continue; + lastFullVersionSent.put(node.getId(), clusterState.version()); + } finally { + versionMapLock.unlock(); + } + bytes = serializedStates.get(node.version()); + if (bytes == null) { + try { + bytes = serializeFullClusterState(clusterState, node.version()); + serializedStates.put(node.version(), bytes); + } catch (Throwable e) { + logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); + publishResponseHandler.onFailure(node, e); + continue; + } + } + } else { + bytes = serializedDiffs.get(node.version()); + if (bytes == null) { + try { + bytes = serializeDiffClusterState(clusterState, node.version()); + serializedDiffs.put(node.version(), bytes); + } catch (Throwable e) { + logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); + publishResponseHandler.onFailure(node, e); + continue; + } } } try { @@ -137,8 +182,13 @@ public void handleResponse(TransportResponse.Empty response) { @Override public void handleException(TransportException exp) { - logger.debug("failed to send cluster state to {}", exp, node); - publishResponseHandler.onFailure(node, exp); + if (exp.unwrapCause() instanceof IncompatibleClusterStateVersionException) { + logger.debug("resending full cluster state to node {} reason {}", node, exp.getDetailedMessage()); + resendFullClusterState(clusterState, node, timedOutWaitingForNodes, publishResponseHandler); + } else { + logger.debug("failed to send cluster state to {}", exp, node); + publishResponseHandler.onFailure(node, exp); + } } }); } catch (Throwable t) { @@ -146,7 +196,7 @@ public void handleException(TransportException exp) { publishResponseHandler.onFailure(node, t); } } - + lastProcessedClusterState = clusterState; if (publishTimeout.millis() > 0) { // only wait if the publish timeout is configured... try { @@ -161,6 +211,95 @@ public void handleException(TransportException exp) { } } + private void resendFullClusterState(final ClusterState clusterState, final DiscoveryNode node, final AtomicBoolean timedOutWaitingForNodes, final ClusterStatePublishResponseHandler publishResponseHandler) { + final TimeValue publishTimeout = discoverySettings.getPublishTimeout(); + final ClusterState currentClusterState = lastProcessedClusterState; + final ClusterState clusterStateToSend; + if (currentClusterState == null || currentClusterState.version() < clusterState.version()) { + clusterStateToSend = clusterState; + } else { + clusterStateToSend = currentClusterState; + } + boolean shouldSendAnUpdate = true; + final Long lastVersionSent; + versionMapLock.lock(); + try { + lastVersionSent = lastFullVersionSent.get(node.getId()); + if (lastVersionSent != null && clusterStateToSend.version() <= lastVersionSent) { + //we already sent a newer full version of cluster state to this node - no reason to send it again + shouldSendAnUpdate = false; + } else { + lastFullVersionSent.put(node.getId(), clusterStateToSend.version()); + } + } finally { + versionMapLock.unlock(); + } + + if (shouldSendAnUpdate) { + + BytesReference bytes; + try { + bytes = serializeFullClusterState(clusterStateToSend, node.version()); + } catch (Throwable e) { + logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); + publishResponseHandler.onFailure(node, e); + return; + } + try { + TransportRequestOptions options = TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withCompress(false); + // no need to put a timeout on the options here, because we want the response to eventually be received + // and not log an error if it arrives after the timeout + transportService.sendRequest(node, ACTION_NAME, + new BytesTransportRequest(bytes, node.version()), + options, // no need to compress, we already compressed the bytes + + new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + + @Override + public void handleResponse(TransportResponse.Empty response) { + if (timedOutWaitingForNodes.get()) { + logger.debug("node {} responded for cluster state [{}] (took longer than [{}])", node, clusterState.version(), publishTimeout); + } + publishResponseHandler.onResponse(node); + } + + @Override + public void handleException(TransportException exp) { + logger.debug("failed to send cluster state to {}", exp, node); + publishResponseHandler.onFailure(node, exp); + } + }); + } catch (Throwable t) { + logger.debug("error sending cluster state to {}", t, node); + publishResponseHandler.onFailure(node, t); + } + } else { + logger.debug("skipping sending version {} to node {}, node already recieved version {}", clusterStateToSend.version(), node, lastVersionSent); + // TODO: Should this be a failure since we are not really sure if this node got the cluster state or not at this moment + publishResponseHandler.onResponse(node); + } + } + + private BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException { + BytesStreamOutput bStream = new BytesStreamOutput(); + StreamOutput stream = new HandlesStreamOutput(CompressorFactory.defaultCompressor().streamOutput(bStream)); + stream.setVersion(nodeVersion); + stream.writeBoolean(true); + ClusterState.Builder.writeTo(clusterState, stream); + stream.close(); + return bStream.bytes(); + } + + private BytesReference serializeDiffClusterState(ClusterState clusterState, Version nodeVersion) throws IOException { + BytesStreamOutput bStream = new BytesStreamOutput(); + StreamOutput stream = new HandlesStreamOutput(CompressorFactory.defaultCompressor().streamOutput(bStream)); + stream.setVersion(nodeVersion); + stream.writeBoolean(false); + ClusterState.Builder.diff(lastProcessedClusterState, clusterState).writeTo(stream); + stream.close(); + return bStream.bytes(); + } + private class PublishClusterStateRequestHandler extends BaseTransportRequestHandler { @Override @@ -178,9 +317,27 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne in = CachedStreamInput.cachedHandles(request.bytes().streamInput()); } in.setVersion(request.version()); - ClusterState clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode(), clusterName); + final ClusterState clusterState; + if (in.readBoolean()) { + clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode(), clusterName); + logger.debug("received full cluster state version {}", clusterState.version()); + } else { + if(lastProcessedClusterState == null) { + logger.debug("received diff cluster state version but don't have any local cluster state - requesting full state"); + throw new IncompatibleClusterStateVersionException("have no local cluster state"); + } + ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode(), clusterName); + if (lastProcessedClusterState.version() >= diff.version()) { + logger.debug("got diffs for obsolete version {}, current version {}, ignoring the diff", diff.version(), lastProcessedClusterState.version()); + return; + } + clusterState = diff.apply(lastProcessedClusterState); + logger.debug("received diff cluster state version {}", clusterState.version()); + } + lastProcessedClusterState = clusterState; + clusterState.status(ClusterState.ClusterStateStatus.RECEIVED); - logger.debug("received cluster state version {}", clusterState.version()); +// logger.debug("received cluster state version {}", clusterState.version()); listener.onNewClusterState(clusterState, new NewClusterStateListener.NewStateProcessed() { @Override public void onNewClusterStateProcessed() { diff --git a/src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java b/src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java new file mode 100644 index 0000000000000..3f3e01dacc3b1 --- /dev/null +++ b/src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.cluster.ClusterState.Builder; +import org.elasticsearch.cluster.ClusterState.ClusterStateDiff; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.lessThan; + +public class ClusterStateDiffTests extends ElasticsearchIntegrationTest { + + @Before + public void indexData() throws Exception { + //TODO: randomize this + index("foo", "bar", "1", XContentFactory.jsonBuilder().startObject().field("foo", "foo").endObject()); + index("fuu", "buu", "1", XContentFactory.jsonBuilder().startObject().field("fuu", "fuu").endObject()); + index("baz", "baz", "1", XContentFactory.jsonBuilder().startObject().field("baz", "baz").endObject()); + refresh(); + } + + @Test + public void testSimpleClusterStateModifications() throws Exception { + ClusterState clusterState0 = client().admin().cluster().prepareState().get().getState(); + int clusterNameSize = clusterState0.getClusterName().value().length(); + + // Assert that we don't send entire cluster state + ClusterState clusterState1 = ClusterState.builder(clusterState0).incrementVersion().build(); + assertThat(Builder.toDiffBytes(clusterState0, clusterState1).length, lessThan(clusterNameSize + 100)); + + ClusterState clusterState2 = ClusterState.builder(clusterState1).incrementVersion() + .blocks(ClusterBlocks.builder().addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)).build(); + + int fullClusterStateSize = Builder.toBytes(clusterState2).length; + int diffSize = Builder.toDiffBytes(clusterState1, clusterState2).length; + logger.info("Complete cluster state {}, diff size {}", fullClusterStateSize, diffSize ); + assertThat(diffSize, lessThan(fullClusterStateSize / 20)); + + ClusterState clusterState3 = Builder.diff(clusterState1, clusterState2).apply(clusterState1); + assertThat(clusterState3.blocks().global(), contains(MetaData.CLUSTER_READ_ONLY_BLOCK)); + + } + +} From 093c12f700f6c6c900ece952341b4d9e7546649d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 26 Dec 2014 13:39:53 -0500 Subject: [PATCH 02/20] Refactor out ClusterStateParts --- .../reroute/ClusterRerouteResponse.java | 2 +- .../cluster/state/ClusterStateResponse.java | 2 +- .../elasticsearch/cluster/ClusterName.java | 37 +++- .../elasticsearch/cluster/ClusterState.java | 176 ++++++++---------- .../cluster/ClusterStateParts.java | 71 ------- .../cluster/CompositeClusterStatePart.java | 173 ++++++++++------- .../discovery/local/LocalDiscovery.java | 4 +- .../zen/membership/MembershipAction.java | 19 +- .../publish/PublishClusterStateAction.java | 4 +- .../ClusterSerializationTests.java | 2 +- 10 files changed, 233 insertions(+), 257 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/cluster/ClusterStateParts.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index ee6c6d35af939..79b31f620d541 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -60,7 +60,7 @@ public RoutingExplanations getExplanations() { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - state = ClusterState.Builder.readFrom(in, null, null); + state = ClusterState.Builder.readFrom(in, null); readAcknowledged(in); explanations = RoutingExplanations.readFrom(in); } diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java index 2a05ec7d1197e..861a84a9e71b8 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java @@ -55,7 +55,7 @@ public ClusterName getClusterName() { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); clusterName = ClusterName.readClusterName(in); - clusterState = ClusterState.Builder.readFrom(in, null, clusterName); + clusterState = ClusterState.Builder.readFrom(in, null); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/ClusterName.java b/src/main/java/org/elasticsearch/cluster/ClusterName.java index 3a9dd82732c0d..90dd65f0f1e5d 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -23,13 +23,19 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; /** * */ -public class ClusterName implements Streamable { +public class ClusterName extends AbstractClusterStatePart implements Streamable { + + public static final String TYPE = "cluster_name"; + + public static final Factory FACTORY = new Factory(); public static final String SETTING = "cluster.name"; @@ -64,6 +70,11 @@ public void readFrom(StreamInput in) throws IOException { value = in.readString().intern(); } + @Override + public String type() { + return TYPE; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(value); @@ -90,4 +101,28 @@ public int hashCode() { public String toString() { return "Cluster [" + value + "]"; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("cluster_name", value); + return builder; + } + + public static class Factory extends AbstractFactory { + + @Override + public String type() { + return TYPE; + } + + @Override + public ClusterName readFrom(StreamInput in) throws IOException { + return readClusterName(in); + } + + @Override + public ClusterName fromXContent(XContentParser parser) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } + } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 78d9c3567fc52..71bb5cd0f1f94 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -47,7 +47,7 @@ /** * */ -public class ClusterState implements ToXContent { +public class ClusterState extends CompositeClusterStatePart { public static enum ClusterStateStatus { UNKNOWN((byte) 0), @@ -68,30 +68,31 @@ public byte id() { public static final long UNKNOWN_VERSION = -1; - private final long version; + private final RoutingTable routingTable; - private final String uuid; + private final DiscoveryNodes nodes; + + private final MetaData metaData; + + private final ClusterBlocks blocks; private final ClusterName clusterName; - private final ClusterStateParts parts; - // built on demand private volatile RoutingNodes routingNodes; private volatile ClusterStateStatus status; - public ClusterState(ClusterName clusterName, long version, String uuid, ClusterStateParts parts) { - this.version = version; - this.clusterName = clusterName; - this.parts = parts; + public ClusterState(long version, String uuid, ImmutableOpenMap parts) { + super(version, uuid, parts); + this.clusterName = get(ClusterName.TYPE); + this.routingTable = get(RoutingTable.TYPE); + this.metaData = get(MetaData.TYPE); + this.nodes = get(DiscoveryNodes.TYPE); + this.blocks = get(ClusterBlocks.TYPE); this.status = ClusterStateStatus.UNKNOWN; - this.uuid = uuid; } - public ClusterState(long version, String uuid, ClusterState state) { - this(state.clusterName, version, uuid, state.parts); - } public ClusterStateStatus status() { return status; @@ -111,7 +112,7 @@ public long getVersion() { } public DiscoveryNodes nodes() { - return (DiscoveryNodes)parts.get(DiscoveryNodes.TYPE); + return this.nodes; } public DiscoveryNodes getNodes() { @@ -119,7 +120,7 @@ public DiscoveryNodes getNodes() { } public MetaData metaData() { - return (MetaData)parts.get(MetaData.TYPE); + return this.metaData; } public MetaData getMetaData() { @@ -127,7 +128,7 @@ public MetaData getMetaData() { } public RoutingTable routingTable() { - return (RoutingTable)parts.get(RoutingTable.TYPE); + return routingTable; } public RoutingTable getRoutingTable() { @@ -135,7 +136,7 @@ public RoutingTable getRoutingTable() { } public RoutingNodes routingNodes() { - return routingTable().routingNodes(this); + return routingTable.routingNodes(this); } public RoutingNodes getRoutingNodes() { @@ -143,7 +144,7 @@ public RoutingNodes getRoutingNodes() { } public ClusterBlocks blocks() { - return (ClusterBlocks)parts.get(ClusterBlocks.TYPE); + return this.blocks; } public ClusterBlocks getBlocks() { @@ -151,6 +152,9 @@ public ClusterBlocks getBlocks() { } public ClusterName getClusterName() { + return clusterName(); + } + public ClusterName clusterName() { return this.clusterName; } @@ -162,7 +166,7 @@ public RoutingNodes readOnlyRoutingNodes() { if (routingNodes != null) { return routingNodes; } - routingNodes = routingTable().routingNodes(this); + routingNodes = routingTable.routingNodes(this); return routingNodes; } @@ -174,7 +178,7 @@ public ClusterState settingsFilter(SettingsFilter settingsFilter) { public String prettyPrint() { StringBuilder sb = new StringBuilder(); sb.append("version: ").append(version).append("\n"); - sb.append("meta data version: ").append(metaData().version()).append("\n"); + sb.append("meta data version: ").append(metaData.version()).append("\n"); sb.append(nodes().prettyPrint()); sb.append(routingTable().prettyPrint()); sb.append(readOnlyRoutingNodes().prettyPrint()); @@ -262,7 +266,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("master_node", nodes().masterNodeId()); } - for(ObjectObjectCursor partIter : parts.parts()) { + for(ObjectObjectCursor partIter : parts) { if (metricStrings.contains(partIter.key)) { builder.startObject(partIter.key); partIter.value.toXContent(builder, params); @@ -304,17 +308,15 @@ public static Builder builder(ClusterState state) { } public static class Builder { - - private final ClusterName clusterName; private long version = 0; private String uuid = null; private final ImmutableOpenMap.Builder parts; public Builder(ClusterState state) { - this.clusterName = state.clusterName; this.version = state.version(); - this.parts = ImmutableOpenMap.builder(state.parts.parts); + this.parts = ImmutableOpenMap.builder(state.parts); + putPart(ClusterName.TYPE, state.getClusterName()); putPart(MetaData.TYPE, state.metaData()); putPart(RoutingTable.TYPE, state.routingTable()); putPart(DiscoveryNodes.TYPE, state.nodes()); @@ -322,8 +324,8 @@ public Builder(ClusterState state) { } public Builder(ClusterName clusterName) { - this.clusterName = clusterName; parts = ImmutableOpenMap.builder(); + putPart(ClusterName.TYPE, clusterName); putPart(MetaData.TYPE, MetaData.EMPTY_META_DATA); putPart(RoutingTable.TYPE, RoutingTable.EMPTY_ROUTING_TABLE); putPart(DiscoveryNodes.TYPE, DiscoveryNodes.EMPTY_NODES); @@ -406,7 +408,7 @@ public ClusterState build() { if (uuid == null) { uuid = Strings.randomBase64UUID(); } - return new ClusterState(clusterName, version, uuid, new ClusterStateParts(parts.build())); + return new ClusterState(version, uuid, parts.build()); } public static byte[] toBytes(ClusterState state) throws IOException { @@ -418,61 +420,38 @@ public static byte[] toBytes(ClusterState state) throws IOException { /** * @param data input bytes * @param localNode used to set the local node in the cluster state. - * @param defaultClusterName this cluster name will be used of if the deserialized cluster state does not have a name set - * (which is only introduced in version 1.1.1) */ - public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode, ClusterName defaultClusterName) throws IOException { - return readFrom(new BytesStreamInput(data, false), localNode, defaultClusterName); + public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException { + return readFrom(new BytesStreamInput(data, false), localNode); } public static void writeTo(ClusterState state, StreamOutput out) throws IOException { - out.writeBoolean(state.clusterName != null); - if (state.clusterName != null) { - state.clusterName.writeTo(out); - } - out.writeLong(state.version); - out.writeString(state.uuid); - state.parts.writeTo(out); + state.writeTo(out); } /** * @param in input stream * @param localNode used to set the local node in the cluster state. can be null. - * @param defaultClusterName this cluster name will be used of receiving a cluster state from a node on version older than 1.1.1 - * or if the sending node did not set a cluster name */ - public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode, @Nullable ClusterName defaultClusterName) throws IOException { - ClusterName clusterName = defaultClusterName; - if (in.readBoolean()) { - clusterName = ClusterName.readClusterName(in); - } - Builder builder = new Builder(clusterName); - builder.version = in.readLong(); - builder.uuid = in.readString(); - builder.parts.putAll(ClusterStateParts.FACTORY.readFrom(in).parts()); + public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { + ClusterState clusterState = FACTORY.readFrom(in); //TODO: Hack!!!! Need to find a better way to handle localNode - DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); if (localNode != null) { + Builder builder = new Builder(clusterState); + DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); + return builder.build(); } - return builder.build(); + return clusterState; } - public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode, @Nullable ClusterName defaultClusterName) throws IOException { - ClusterName clusterName = defaultClusterName; - if (in.readBoolean()) { - clusterName = ClusterName.readClusterName(in); - } - long version = in.readLong(); - String previousUuid = in.readString(); - String newUuid = in.readString(); - ClusterStatePart.Diff diff = ClusterStateParts.FACTORY.readDiffFrom(in); - return new ClusterStateDiff(clusterName, version, previousUuid, newUuid, localNode, diff); - + public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { + long version = in.readVLong(); + return new ClusterStateDiff(version, localNode, FACTORY.readDiffFrom(in)); } public static ClusterStateDiff diff(ClusterState before, ClusterState after) { - return new ClusterStateDiff(after.clusterName, after.version, before.uuid, after.uuid, after.nodes().localNode(), ClusterStateParts.FACTORY.diff(before.parts, after.parts) ); + return new ClusterStateDiff(after.version(), null, ClusterState.FACTORY.diff(before, after) ); } public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws IOException { @@ -481,60 +460,67 @@ public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws return os.bytes().toBytes(); } - public static ClusterState fromDiffBytes(ClusterState before, byte[] data, DiscoveryNode localNode, ClusterName defaultClusterName) throws IOException { - ClusterStateDiff diff = readDiffFrom(new BytesStreamInput(data, false), localNode, defaultClusterName); + public static ClusterState fromDiffBytes(ClusterState before, byte[] data, DiscoveryNode localNode) throws IOException { + ClusterStateDiff diff = readDiffFrom(new BytesStreamInput(data, false), localNode); return diff.apply(before); } + } + + public static final String TYPE = "cluster"; + + public static final Factory FACTORY = new Factory(); + + public static class Factory extends AbstractCompositeClusterStatePartFactory { + + @Override + public String type() { + return TYPE; + } + @Override + public ClusterState fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { + return new ClusterState(version, uuid, parts.build()); + } + } + static { + registerFactory(ClusterName.TYPE, ClusterName.FACTORY); + registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); + registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); + registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); + registerFactory(MetaData.TYPE, MetaData.FACTORY); + } + @Override + public String type() { + return TYPE; } public static class ClusterStateDiff { - private ClusterName clusterName; - private long version; - private String previousUuid; - private String newUuid; private DiscoveryNode localNode; - private ClusterStatePart.Diff diff; + private long version; + private ClusterState.Diff diff; - public ClusterStateDiff(ClusterName clusterName, long version, String previousUuid, String newUuid, @Nullable DiscoveryNode localNode, ClusterStatePart.Diff diff) { - this.clusterName = clusterName; + public ClusterStateDiff(long version, @Nullable DiscoveryNode localNode, ClusterState.Diff diff) { this.version = version; - this.previousUuid = previousUuid; - this.newUuid = newUuid; this.localNode = localNode; this.diff = diff; } public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { - if (!previousUuid.equals(previous.uuid)) { - throw new IncompatibleClusterStateVersionException("Expected version " + (previous.version + 1) +"/" + previous.uuid + " got version " + version + "/" + previousUuid); - } - Builder builder = new Builder(clusterName); - builder.version = version; - builder.uuid = newUuid; - builder.parts.putAll(diff.apply(previous.parts).parts); - //TODO: Is there a better way to handle it? - DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); - if (localNode != null) { + ClusterState newState = diff.apply(previous); + if (!newState.equals(previous) && localNode != null) { + Builder builder = new Builder(newState); + DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); + return builder.build(); } - return builder.build(); + return newState; } public void writeTo(StreamOutput out) throws IOException { - if (clusterName != null) { - out.writeBoolean(true); - clusterName.writeTo(out); - } else { - out.writeBoolean(false); - } - out.writeLong(version); - out.writeString(previousUuid); - out.writeString(newUuid); + out.writeVLong(version); diff.writeTo(out); - } public long version() { diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java b/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java deleted file mode 100644 index 63076df64b21d..0000000000000 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateParts.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cluster; - -import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.common.collect.ImmutableOpenMap; - -/** - */ -public class ClusterStateParts extends CompositeClusterStatePart { - public static final String TYPE = "cluster"; - - public static final Factory FACTORY = new Factory(); - - public static class Factory extends AbstractCompositeClusterStatePartFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - public ClusterStateParts fromParts(ImmutableOpenMap.Builder parts) { - return new ClusterStateParts(parts.build()); - } - } - - static { - registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); - registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); - registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); - registerFactory(MetaData.TYPE, MetaData.FACTORY); - } - - @Override - public String type() { - return TYPE; - } - - static { - registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); - registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); - registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); - registerFactory(MetaData.TYPE, MetaData.FACTORY); - } - - public ClusterStateParts(ImmutableOpenMap parts) { - super(parts); - } - -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 91ca827ca8f1d..cc62f3c3f42b6 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -20,7 +20,6 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import com.google.common.collect.ImmutableList; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -39,12 +38,42 @@ import static com.google.common.collect.Maps.newHashMap; /** + * Represents a map of cluster state parts with different types. + *

+ * Only one instance of each type can be present in the composite part. The key of the map is the part's type. */ public abstract class CompositeClusterStatePart extends AbstractClusterStatePart { + private final static Map partFactories = new HashMap<>(); + protected final long version; + protected final String uuid; protected final ImmutableOpenMap parts; - private final static Map partFactories = new HashMap<>(); + protected CompositeClusterStatePart(long version, String uuid, ImmutableOpenMap parts) { + this.version = version; + this.uuid = uuid; + this.parts = parts; + } + + /** + * Register a custom index meta data factory. Make sure to call it from a static block. + */ + public static void registerFactory(String type, ClusterStatePart.Factory factory) { + partFactories.put(type, factory); + } + + @Nullable + public static ClusterStatePart.Factory lookupFactory(String type) { + return partFactories.get(type); + } + + public static ClusterStatePart.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { + ClusterStatePart.Factory factory = lookupFactory(type); + if (factory == null) { + throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); + } + return factory; + } @Override public boolean equals(Object o) { @@ -63,10 +92,58 @@ public int hashCode() { return parts != null ? parts.hashCode() : 0; } - public static abstract class AbstractCompositeClusterStatePartFactory extends AbstractClusterStatePart.AbstractFactory{ + public ImmutableOpenMap parts() { + return parts; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(version); + out.writeString(uuid); + out.writeVInt(parts().size()); + for (ObjectObjectCursor cursor : parts()) { + out.writeString(cursor.key); + cursor.value.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("version", version); + builder.field("uuid", uuid); + for (ObjectObjectCursor partIter : parts) { + builder.startObject(partIter.key); + partIter.value.toXContent(builder, params); + } + return builder; + } + + public T get(String type) { + return (T) parts.get(type); + } + + public long getVersion() { + return version; + } + + public String getUuid() { + return uuid; + } + + public long version() { + return version; + } + + public String uuid() { + return uuid; + } + + public static abstract class AbstractCompositeClusterStatePartFactory extends AbstractClusterStatePart.AbstractFactory { @Override public T readFrom(StreamInput in) throws IOException { + long version = in.readVLong(); + String uuid = in.readString(); ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); int partsSize = in.readVInt(); for (int i = 0; i < partsSize; i++) { @@ -74,7 +151,7 @@ public T readFrom(StreamInput in) throws IOException { ClusterStatePart part = lookupFactorySafe(type).readFrom(in); builder.put(type, part); } - return fromParts(builder); + return fromParts(version, uuid, builder); } @Override @@ -82,7 +159,7 @@ public T fromXContent(XContentParser parser) throws IOException { return null; } - public abstract T fromParts(ImmutableOpenMap.Builder parts); + public abstract T fromParts(long version, String uuid, ImmutableOpenMap.Builder parts); @Override public Diff diff(T before, T after) { @@ -102,26 +179,29 @@ public Diff diff(T before, T after) { ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); diffs.put(partIter.key, factory.diff(beforeParts.get(partIter.key), partIter.value)); } - return new CompositeDiff<>(this, deletes, diffs); + return new CompositeDiff<>(this, after.version(), before.uuid(), after.uuid(), deletes, diffs); } } @Override public Diff readDiffFrom(StreamInput in) throws IOException { if (in.readBoolean()) { + long version = in.readVLong(); + String previousUuid = in.readString(); + String uuid = in.readString(); int deletesSize = in.readVInt(); List deletes = new ArrayList<>(); - for (int i=0; i> diffs = newHashMap(); - for (int i=0; i(this, deletes, diffs); + return new CompositeDiff<>(this, version, previousUuid, uuid, deletes, diffs); } else { return new NoDiff(); @@ -131,11 +211,17 @@ public Diff readDiffFrom(StreamInput in) throws IOException { private static class CompositeDiff implements Diff { - Map> diffs; - List deletes; - AbstractCompositeClusterStatePartFactory factory; - - private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, List deletes, Map> diffs) { + private long version; + private String previousUuid; + private String uuid; + private Map> diffs; + private List deletes; + private AbstractCompositeClusterStatePartFactory factory; + + private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, long version, String previousUuid, String uuid, List deletes, Map> diffs) { + this.version = version; + this.previousUuid = previousUuid; + this.uuid = uuid; this.diffs = diffs; this.deletes = deletes; this.factory = factory; @@ -143,6 +229,9 @@ private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, List< @Override public T apply(T part) { + if(!previousUuid.equals(part.getUuid())) { + throw new IncompatibleClusterStateVersionException("Expected diffs for version " + part.version + " with uuid " + part.uuid + " got uuid " + previousUuid); + } ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); parts.putAll(part.parts); for (String delete : deletes) { @@ -152,14 +241,17 @@ public T apply(T part) { for (Map.Entry> entry : diffs.entrySet()) { parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); } - return factory.fromParts(parts); + return factory.fromParts(version, uuid, parts); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(true); // We have diffs + out.writeVLong(version); + out.writeString(previousUuid); + out.writeString(uuid); out.writeVInt(deletes.size()); - for(String delete : deletes) { + for (String delete : deletes) { out.writeString(delete); } @@ -171,54 +263,5 @@ public void writeTo(StreamOutput out) throws IOException { } } - protected CompositeClusterStatePart(ImmutableOpenMap parts) { - this.parts = parts; - } - - public ImmutableOpenMap parts() { - return parts; - } - - /** - * Register a custom index meta data factory. Make sure to call it from a static block. - */ - public static void registerFactory(String type, ClusterStatePart.Factory factory) { - partFactories.put(type, factory); - } - - @Nullable - public static ClusterStatePart.Factory lookupFactory(String type) { - return partFactories.get(type); - } - - public static ClusterStatePart.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { - ClusterStatePart.Factory factory = partFactories.get(type); - if (factory == null) { - throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); - } - return factory; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(parts().size()); - for (ObjectObjectCursor cursor : parts()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - for (ObjectObjectCursor partIter : parts) { - builder.startObject(partIter.key); - partIter.value.toXContent(builder, params); - } - return builder; - } - - public T get(String type) { - return (T) parts.get(type); - } } diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 1accdf2d5d3e8..b2fcd824d4ec1 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -311,12 +311,12 @@ private void publish(LocalDiscovery[] members, ClusterState clusterState, final if (clusterStateDiffBytes == null) { clusterStateDiffBytes = Builder.toDiffBytes(lastProcessedClusterState, clusterState); } - nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode, clusterName ); + nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode); } else { if (clusterStateBytes == null) { clusterStateBytes = Builder.toBytes(clusterState); } - nodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode, clusterName); + nodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode); } nodeSpecificClusterState.status(ClusterState.ClusterStateStatus.RECEIVED); // ignore cluster state messages that do not include "me", not in the game yet... diff --git a/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java b/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java index e0360e2bc31d4..e411cb6c66869 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java @@ -160,7 +160,7 @@ class JoinResponse extends TransportResponse { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); // we don't care about cluster name. This cluster state is never used. - clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode(), null); + clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); } @Override @@ -215,23 +215,6 @@ class ValidateJoinRequest extends TransportRequest { ValidateJoinRequest() { } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - if (in.getVersion().before(Version.V_1_4_0_Beta1)) { - // cluster name doesn't matter... - ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode(), null); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - if (out.getVersion().before(Version.V_1_4_0_Beta1)) { - ClusterState.Builder.writeTo(clusterService.state(), out); - } - } } private class ValidateJoinRequestRequestHandler extends BaseTransportRequestHandler { diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 7038731b51f21..6abe328e4612d 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -319,14 +319,14 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne in.setVersion(request.version()); final ClusterState clusterState; if (in.readBoolean()) { - clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode(), clusterName); + clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); logger.debug("received full cluster state version {}", clusterState.version()); } else { if(lastProcessedClusterState == null) { logger.debug("received diff cluster state version but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); } - ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode(), clusterName); + ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode()); if (lastProcessedClusterState.version() >= diff.version()) { logger.debug("got diffs for obsolete version {}, current version {}, ignoring the diff", diff.version(), lastProcessedClusterState.version()); return; diff --git a/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index b6513e878fe7a..e767ecd43d5a5 100644 --- a/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -56,7 +56,7 @@ public void testClusterStateSerialization() throws Exception { AllocationService strategy = createAllocationService(); clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState).routingTable()).build(); - ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1"), new ClusterName("clusterName2")); + ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1")); assertThat(serializedClusterState.getClusterName().value(), equalTo(clusterState.getClusterName().value())); assertThat(serializedClusterState.routingTable().prettyPrint(), equalTo(clusterState.routingTable().prettyPrint())); From 26233445afa691a593bedb9d8bf90160b93bfe54 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 28 Dec 2014 20:26:13 -0500 Subject: [PATCH 03/20] Add local context for handling local discovery node injection to deserialized cluster state --- .../cluster/AbstractClusterStatePart.java | 10 ++++- .../elasticsearch/cluster/ClusterName.java | 6 +-- .../elasticsearch/cluster/ClusterState.java | 26 +++---------- .../cluster/ClusterStatePart.java | 6 +-- .../cluster/CompositeClusterStatePart.java | 13 ++----- .../elasticsearch/cluster/LocalContext.java | 38 +++++++++++++++++++ .../cluster/block/ClusterBlocks.java | 8 +--- .../cluster/metadata/MetaData.java | 5 ++- .../cluster/node/DiscoveryNodes.java | 10 ++--- .../cluster/routing/RoutingTable.java | 5 ++- .../service/InternalClusterService.java | 2 +- .../discovery/local/LocalDiscovery.java | 20 +++++++--- 12 files changed, 86 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cluster/LocalContext.java diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 944997c35c7b5..298802144db75 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.EnumSet; @@ -86,10 +87,10 @@ public Diff diff(T before, T after) { } @Override - public Diff readDiffFrom(StreamInput in) throws IOException { + public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { if(in.readBoolean()) { if (in.readBoolean()) { - T part = readFrom(in); + T part = readFrom(in, context); return new CompleteDiff(part); } else { return new CompleteDiff(null); @@ -103,6 +104,11 @@ public Diff readDiffFrom(StreamInput in) throws IOException { public Version addedIn() { return null; } + + @Override + public T fromXContent(XContentParser parser, LocalContext context) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); + } } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterName.java b/src/main/java/org/elasticsearch/cluster/ClusterName.java index 90dd65f0f1e5d..3093679d2c877 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -116,13 +116,9 @@ public String type() { } @Override - public ClusterName readFrom(StreamInput in) throws IOException { + public ClusterName readFrom(StreamInput in, LocalContext context) throws IOException { return readClusterName(in); } - @Override - public ClusterName fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); - } } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 71bb5cd0f1f94..9e4b1a670a3ae 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -315,6 +315,7 @@ public static class Builder { public Builder(ClusterState state) { this.version = state.version(); + this.uuid = state.uuid(); this.parts = ImmutableOpenMap.builder(state.parts); putPart(ClusterName.TYPE, state.getClusterName()); putPart(MetaData.TYPE, state.metaData()); @@ -434,24 +435,17 @@ public static void writeTo(ClusterState state, StreamOutput out) throws IOExcept * @param localNode used to set the local node in the cluster state. can be null. */ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { - ClusterState clusterState = FACTORY.readFrom(in); - //TODO: Hack!!!! Need to find a better way to handle localNode - if (localNode != null) { - Builder builder = new Builder(clusterState); - DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); - builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); - return builder.build(); - } - return clusterState; + return FACTORY.readFrom(in, new LocalContext(localNode)); } public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { long version = in.readVLong(); - return new ClusterStateDiff(version, localNode, FACTORY.readDiffFrom(in)); + LocalContext localContext = new LocalContext(localNode); + return new ClusterStateDiff(version, FACTORY.readDiffFrom(in, localContext)); } public static ClusterStateDiff diff(ClusterState before, ClusterState after) { - return new ClusterStateDiff(after.version(), null, ClusterState.FACTORY.diff(before, after) ); + return new ClusterStateDiff(after.version(), ClusterState.FACTORY.diff(before, after) ); } public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws IOException { @@ -497,24 +491,16 @@ public String type() { } public static class ClusterStateDiff { - private DiscoveryNode localNode; private long version; private ClusterState.Diff diff; - public ClusterStateDiff(long version, @Nullable DiscoveryNode localNode, ClusterState.Diff diff) { + public ClusterStateDiff(long version, ClusterState.Diff diff) { this.version = version; - this.localNode = localNode; this.diff = diff; } public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { ClusterState newState = diff.apply(previous); - if (!newState.equals(previous) && localNode != null) { - Builder builder = new Builder(newState); - DiscoveryNodes discoveryNodes = builder.getPart(DiscoveryNodes.TYPE); - builder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNode.id())); - return builder.build(); - } return newState; } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index cf9e9d69751d7..5dc7c61cf3a15 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -63,11 +63,11 @@ interface Factory { Diff diff(T before, T after); - Diff readDiffFrom(StreamInput in) throws IOException; + Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException; - T readFrom(StreamInput in) throws IOException; + T readFrom(StreamInput in, LocalContext context) throws IOException; - T fromXContent(XContentParser parser) throws IOException; + T fromXContent(XContentParser parser, LocalContext context) throws IOException; Version addedIn(); diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index cc62f3c3f42b6..f5ff1c2ebdeb8 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -141,24 +141,19 @@ public String uuid() { public static abstract class AbstractCompositeClusterStatePartFactory extends AbstractClusterStatePart.AbstractFactory { @Override - public T readFrom(StreamInput in) throws IOException { + public T readFrom(StreamInput in, LocalContext context) throws IOException { long version = in.readVLong(); String uuid = in.readString(); ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); int partsSize = in.readVInt(); for (int i = 0; i < partsSize; i++) { String type = in.readString(); - ClusterStatePart part = lookupFactorySafe(type).readFrom(in); + ClusterStatePart part = lookupFactorySafe(type).readFrom(in, context); builder.put(type, part); } return fromParts(version, uuid, builder); } - @Override - public T fromXContent(XContentParser parser) throws IOException { - return null; - } - public abstract T fromParts(long version, String uuid, ImmutableOpenMap.Builder parts); @Override @@ -184,7 +179,7 @@ public Diff diff(T before, T after) { } @Override - public Diff readDiffFrom(StreamInput in) throws IOException { + public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { if (in.readBoolean()) { long version = in.readVLong(); String previousUuid = in.readString(); @@ -199,7 +194,7 @@ public Diff readDiffFrom(StreamInput in) throws IOException { Map> diffs = newHashMap(); for (int i = 0; i < diffsSize; i++) { String key = in.readString(); - diffs.put(key, lookupFactorySafe(key).readDiffFrom(in)); + diffs.put(key, lookupFactorySafe(key).readDiffFrom(in, context)); } return new CompositeDiff<>(this, version, previousUuid, uuid, deletes, diffs); diff --git a/src/main/java/org/elasticsearch/cluster/LocalContext.java b/src/main/java/org/elasticsearch/cluster/LocalContext.java new file mode 100644 index 0000000000000..798416ecf289b --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/LocalContext.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import org.elasticsearch.cluster.node.DiscoveryNode; + +/** + * Local context needed for reading cluster state + */ +public class LocalContext { + + private final DiscoveryNode localNode; + + public LocalContext(DiscoveryNode localNode) { + this.localNode = localNode; + } + + public DiscoveryNode getLocalNode() { + return localNode; + } +} diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index c2494ef9ce16c..fe570fdead774 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -25,6 +25,7 @@ import com.google.common.collect.Sets; import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.io.stream.StreamInput; @@ -93,14 +94,9 @@ public String type() { } @Override - public ClusterBlocks readFrom(StreamInput in) throws IOException { + public ClusterBlocks readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readClusterBlocks(in); } - - @Override - public ClusterBlocks fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); - } } public ImmutableSet global() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 9668ceec094dc..452dc20d24694 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.Nullable; @@ -77,12 +78,12 @@ public String type() { } @Override - public MetaData readFrom(StreamInput in) throws IOException { + public MetaData readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in); } @Override - public MetaData fromXContent(XContentParser parser) throws IOException { + public MetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { throw new UnsupportedOperationException("Not implemented yet"); } } diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index e130f4f73201d..742a7b30e1636 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -28,6 +28,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -76,13 +77,8 @@ public String type() { } @Override - public DiscoveryNodes readFrom(StreamInput in) throws IOException { - return Builder.readFrom(in, null); - } - - @Override - public DiscoveryNodes fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); + public DiscoveryNodes readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in, context.getLocalNode()); } } diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index d46067a13356a..0ff703ac37fc8 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; @@ -76,12 +77,12 @@ public String type() { } @Override - public RoutingTable readFrom(StreamInput in) throws IOException { + public RoutingTable readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in); } @Override - public RoutingTable fromXContent(XContentParser parser) throws IOException { + public RoutingTable fromXContent(XContentParser parser, LocalContext context) throws IOException { throw new UnsupportedOperationException("Not implemented yet"); } } diff --git a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index 8f2dd44c4d761..c21d9b0b17c9e 100644 --- a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -481,7 +481,7 @@ public void run() { logger.debug("processing [{}]: done applying updated cluster_state (version: {})", source, newClusterState.version()); } catch (Throwable t) { - StringBuilder sb = new StringBuilder("failed to apply updated cluster state:\nversion [").append(newClusterState.version()).append("], source [").append(source).append("]\n"); + StringBuilder sb = new StringBuilder("failed to apply updated cluster state:\nversion [").append(newClusterState.version()).append("], uuid [").append(newClusterState.uuid()).append("], source [").append(source).append("]\n"); sb.append(newClusterState.nodes().prettyPrint()); sb.append(newClusterState.routingTable().prettyPrint()); sb.append(newClusterState.readOnlyRoutingNodes().prettyPrint()); diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index b2fcd824d4ec1..78afffb1fa375 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.internal.Nullable; +import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -40,8 +41,6 @@ import org.elasticsearch.node.service.NodeService; import org.elasticsearch.transport.TransportService; -import java.io.IOException; -import java.util.Arrays; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -305,19 +304,28 @@ private void publish(LocalDiscovery[] members, ClusterState clusterState, final if (discovery.master) { continue; } - final ClusterState nodeSpecificClusterState; + ClusterState newNodeSpecificClusterState = null; // we do the marshaling intentionally, to check it works well... if (lastProcessedClusterState != null && lastProcessedClusterState.nodes().nodeExists(discovery.localNode.id())) { if (clusterStateDiffBytes == null) { clusterStateDiffBytes = Builder.toDiffBytes(lastProcessedClusterState, clusterState); } - nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode); - } else { +// nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode); + ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(new BytesStreamInput(clusterStateDiffBytes, false), discovery.localNode); + try { + newNodeSpecificClusterState = diff.apply(discovery.clusterService.state()); + } catch (IncompatibleClusterStateVersionException ex) { + logger.debug("incompatible cluster state version - resending complete cluster state", ex); + } + } + if (newNodeSpecificClusterState == null) { if (clusterStateBytes == null) { clusterStateBytes = Builder.toBytes(clusterState); } - nodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode); + newNodeSpecificClusterState = ClusterState.Builder.fromBytes(clusterStateBytes, discovery.localNode); } + final ClusterState nodeSpecificClusterState = newNodeSpecificClusterState; + nodeSpecificClusterState.status(ClusterState.ClusterStateStatus.RECEIVED); // ignore cluster state messages that do not include "me", not in the game yet... if (nodeSpecificClusterState.nodes().localNode() != null) { From 51712e4adc29658f30a7c58d74578cc6853fa6e1 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 28 Dec 2014 21:28:04 -0500 Subject: [PATCH 04/20] Add MapClusterStatePart --- .../elasticsearch/cluster/ClusterName.java | 11 --- .../elasticsearch/cluster/ClusterState.java | 11 --- .../cluster/ClusterStatePart.java | 4 - .../cluster/ClusterStateSettingsPart.java | 72 +++++++++++++++ .../cluster/MapClusterStatePart.java | 90 +++++++++++++++++++ .../cluster/MapItemClusterStatePart.java | 27 ++++++ .../cluster/block/ClusterBlocks.java | 13 --- .../cluster/metadata/MetaData.java | 13 +-- .../cluster/node/DiscoveryNodes.java | 14 --- .../cluster/routing/RoutingTable.java | 12 --- 10 files changed, 190 insertions(+), 77 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java create mode 100644 src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java create mode 100644 src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java diff --git a/src/main/java/org/elasticsearch/cluster/ClusterName.java b/src/main/java/org/elasticsearch/cluster/ClusterName.java index 3093679d2c877..f83e311c66e25 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -70,11 +69,6 @@ public void readFrom(StreamInput in) throws IOException { value = in.readString().intern(); } - @Override - public String type() { - return TYPE; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(value); @@ -110,11 +104,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static class Factory extends AbstractFactory { - @Override - public String type() { - return TYPE; - } - @Override public ClusterName readFrom(StreamInput in, LocalContext context) throws IOException { return readClusterName(in); diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 9e4b1a670a3ae..508487b2cfd20 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -466,11 +465,6 @@ public static ClusterState fromDiffBytes(ClusterState before, byte[] data, Disco public static class Factory extends AbstractCompositeClusterStatePartFactory { - @Override - public String type() { - return TYPE; - } - @Override public ClusterState fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { return new ClusterState(version, uuid, parts.build()); @@ -485,11 +479,6 @@ public ClusterState fromParts(long version, String uuid, ImmutableOpenMap.Builde registerFactory(MetaData.TYPE, MetaData.FACTORY); } - @Override - public String type() { - return TYPE; - } - public static class ClusterStateDiff { private long version; private ClusterState.Diff diff; diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index 5dc7c61cf3a15..e8f5bd903f484 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -51,16 +51,12 @@ public enum XContentContext { public static EnumSet API_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT); public static EnumSet API_GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.GATEWAY, XContentContext.SNAPSHOT); - String type(); - void writeTo(StreamOutput out) throws IOException; EnumSet context(); interface Factory { - String type(); - Diff diff(T before, T after); Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException; diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java new file mode 100644 index 0000000000000..facb8526aa881 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.loader.SettingsLoader; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; +import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; + +/** + */ +public class ClusterStateSettingsPart extends AbstractClusterStatePart { + + private final Settings settings; + + public ClusterStateSettingsPart(Settings settings) { + this.settings = settings; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeSettingsToStream(settings, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (Map.Entry entry : settings.getAsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + return builder; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public ClusterStateSettingsPart readFrom(StreamInput in, LocalContext context) throws IOException { + return new ClusterStateSettingsPart(readSettingsFromStream(in)); + } + + @Override + public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext context) throws IOException { + Settings settings = ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).build(); + return new ClusterStateSettingsPart(settings); + } + } +} diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java new file mode 100644 index 0000000000000..db20147d7d249 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * Represents a map of cluster state parts with the same type. + */ +public class MapClusterStatePart extends AbstractClusterStatePart { + + private final ImmutableOpenMap parts; + + public MapClusterStatePart(ImmutableOpenMap parts) { + this.parts = parts; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(parts.size()); + for (ObjectObjectCursor part : parts) { + out.writeString(part.key); + part.value.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (ObjectCursor part : parts.values()) { + part.value.toXContent(builder, params); + } + return builder; + } + + public static class Factory extends AbstractFactory> { + + private final ClusterStatePart.Factory factory; + + public Factory(ClusterStatePart.Factory factory) { + this.factory = factory; + } + + @Override + public MapClusterStatePart readFrom(StreamInput in, LocalContext context) throws IOException { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + int size = in.readVInt(); + for (int i = 0; i < size; i++) { + T part = factory.readFrom(in, context); + builder.put(part.key(), part); + } + return new MapClusterStatePart<>(builder.build()); + } + + @Override + public MapClusterStatePart fromXContent(XContentParser parser, LocalContext context) throws IOException { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + T part = factory.fromXContent(parser, context); + builder.put(part.key(), part); + } + return new MapClusterStatePart<>(builder.build()); + } + } + +} diff --git a/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java new file mode 100644 index 0000000000000..067a024895a12 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +/** + * All cluster state parts that can be put into a map should implement this interface + */ +public interface MapItemClusterStatePart extends ClusterStatePart { + String key(); +} diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index fe570fdead774..dc264b8aa4cc1 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -24,15 +24,12 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -88,11 +85,6 @@ public class ClusterBlocks extends AbstractClusterStatePart { public static class Factory extends AbstractFactory { - @Override - public String type() { - return TYPE; - } - @Override public ClusterBlocks readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readClusterBlocks(in); @@ -226,11 +218,6 @@ public ClusterBlockException indicesBlockedException(ClusterBlockLevel level, St return new ClusterBlockException(builder.build()); } - @Override - public String type() { - return TYPE; - } - @Override public void writeTo(StreamOutput out) throws IOException { Builder.writeClusterBlocks(this, out); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 452dc20d24694..c20e2aef2e235 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -28,7 +28,6 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -72,11 +71,6 @@ public class MetaData extends AbstractClusterStatePart implements Iterable { - @Override - public String type() { - return TYPE; - } - @Override public MetaData readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in); @@ -88,11 +82,6 @@ public MetaData fromXContent(XContentParser parser, LocalContext context) throws } } - @Override - public String type() { - return TYPE; - } - @Override public void writeTo(StreamOutput out) throws IOException { Builder.writeTo(this, out); @@ -264,7 +253,7 @@ public static Custom.Factory lookupFactorySafe(String type private final Settings transientSettings; private final Settings persistentSettings; - private final Settings settings; + private final Settings settings; // Generated on the fly from transientSettings and persistentSettings private final ImmutableOpenMap indices; private final ImmutableOpenMap templates; private final ImmutableOpenMap customs; diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 742a7b30e1636..1bdace3b93963 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -27,7 +27,6 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; @@ -36,12 +35,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -71,11 +67,6 @@ public class DiscoveryNodes extends AbstractClusterStatePart implements Iterable public static class Factory extends AbstractFactory { - @Override - public String type() { - return TYPE; - } - @Override public DiscoveryNodes readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in, context.getLocalNode()); @@ -491,11 +482,6 @@ public Delta emptyDelta() { return new Delta(null, null, localNodeId, DiscoveryNode.EMPTY_LIST, DiscoveryNode.EMPTY_LIST); } - @Override - public String type() { - return DiscoveryNodes.TYPE; - } - @Override public void writeTo(StreamOutput out) throws IOException { DiscoveryNodes.Builder.writeTo(this, out); diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 0ff703ac37fc8..bbadba753befd 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -23,13 +23,11 @@ import com.google.common.collect.*; import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; @@ -71,11 +69,6 @@ public class RoutingTable extends AbstractClusterStatePart implements Iterable { - @Override - public String type() { - return TYPE; - } - @Override public RoutingTable readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in); @@ -323,11 +316,6 @@ public static Builder builder(RoutingTable routingTable) { return new Builder(routingTable); } - @Override - public String type() { - return TYPE; - } - @Override public void writeTo(StreamOutput out) throws IOException { Builder.writeTo(this, out); From dceedd03bf802dbce2ef039d6e096744c70a6769 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 29 Dec 2014 08:56:45 -0500 Subject: [PATCH 05/20] First pass through adding diffs to MetaData --- .../state/TransportClusterStateAction.java | 9 +- .../cluster/AbstractClusterStatePart.java | 25 +- .../cluster/ClusterStateSettingsPart.java | 4 + .../cluster/CompositeClusterStatePart.java | 38 +- .../cluster/MapClusterStatePart.java | 5 +- .../cluster/metadata/BenchmarkMetaData.java | 75 +- .../cluster/metadata/IndexMetaData.java | 39 +- .../metadata/IndexTemplateMetaData.java | 39 +- .../cluster/metadata/MetaData.java | 160 ++--- .../metadata/RepositoriesMetaData.java | 106 ++- .../cluster/metadata/RestoreMetaData.java | 165 ++--- .../cluster/metadata/SnapshotMetaData.java | 168 +++-- .../get/RestGetRepositoriesAction.java | 2 +- .../snapshots/RestoreService.java | 2 +- .../DedicatedClusterSnapshotRestoreTests.java | 642 +++++++++--------- 15 files changed, 750 insertions(+), 729 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index e3dedfe6252ec..6f63caf8f0b72 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -28,11 +28,11 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.MetaData.Custom; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -127,10 +127,9 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS } // Filter our metadata that shouldn't be returned by API - for(ObjectCursor type : currentState.metaData().customs().keys()) { - Custom.Factory factory = lookupFactorySafe(type.value); - if(!factory.context().contains(MetaData.XContentContext.API)) { - mdBuilder.removeCustom(type.value); + for(ObjectObjectCursor cursor : currentState.metaData().customs()) { + if(!cursor.value.context().contains(MetaData.XContentContext.API)) { + mdBuilder.removeCustom(cursor.key); } } diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 298802144db75..ea01e236377f6 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentParser; @@ -50,12 +51,7 @@ public T apply(T state) { public void writeTo(StreamOutput out) throws IOException{ out.writeBoolean(true); - if (part != null) { - out.writeBoolean(true); - part.writeTo(out); - } else { - out.writeBoolean(false); - } + part.writeTo(out); } } @@ -78,23 +74,20 @@ public void writeTo(StreamOutput out) throws IOException { protected static abstract class AbstractFactory implements ClusterStatePart.Factory { @Override - public Diff diff(T before, T after) { - if (before.equals(after)) { - return new NoDiff(); + public Diff diff(@Nullable T before, T after) { + assert after != null; + if (after.equals(before)) { + return new NoDiff<>(); } else { - return new CompleteDiff(after); + return new CompleteDiff<>(after); } } @Override public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { if(in.readBoolean()) { - if (in.readBoolean()) { - T part = readFrom(in, context); - return new CompleteDiff(part); - } else { - return new CompleteDiff(null); - } + T part = readFrom(in, context); + return new CompleteDiff(part); } else { return new NoDiff(); } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index facb8526aa881..9de327766057a 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -56,6 +56,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public Settings getSettings() { + return settings; + } + public static class Factory extends AbstractClusterStatePart.AbstractFactory { @Override diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index f5ff1c2ebdeb8..7a201d744d81d 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -19,6 +19,8 @@ package org.elasticsearch.cluster; +import com.carrotsearch.hppc.ObjectLookupContainer; +import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; @@ -157,25 +159,35 @@ public T readFrom(StreamInput in, LocalContext context) throws IOException { public abstract T fromParts(long version, String uuid, ImmutableOpenMap.Builder parts); @Override - public Diff diff(T before, T after) { - ImmutableOpenMap beforeParts = before.parts(); - ImmutableOpenMap afterParts = after.parts(); - if (before.equals(after)) { - return new NoDiff(); - } else { - Map> diffs = newHashMap(); - List deletes = newArrayList(); - for (ObjectObjectCursor partIter : beforeParts) { - if (!afterParts.containsKey(partIter.key)) { - deletes.add(partIter.key); + public Diff diff(@Nullable T before, T after) { + assert after != null; + Map> diffs = newHashMap(); + List deletes = newArrayList(); + if (before != null) { + ImmutableOpenMap beforeParts = before.parts(); + ImmutableOpenMap afterParts = after.parts(); + if (before.equals(after)) { + return new NoDiff(); + } else { + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); + } + } + for (ObjectObjectCursor partIter : afterParts) { + ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); + ClusterStatePart beforePart = beforeParts.get(partIter.key); + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); } } + } else { + ImmutableOpenMap afterParts = after.parts(); for (ObjectObjectCursor partIter : afterParts) { ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); - diffs.put(partIter.key, factory.diff(beforeParts.get(partIter.key), partIter.value)); + diffs.put(partIter.key, factory.diff(null, partIter.value)); } - return new CompositeDiff<>(this, after.version(), before.uuid(), after.uuid(), deletes, diffs); } + return new CompositeDiff<>(this, after.version(), before.uuid(), after.uuid(), deletes, diffs); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index db20147d7d249..90b5076440098 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -44,7 +44,6 @@ public MapClusterStatePart(ImmutableOpenMap parts) { public void writeTo(StreamOutput out) throws IOException { out.writeVInt(parts.size()); for (ObjectObjectCursor part : parts) { - out.writeString(part.key); part.value.writeTo(out); } } @@ -57,6 +56,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public ImmutableOpenMap getParts() { + return parts; + } + public static class Factory extends AbstractFactory> { private final ClusterStatePart.Factory factory; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java index 6638efda76f86..6cd899994d890 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java @@ -20,6 +20,8 @@ import com.google.common.collect.ImmutableList; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -31,7 +33,7 @@ /** * Meta data about benchmarks that are currently executing */ -public class BenchmarkMetaData implements MetaData.Custom { +public class BenchmarkMetaData extends AbstractClusterStatePart { public static final String TYPE = "benchmark"; public static final Factory FACTORY = new Factory(); @@ -154,16 +156,41 @@ public ImmutableList entries() { return this.entries; } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(entries.size()); + for (Entry entry : entries) { + out.writeString(entry.benchmarkId()); + out.writeByte(entry.state().id()); + out.writeStringArray(entry.nodes()); + } + } - public static class Factory extends MetaData.Custom.Factory { + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray("benchmarks"); + for (Entry entry : entries) { + toXContent(entry, builder, params); + } + builder.endArray(); + return builder; + } - @Override - public String type() { - return TYPE; + public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field("id", entry.benchmarkId()); + builder.field("state", entry.state()); + builder.startArray("on_nodes"); + for (String nodeid : entry.nodes()) { + builder.value(nodeid); } + builder.endArray(); + builder.endObject(); + } + public static class Factory extends AbstractClusterStatePart.AbstractFactory { @Override - public BenchmarkMetaData readFrom(StreamInput in) throws IOException { + public BenchmarkMetaData readFrom(StreamInput in, LocalContext context) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { String benchmarkId = in.readString(); @@ -173,42 +200,6 @@ public BenchmarkMetaData readFrom(StreamInput in) throws IOException { } return new BenchmarkMetaData(entries); } - - @Override - public void writeTo(BenchmarkMetaData repositories, StreamOutput out) throws IOException { - out.writeVInt(repositories.entries().size()); - for (Entry entry : repositories.entries()) { - out.writeString(entry.benchmarkId()); - out.writeByte(entry.state().id()); - out.writeStringArray(entry.nodes()); - } - } - - @Override - public BenchmarkMetaData fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void toXContent(BenchmarkMetaData customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray("benchmarks"); - for (Entry entry : customIndexMetaData.entries()) { - toXContent(entry, builder, params); - } - builder.endArray(); - } - - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(); - builder.field("id", entry.benchmarkId()); - builder.field("state", entry.state()); - builder.startArray("on_nodes"); - for (String nodeid : entry.nodes()) { - builder.value(nodeid); - } - builder.endArray(); - builder.endObject(); - } } public boolean contains(String benchmarkId) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 1d2e9646ae4b4..7c5267e49df22 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -25,6 +25,9 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.MapItemClusterStatePart; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; @@ -44,6 +47,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.warmer.IndexWarmersMetaData; @@ -61,8 +65,11 @@ /** * */ -public class IndexMetaData { +public class IndexMetaData extends AbstractClusterStatePart implements MapItemClusterStatePart { + public static String TYPE = "index_metadata"; + + public static Factory FACTORY = new Factory(); public interface Custom { @@ -784,4 +791,34 @@ public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws } } } + + @Override + public String key() { + return index; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(this, builder, params); + return builder; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public IndexMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in); + } + + @Override + public IndexMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return Builder.fromXContent(parser); + } + } + } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 6198f35cd52cf..b58b13ef6e74a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -22,6 +22,9 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.Sets; import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.MapItemClusterStatePart; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedString; @@ -42,7 +45,11 @@ /** * */ -public class IndexTemplateMetaData { +public class IndexTemplateMetaData extends AbstractClusterStatePart implements MapItemClusterStatePart { + + public static String TYPE = "index_template"; + + public static Factory FACTORY = new Factory(); private final String name; @@ -451,4 +458,34 @@ public static void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOu } } + @Override + public String key() { + return template; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(this, builder, params); + return builder; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public IndexTemplateMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in); + } + + @Override + public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return Builder.fromXContent(parser, parser.currentName()); + } + } + + } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index c20e2aef2e235..4ac50c2851994 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -27,8 +27,7 @@ import com.google.common.collect.*; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.Nullable; @@ -60,22 +59,31 @@ /** * */ -public class MetaData extends AbstractClusterStatePart implements Iterable { +public class MetaData extends CompositeClusterStatePart implements Iterable { public static final String TYPE = "metadata"; + public static final String PERSISTENT_SETTINGS_TYPE = "persistent_settings"; + + public static final String TRANSIENT_SETTINGS_TYPE = "transient_settings"; + public static final Factory FACTORY = new Factory(); public static final String ALL = "_all"; - public static class Factory extends AbstractFactory { + public static class Factory extends AbstractCompositeClusterStatePartFactory { @Override public MetaData readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in); } + @Override + public MetaData fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { + return new MetaData(version, uuid, parts.build()); + } + @Override public MetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { throw new UnsupportedOperationException("Not implemented yet"); @@ -162,52 +170,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endObject(); - for (ObjectObjectCursor cursor : customs()) { + for (ObjectObjectCursor cursor : customs()) { builder.startObject(cursor.key); - MetaData.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); + cursor.value.toXContent(builder, params); builder.endObject(); } return builder; } - public enum XContentContext { - /* Custom metadata should be returns as part of API call */ - API, - - /* Custom metadata should be stored as part of the persistent cluster state */ - GATEWAY, - - /* Custom metadata should be stored as part of a snapshot */ - SNAPSHOT; - } - - public static EnumSet API_ONLY = EnumSet.of(XContentContext.API); - public static EnumSet API_AND_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY); - public static EnumSet API_AND_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT); - - public interface Custom { - - abstract class Factory { - - public abstract String type(); - - public abstract T readFrom(StreamInput in) throws IOException; - - public abstract void writeTo(T customIndexMetaData, StreamOutput out) throws IOException; - - public abstract T fromXContent(XContentParser parser) throws IOException; - - public abstract void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException; - - public EnumSet context() { - return API_ONLY; - } - } - } - - public static Map customFactories = new HashMap<>(); - static { + registerFactory(TRANSIENT_SETTINGS_TYPE, new ClusterStateSettingsPart.Factory()); + registerFactory(PERSISTENT_SETTINGS_TYPE, new ClusterStateSettingsPart.Factory()); + registerFactory(IndexMetaData.TYPE, new MapClusterStatePart.Factory<>(IndexMetaData.FACTORY)); + registerFactory(IndexTemplateMetaData.TYPE, new MapClusterStatePart.Factory<>(IndexTemplateMetaData.FACTORY)); + // register non plugin custom metadata registerFactory(RepositoriesMetaData.TYPE, RepositoriesMetaData.FACTORY); registerFactory(SnapshotMetaData.TYPE, SnapshotMetaData.FACTORY); @@ -215,27 +191,6 @@ public EnumSet context() { registerFactory(BenchmarkMetaData.TYPE, BenchmarkMetaData.FACTORY); } - /** - * Register a custom index meta data factory. Make sure to call it from a static block. - */ - public static void registerFactory(String type, Custom.Factory factory) { - customFactories.put(type, factory); - } - - @Nullable - public static Custom.Factory lookupFactory(String type) { - return customFactories.get(type); - } - - public static Custom.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { - Custom.Factory factory = customFactories.get(type); - if (factory == null) { - throw new ElasticsearchIllegalArgumentException("No custom index metadata factory registered for type [" + type + "]"); - } - return factory; - } - - public static final String SETTING_READ_ONLY = "cluster.blocks.read_only"; public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK = new ClusterBlock(6, "cluster read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA)); @@ -256,7 +211,7 @@ public static Custom.Factory lookupFactorySafe(String type private final Settings settings; // Generated on the fly from transientSettings and persistentSettings private final ImmutableOpenMap indices; private final ImmutableOpenMap templates; - private final ImmutableOpenMap customs; + private final ImmutableOpenMap customs; private final transient int totalNumberOfShards; // Transient ? not serializable anyway? private final int numberOfShards; @@ -272,16 +227,27 @@ public static Custom.Factory lookupFactorySafe(String type //TODO: This is a hack - needed for plugins. We should refactor it using params private SettingsFilter settingsFilter; + // Parts: Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs + @SuppressWarnings("unchecked") - MetaData(String uuid, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs) { + MetaData(long version, String uuid, ImmutableOpenMap parts) { + super(version, uuid, parts); this.uuid = uuid; this.version = version; - this.transientSettings = transientSettings; - this.persistentSettings = persistentSettings; + this.transientSettings = ((ClusterStateSettingsPart)get(TRANSIENT_SETTINGS_TYPE)).getSettings(); + this.persistentSettings = ((ClusterStateSettingsPart)get(PERSISTENT_SETTINGS_TYPE)).getSettings(); this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build(); - this.indices = indices; - this.customs = customs; - this.templates = templates; + this.indices = ((MapClusterStatePart)get(IndexMetaData.TYPE)).getParts(); + this.templates = ((MapClusterStatePart)get(IndexTemplateMetaData.TYPE)).getParts(); + //TODO: Hack, for now to make things running + ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(); + customsBuilder.putAll(parts); + customsBuilder.remove(TRANSIENT_SETTINGS_TYPE); + customsBuilder.remove(PERSISTENT_SETTINGS_TYPE); + customsBuilder.remove(IndexMetaData.TYPE); + customsBuilder.remove(IndexTemplateMetaData.TYPE); + this.customs = customsBuilder.build(); + int totalNumberOfShards = 0; int numberOfShards = 0; int numAliases = 0; @@ -1066,15 +1032,15 @@ public ImmutableOpenMap getTemplates() { return this.templates; } - public ImmutableOpenMap customs() { + public ImmutableOpenMap customs() { return this.customs; } - public ImmutableOpenMap getCustoms() { + public ImmutableOpenMap getCustoms() { return this.customs; } - public T custom(String type) { + public T custom(String type) { return (T) customs.get(type); } @@ -1249,15 +1215,15 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 } // Check if any persistent metadata needs to be saved int customCount1 = 0; - for (ObjectObjectCursor cursor : metaData1.customs) { - if (customFactories.get(cursor.key).context().contains(XContentContext.GATEWAY)) { + for (ObjectObjectCursor cursor : metaData1.customs) { + if (cursor.value.context().contains(XContentContext.GATEWAY)) { if (!cursor.value.equals(metaData2.custom(cursor.key))) return false; customCount1++; } } int customCount2 = 0; - for (ObjectObjectCursor cursor : metaData2.customs) { - if (customFactories.get(cursor.key).context().contains(XContentContext.GATEWAY)) { + for (ObjectObjectCursor cursor : metaData2.customs) { + if (cursor.value.context().contains(XContentContext.GATEWAY)) { customCount2++; } } @@ -1283,7 +1249,7 @@ public static class Builder { private final ImmutableOpenMap.Builder indices; private final ImmutableOpenMap.Builder templates; - private final ImmutableOpenMap.Builder customs; + private final ImmutableOpenMap.Builder customs; public Builder() { uuid = "_na_"; @@ -1350,11 +1316,11 @@ public Builder removeTemplate(String templateName) { return this; } - public Custom getCustom(String type) { + public ClusterStatePart getCustom(String type) { return customs.get(type); } - public Builder putCustom(String type, Custom custom) { + public Builder putCustom(String type, ClusterStatePart custom) { customs.put(type, custom); return this; } @@ -1423,8 +1389,19 @@ public Builder generateUuidIfNeeded() { return this; } + + private static ImmutableOpenMap buildParts(Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs) { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + builder.put(TRANSIENT_SETTINGS_TYPE, new ClusterStateSettingsPart(transientSettings)); + builder.put(PERSISTENT_SETTINGS_TYPE, new ClusterStateSettingsPart(persistentSettings)); + builder.put(IndexMetaData.TYPE, new MapClusterStatePart<>(indices)); + builder.put(IndexTemplateMetaData.TYPE, new MapClusterStatePart<>(templates)); + builder.putAll(customs); + return builder.build(); + } + public MetaData build() { - return new MetaData(uuid, version, transientSettings, persistentSettings, indices.build(), templates.build(), customs.build()); + return new MetaData(version, uuid, buildParts(transientSettings, persistentSettings, indices.build(), templates.build(), customs.build())); } public static String toXContent(MetaData metaData) throws IOException { @@ -1473,11 +1450,10 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon builder.endObject(); } - for (ObjectObjectCursor cursor : metaData.customs()) { - Custom.Factory factory = lookupFactorySafe(cursor.key); - if (factory.context().contains(context)) { + for (ObjectObjectCursor cursor : metaData.customs()) { + if (cursor.value.context().contains(context)) { builder.startObject(cursor.key); - factory.toXContent(cursor.value, builder, params); + cursor.value.toXContent(builder, params); builder.endObject(); } } @@ -1521,12 +1497,13 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { } } else { // check if its a custom index metadata - Custom.Factory factory = lookupFactory(currentFieldName); + ClusterStatePart.Factory factory = lookupFactory(currentFieldName); if (factory == null) { //TODO warn parser.skipChildren(); } else { - builder.putCustom(factory.type(), factory.fromXContent(parser)); + // TODO: context + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } } else if (token.isValue()) { @@ -1557,7 +1534,8 @@ public static MetaData readFrom(StreamInput in) throws IOException { int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { String type = in.readString(); - Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in); + // TODO: context + ClusterStatePart customIndexMetaData = lookupFactorySafe(type).readFrom(in, null); builder.putCustom(type, customIndexMetaData); } return builder.build(); @@ -1577,9 +1555,9 @@ public static void writeTo(MetaData metaData, StreamOutput out) throws IOExcepti IndexTemplateMetaData.Builder.writeTo(cursor.value, out); } out.writeVInt(metaData.customs().size()); - for (ObjectObjectCursor cursor : metaData.customs()) { + for (ObjectObjectCursor cursor : metaData.customs()) { out.writeString(cursor.key); - lookupFactorySafe(cursor.key).writeTo(cursor.value, out); + cursor.value.writeTo(out); } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index 81b11fc14b1b6..73547e65be59f 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -21,6 +21,9 @@ import com.google.common.collect.ImmutableList; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.ImmutableSettings; @@ -39,7 +42,7 @@ /** * Contains metadata about registered snapshot repositories */ -public class RepositoriesMetaData implements MetaData.Custom { +public class RepositoriesMetaData extends AbstractClusterStatePart { public static final String TYPE = "repositories"; @@ -80,24 +83,57 @@ public RepositoryMetaData repository(String name) { return null; } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(repositories.size()); + for (RepositoryMetaData repository : repositories) { + repository.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (RepositoryMetaData repository : repositories()) { + toXContent(repository, builder, params); + } + return builder; + } + /** - * Repository metadata factory + * Serializes information about a single repository + * + * @param repository repository metadata + * @param builder XContent builder + * @param params serialization parameters + * @throws IOException */ - public static class Factory extends MetaData.Custom.Factory { - - /** - * {@inheritDoc} - */ - @Override - public String type() { - return TYPE; + public static void toXContent(RepositoryMetaData repository, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(repository.name(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("type", repository.type()); + builder.startObject("settings"); + for (Map.Entry settingEntry : repository.settings().getAsMap().entrySet()) { + builder.field(settingEntry.getKey(), settingEntry.getValue()); } + builder.endObject(); + + builder.endObject(); + } + + @Override + public EnumSet context() { + return ClusterStatePart.API_GATEWAY; + } + + /** + * Repository metadata factory + */ + public static class Factory extends AbstractClusterStatePart.AbstractFactory { /** * {@inheritDoc} */ @Override - public RepositoriesMetaData readFrom(StreamInput in) throws IOException { + public RepositoriesMetaData readFrom(StreamInput in, LocalContext context) throws IOException { RepositoryMetaData[] repository = new RepositoryMetaData[in.readVInt()]; for (int i = 0; i < repository.length; i++) { repository[i] = RepositoryMetaData.readFrom(in); @@ -109,18 +145,7 @@ public RepositoriesMetaData readFrom(StreamInput in) throws IOException { * {@inheritDoc} */ @Override - public void writeTo(RepositoriesMetaData repositories, StreamOutput out) throws IOException { - out.writeVInt(repositories.repositories().size()); - for (RepositoryMetaData repository : repositories.repositories()) { - repository.writeTo(out); - } - } - - /** - * {@inheritDoc} - */ - @Override - public RepositoriesMetaData fromXContent(XContentParser parser) throws IOException { + public RepositoriesMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { XContentParser.Token token; List repository = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -161,41 +186,6 @@ public RepositoriesMetaData fromXContent(XContentParser parser) throws IOExcepti } return new RepositoriesMetaData(repository.toArray(new RepositoryMetaData[repository.size()])); } - - /** - * {@inheritDoc} - */ - @Override - public void toXContent(RepositoriesMetaData customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - for (RepositoryMetaData repository : customIndexMetaData.repositories()) { - toXContent(repository, builder, params); - } - } - - @Override - public EnumSet context() { - return MetaData.API_AND_GATEWAY; - } - - /** - * Serializes information about a single repository - * - * @param repository repository metadata - * @param builder XContent builder - * @param params serialization parameters - * @throws IOException - */ - public void toXContent(RepositoryMetaData repository, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(repository.name(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("type", repository.type()); - builder.startObject("settings"); - for (Map.Entry settingEntry : repository.settings().getAsMap().entrySet()) { - builder.field(settingEntry.getKey(), settingEntry.getValue()); - } - builder.endObject(); - - builder.endObject(); - } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java index 373d5ff858c9e..33aad9ec51579 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java @@ -22,6 +22,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -35,7 +37,7 @@ /** * Meta data about restore processes that are currently executing */ -public class RestoreMetaData implements MetaData.Custom { +public class RestoreMetaData extends AbstractClusterStatePart { public static final String TYPE = "restore"; @@ -395,23 +397,86 @@ public static State fromValue(byte value) { } /** - * Restore metadata factory + * {@inheritDoc} */ - public static class Factory extends MetaData.Custom.Factory { + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(entries.size()); + for (Entry entry : entries) { + entry.snapshotId().writeTo(out); + out.writeByte(entry.state().value()); + out.writeVInt(entry.indices().size()); + for (String index : entry.indices()) { + out.writeString(index); + } + out.writeVInt(entry.shards().size()); + for (Map.Entry shardEntry : entry.shards().entrySet()) { + shardEntry.getKey().writeTo(out); + shardEntry.getValue().writeTo(out); + } + } + } - /** - * {@inheritDoc} - */ - @Override - public String type() { - return TYPE; + /** + * {@inheritDoc} + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray("snapshots"); + for (Entry entry : entries) { + toXContent(entry, builder, params); } + builder.endArray(); + return builder; + } + /** + * Serializes single restore operation + * + * @param entry restore operation metadata + * @param builder XContent builder + * @param params serialization parameters + * @throws IOException + */ + public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field("snapshot", entry.snapshotId().getSnapshot()); + builder.field("repository", entry.snapshotId().getRepository()); + builder.field("state", entry.state()); + builder.startArray("indices"); + { + for (String index : entry.indices()) { + builder.value(index); + } + } + builder.endArray(); + builder.startArray("shards"); + { + for (Map.Entry shardEntry : entry.shards.entrySet()) { + ShardId shardId = shardEntry.getKey(); + ShardRestoreStatus status = shardEntry.getValue(); + builder.startObject(); + { + builder.field("index", shardId.getIndex()); + builder.field("shard", shardId.getId()); + builder.field("state", status.state()); + } + builder.endObject(); + } + } + + builder.endArray(); + builder.endObject(); + } + /** + * Restore metadata factory + */ + public static class Factory extends AbstractClusterStatePart.AbstractFactory { /** * {@inheritDoc} */ @Override - public RestoreMetaData readFrom(StreamInput in) throws IOException { + public RestoreMetaData readFrom(StreamInput in, LocalContext context) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { SnapshotId snapshotId = SnapshotId.readSnapshotId(in); @@ -432,86 +497,6 @@ public RestoreMetaData readFrom(StreamInput in) throws IOException { } return new RestoreMetaData(entries); } - - /** - * {@inheritDoc} - */ - @Override - public void writeTo(RestoreMetaData repositories, StreamOutput out) throws IOException { - out.writeVInt(repositories.entries().size()); - for (Entry entry : repositories.entries()) { - entry.snapshotId().writeTo(out); - out.writeByte(entry.state().value()); - out.writeVInt(entry.indices().size()); - for (String index : entry.indices()) { - out.writeString(index); - } - out.writeVInt(entry.shards().size()); - for (Map.Entry shardEntry : entry.shards().entrySet()) { - shardEntry.getKey().writeTo(out); - shardEntry.getValue().writeTo(out); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public RestoreMetaData fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException(); - } - - /** - * {@inheritDoc} - */ - @Override - public void toXContent(RestoreMetaData customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray("snapshots"); - for (Entry entry : customIndexMetaData.entries()) { - toXContent(entry, builder, params); - } - builder.endArray(); - } - - /** - * Serializes single restore operation - * - * @param entry restore operation metadata - * @param builder XContent builder - * @param params serialization parameters - * @throws IOException - */ - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(); - builder.field("snapshot", entry.snapshotId().getSnapshot()); - builder.field("repository", entry.snapshotId().getRepository()); - builder.field("state", entry.state()); - builder.startArray("indices"); - { - for (String index : entry.indices()) { - builder.value(index); - } - } - builder.endArray(); - builder.startArray("shards"); - { - for (Map.Entry shardEntry : entry.shards.entrySet()) { - ShardId shardId = shardEntry.getKey(); - ShardRestoreStatus status = shardEntry.getValue(); - builder.startObject(); - { - builder.field("index", shardId.getIndex()); - builder.field("shard", shardId.getId()); - builder.field("state", status.state()); - } - builder.endObject(); - } - } - - builder.endArray(); - builder.endObject(); - } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java index b759fe5daebaf..747703eb35196 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java @@ -22,6 +22,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -38,7 +40,7 @@ /** * Meta data about snapshots that are currently executing */ -public class SnapshotMetaData implements MetaData.Custom { +public class SnapshotMetaData extends AbstractClusterStatePart { public static final String TYPE = "snapshots"; public static final Factory FACTORY = new Factory(); @@ -330,16 +332,88 @@ public Entry snapshot(SnapshotId snapshotId) { return null; } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(entries.size()); + for (Entry entry : entries) { + entry.snapshotId().writeTo(out); + out.writeBoolean(entry.includeGlobalState()); + out.writeByte(entry.state().value()); + out.writeVInt(entry.indices().size()); + for (String index : entry.indices()) { + out.writeString(index); + } + out.writeLong(entry.startTime()); + out.writeVInt(entry.shards().size()); + for (Map.Entry shardEntry : entry.shards().entrySet()) { + shardEntry.getKey().writeTo(out); + out.writeOptionalString(shardEntry.getValue().nodeId()); + out.writeByte(shardEntry.getValue().state().value()); + } + } + } - public static class Factory extends MetaData.Custom.Factory { + static final class Fields { + static final XContentBuilderString REPOSITORY = new XContentBuilderString("repository"); + static final XContentBuilderString SNAPSHOTS = new XContentBuilderString("snapshots"); + static final XContentBuilderString SNAPSHOT = new XContentBuilderString("snapshot"); + static final XContentBuilderString INCLUDE_GLOBAL_STATE = new XContentBuilderString("include_global_state"); + static final XContentBuilderString STATE = new XContentBuilderString("state"); + static final XContentBuilderString INDICES = new XContentBuilderString("indices"); + static final XContentBuilderString START_TIME_MILLIS = new XContentBuilderString("start_time_millis"); + static final XContentBuilderString START_TIME = new XContentBuilderString("start_time"); + static final XContentBuilderString SHARDS = new XContentBuilderString("shards"); + static final XContentBuilderString INDEX = new XContentBuilderString("index"); + static final XContentBuilderString SHARD = new XContentBuilderString("shard"); + static final XContentBuilderString NODE = new XContentBuilderString("node"); + } - @Override - public String type() { - return TYPE; //To change body of implemented methods use File | Settings | File Templates. + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray(Fields.SNAPSHOTS); + for (Entry entry : entries) { + toXContent(entry, builder, params); } + builder.endArray(); + return builder; + } + + public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository()); + builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot()); + builder.field(Fields.INCLUDE_GLOBAL_STATE, entry.includeGlobalState()); + builder.field(Fields.STATE, entry.state()); + builder.startArray(Fields.INDICES); + { + for (String index : entry.indices()) { + builder.value(index); + } + } + builder.endArray(); + builder.timeValueField(Fields.START_TIME_MILLIS, Fields.START_TIME, entry.startTime()); + builder.startArray(Fields.SHARDS); + { + for (Map.Entry shardEntry : entry.shards.entrySet()) { + ShardId shardId = shardEntry.getKey(); + ShardSnapshotStatus status = shardEntry.getValue(); + builder.startObject(); + { + builder.field(Fields.INDEX, shardId.getIndex()); + builder.field(Fields.SHARD, shardId.getId()); + builder.field(Fields.STATE, status.state()); + builder.field(Fields.NODE, status.nodeId()); + } + builder.endObject(); + } + } + builder.endArray(); + builder.endObject(); + } + public static class Factory extends AbstractClusterStatePart.AbstractFactory { @Override - public SnapshotMetaData readFrom(StreamInput in) throws IOException { + public SnapshotMetaData readFrom(StreamInput in, LocalContext context) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { SnapshotId snapshotId = SnapshotId.readSnapshotId(in); @@ -364,88 +438,6 @@ public SnapshotMetaData readFrom(StreamInput in) throws IOException { return new SnapshotMetaData(entries); } - @Override - public void writeTo(SnapshotMetaData repositories, StreamOutput out) throws IOException { - out.writeVInt(repositories.entries().size()); - for (Entry entry : repositories.entries()) { - entry.snapshotId().writeTo(out); - out.writeBoolean(entry.includeGlobalState()); - out.writeByte(entry.state().value()); - out.writeVInt(entry.indices().size()); - for (String index : entry.indices()) { - out.writeString(index); - } - out.writeLong(entry.startTime()); - out.writeVInt(entry.shards().size()); - for (Map.Entry shardEntry : entry.shards().entrySet()) { - shardEntry.getKey().writeTo(out); - out.writeOptionalString(shardEntry.getValue().nodeId()); - out.writeByte(shardEntry.getValue().state().value()); - } - } - } - - @Override - public SnapshotMetaData fromXContent(XContentParser parser) throws IOException { - throw new UnsupportedOperationException(); - } - - static final class Fields { - static final XContentBuilderString REPOSITORY = new XContentBuilderString("repository"); - static final XContentBuilderString SNAPSHOTS = new XContentBuilderString("snapshots"); - static final XContentBuilderString SNAPSHOT = new XContentBuilderString("snapshot"); - static final XContentBuilderString INCLUDE_GLOBAL_STATE = new XContentBuilderString("include_global_state"); - static final XContentBuilderString STATE = new XContentBuilderString("state"); - static final XContentBuilderString INDICES = new XContentBuilderString("indices"); - static final XContentBuilderString START_TIME_MILLIS = new XContentBuilderString("start_time_millis"); - static final XContentBuilderString START_TIME = new XContentBuilderString("start_time"); - static final XContentBuilderString SHARDS = new XContentBuilderString("shards"); - static final XContentBuilderString INDEX = new XContentBuilderString("index"); - static final XContentBuilderString SHARD = new XContentBuilderString("shard"); - static final XContentBuilderString NODE = new XContentBuilderString("node"); - } - - @Override - public void toXContent(SnapshotMetaData customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray(Fields.SNAPSHOTS); - for (Entry entry : customIndexMetaData.entries()) { - toXContent(entry, builder, params); - } - builder.endArray(); - } - - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(); - builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository()); - builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot()); - builder.field(Fields.INCLUDE_GLOBAL_STATE, entry.includeGlobalState()); - builder.field(Fields.STATE, entry.state()); - builder.startArray(Fields.INDICES); - { - for (String index : entry.indices()) { - builder.value(index); - } - } - builder.endArray(); - builder.timeValueField(Fields.START_TIME_MILLIS, Fields.START_TIME, entry.startTime()); - builder.startArray(Fields.SHARDS); - { - for (Map.Entry shardEntry : entry.shards.entrySet()) { - ShardId shardId = shardEntry.getKey(); - ShardSnapshotStatus status = shardEntry.getValue(); - builder.startObject(); - { - builder.field(Fields.INDEX, shardId.getIndex()); - builder.field(Fields.SHARD, shardId.getId()); - builder.field(Fields.STATE, status.state()); - builder.field(Fields.NODE, status.nodeId()); - } - builder.endObject(); - } - } - builder.endArray(); - builder.endObject(); - } } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java index be4e1b4e3f3c6..85b46925b5fdc 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java @@ -58,7 +58,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, public RestResponse buildResponse(GetRepositoriesResponse response, XContentBuilder builder) throws Exception { builder.startObject(); for (RepositoryMetaData repositoryMetaData : response.repositories()) { - RepositoriesMetaData.FACTORY.toXContent(repositoryMetaData, builder, request); + RepositoriesMetaData.toXContent(repositoryMetaData, builder, request); } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 870d69c8ba4f9..32a8167eca4ae 100644 --- a/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -292,7 +292,7 @@ private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) { } } if (metaData.customs() != null) { - for (ObjectObjectCursor cursor : metaData.customs()) { + for (ObjectObjectCursor cursor : metaData.customs()) { if (!RepositoriesMetaData.TYPE.equals(cursor.key)) { // Don't restore repositories while we are working with them // TODO: Should we restore them at the end? diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index fad835d5e24f5..b67f2bfba5bb4 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -115,108 +115,108 @@ public void restorePersistentSettingsTest() throws Exception { .getMetaData().persistentSettings().get(ThreadPool.THREADPOOL_GROUP + "dummy.value"), equalTo(settingValue)); } - @Test - public void restoreCustomMetadata() throws Exception { - Path tempDir = newTempDirPath(); - - logger.info("--> start node"); - internalCluster().startNode(settingsBuilder().put("gateway.type", "local")); - Client client = client(); - createIndex("test-idx"); - ensureYellow(); - logger.info("--> add custom persistent metadata"); - updateClusterState(new ClusterStateUpdater() { - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState.Builder builder = ClusterState.builder(currentState); - MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); - metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s")); - metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns")); - metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw")); - metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw")); - metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi")); - builder.metaData(metadataBuilder); - return builder.build(); - } - }); - - logger.info("--> create repository"); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - logger.info("--> start snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); - assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - - logger.info("--> change custom persistent metadata"); - updateClusterState(new ClusterStateUpdater() { - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - ClusterState.Builder builder = ClusterState.builder(currentState); - MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); - if (randomBoolean()) { - metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s")); - } else { - metadataBuilder.removeCustom(SnapshottableMetadata.TYPE); - } - metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns")); - if (randomBoolean()) { - metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw")); - } else { - metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE); - } - metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw")); - metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE); - builder.metaData(metadataBuilder); - return builder.build(); - } - }); - - logger.info("--> delete repository"); - assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo")); - - logger.info("--> create repository"); - putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2") - .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - logger.info("--> restore snapshot"); - client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet(); - - logger.info("--> make sure old repository wasn't restored"); - assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class); - assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); - - logger.info("--> check that custom persistent metadata was restored"); - ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); - logger.info("Cluster state: {}", clusterState); - MetaData metaData = clusterState.getMetaData(); - assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); - assertThat(((NonSnapshottableMetadata) metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns")); - assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); - assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); - - logger.info("--> restart all nodes"); - internalCluster().fullRestart(); - ensureYellow(); - - logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); - clusterState = client().admin().cluster().prepareState().get().getState(); - logger.info("Cluster state: {}", clusterState); - metaData = clusterState.getMetaData(); - assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); - assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue()); - assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); - assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); - // Shouldn't be returned as part of API response - assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue()); - // But should still be in state - metaData = internalCluster().getInstance(ClusterService.class).state().metaData(); - assertThat(((SnapshotableGatewayNoApiMetadata) metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi")); - } +// @Test +// public void restoreCustomMetadata() throws Exception { +// Path tempDir = newTempDirPath(); +// +// logger.info("--> start node"); +// internalCluster().startNode(settingsBuilder().put("gateway.type", "local")); +// Client client = client(); +// createIndex("test-idx"); +// ensureYellow(); +// logger.info("--> add custom persistent metadata"); +// updateClusterState(new ClusterStateUpdater() { +// @Override +// public ClusterState execute(ClusterState currentState) throws Exception { +// ClusterState.Builder builder = ClusterState.builder(currentState); +// MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); +// metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s")); +// metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns")); +// metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw")); +// metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw")); +// metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi")); +// builder.metaData(metadataBuilder); +// return builder.build(); +// } +// }); +// +// logger.info("--> create repository"); +// PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") +// .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); +// assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); +// +// logger.info("--> start snapshot"); +// CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); +// assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0)); +// assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); +// assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); +// +// logger.info("--> change custom persistent metadata"); +// updateClusterState(new ClusterStateUpdater() { +// @Override +// public ClusterState execute(ClusterState currentState) throws Exception { +// ClusterState.Builder builder = ClusterState.builder(currentState); +// MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); +// if (randomBoolean()) { +// metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s")); +// } else { +// metadataBuilder.removeCustom(SnapshottableMetadata.TYPE); +// } +// metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns")); +// if (randomBoolean()) { +// metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw")); +// } else { +// metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE); +// } +// metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw")); +// metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE); +// builder.metaData(metadataBuilder); +// return builder.build(); +// } +// }); +// +// logger.info("--> delete repository"); +// assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo")); +// +// logger.info("--> create repository"); +// putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2") +// .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); +// assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); +// +// logger.info("--> restore snapshot"); +// client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet(); +// +// logger.info("--> make sure old repository wasn't restored"); +// assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class); +// assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); +// +// logger.info("--> check that custom persistent metadata was restored"); +// ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); +// logger.info("Cluster state: {}", clusterState); +// MetaData metaData = clusterState.getMetaData(); +// assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); +// assertThat(((NonSnapshottableMetadata) metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns")); +// assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); +// assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); +// +// logger.info("--> restart all nodes"); +// internalCluster().fullRestart(); +// ensureYellow(); +// +// logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); +// clusterState = client().admin().cluster().prepareState().get().getState(); +// logger.info("Cluster state: {}", clusterState); +// metaData = clusterState.getMetaData(); +// assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); +// assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue()); +// assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); +// assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); +// // Shouldn't be returned as part of API response +// assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue()); +// // But should still be in state +// metaData = internalCluster().getInstance(ClusterService.class).state().metaData(); +// assertThat(((SnapshotableGatewayNoApiMetadata) metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi")); +// } private void updateClusterState(final ClusterStateUpdater updater) throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); @@ -686,223 +686,223 @@ private void createTestIndex(String name) { )); } - public static abstract class TestCustomMetaData implements MetaData.Custom { - private final String data; - - protected TestCustomMetaData(String data) { - this.data = data; - } - - public String getData() { - return data; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TestCustomMetaData that = (TestCustomMetaData) o; - - if (!data.equals(that.data)) return false; - - return true; - } - - @Override - public int hashCode() { - return data.hashCode(); - } - - public static abstract class TestCustomMetaDataFactory extends MetaData.Custom.Factory { - - protected abstract TestCustomMetaData newTestCustomMetaData(String data); - - @Override - public T readFrom(StreamInput in) throws IOException { - return (T) newTestCustomMetaData(in.readString()); - } - - @Override - public void writeTo(T metadata, StreamOutput out) throws IOException { - out.writeString(metadata.getData()); - } - - @Override - public T fromXContent(XContentParser parser) throws IOException { - XContentParser.Token token; - String data = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String currentFieldName = parser.currentName(); - if ("data".equals(currentFieldName)) { - if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { - throw new ElasticsearchParseException("failed to parse snapshottable metadata, invalid data type"); - } - data = parser.text(); - } else { - throw new ElasticsearchParseException("failed to parse snapshottable metadata, unknown field [" + currentFieldName + "]"); - } - } else { - throw new ElasticsearchParseException("failed to parse snapshottable metadata"); - } - } - if (data == null) { - throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found"); - } - return (T) newTestCustomMetaData(data); - } - - @Override - public void toXContent(T metadata, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.field("data", metadata.getData()); - } - } - } - - static { - MetaData.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY); - MetaData.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY); - MetaData.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY); - MetaData.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY); - MetaData.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY); - } - - public static class SnapshottableMetadata extends TestCustomMetaData { - public static final String TYPE = "test_snapshottable"; - - public static final Factory FACTORY = new Factory(); - - public SnapshottableMetadata(String data) { - super(data); - } - - private static class Factory extends TestCustomMetaDataFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - protected TestCustomMetaData newTestCustomMetaData(String data) { - return new SnapshottableMetadata(data); - } - - @Override - public EnumSet context() { - return MetaData.API_AND_SNAPSHOT; - } - } - } - - public static class NonSnapshottableMetadata extends TestCustomMetaData { - public static final String TYPE = "test_non_snapshottable"; - - public static final Factory FACTORY = new Factory(); - - public NonSnapshottableMetadata(String data) { - super(data); - } - - private static class Factory extends TestCustomMetaDataFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - protected NonSnapshottableMetadata newTestCustomMetaData(String data) { - return new NonSnapshottableMetadata(data); - } - } - } - - public static class SnapshottableGatewayMetadata extends TestCustomMetaData { - public static final String TYPE = "test_snapshottable_gateway"; - - public static final Factory FACTORY = new Factory(); - - public SnapshottableGatewayMetadata(String data) { - super(data); - } - - private static class Factory extends TestCustomMetaDataFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - protected TestCustomMetaData newTestCustomMetaData(String data) { - return new SnapshottableGatewayMetadata(data); - } - - @Override - public EnumSet context() { - return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.SNAPSHOT, MetaData.XContentContext.GATEWAY); - } - } - } - - public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData { - public static final String TYPE = "test_non_snapshottable_gateway"; - - public static final Factory FACTORY = new Factory(); - - public NonSnapshottableGatewayMetadata(String data) { - super(data); - } - - private static class Factory extends TestCustomMetaDataFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) { - return new NonSnapshottableGatewayMetadata(data); - } - - @Override - public EnumSet context() { - return MetaData.API_AND_GATEWAY; - } - - } - } - - public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData { - public static final String TYPE = "test_snapshottable_gateway_no_api"; - - public static final Factory FACTORY = new Factory(); - - public SnapshotableGatewayNoApiMetadata(String data) { - super(data); - } - - private static class Factory extends TestCustomMetaDataFactory { - - @Override - public String type() { - return TYPE; - } - - @Override - protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) { - return new SnapshotableGatewayNoApiMetadata(data); - } - - @Override - public EnumSet context() { - return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT); - } - - } - } - +// public static abstract class TestCustomMetaData implements MetaData.Custom { +// private final String data; +// +// protected TestCustomMetaData(String data) { +// this.data = data; +// } +// +// public String getData() { +// return data; +// } +// +// @Override +// public boolean equals(Object o) { +// if (this == o) return true; +// if (o == null || getClass() != o.getClass()) return false; +// +// TestCustomMetaData that = (TestCustomMetaData) o; +// +// if (!data.equals(that.data)) return false; +// +// return true; +// } +// +// @Override +// public int hashCode() { +// return data.hashCode(); +// } +// +// public static abstract class TestCustomMetaDataFactory extends MetaData.Custom.Factory { +// +// protected abstract TestCustomMetaData newTestCustomMetaData(String data); +// +// @Override +// public T readFrom(StreamInput in) throws IOException { +// return (T) newTestCustomMetaData(in.readString()); +// } +// +// @Override +// public void writeTo(T metadata, StreamOutput out) throws IOException { +// out.writeString(metadata.getData()); +// } +// +// @Override +// public T fromXContent(XContentParser parser) throws IOException { +// XContentParser.Token token; +// String data = null; +// while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { +// if (token == XContentParser.Token.FIELD_NAME) { +// String currentFieldName = parser.currentName(); +// if ("data".equals(currentFieldName)) { +// if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { +// throw new ElasticsearchParseException("failed to parse snapshottable metadata, invalid data type"); +// } +// data = parser.text(); +// } else { +// throw new ElasticsearchParseException("failed to parse snapshottable metadata, unknown field [" + currentFieldName + "]"); +// } +// } else { +// throw new ElasticsearchParseException("failed to parse snapshottable metadata"); +// } +// } +// if (data == null) { +// throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found"); +// } +// return (T) newTestCustomMetaData(data); +// } +// +// @Override +// public void toXContent(T metadata, XContentBuilder builder, ToXContent.Params params) throws IOException { +// builder.field("data", metadata.getData()); +// } +// } +// } +// +// static { +// MetaData.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY); +// MetaData.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY); +// MetaData.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY); +// MetaData.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY); +// MetaData.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY); +// } +// +// public static class SnapshottableMetadata extends TestCustomMetaData { +// public static final String TYPE = "test_snapshottable"; +// +// public static final Factory FACTORY = new Factory(); +// +// public SnapshottableMetadata(String data) { +// super(data); +// } +// +// private static class Factory extends TestCustomMetaDataFactory { +// +// @Override +// public String type() { +// return TYPE; +// } +// +// @Override +// protected TestCustomMetaData newTestCustomMetaData(String data) { +// return new SnapshottableMetadata(data); +// } +// +// @Override +// public EnumSet context() { +// return MetaData.API_AND_SNAPSHOT; +// } +// } +// } +// +// public static class NonSnapshottableMetadata extends TestCustomMetaData { +// public static final String TYPE = "test_non_snapshottable"; +// +// public static final Factory FACTORY = new Factory(); +// +// public NonSnapshottableMetadata(String data) { +// super(data); +// } +// +// private static class Factory extends TestCustomMetaDataFactory { +// +// @Override +// public String type() { +// return TYPE; +// } +// +// @Override +// protected NonSnapshottableMetadata newTestCustomMetaData(String data) { +// return new NonSnapshottableMetadata(data); +// } +// } +// } +// +// public static class SnapshottableGatewayMetadata extends TestCustomMetaData { +// public static final String TYPE = "test_snapshottable_gateway"; +// +// public static final Factory FACTORY = new Factory(); +// +// public SnapshottableGatewayMetadata(String data) { +// super(data); +// } +// +// private static class Factory extends TestCustomMetaDataFactory { +// +// @Override +// public String type() { +// return TYPE; +// } +// +// @Override +// protected TestCustomMetaData newTestCustomMetaData(String data) { +// return new SnapshottableGatewayMetadata(data); +// } +// +// @Override +// public EnumSet context() { +// return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.SNAPSHOT, MetaData.XContentContext.GATEWAY); +// } +// } +// } +// +// public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData { +// public static final String TYPE = "test_non_snapshottable_gateway"; +// +// public static final Factory FACTORY = new Factory(); +// +// public NonSnapshottableGatewayMetadata(String data) { +// super(data); +// } +// +// private static class Factory extends TestCustomMetaDataFactory { +// +// @Override +// public String type() { +// return TYPE; +// } +// +// @Override +// protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) { +// return new NonSnapshottableGatewayMetadata(data); +// } +// +// @Override +// public EnumSet context() { +// return MetaData.API_AND_GATEWAY; +// } +// +// } +// } +// +// public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData { +// public static final String TYPE = "test_snapshottable_gateway_no_api"; +// +// public static final Factory FACTORY = new Factory(); +// +// public SnapshotableGatewayNoApiMetadata(String data) { +// super(data); +// } +// +// private static class Factory extends TestCustomMetaDataFactory { +// +// @Override +// public String type() { +// return TYPE; +// } +// +// @Override +// protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) { +// return new SnapshotableGatewayNoApiMetadata(data); +// } +// +// @Override +// public EnumSet context() { +// return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT); +// } +// +// } +// } +// } From b671b59afb29bdd62995f250ca6c4b503f11c9f1 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 29 Dec 2014 09:51:31 -0500 Subject: [PATCH 06/20] Fix bug in handling index templates --- .../elasticsearch/cluster/metadata/IndexTemplateMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index b58b13ef6e74a..b2abd85663300 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -460,7 +460,7 @@ public static void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOu @Override public String key() { - return template; + return name; } @Override From c1e91119f3ba773f61f0bb3caf887e8cac80c5e8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 29 Dec 2014 13:29:32 -0500 Subject: [PATCH 07/20] Added diffs on map level --- .../cluster/CompositeClusterStatePart.java | 14 +-- .../cluster/MapClusterStatePart.java | 107 +++++++++++++++++- .../cluster/MapItemClusterStatePart.java | 1 + .../cluster/metadata/IndexMetaData.java | 5 + .../cluster/metadata/MetaData.java | 4 +- 5 files changed, 121 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 7a201d744d81d..58cefacd30957 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -167,7 +167,7 @@ public Diff diff(@Nullable T before, T after) { ImmutableOpenMap beforeParts = before.parts(); ImmutableOpenMap afterParts = after.parts(); if (before.equals(after)) { - return new NoDiff(); + return new NoDiff<>(); } else { for (ObjectObjectCursor partIter : beforeParts) { if (!afterParts.containsKey(partIter.key)) { @@ -218,12 +218,12 @@ public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOExcep private static class CompositeDiff implements Diff { - private long version; - private String previousUuid; - private String uuid; - private Map> diffs; - private List deletes; - private AbstractCompositeClusterStatePartFactory factory; + private final long version; + private final String previousUuid; + private final String uuid; + private final Map> diffs; + private final List deletes; + private final AbstractCompositeClusterStatePartFactory factory; private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, long version, String previousUuid, String uuid, List deletes, Map> diffs) { this.version = version; diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index 90b5076440098..ee9ce4f8d3ef8 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -28,6 +29,12 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; /** * Represents a map of cluster state parts with the same type. @@ -56,10 +63,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public ImmutableOpenMap getParts() { + public ImmutableOpenMap parts() { return parts; } + public T get(String type) { + return parts.get(type); + } + + public static class Factory extends AbstractFactory> { private final ClusterStatePart.Factory factory; @@ -88,6 +100,99 @@ public MapClusterStatePart fromXContent(XContentParser parser, LocalContext c } return new MapClusterStatePart<>(builder.build()); } + + @Override + public Diff> diff(@Nullable MapClusterStatePart before, MapClusterStatePart after) { + assert after != null; + Map> diffs = newHashMap(); + List deletes = newArrayList(); + if (before != null) { + ImmutableOpenMap beforeParts = before.parts(); + ImmutableOpenMap afterParts = after.parts(); + if (before.equals(after)) { + return new NoDiff<>(); + } else { + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); + } + } + for (ObjectObjectCursor partIter : afterParts) { + T beforePart = beforeParts.get(partIter.key); + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + } + } + } else { + ImmutableOpenMap afterParts = after.parts(); + for (ObjectObjectCursor partIter : afterParts) { + diffs.put(partIter.key, factory.diff(null, partIter.value)); + } + } + return new MapDiff<>(deletes, diffs); + } + + @Override + public Diff> readDiffFrom(StreamInput in, LocalContext context) throws IOException { + if (in.readBoolean()) { + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } + + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String key = in.readString(); + diffs.put(key, factory.readDiffFrom(in, context)); + } + return new MapDiff<>(deletes, diffs); + + } else { + return new NoDiff<>(); + } + } + + } + + private static class MapDiff implements Diff> { + + private final Map> diffs; + private final List deletes; + + private MapDiff(List deletes, Map> diffs) { + this.diffs = diffs; + this.deletes = deletes; + } + + @Override + public MapClusterStatePart apply(MapClusterStatePart part) { + ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); + parts.putAll(part.parts); + for (String delete : deletes) { + parts.remove(delete); + } + + for (Map.Entry> entry : diffs.entrySet()) { + parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); + } + return new MapClusterStatePart(parts.build()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); // We have diffs + out.writeVInt(deletes.size()); + for (String delete : deletes) { + out.writeString(delete); + } + + out.writeVInt(diffs.size()); + for (Map.Entry> entry : diffs.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } } } diff --git a/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java index 067a024895a12..d1d0d4f3332fc 100644 --- a/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java @@ -24,4 +24,5 @@ */ public interface MapItemClusterStatePart extends ClusterStatePart { String key(); + } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 7c5267e49df22..7891c6dfcd0dd 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -459,6 +459,10 @@ public boolean equals(Object o) { return false; } + if (!customs.equals(that.customs)) { + return false; + } + return true; } @@ -469,6 +473,7 @@ public int hashCode() { result = 31 * result + aliases.hashCode(); result = 31 * result + settings.hashCode(); result = 31 * result + mappings.hashCode(); + result = 31 * result + customs.hashCode(); return result; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 4ac50c2851994..19b2f09212ead 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -237,8 +237,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws this.transientSettings = ((ClusterStateSettingsPart)get(TRANSIENT_SETTINGS_TYPE)).getSettings(); this.persistentSettings = ((ClusterStateSettingsPart)get(PERSISTENT_SETTINGS_TYPE)).getSettings(); this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build(); - this.indices = ((MapClusterStatePart)get(IndexMetaData.TYPE)).getParts(); - this.templates = ((MapClusterStatePart)get(IndexTemplateMetaData.TYPE)).getParts(); + this.indices = ((MapClusterStatePart)get(IndexMetaData.TYPE)).parts(); + this.templates = ((MapClusterStatePart)get(IndexTemplateMetaData.TYPE)).parts(); //TODO: Hack, for now to make things running ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(); customsBuilder.putAll(parts); From 357343d59c35058969ac061ce59eca8a301e1e44 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 29 Dec 2014 21:08:05 -0500 Subject: [PATCH 08/20] Added diffs on routing table level --- .../cluster/CompositeClusterStatePart.java | 4 +- .../cluster/MapClusterStatePart.java | 4 +- .../cluster/routing/IndexRoutingTable.java | 89 ++++++++++++++-- .../routing/IndexShardRoutingTable.java | 22 ++++ .../cluster/routing/RoutingTable.java | 100 ++++++++++++++++++ .../publish/PublishClusterStateAction.java | 8 +- 6 files changed, 213 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 58cefacd30957..f2d537e91becd 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -177,7 +177,9 @@ public Diff diff(@Nullable T before, T after) { for (ObjectObjectCursor partIter : afterParts) { ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); ClusterStatePart beforePart = beforeParts.get(partIter.key); - diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + } } } } else { diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index ee9ce4f8d3ef8..e9194fc823aa4 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -119,7 +119,9 @@ public Diff> diff(@Nullable MapClusterStatePart before } for (ObjectObjectCursor partIter : afterParts) { T beforePart = beforeParts.get(partIter.key); - diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + } } } } else { diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 5f0356d357254..9c92ac83c0364 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -22,24 +22,27 @@ import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.cursors.IntCursor; import com.carrotsearch.hppc.cursors.IntObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectCursor; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.ElasticsearchIllegalStateException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.ThreadLocalRandom; import static com.google.common.collect.Lists.newArrayList; +import static org.apache.lucene.util.CollectionUtil.timSort; /** * The {@link IndexRoutingTable} represents routing information for a single @@ -56,7 +59,12 @@ * represented as {@link ShardRouting}. *

*/ -public class IndexRoutingTable implements Iterable { +public class IndexRoutingTable extends AbstractClusterStatePart implements Iterable { + + public static final String TYPE = "routing_table"; + + public static final Factory FACTORY = new Factory(); + private final String index; private final ShardShuffler shuffler; @@ -86,7 +94,51 @@ public class IndexRoutingTable implements Iterable { this.allActiveShards = allActiveShards.build(); } - /** + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject("shards"); + for (ObjectCursor indexShardRoutingTable : shards.values()) { + builder.startArray(Integer.toString(indexShardRoutingTable.value.shardId().id())); + for (ShardRouting shardRouting : indexShardRoutingTable.value) { + shardRouting.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IndexRoutingTable that = (IndexRoutingTable) o; + + if (!index.equals(that.index)) return false; + if (!shards.equals(that.shards)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = index.hashCode(); + result = 31 * result + shards.hashCode(); + return result; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public IndexRoutingTable readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in); + } + + } + /** * Return the index id * * @return id of the index @@ -111,11 +163,20 @@ public String getIndex() { * @return new {@link IndexRoutingTable} */ public IndexRoutingTable normalizeVersions() { + boolean changed = false; IndexRoutingTable.Builder builder = new Builder(this.index); for (IntObjectCursor cursor : shards) { - builder.addIndexShard(cursor.value.normalizeVersions()); + IndexShardRoutingTable normalizedShard = cursor.value.normalizeVersions(); + if (cursor.value != normalizedShard) { + changed = true; + } + builder.addIndexShard(normalizedShard); + } + if(changed) { + return builder.build(); + } else { + return this; } - return builder.build(); } public void validate(RoutingTableValidation validation, MetaData metaData) { @@ -322,6 +383,11 @@ public static Builder builder(String index) { return new Builder(index); } + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + public static class Builder { private final String index; @@ -519,7 +585,14 @@ public IndexRoutingTable build() throws RoutingValidationException { public String prettyPrint() { StringBuilder sb = new StringBuilder("-- index [" + index + "]\n"); - for (IndexShardRoutingTable indexShard : this) { + List indexShards = newArrayList(this); + timSort(indexShards, new Comparator() { + @Override + public int compare(IndexShardRoutingTable o1, IndexShardRoutingTable o2) { + return o1.getShardId().compareTo(o2.getShardId()); + } + }); + for (IndexShardRoutingTable indexShard : indexShards) { sb.append("----shard_id [").append(indexShard.shardId().index().name()).append("][").append(indexShard.shardId().id()).append("]\n"); for (ShardRouting shard : indexShard) { sb.append("--------").append(shard.shortSummary()).append("\n"); diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index 00e50b76129b9..ec4a561ae5f0e 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -354,6 +354,28 @@ public boolean allShardsStarted() { return allShardsStarted; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IndexShardRoutingTable that = (IndexShardRoutingTable) o; + + if (primaryAllocatedPostApi != that.primaryAllocatedPostApi) return false; + if (!shardId.equals(that.shardId)) return false; + if (!shards.equals(that.shards)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = shardId.hashCode(); + result = 31 * result + shards.hashCode(); + result = 31 * result + (primaryAllocatedPostApi ? 1 : 0); + return result; + } + static class AttributesKey { final String[] attributes; diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index bbadba753befd..8af05d31116e7 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -78,6 +79,105 @@ public RoutingTable readFrom(StreamInput in, LocalContext context) throws IOExce public RoutingTable fromXContent(XContentParser parser, LocalContext context) throws IOException { throw new UnsupportedOperationException("Not implemented yet"); } + + @Override + public Diff diff(@Nullable RoutingTable before, RoutingTable after) { + assert after != null; + Map> diffs = newHashMap(); + List deletes = newArrayList(); + if (before != null) { + ImmutableMap beforeParts = before.indicesRouting; + ImmutableMap afterParts = after.indicesRouting; + if (before.equals(after)) { + return new NoDiff<>(); + } else { + for (String key : beforeParts.keySet()) { + if (!afterParts.containsKey(key)) { + deletes.add(key); + } + } + for (Map.Entry part : afterParts.entrySet()) { + IndexRoutingTable beforePart = beforeParts.get(part.getKey()); + if (!part.getValue().equals(beforePart)) { + diffs.put(part.getKey(), IndexRoutingTable.FACTORY.diff(beforePart, part.getValue())); + } + } + } + } else { + ImmutableMap afterParts = after.indicesRouting; + for (Map.Entry part : afterParts.entrySet()) { + diffs.put(part.getKey(), IndexRoutingTable.FACTORY.diff(null, part.getValue())); + } + } + return new RoutingTableDiff(after.version(), deletes, diffs); + } + + @Override + public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { + if (in.readBoolean()) { + long version = in.readVLong(); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } + + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String key = in.readString(); + diffs.put(key, IndexRoutingTable.FACTORY.readDiffFrom(in, context)); + } + return new RoutingTableDiff(version, deletes, diffs); + + } else { + return new NoDiff<>(); + } + } + + } + + private static class RoutingTableDiff implements Diff { + + private final Map> diffs; + private final List deletes; + private long version; + + private RoutingTableDiff(long version, List deletes, Map> diffs) { + this.version = version; + this.diffs = diffs; + this.deletes = deletes; + } + + @Override + public RoutingTable apply(RoutingTable part) { + Map parts = newHashMap(); + parts.putAll(part.indicesRouting); + for (String delete : deletes) { + parts.remove(delete); + } + + for (Map.Entry> entry : diffs.entrySet()) { + parts.put(entry.getKey(), entry.getValue().apply(part.indicesRouting.get(entry.getKey()))); + } + return new RoutingTable(version, parts); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); // We have diffs + out.writeVLong(version); + out.writeVInt(deletes.size()); + for (String delete : deletes) { + out.writeString(delete); + } + + out.writeVInt(diffs.size()); + for (Map.Entry> entry : diffs.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } } /** * Returns the version of the {@link RoutingTable}. diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 6abe328e4612d..ec6a27b077fe5 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -274,7 +274,7 @@ public void handleException(TransportException exp) { publishResponseHandler.onFailure(node, t); } } else { - logger.debug("skipping sending version {} to node {}, node already recieved version {}", clusterStateToSend.version(), node, lastVersionSent); + logger.debug("skipping sending version {} to node {}, node already received version {}", clusterStateToSend.version(), node, lastVersionSent); // TODO: Should this be a failure since we are not really sure if this node got the cluster state or not at this moment publishResponseHandler.onResponse(node); } @@ -320,10 +320,10 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne final ClusterState clusterState; if (in.readBoolean()) { clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); - logger.debug("received full cluster state version {}", clusterState.version()); + logger.debug("received full cluster state version {} with size {}", clusterState.version(), request.bytes().length()); } else { if(lastProcessedClusterState == null) { - logger.debug("received diff cluster state version but don't have any local cluster state - requesting full state"); + logger.debug("received diff cluster state version {} but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); } ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode()); @@ -332,7 +332,7 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne return; } clusterState = diff.apply(lastProcessedClusterState); - logger.debug("received diff cluster state version {}", clusterState.version()); + logger.debug("received diff cluster state version {} with size {}", clusterState.version(), request.bytes().length()); } lastProcessedClusterState = clusterState; From ef4d60407d0f4a194ee8f5595caaac035bfc97b8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 30 Dec 2014 21:08:37 -0500 Subject: [PATCH 09/20] Switch to generic xcontent serialization of MetaData --- .../cluster/node/info/NodesInfoResponse.java | 9 +- .../get/TransportGetSettingsAction.java | 2 +- .../cluster/AbstractClusterStatePart.java | 2 +- .../elasticsearch/cluster/ClusterState.java | 5 - .../cluster/ClusterStatePart.java | 9 +- .../cluster/ClusterStateSettingsPart.java | 28 ++- .../cluster/CompositeClusterStatePart.java | 8 +- .../cluster/MapClusterStatePart.java | 35 ++- .../cluster/metadata/IndexMetaData.java | 38 +++- .../metadata/IndexTemplateMetaData.java | 9 +- .../cluster/metadata/MetaData.java | 209 ++++-------------- .../common/settings/ImmutableSettings.java | 5 +- .../common/settings/SettingsFilter.java | 71 +++++- .../common/xcontent/ToXContent.java | 91 ++++++++ .../repositories/RepositoriesService.java | 4 +- .../node/info/RestNodesInfoAction.java | 3 +- .../reroute/RestClusterRerouteAction.java | 2 +- .../cluster/state/RestClusterStateAction.java | 2 +- .../snapshots/RestoreService.java | 24 +- 19 files changed, 321 insertions(+), 235 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 0eb8ac2176faf..2b16962b46136 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -37,8 +37,6 @@ */ public class NodesInfoResponse extends NodesOperationResponse implements ToXContent { - private SettingsFilter settingsFilter; - public NodesInfoResponse() { } @@ -64,11 +62,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - public NodesInfoResponse settingsFilter(SettingsFilter settingsFilter) { - this.settingsFilter = settingsFilter; - return this; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("cluster_name", getClusterName().value(), XContentBuilder.FieldCaseConversion.NONE); @@ -102,7 +95,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (nodeInfo.getSettings() != null) { builder.startObject("settings"); - Settings settings = settingsFilter != null ? settingsFilter.filterSettings(nodeInfo.getSettings()) : nodeInfo.getSettings(); + Settings settings = nodeInfo.getSettings(); settings.toXContent(builder, params); builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 1ad60c477a962..022acd51b0282 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -86,7 +86,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A continue; } - Settings settings = settingsFilter.filterSettings(indexMetaData.settings()); + Settings settings = settingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.settings()); if (!CollectionUtils.isEmpty(request.names())) { ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder(); for (Map.Entry entry : settings.getAsMap().entrySet()) { diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index ea01e236377f6..36f4eac21bb81 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -34,7 +34,7 @@ public abstract class AbstractClusterStatePart implements ClusterStatePart { @Override public EnumSet context() { - return API_ONLY; + return API; } public static class CompleteDiff implements Diff { diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 508487b2cfd20..e2e68921f8fa5 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -169,11 +169,6 @@ public RoutingNodes readOnlyRoutingNodes() { return routingNodes; } - public ClusterState settingsFilter(SettingsFilter settingsFilter) { - metaData().settingsFilter(settingsFilter); - return this; - } - public String prettyPrint() { StringBuilder sb = new StringBuilder(); sb.append("version: ").append(version).append("\n"); diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index e8f5bd903f484..88e2f417f5d51 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -32,7 +32,8 @@ */ public interface ClusterStatePart extends ToXContent { - public static final String ALL = "_all"; + public static final String CONTEXT_MODE_PARAM = "context_mode"; + public enum XContentContext { /* Custom metadata should be returns as part of API call */ @@ -45,11 +46,13 @@ public enum XContentContext { SNAPSHOT; } - public static EnumSet API_ONLY = EnumSet.of(XContentContext.API); - public static EnumSet GATEWAY_ONLY = EnumSet.of(XContentContext.GATEWAY); + public static EnumSet NONE = EnumSet.noneOf(XContentContext.class); + public static EnumSet API = EnumSet.of(XContentContext.API); public static EnumSet API_GATEWAY = EnumSet.of(XContentContext.API, XContentContext.GATEWAY); public static EnumSet API_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT); public static EnumSet API_GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.API, XContentContext.GATEWAY, XContentContext.SNAPSHOT); + public static EnumSet GATEWAY = EnumSet.of(XContentContext.GATEWAY); + public static EnumSet GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.GATEWAY, XContentContext.SNAPSHOT); void writeTo(StreamOutput out) throws IOException; diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index 9de327766057a..b875a9d0ee279 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -23,11 +23,11 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.EnumSet; import java.util.Map; import static org.elasticsearch.common.settings.ImmutableSettings.readSettingsFromStream; @@ -38,9 +38,11 @@ public class ClusterStateSettingsPart extends AbstractClusterStatePart { private final Settings settings; + private final EnumSet xContentContext; - public ClusterStateSettingsPart(Settings settings) { + public ClusterStateSettingsPart(Settings settings, EnumSet xContentContext) { this.settings = settings; + this.xContentContext = xContentContext; } @Override @@ -60,17 +62,35 @@ public Settings getSettings() { return settings; } + @Override + public EnumSet context() { + return xContentContext; + } + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + private final EnumSet xContentContext; + + public Factory() { + xContentContext = API; + } + + public Factory(EnumSet xContentContext) { + this.xContentContext = xContentContext; + } @Override public ClusterStateSettingsPart readFrom(StreamInput in, LocalContext context) throws IOException { - return new ClusterStateSettingsPart(readSettingsFromStream(in)); + return new ClusterStateSettingsPart(readSettingsFromStream(in), xContentContext); } @Override public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext context) throws IOException { Settings settings = ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).build(); - return new ClusterStateSettingsPart(settings); + return new ClusterStateSettingsPart(settings, xContentContext); + } + + public ClusterStateSettingsPart fromSettings(Settings settings) { + return new ClusterStateSettingsPart(settings, xContentContext); } } } diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index f2d537e91becd..99070521a6513 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -111,11 +111,15 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); builder.field("version", version); builder.field("uuid", uuid); for (ObjectObjectCursor partIter : parts) { - builder.startObject(partIter.key); - partIter.value.toXContent(builder, params); + if (partIter.value.context().contains(context)) { + builder.startObject(partIter.key); + partIter.value.toXContent(builder, params); + builder.endObject(); + } } return builder; } diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index e9194fc823aa4..af4bb9639d88b 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -42,9 +43,11 @@ public class MapClusterStatePart extends AbstractClusterStatePart { private final ImmutableOpenMap parts; + private final EnumSet xContentContext; - public MapClusterStatePart(ImmutableOpenMap parts) { + public MapClusterStatePart(ImmutableOpenMap parts, EnumSet xContentContext) { this.parts = parts; + this.xContentContext = xContentContext; } @Override @@ -71,13 +74,25 @@ public T get(String type) { return parts.get(type); } + @Override + public EnumSet context() { + return xContentContext; + } + public static class Factory extends AbstractFactory> { + private final EnumSet xContentContext; + private final ClusterStatePart.Factory factory; public Factory(ClusterStatePart.Factory factory) { + this(factory, API); + } + + public Factory(ClusterStatePart.Factory factory, EnumSet xContentContext) { this.factory = factory; + this.xContentContext = xContentContext; } @Override @@ -88,7 +103,7 @@ public MapClusterStatePart readFrom(StreamInput in, LocalContext context) thr T part = factory.readFrom(in, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(builder.build()); + return new MapClusterStatePart<>(builder.build(), xContentContext); } @Override @@ -98,7 +113,7 @@ public MapClusterStatePart fromXContent(XContentParser parser, LocalContext c T part = factory.fromXContent(parser, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(builder.build()); + return new MapClusterStatePart<>(builder.build(), xContentContext); } @Override @@ -130,7 +145,7 @@ public Diff> diff(@Nullable MapClusterStatePart before diffs.put(partIter.key, factory.diff(null, partIter.value)); } } - return new MapDiff<>(deletes, diffs); + return new MapDiff<>(deletes, diffs, xContentContext); } @Override @@ -148,23 +163,29 @@ public Diff> readDiffFrom(StreamInput in, LocalContext co String key = in.readString(); diffs.put(key, factory.readDiffFrom(in, context)); } - return new MapDiff<>(deletes, diffs); + return new MapDiff<>(deletes, diffs, xContentContext); } else { return new NoDiff<>(); } } + public MapClusterStatePart fromMap(ImmutableOpenMap map) { + return new MapClusterStatePart<>(map, xContentContext); + } + } private static class MapDiff implements Diff> { + private final EnumSet xContentContext; private final Map> diffs; private final List deletes; - private MapDiff(List deletes, Map> diffs) { + private MapDiff(List deletes, Map> diffs, EnumSet xContentContext) { this.diffs = diffs; this.deletes = deletes; + this.xContentContext = xContentContext; } @Override @@ -178,7 +199,7 @@ public MapClusterStatePart apply(MapClusterStatePart part) { for (Map.Entry> entry : diffs.entrySet()) { parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); } - return new MapClusterStatePart(parts.build()); + return new MapClusterStatePart(parts.build(), xContentContext); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 7891c6dfcd0dd..43e0cf3c631f2 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -651,24 +651,42 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build boolean binary = params.paramAsBoolean("binary", false); builder.startObject("settings"); + Settings settings = indexMetaData.settings(); + settings.toXContent(builder, params); for (Map.Entry entry : indexMetaData.settings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); - builder.startArray("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - if (binary) { - builder.value(cursor.value.source().compressed()); - } else { - byte[] data = cursor.value.source().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrdered(); - parser.close(); + if (params.paramAsBoolean("reduce_mappings", false)) { + builder.startObject("mappings"); + for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + byte[] mappingSource = cursor.value.source().uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(cursor.key); + } + builder.field(cursor.key); builder.map(mapping); } + builder.endObject(); + } else { + builder.startArray("mappings"); + for (ObjectObjectCursor cursor : indexMetaData.mappings()) { + if (binary) { + builder.value(cursor.value.source().compressed()); + } else { + byte[] data = cursor.value.source().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrdered(); + parser.close(); + builder.map(mapping); + } + } + builder.endArray(); } - builder.endArray(); for (ObjectObjectCursor cursor : indexMetaData.customs()) { builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index b2abd85663300..5e4c88a2bb27b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -23,6 +23,7 @@ import com.google.common.collect.Sets; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.MapItemClusterStatePart; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -39,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; +import java.util.EnumSet; import java.util.Map; import java.util.Set; @@ -47,7 +49,7 @@ */ public class IndexTemplateMetaData extends AbstractClusterStatePart implements MapItemClusterStatePart { - public static String TYPE = "index_template"; + public static String TYPE = "template"; public static Factory FACTORY = new Factory(); @@ -168,6 +170,11 @@ public int hashCode() { return result; } + @Override + public EnumSet context() { + return API_GATEWAY_SNAPSHOT; + } + public static class Builder { private static final Set VALID_FIELDS = Sets.newHashSet("template", "order", "mappings", "settings"); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 19b2f09212ead..14adcaf20c4f4 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -63,10 +63,6 @@ public class MetaData extends CompositeClusterStatePart implements Ite public static final String TYPE = "metadata"; - public static final String PERSISTENT_SETTINGS_TYPE = "persistent_settings"; - - public static final String TRANSIENT_SETTINGS_TYPE = "transient_settings"; - public static final Factory FACTORY = new Factory(); public static final String ALL = "_all"; @@ -97,92 +93,30 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("templates"); - for (ObjectCursor cursor : templates().values()) { - IndexTemplateMetaData templateMetaData = cursor.value; - builder.startObject(templateMetaData.name(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("template", templateMetaData.template()); - builder.field("order", templateMetaData.order()); - - builder.startObject("settings"); - Settings settings = templateMetaData.settings(); - if (settingsFilter != null) { - settings = settingsFilter.filterSettings(settings); - } - settings.toXContent(builder, params); - builder.endObject(); + return super.toXContent(builder, new DelegatingBooleanParams("reduce_mappings", true, params)); + } - builder.startObject("mappings"); - for (ObjectObjectCursor cursor1 : templateMetaData.mappings()) { - byte[] mappingSource = cursor1.value.uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor1.key); - } - builder.field(cursor1.key); - builder.map(mapping); - } - builder.endObject(); + public static final String PERSISTENT_SETTINGS_TYPE = "settings"; + public static final ClusterStateSettingsPart.Factory PERSISTENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(API_GATEWAY_SNAPSHOT); - builder.endObject(); - } - builder.endObject(); + public static final String TRANSIENT_SETTINGS_TYPE = "transient_settings"; - builder.startObject("indices"); - for (IndexMetaData indexMetaData : this) { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + public static final ClusterStateSettingsPart.Factory TRANSIENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(API); - builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); + public static final String INDICES_TYPE = "indices"; - builder.startObject("settings"); - Settings settings = indexMetaData.settings(); - if (settingsFilter != null) { - settings = settingsFilter.filterSettings(settings); - } - settings.toXContent(builder, params); - builder.endObject(); + public static final MapClusterStatePart.Factory INDICES_FACTORY = new MapClusterStatePart.Factory<>(IndexMetaData.FACTORY); - builder.startObject("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - byte[] mappingSource = cursor.value.source().uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor.key); - } - builder.field(cursor.key); - builder.map(mapping); - } - builder.endObject(); + public static final String TEMPLATES_TYPE = "templates"; - builder.startArray("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().keys()) { - builder.value(cursor.value); - } - builder.endArray(); - - builder.endObject(); - } - builder.endObject(); - - for (ObjectObjectCursor cursor : customs()) { - builder.startObject(cursor.key); - cursor.value.toXContent(builder, params); - builder.endObject(); - } - return builder; - } + public static final MapClusterStatePart.Factory TEMPLATES_FACTORY = new MapClusterStatePart.Factory<>(IndexTemplateMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { - registerFactory(TRANSIENT_SETTINGS_TYPE, new ClusterStateSettingsPart.Factory()); - registerFactory(PERSISTENT_SETTINGS_TYPE, new ClusterStateSettingsPart.Factory()); - registerFactory(IndexMetaData.TYPE, new MapClusterStatePart.Factory<>(IndexMetaData.FACTORY)); - registerFactory(IndexTemplateMetaData.TYPE, new MapClusterStatePart.Factory<>(IndexTemplateMetaData.FACTORY)); + registerFactory(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY); + registerFactory(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY); + registerFactory(INDICES_TYPE, INDICES_FACTORY); + registerFactory(TEMPLATES_TYPE, TEMPLATES_FACTORY); // register non plugin custom metadata registerFactory(RepositoriesMetaData.TYPE, RepositoriesMetaData.FACTORY); @@ -197,8 +131,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static final MetaData EMPTY_META_DATA = builder().build(); - public static final String CONTEXT_MODE_PARAM = "context_mode"; - public static final String CONTEXT_MODE_SNAPSHOT = XContentContext.SNAPSHOT.toString(); public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString(); @@ -211,7 +143,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private final Settings settings; // Generated on the fly from transientSettings and persistentSettings private final ImmutableOpenMap indices; private final ImmutableOpenMap templates; - private final ImmutableOpenMap customs; private final transient int totalNumberOfShards; // Transient ? not serializable anyway? private final int numberOfShards; @@ -224,9 +155,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private final ImmutableOpenMap> aliases; private final ImmutableOpenMap aliasAndIndexToIndexMap; - //TODO: This is a hack - needed for plugins. We should refactor it using params - private SettingsFilter settingsFilter; - // Parts: Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs @SuppressWarnings("unchecked") @@ -237,16 +165,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws this.transientSettings = ((ClusterStateSettingsPart)get(TRANSIENT_SETTINGS_TYPE)).getSettings(); this.persistentSettings = ((ClusterStateSettingsPart)get(PERSISTENT_SETTINGS_TYPE)).getSettings(); this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build(); - this.indices = ((MapClusterStatePart)get(IndexMetaData.TYPE)).parts(); - this.templates = ((MapClusterStatePart)get(IndexTemplateMetaData.TYPE)).parts(); - //TODO: Hack, for now to make things running - ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(); - customsBuilder.putAll(parts); - customsBuilder.remove(TRANSIENT_SETTINGS_TYPE); - customsBuilder.remove(PERSISTENT_SETTINGS_TYPE); - customsBuilder.remove(IndexMetaData.TYPE); - customsBuilder.remove(IndexTemplateMetaData.TYPE); - this.customs = customsBuilder.build(); + this.indices = ((MapClusterStatePart)get(INDICES_TYPE)).parts(); + this.templates = ((MapClusterStatePart)get(TEMPLATES_TYPE)).parts(); int totalNumberOfShards = 0; int numberOfShards = 0; @@ -333,11 +253,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws this.aliasAndIndexToIndexMap = aliasAndIndexToIndexMap.cast().build(); } - public MetaData settingsFilter(SettingsFilter settingsFilter) { - this.settingsFilter = settingsFilter; - return this; - } - public long version() { return this.version; } @@ -1032,16 +947,20 @@ public ImmutableOpenMap getTemplates() { return this.templates; } + /** + * Use parts() instead + */ + @Deprecated public ImmutableOpenMap customs() { - return this.customs; - } - - public ImmutableOpenMap getCustoms() { - return this.customs; + return this.parts; } + /** + * Use get(String type) instead + */ + @Deprecated public T custom(String type) { - return (T) customs.get(type); + return (T) parts.get(type); } public int totalNumberOfShards() { @@ -1215,14 +1134,14 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2 } // Check if any persistent metadata needs to be saved int customCount1 = 0; - for (ObjectObjectCursor cursor : metaData1.customs) { + for (ObjectObjectCursor cursor : metaData1.parts()) { if (cursor.value.context().contains(XContentContext.GATEWAY)) { - if (!cursor.value.equals(metaData2.custom(cursor.key))) return false; + if (!cursor.value.equals(metaData2.get(cursor.key))) return false; customCount1++; } } int customCount2 = 0; - for (ObjectObjectCursor cursor : metaData2.customs) { + for (ObjectObjectCursor cursor : metaData2.parts()) { if (cursor.value.context().contains(XContentContext.GATEWAY)) { customCount2++; } @@ -1265,7 +1184,11 @@ public Builder(MetaData metaData) { this.version = metaData.version; this.indices = ImmutableOpenMap.builder(metaData.indices); this.templates = ImmutableOpenMap.builder(metaData.templates); - this.customs = ImmutableOpenMap.builder(metaData.customs); + this.customs = ImmutableOpenMap.builder(metaData.parts); + customs.remove(TRANSIENT_SETTINGS_TYPE); + customs.remove(PERSISTENT_SETTINGS_TYPE); + customs.remove(INDICES_TYPE); + customs.remove(TEMPLATES_TYPE); } public Builder put(IndexMetaData.Builder indexMetaDataBuilder) { @@ -1392,10 +1315,10 @@ public Builder generateUuidIfNeeded() { private static ImmutableOpenMap buildParts(Settings transientSettings, Settings persistentSettings, ImmutableOpenMap indices, ImmutableOpenMap templates, ImmutableOpenMap customs) { ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); - builder.put(TRANSIENT_SETTINGS_TYPE, new ClusterStateSettingsPart(transientSettings)); - builder.put(PERSISTENT_SETTINGS_TYPE, new ClusterStateSettingsPart(persistentSettings)); - builder.put(IndexMetaData.TYPE, new MapClusterStatePart<>(indices)); - builder.put(IndexTemplateMetaData.TYPE, new MapClusterStatePart<>(templates)); + builder.put(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY.fromSettings(transientSettings)); + builder.put(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY.fromSettings(persistentSettings)); + builder.put(INDICES_TYPE, INDICES_FACTORY.fromMap(indices)); + builder.put(TEMPLATES_TYPE, TEMPLATES_FACTORY.fromMap(templates)); builder.putAll(customs); return builder.build(); } @@ -1407,56 +1330,15 @@ public MetaData build() { public static String toXContent(MetaData metaData) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject(); + metaData.toXContent(builder, ToXContent.EMPTY_PARAMS); toXContent(metaData, builder, ToXContent.EMPTY_PARAMS); builder.endObject(); return builder.string(); } public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API")); - builder.startObject("meta-data"); - - builder.field("version", metaData.version()); - builder.field("uuid", metaData.uuid); - - if (!metaData.persistentSettings().getAsMap().isEmpty()) { - builder.startObject("settings"); - for (Map.Entry entry : metaData.persistentSettings().getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - } - - if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) { - builder.startObject("transient_settings"); - for (Map.Entry entry : metaData.transientSettings().getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - } - - builder.startObject("templates"); - for (ObjectCursor cursor : metaData.templates().values()) { - IndexTemplateMetaData.Builder.toXContent(cursor.value, builder, params); - } - builder.endObject(); - - if (context == XContentContext.API && !metaData.indices().isEmpty()) { - builder.startObject("indices"); - for (IndexMetaData indexMetaData : metaData) { - IndexMetaData.Builder.toXContent(indexMetaData, builder, params); - } - builder.endObject(); - } - - for (ObjectObjectCursor cursor : metaData.customs()) { - if (cursor.value.context().contains(context)) { - builder.startObject(cursor.key); - cursor.value.toXContent(builder, params); - builder.endObject(); - } - } + metaData.toXContent(builder, params); builder.endObject(); } @@ -1554,10 +1436,13 @@ public static void writeTo(MetaData metaData, StreamOutput out) throws IOExcepti for (ObjectCursor cursor : metaData.templates.values()) { IndexTemplateMetaData.Builder.writeTo(cursor.value, out); } - out.writeVInt(metaData.customs().size()); - for (ObjectObjectCursor cursor : metaData.customs()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); + //TODO: Hack - generalize the toXContent + out.writeVInt(metaData.parts().size() - 4); + for (ObjectObjectCursor cursor : metaData.parts()) { + if (!cursor.key.equals(TRANSIENT_SETTINGS_TYPE) && !cursor.key.equals(PERSISTENT_SETTINGS_TYPE) && !cursor.key.equals(INDICES_TYPE) && !cursor.key.equals(TEMPLATES_TYPE)) { + out.writeString(cursor.key); + cursor.value.writeTo(out); + } } } } diff --git a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index 50482c6ab2d2d..2dd0936be2368 100644 --- a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -648,12 +648,13 @@ public static Builder settingsBuilder() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Settings settings = SettingsFilter.filterSettings(params, this); if (!params.paramAsBoolean("flat_settings", false)) { - for (Map.Entry entry : getAsStructuredMap().entrySet()) { + for (Map.Entry entry : settings.getAsStructuredMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } } else { - for (Map.Entry entry : getAsMap().entrySet()) { + for (Map.Entry entry : settings.getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue(), XContentBuilder.FieldCaseConversion.NONE); } } diff --git a/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java b/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java index d7332f0072f81..680c15134341e 100644 --- a/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java +++ b/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java @@ -18,40 +18,95 @@ */ package org.elasticsearch.common.settings; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; +import static com.google.common.collect.Lists.newArrayList; + /** * */ public class SettingsFilter extends AbstractComponent { + /** + * Can be used to specify settings filter that will be used to filter out matching settings in toXContent method + */ + public static String SETTINGS_FILTER_PARAM = "settings_filter"; + public static interface Filter { void filter(ImmutableSettings.Builder settings); } - private final CopyOnWriteArrayList filters = new CopyOnWriteArrayList<>(); + private final CopyOnWriteArrayList pattens = new CopyOnWriteArrayList<>(); @Inject public SettingsFilter(Settings settings) { super(settings); } - public void addFilter(Filter filter) { - filters.add(filter); + /** + * Adds a new simple pattern to the list of filters + * @param pattern + */ + public void addFilter(String pattern) { + pattens.add(pattern); + } + + /** + * Removes a simple pattern from the list of filters + * @param pattern + */ + public void removeFilter(String pattern) { + pattens.remove(pattern); + } + + public String getPatterns() { + return Strings.collectionToDelimitedString(pattens, ","); } - public void removeFilter(Filter filter) { - filters.remove(filter); + public ToXContent.Params withFilterSettingParams(Params delegate) { + return new ToXContent.DelegatingStringParams(SETTINGS_FILTER_PARAM, getPatterns(), delegate); } - public Settings filterSettings(Settings settings) { + public static Settings filterSettings(Params params, Settings settings) { + String patterns = params.param(SETTINGS_FILTER_PARAM); + Settings filteredSettings = settings; + if (patterns != null) { + filteredSettings = SettingsFilter.filterSettings(patterns, filteredSettings); + } + return filteredSettings; + } + + public static Settings filterSettings(String patterns, Settings settings) { + String[] patternArray = Strings.delimitedListToStringArray(patterns, ","); ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder().put(settings); - for (Filter filter : filters) { - filter.filter(builder); + ArrayList simpleMatchPatternList = newArrayList(); + for (String pattern : patternArray) { + if (Regex.isSimpleMatchPattern(pattern)) { + simpleMatchPatternList.add(pattern); + } else { + builder.remove(pattern); + } + } + if (!simpleMatchPatternList.isEmpty()) { + String[] simpleMatchPatterns = simpleMatchPatternList.toArray(new String[simpleMatchPatternList.size()]); + Iterator> iterator = builder.internalMap().entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry current = iterator.next(); + if (Regex.simpleMatch(simpleMatchPatterns, current.getKey())) { + iterator.remove(); + } + } } return builder.build(); } diff --git a/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java index b4bfb668ccf7a..a892ab8be2373 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java +++ b/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java @@ -146,5 +146,96 @@ public Boolean paramAsBooleanOptional(String key, Boolean defaultValue) { } } + public static class DelegatingStringParams implements Params { + + private final String key; + private final String value; + private final Params delegate; + + public DelegatingStringParams(String key, String value, Params delegate) { + this.key = key; + this.value = value; + this.delegate = delegate; + } + + @Override + public String param(String key) { + if (this.key.equals(key)) { + return value; + } + return delegate.param(key); + } + + @Override + public String param(String key, String defaultValue) { + if (this.key.equals(key)) { + return value; + } + return delegate.param(key, defaultValue); + } + + @Override + public boolean paramAsBoolean(String key, boolean defaultValue) { + return delegate.paramAsBoolean(key, defaultValue); + } + + @Override + public Boolean paramAsBoolean(String key, Boolean defaultValue) { + return delegate.paramAsBoolean(key, defaultValue); + } + + @Override + public Boolean paramAsBooleanOptional(String key, Boolean defaultValue) { + return delegate.paramAsBooleanOptional(key, defaultValue); + } + } + + public static class DelegatingBooleanParams implements Params { + + private final String key; + private final boolean value; + private final Params delegate; + + public DelegatingBooleanParams(String key, boolean value, Params delegate) { + this.key = key; + this.value = value; + this.delegate = delegate; + } + + @Override + public String param(String key) { + return delegate.param(key); + } + + @Override + public String param(String key, String defaultValue) { + return delegate.param(key, defaultValue); + } + + @Override + public boolean paramAsBoolean(String key, boolean defaultValue) { + if (this.key.equals(key)) { + return value; + } + return delegate.paramAsBoolean(key, defaultValue); + } + + @Override + public Boolean paramAsBoolean(String key, Boolean defaultValue) { + if (this.key.equals(key)) { + return value; + } + return delegate.paramAsBoolean(key, defaultValue); + } + + @Override + public Boolean paramAsBooleanOptional(String key, Boolean defaultValue) { + if (this.key.equals(key)) { + return value; + } + return delegate.paramAsBooleanOptional(key, defaultValue); + } + } + XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; } diff --git a/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 638b22c21f574..22430e4dc5e32 100644 --- a/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -115,7 +115,7 @@ public ClusterState execute(ClusterState currentState) throws IOException { } MetaData metaData = currentState.metaData(); MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - RepositoriesMetaData repositories = metaData.custom(RepositoriesMetaData.TYPE); + RepositoriesMetaData repositories = metaData.get(RepositoriesMetaData.TYPE); if (repositories == null) { logger.info("put repository [{}]", request.name); repositories = new RepositoriesMetaData(new RepositoryMetaData(request.name, request.type, request.settings)); @@ -175,7 +175,7 @@ public ClusterState execute(ClusterState currentState) { ensureRepositoryNotInUse(currentState, request.name); MetaData metaData = currentState.metaData(); MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - RepositoriesMetaData repositories = metaData.custom(RepositoriesMetaData.TYPE); + RepositoriesMetaData repositories = metaData.get(RepositoriesMetaData.TYPE); if (repositories != null && repositories.repositories().size() > 0) { List repositoriesMetaData = new ArrayList<>(repositories.repositories().size()); boolean changed = false; diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java index e058f2cf2166c..22c6fddeaee06 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java @@ -103,9 +103,8 @@ public void handleRequest(final RestRequest request, final RestChannel channel, @Override public RestResponse buildResponse(NodesInfoResponse response, XContentBuilder builder) throws Exception { - response.settingsFilter(settingsFilter); builder.startObject(); - response.toXContent(builder, request); + response.toXContent(builder, settingsFilter.withFilterSettingParams(request)); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java index 43f35a7d6b91a..5000f29e81ed7 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java @@ -71,7 +71,7 @@ protected void addCustomFields(XContentBuilder builder, ClusterRerouteResponse r if (request.param("metric") == null) { request.params().put("metric", DEFAULT_METRICS); } - response.getState().settingsFilter(settingsFilter).toXContent(builder, request); + response.getState().toXContent(builder, settingsFilter.withFilterSettingParams(request)); builder.endObject(); if (clusterRerouteRequest.explain()) { assert response.getExplanations() != null; diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java index ff3218f203962..f7042bdd0bb63 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java @@ -80,7 +80,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder builder) throws Exception { builder.startObject(); builder.field(Fields.CLUSTER_NAME, response.getClusterName().value()); - response.getState().settingsFilter(settingsFilter).toXContent(builder, request); + response.getState().toXContent(builder, settingsFilter.withFilterSettingParams(request)); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); } diff --git a/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 32a8167eca4ae..c42363a56a315 100644 --- a/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -229,7 +229,7 @@ public ClusterState execute(ClusterState currentState) { shards.size(), shards.size() - failedShards(shards)); } - ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocks).routingTable(rtBuilder).build(); + ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocks).build(); RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(updatedState).routingTable(rtBuilder).build()); return ClusterState.builder(updatedState).routingResult(routingResult).build(); } @@ -282,20 +282,14 @@ private void validateExistingIndex(IndexMetaData currentIndexMetaData, IndexMeta private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) { if (request.includeGlobalState()) { - if (metaData.persistentSettings() != null) { - mdBuilder.persistentSettings(metaData.persistentSettings()); - } - if (metaData.templates() != null) { - // TODO: Should all existing templates be deleted first? - for (ObjectCursor cursor : metaData.templates().values()) { - mdBuilder.put(cursor.value); - } - } - if (metaData.customs() != null) { - for (ObjectObjectCursor cursor : metaData.customs()) { - if (!RepositoriesMetaData.TYPE.equals(cursor.key)) { - // Don't restore repositories while we are working with them - // TODO: Should we restore them at the end? + for (ObjectObjectCursor cursor : metaData.parts()) { + if (cursor.value.context().contains(ClusterStatePart.XContentContext.SNAPSHOT)) { + if (cursor.key.equals(IndexTemplateMetaData.TYPE)) { + // Merge templates instead of replacing them + for (ObjectCursor cursor2 : metaData.templates().values()) { + mdBuilder.put(cursor2.value); + } + } else { mdBuilder.putCustom(cursor.key, cursor.value); } } From 7f9477dec3370eb40576ec6716cbec5987ae0c6b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 30 Dec 2014 21:38:47 -0500 Subject: [PATCH 10/20] Switch to generic xcontent parsing of MetaData --- .../cluster/CompositeClusterStatePart.java | 30 ++++++++++++ .../cluster/metadata/MetaData.java | 48 ++----------------- 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 99070521a6513..68a2a964e89a5 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -220,6 +220,36 @@ public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOExcep return new NoDiff(); } } + + @Override + public T fromXContent(XContentParser parser, LocalContext context) throws IOException { + XContentParser.Token token; + long version = -1; + String uuid = "_na_"; + ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + // check if its a custom index metadata + ClusterStatePart.Factory factory = lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn? + parser.skipChildren(); + } else { + parts.put(currentFieldName, factory.fromXContent(parser, context)); + } + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + version = parser.longValue(); + } else if ("uuid".equals(currentFieldName)) { + uuid = parser.text(); + } + } + } + return fromParts(version, uuid, parts); + } } private static class CompositeDiff implements Diff { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 14adcaf20c4f4..37f4844f00275 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -79,11 +79,6 @@ public MetaData readFrom(StreamInput in, LocalContext context) throws IOExceptio public MetaData fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { return new MetaData(version, uuid, parts.build()); } - - @Override - public MetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); - } } @Override @@ -162,11 +157,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws super(version, uuid, parts); this.uuid = uuid; this.version = version; - this.transientSettings = ((ClusterStateSettingsPart)get(TRANSIENT_SETTINGS_TYPE)).getSettings(); - this.persistentSettings = ((ClusterStateSettingsPart)get(PERSISTENT_SETTINGS_TYPE)).getSettings(); + this.transientSettings = parts.containsKey(TRANSIENT_SETTINGS_TYPE) ? ((ClusterStateSettingsPart)get(TRANSIENT_SETTINGS_TYPE)).getSettings() : ImmutableSettings.EMPTY; + this.persistentSettings = parts.containsKey(PERSISTENT_SETTINGS_TYPE) ? ((ClusterStateSettingsPart)get(PERSISTENT_SETTINGS_TYPE)).getSettings() : ImmutableSettings.EMPTY; this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build(); - this.indices = ((MapClusterStatePart)get(INDICES_TYPE)).parts(); - this.templates = ((MapClusterStatePart)get(TEMPLATES_TYPE)).parts(); + this.indices = parts.containsKey(INDICES_TYPE) ? ((MapClusterStatePart)get(INDICES_TYPE)).parts() : ImmutableOpenMap.of(); + this.templates = parts.containsKey(TEMPLATES_TYPE) ? ((MapClusterStatePart)get(TEMPLATES_TYPE)).parts() : ImmutableOpenMap.of(); int totalNumberOfShards = 0; int numberOfShards = 0; @@ -1363,40 +1358,7 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { } } - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("settings".equals(currentFieldName)) { - builder.persistentSettings(ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).build()); - } else if ("indices".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.put(IndexMetaData.Builder.fromXContent(parser), false); - } - } else if ("templates".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName())); - } - } else { - // check if its a custom index metadata - ClusterStatePart.Factory factory = lookupFactory(currentFieldName); - if (factory == null) { - //TODO warn - parser.skipChildren(); - } else { - // TODO: context - builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); - } - } - } else if (token.isValue()) { - if ("version".equals(currentFieldName)) { - builder.version = parser.longValue(); - } else if ("uuid".equals(currentFieldName)) { - builder.uuid = parser.text(); - } - } - } - return builder.build(); + return FACTORY.fromXContent(parser, new LocalContext(null)); } public static MetaData readFrom(StreamInput in) throws IOException { From 474bcae3145a5227d3b89cb718ad04691a1d2d2e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 5 Jan 2015 12:22:42 -0500 Subject: [PATCH 11/20] Added diffs on index metadata level --- .../state/TransportClusterStateAction.java | 2 - .../CreateIndexClusterStateUpdateRequest.java | 7 +- .../indices/create/CreateIndexRequest.java | 20 +- .../create/CreateIndexRequestBuilder.java | 4 +- .../template/put/PutIndexTemplateRequest.java | 20 +- .../cluster/AbstractClusterStatePart.java | 19 ++ .../elasticsearch/cluster/ClusterName.java | 9 + .../elasticsearch/cluster/ClusterState.java | 99 +++--- .../cluster/ClusterStatePart.java | 8 +- .../cluster/ClusterStateSettingsPart.java | 37 +- .../cluster/CompositeClusterStatePart.java | 50 ++- .../cluster/MapClusterStatePart.java | 72 ++-- ...tePart.java => NamedClusterStatePart.java} | 2 +- .../NamedCompositeClusterStatePart.java | 321 ++++++++++++++++++ .../cluster/block/ClusterBlocks.java | 10 + .../cluster/metadata/AliasMetaData.java | 49 ++- .../cluster/metadata/BenchmarkMetaData.java | 10 + .../metadata/IndexClusterStatePart.java | 33 ++ .../cluster/metadata/IndexMetaData.java | 304 +++++++---------- .../metadata/IndexTemplateMetaData.java | 51 +-- .../cluster/metadata/MappingMetaData.java | 77 ++++- .../cluster/metadata/MetaData.java | 45 +-- .../metadata/MetaDataCreateIndexService.java | 14 +- .../MetaDataIndexTemplateService.java | 6 +- .../metadata/RepositoriesMetaData.java | 11 + .../cluster/metadata/RestoreMetaData.java | 10 + .../cluster/metadata/SnapshotMetaData.java | 9 + .../cluster/node/DiscoveryNodes.java | 10 + .../cluster/routing/IndexRoutingTable.java | 10 + .../cluster/routing/RoutingTable.java | 11 +- .../discovery/local/LocalDiscovery.java | 1 + .../indices/get/RestGetIndicesAction.java | 2 +- .../warmer/get/RestGetWarmerAction.java | 2 +- .../search/warmer/IndexWarmersMetaData.java | 169 +++++---- .../warmer/SimpleIndicesWarmerTests.java | 18 +- 35 files changed, 1057 insertions(+), 465 deletions(-) rename src/main/java/org/elasticsearch/cluster/{MapItemClusterStatePart.java => NamedClusterStatePart.java} (93%) create mode 100644 src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java create mode 100644 src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index 6f63caf8f0b72..a5c2a85a43870 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -41,8 +41,6 @@ import java.util.List; -import static com.google.common.collect.Lists.newArrayList; -import static org.elasticsearch.cluster.metadata.MetaData.lookupFactorySafe; /** * diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 2e32c48321451..5f594eeae6814 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; @@ -51,7 +52,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private final Set aliases = Sets.newHashSet(); - private final Map customs = newHashMap(); + private final Map customs = newHashMap(); private final Set blocks = Sets.newHashSet(); @@ -77,7 +78,7 @@ public CreateIndexClusterStateUpdateRequest aliases(Set aliases) { return this; } - public CreateIndexClusterStateUpdateRequest customs(Map customs) { + public CreateIndexClusterStateUpdateRequest customs(Map customs) { this.customs.putAll(customs); return this; } @@ -120,7 +121,7 @@ public Set aliases() { return aliases; } - public Map customs() { + public Map customs() { return customs; } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index bd1aaba4a4605..79313a8869e5f 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -24,7 +24,6 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -32,6 +31,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -73,7 +73,7 @@ public class CreateIndexRequest extends AcknowledgedRequest private final Set aliases = Sets.newHashSet(); - private final Map customs = newHashMap(); + private final Map customs = newHashMap(); CreateIndexRequest() { } @@ -406,11 +406,11 @@ public CreateIndexRequest source(Map source) { aliases((Map) entry.getValue()); } else { // maybe custom? - IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name); + IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactory(name); if (factory != null) { found = true; try { - customs.put(name, factory.fromMap((Map) entry.getValue())); + customs.put(name, factory.fromMap((Map) entry.getValue(), null)); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse custom metadata for [" + name + "]"); } @@ -435,12 +435,12 @@ Set aliases() { /** * Adds custom metadata to the index to be created. */ - public CreateIndexRequest custom(IndexMetaData.Custom custom) { - customs.put(custom.type(), custom); + public CreateIndexRequest custom(IndexClusterStatePart custom) { + customs.put(custom.partType(), custom); return this; } - Map customs() { + Map customs() { return this.customs; } @@ -458,7 +458,7 @@ public void readFrom(StreamInput in) throws IOException { int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in); + IndexClusterStatePart customIndexMetaData = IndexMetaData.FACTORY.lookupFactorySafe(type).readFrom(in, null); customs.put(type, customIndexMetaData); } int aliasesSize = in.readVInt(); @@ -480,9 +480,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getValue()); } out.writeVInt(customs.size()); - for (Map.Entry entry : customs.entrySet()) { + for (Map.Entry entry : customs.entrySet()) { out.writeString(entry.getKey()); - IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out); + entry.getValue().writeTo(out); } out.writeVInt(aliases.size()); for (Alias alias : aliases) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 83f1cc104bbc8..6ef84b2437673 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -232,7 +232,7 @@ public CreateIndexRequestBuilder setSource(Map source) { /** * Adds custom metadata to the index to be created. */ - public CreateIndexRequestBuilder addCustom(IndexMetaData.Custom custom) { + public CreateIndexRequestBuilder addCustom(IndexClusterStatePart custom) { request.custom(custom); return this; } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 64d268f5d32b4..de1144bf5a9e7 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeOperationRequest; +import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -71,7 +73,7 @@ public class PutIndexTemplateRequest extends MasterNodeOperationRequest aliases = newHashSet(); - private Map customs = newHashMap(); + private Map customs = newHashMap(); PutIndexTemplateRequest() { } @@ -294,10 +296,10 @@ public PutIndexTemplateRequest source(Map templateSource) { aliases((Map) entry.getValue()); } else { // maybe custom? - IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name); + ClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactory(name); if (factory != null) { try { - customs.put(name, factory.fromMap((Map) entry.getValue())); + customs.put(name, factory.fromMap((Map) entry.getValue(), null)); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse custom metadata for [" + name + "]"); } @@ -347,12 +349,12 @@ public PutIndexTemplateRequest source(BytesReference source) { } } - public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { - customs.put(custom.type(), custom); + public PutIndexTemplateRequest custom(IndexClusterStatePart custom) { + customs.put(custom.partType(), custom); return this; } - Map customs() { + Map customs() { return this.customs; } @@ -442,7 +444,7 @@ public void readFrom(StreamInput in) throws IOException { int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in); + IndexClusterStatePart customIndexMetaData = IndexMetaData.FACTORY.lookupFactorySafe(type).readFrom(in, null); customs.put(type, customIndexMetaData); } int aliasesSize = in.readVInt(); @@ -466,9 +468,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getValue()); } out.writeVInt(customs.size()); - for (Map.Entry entry : customs.entrySet()) { + for (Map.Entry entry : customs.entrySet()) { out.writeString(entry.getKey()); - IndexMetaData.lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out); + entry.getValue().writeTo(out); } out.writeVInt(aliases.size()); for (Alias alias : aliases) { diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 36f4eac21bb81..39e989288d00b 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -23,10 +23,14 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.EnumSet; +import java.util.Map; /** */ @@ -102,6 +106,21 @@ public Version addedIn() { public T fromXContent(XContentParser parser, LocalContext context) throws IOException { throw new UnsupportedOperationException("Not implemented yet"); } + + @Override + public T fromMap(Map map, LocalContext context) throws IOException { + // if it starts with the type, remove it + if (map.size() == 1 && map.containsKey(partType())) { + map = (Map) map.values().iterator().next(); + } + XContentBuilder builder = XContentFactory.smileBuilder().map(map); + try (XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.bytes())) { + // move to START_OBJECT + parser.nextToken(); + return fromXContent(parser, context); + } + } + } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterName.java b/src/main/java/org/elasticsearch/cluster/ClusterName.java index f83e311c66e25..9031cb056a61b 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -86,6 +86,11 @@ public boolean equals(Object o) { return true; } + @Override + public String partType() { + return TYPE; + } + @Override public int hashCode() { return value != null ? value.hashCode() : 0; @@ -109,5 +114,9 @@ public ClusterName readFrom(StreamInput in, LocalContext context) throws IOExcep return readClusterName(in); } + @Override + public String partType() { + return TYPE; + } } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index e2e68921f8fa5..597f6e68f8826 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -48,6 +47,55 @@ */ public class ClusterState extends CompositeClusterStatePart { + public static final String TYPE = "cluster"; + + public static final Factory FACTORY = new Factory(); + + public static class Factory extends AbstractCompositeFactory { + + @Override + public ClusterState fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { + return new ClusterState(version, uuid, parts.build()); + } + + @Override + public String partType() { + return TYPE; + } + } + + static { + FACTORY.registerFactory(ClusterName.TYPE, ClusterName.FACTORY); + FACTORY.registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); + FACTORY.registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); + FACTORY.registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); + FACTORY.registerFactory(MetaData.TYPE, MetaData.FACTORY); + } + + public static class ClusterStateDiff { + private long version; + private ClusterState.Diff diff; + + public ClusterStateDiff(long version, ClusterState.Diff diff) { + this.version = version; + this.diff = diff; + } + + public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { + ClusterState newState = diff.apply(previous); + return newState; + } + + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(version); + diff.writeTo(out); + } + + public long version() { + return version; + } + } + public static enum ClusterStateStatus { UNKNOWN((byte) 0), RECEIVED((byte) 1), @@ -293,6 +341,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @Override + public String partType() { + return TYPE; + } + public static Builder builder(ClusterName clusterName) { return new Builder(clusterName); } @@ -454,48 +507,4 @@ public static ClusterState fromDiffBytes(ClusterState before, byte[] data, Disco } } - public static final String TYPE = "cluster"; - - public static final Factory FACTORY = new Factory(); - - public static class Factory extends AbstractCompositeClusterStatePartFactory { - - @Override - public ClusterState fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { - return new ClusterState(version, uuid, parts.build()); - } - } - - static { - registerFactory(ClusterName.TYPE, ClusterName.FACTORY); - registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); - registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); - registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); - registerFactory(MetaData.TYPE, MetaData.FACTORY); - } - - public static class ClusterStateDiff { - private long version; - private ClusterState.Diff diff; - - public ClusterStateDiff(long version, ClusterState.Diff diff) { - this.version = version; - this.diff = diff; - } - - public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { - ClusterState newState = diff.apply(previous); - return newState; - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(version); - diff.writeTo(out); - } - - public long version() { - return version; - } - } - } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index 88e2f417f5d51..c8f75cf6aff9f 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.EnumSet; +import java.util.Map; /** */ @@ -34,7 +35,6 @@ public interface ClusterStatePart extends ToXContent { public static final String CONTEXT_MODE_PARAM = "context_mode"; - public enum XContentContext { /* Custom metadata should be returns as part of API call */ API, @@ -58,6 +58,8 @@ public enum XContentContext { EnumSet context(); + String partType(); + interface Factory { Diff diff(T before, T after); @@ -68,6 +70,10 @@ interface Factory { T fromXContent(XContentParser parser, LocalContext context) throws IOException; + T fromMap(Map map, LocalContext context) throws IOException; + + String partType(); + Version addedIn(); } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index b875a9d0ee279..61ce77ec8d6e6 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.cluster; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.ImmutableSettings; @@ -35,12 +36,14 @@ /** */ -public class ClusterStateSettingsPart extends AbstractClusterStatePart { +public class ClusterStateSettingsPart extends AbstractClusterStatePart implements IndexClusterStatePart { + private final String type; private final Settings settings; private final EnumSet xContentContext; - public ClusterStateSettingsPart(Settings settings, EnumSet xContentContext) { + public ClusterStateSettingsPart(String type, Settings settings, EnumSet xContentContext) { + this.type = type; this.settings = settings; this.xContentContext = xContentContext; } @@ -67,30 +70,50 @@ public EnumSet context() { return xContentContext; } + @Override + public ClusterStateSettingsPart mergeWith(ClusterStateSettingsPart second) { + return second; + } + + @Override + public String partType() { + return type; + } + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + private final String type; private final EnumSet xContentContext; - public Factory() { + public Factory(String type) { + this.type = type; xContentContext = API; } - public Factory(EnumSet xContentContext) { + public Factory(String type, EnumSet xContentContext) { + this.type = type; this.xContentContext = xContentContext; } @Override public ClusterStateSettingsPart readFrom(StreamInput in, LocalContext context) throws IOException { - return new ClusterStateSettingsPart(readSettingsFromStream(in), xContentContext); + return new ClusterStateSettingsPart(type, readSettingsFromStream(in), xContentContext); } @Override public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext context) throws IOException { Settings settings = ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).build(); - return new ClusterStateSettingsPart(settings, xContentContext); + return new ClusterStateSettingsPart(type, settings, xContentContext); } public ClusterStateSettingsPart fromSettings(Settings settings) { - return new ClusterStateSettingsPart(settings, xContentContext); + return new ClusterStateSettingsPart(type, settings, xContentContext); } + + + @Override + public String partType() { + return type; + } + } } diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 68a2a964e89a5..afb2ef8c1923b 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster; -import com.carrotsearch.hppc.ObjectLookupContainer; -import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; @@ -46,7 +44,6 @@ */ public abstract class CompositeClusterStatePart extends AbstractClusterStatePart { - private final static Map partFactories = new HashMap<>(); protected final long version; protected final String uuid; protected final ImmutableOpenMap parts; @@ -57,26 +54,6 @@ protected CompositeClusterStatePart(long version, String uuid, ImmutableOpenMap< this.parts = parts; } - /** - * Register a custom index meta data factory. Make sure to call it from a static block. - */ - public static void registerFactory(String type, ClusterStatePart.Factory factory) { - partFactories.put(type, factory); - } - - @Nullable - public static ClusterStatePart.Factory lookupFactory(String type) { - return partFactories.get(type); - } - - public static ClusterStatePart.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { - ClusterStatePart.Factory factory = lookupFactory(type); - if (factory == null) { - throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); - } - return factory; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -144,7 +121,28 @@ public String uuid() { return uuid; } - public static abstract class AbstractCompositeClusterStatePartFactory extends AbstractClusterStatePart.AbstractFactory { + public static abstract class AbstractCompositeFactory extends AbstractClusterStatePart.AbstractFactory { + + private final Map partFactories = new HashMap<>(); + /** + * Register a custom index meta data factory. Make sure to call it from a static block. + */ + public void registerFactory(String type, ClusterStatePart.Factory factory) { + partFactories.put(type, factory); + } + + @Nullable + public ClusterStatePart.Factory lookupFactory(String type) { + return partFactories.get(type); + } + + public ClusterStatePart.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { + ClusterStatePart.Factory factory = lookupFactory(type); + if (factory == null) { + throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); + } + return factory; + } @Override public T readFrom(StreamInput in, LocalContext context) throws IOException { @@ -259,9 +257,9 @@ private static class CompositeDiff implemen private final String uuid; private final Map> diffs; private final List deletes; - private final AbstractCompositeClusterStatePartFactory factory; + private final AbstractCompositeFactory factory; - private CompositeDiff(AbstractCompositeClusterStatePartFactory factory, long version, String previousUuid, String uuid, List deletes, Map> diffs) { + private CompositeDiff(AbstractCompositeFactory factory, long version, String previousUuid, String uuid, List deletes, Map> diffs) { this.version = version; this.previousUuid = previousUuid; this.uuid = uuid; diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index af4bb9639d88b..c2729a253a02c 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -40,13 +41,15 @@ /** * Represents a map of cluster state parts with the same type. */ -public class MapClusterStatePart extends AbstractClusterStatePart { +public class MapClusterStatePart extends AbstractClusterStatePart implements IndexClusterStatePart< MapClusterStatePart> { private final ImmutableOpenMap parts; private final EnumSet xContentContext; + private final String type; - public MapClusterStatePart(ImmutableOpenMap parts, EnumSet xContentContext) { + public MapClusterStatePart(String type, ImmutableOpenMap parts, EnumSet xContentContext) { this.parts = parts; + this.type = type; this.xContentContext = xContentContext; } @@ -79,18 +82,31 @@ public EnumSet context() { return xContentContext; } + @Override + public MapClusterStatePart mergeWith(MapClusterStatePart second) { + //TODO: Implement + return null; + } + + @Override + public String partType() { + return type; + } - public static class Factory extends AbstractFactory> { + public static class Factory extends AbstractFactory> { private final EnumSet xContentContext; private final ClusterStatePart.Factory factory; - public Factory(ClusterStatePart.Factory factory) { - this(factory, API); + private final String type; + + public Factory(String type, ClusterStatePart.Factory factory) { + this(type, factory, API); } - public Factory(ClusterStatePart.Factory factory, EnumSet xContentContext) { + public Factory(String type, ClusterStatePart.Factory factory, EnumSet xContentContext) { + this.type = type; this.factory = factory; this.xContentContext = xContentContext; } @@ -103,7 +119,7 @@ public MapClusterStatePart readFrom(StreamInput in, LocalContext context) thr T part = factory.readFrom(in, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(builder.build(), xContentContext); + return new MapClusterStatePart<>(type, builder.build(), xContentContext); } @Override @@ -113,7 +129,7 @@ public MapClusterStatePart fromXContent(XContentParser parser, LocalContext c T part = factory.fromXContent(parser, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(builder.build(), xContentContext); + return new MapClusterStatePart<>(type, builder.build(), xContentContext); } @Override @@ -145,7 +161,7 @@ public Diff> diff(@Nullable MapClusterStatePart before diffs.put(partIter.key, factory.diff(null, partIter.value)); } } - return new MapDiff<>(deletes, diffs, xContentContext); + return new MapDiff<>(type, deletes, diffs, xContentContext); } @Override @@ -163,26 +179,34 @@ public Diff> readDiffFrom(StreamInput in, LocalContext co String key = in.readString(); diffs.put(key, factory.readDiffFrom(in, context)); } - return new MapDiff<>(deletes, diffs, xContentContext); + return new MapDiff<>(type, deletes, diffs, xContentContext); } else { return new NoDiff<>(); } } - public MapClusterStatePart fromMap(ImmutableOpenMap map) { - return new MapClusterStatePart<>(map, xContentContext); + public MapClusterStatePart fromOpenMap(ImmutableOpenMap map) { + return new MapClusterStatePart<>(type, map, xContentContext); + } + + + @Override + public String partType() { + return type; } } - private static class MapDiff implements Diff> { + private static class MapDiff implements Diff> { + private final String type; private final EnumSet xContentContext; private final Map> diffs; private final List deletes; - private MapDiff(List deletes, Map> diffs, EnumSet xContentContext) { + private MapDiff(String type, List deletes, Map> diffs, EnumSet xContentContext) { + this.type = type; this.diffs = diffs; this.deletes = deletes; this.xContentContext = xContentContext; @@ -191,15 +215,21 @@ private MapDiff(List deletes, Map> diffs, EnumSet apply(MapClusterStatePart part) { ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); - parts.putAll(part.parts); - for (String delete : deletes) { - parts.remove(delete); - } + if (part != null) { + parts.putAll(part.parts); + for (String delete : deletes) { + parts.remove(delete); + } - for (Map.Entry> entry : diffs.entrySet()) { - parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); + for (Map.Entry> entry : diffs.entrySet()) { + parts.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); + } + } else { + for (Map.Entry> entry : diffs.entrySet()) { + parts.put(entry.getKey(), entry.getValue().apply(null)); + } } - return new MapClusterStatePart(parts.build(), xContentContext); + return new MapClusterStatePart(type, parts.build(), xContentContext); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java similarity index 93% rename from src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java rename to src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java index d1d0d4f3332fc..3c6feadfb1a1e 100644 --- a/src/main/java/org/elasticsearch/cluster/MapItemClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java @@ -22,7 +22,7 @@ /** * All cluster state parts that can be put into a map should implement this interface */ -public interface MapItemClusterStatePart extends ClusterStatePart { +public interface NamedClusterStatePart extends ClusterStatePart { String key(); } diff --git a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java new file mode 100644 index 0000000000000..b857875371d3a --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java @@ -0,0 +1,321 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.*; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; + +/** + * Represents a map of cluster state parts with different types. + *

+ * Only one instance of each type can be present in the composite part. The key of the map is the part's type. + */ +public abstract class NamedCompositeClusterStatePart extends AbstractClusterStatePart implements NamedClusterStatePart{ + + protected final ImmutableOpenMap parts; + + protected NamedCompositeClusterStatePart(ImmutableOpenMap parts) { + this.parts = parts; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NamedCompositeClusterStatePart that = (NamedCompositeClusterStatePart) o; + + if (parts != null ? !parts.equals(that.parts) : that.parts != null) return false; + + return true; + } + + @Override + public int hashCode() { + return parts != null ? parts.hashCode() : 0; + } + + public ImmutableOpenMap parts() { + return parts; + } + + protected abstract void valuesPartWriteTo(StreamOutput out) throws IOException; + protected abstract void valuesPartToXContent(XContentBuilder builder, Params params) throws IOException; + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(key()); + valuesPartWriteTo(out); + out.writeVInt(parts().size()); + for (ObjectObjectCursor cursor : parts()) { + out.writeString(cursor.key); + cursor.value.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); + valuesPartToXContent(builder, params); + for (ObjectObjectCursor partIter : parts) { + if (partIter.value.context().contains(context)) { + builder.startObject(partIter.key); + partIter.value.toXContent(builder, params); + builder.endObject(); + } + } + return builder; + } + + public T get(String type) { + return (T) parts.get(type); + } + + + public static abstract class Builder> { + + protected ImmutableOpenMap.Builder parts = ImmutableOpenMap.builder(); + + public void putAll(T part) { + parts.putAll(part.parts()); + } + + public void put(String type, E part) { + parts.put(type, part); + } + + public void remove(String type) { + parts.remove(type); + } + + public abstract T build(); + + public abstract String getKey(); + + public abstract void parseValuePart(XContentParser parser, String currentFieldName, LocalContext context) throws IOException; + + public abstract void readValuePartsFrom(StreamInput in, LocalContext context) throws IOException; + + public abstract void writeValuePartsTo(StreamOutput out) throws IOException; + } + + public static abstract class AbstractFactory> extends AbstractClusterStatePart.AbstractFactory { + + private final Map> partFactories = new HashMap<>(); + /** + * Register a custom index meta data factory. Make sure to call it from a static block. + */ + public void registerFactory(String type, Factory factory) { + partFactories.put(type, factory); + } + + public Set availableFactories() { + return partFactories.keySet(); + } + + @Nullable + public Factory lookupFactory(String type) { + return (Factory)partFactories.get(type); + } + + public Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { + Factory factory = lookupFactory(type); + if (factory == null) { + throw new ElasticsearchIllegalArgumentException("No cluster state part factory registered for type [" + type + "]"); + } + return factory; + } + + public abstract Builder builder(String key); + + public abstract Builder builder(T part); + + @Override + public T readFrom(StreamInput in, LocalContext context) throws IOException { + String key = in.readString(); + Builder builder = builder(key); + builder.readValuePartsFrom(in, context); + int partsSize = in.readVInt(); + for (int i = 0; i < partsSize; i++) { + String type = in.readString(); + E part = lookupFactorySafe(type).readFrom(in, context); + builder.put(type, part); + } + return builder.build(); + } + + @Override + public Diff diff(@Nullable T before, T after) { + assert after != null; + Map> diffs = newHashMap(); + List deletes = newArrayList(); + if (before != null) { + ImmutableOpenMap beforeParts = before.parts(); + ImmutableOpenMap afterParts = after.parts(); + if (before.equals(after)) { + return new NoDiff<>(); + } else { + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); + } + } + for (ObjectObjectCursor partIter : afterParts) { + Factory factory = lookupFactorySafe(partIter.key); + E beforePart = beforeParts.get(partIter.key); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + } + } + } + } else { + ImmutableOpenMap afterParts = after.parts(); + for (ObjectObjectCursor partIter : afterParts) { + Factory factory = lookupFactorySafe(partIter.key); + diffs.put(partIter.key, factory.diff(null, partIter.value)); + } + } + return new CompositeDiff<>(builder(after), deletes, diffs); + } + + @Override + public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { + if (in.readBoolean()) { + String key = in.readString(); + Builder builder = builder(key); + builder.readValuePartsFrom(in, context); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } + + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String partKey = in.readString(); + diffs.put(partKey, lookupFactorySafe(partKey).readDiffFrom(in, context)); + } + return new CompositeDiff<>(builder, deletes, diffs); + + } else { + return new NoDiff(); + } + } + + @Override + public T fromXContent(XContentParser parser, LocalContext context) throws IOException { + XContentParser.Token token; + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { // on a start object move to next token + parser.nextToken(); + } + Builder builder = builder(parser.currentName()); + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + // check if its a custom index metadata + Factory factory = lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn? + parser.skipChildren(); + } else { + builder.put(currentFieldName, factory.fromXContent(parser, context)); + } + } else if (token.isValue()) { + builder.parseValuePart(parser, currentFieldName, context); +// if ("version".equals(currentFieldName)) { +// version = parser.longValue(); +// } else if ("uuid".equals(currentFieldName)) { +// uuid = parser.text(); +// } + } + } + return builder.build(); + } + } + + private static class CompositeDiff> implements Diff { + + private final Builder builder; + private final Map> diffs; + private final List deletes; + + private CompositeDiff(Builder builder, List deletes, Map> diffs) { + this.diffs = diffs; + this.deletes = deletes; + this.builder = builder; + } + + @Override + public T apply(T part) { + if (part != null) { + builder.putAll(part); + for (String delete : deletes) { + builder.remove(delete); + } + + for (Map.Entry> entry : diffs.entrySet()) { + builder.put(entry.getKey(), entry.getValue().apply(part.get(entry.getKey()))); + } + } else { + for (Map.Entry> entry : diffs.entrySet()) { + builder.put(entry.getKey(), entry.getValue().apply(null)); + } + } + return builder.build(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); // We have diffs + out.writeString(builder.getKey()); + builder.writeValuePartsTo(out); + out.writeVInt(deletes.size()); + for (String delete : deletes) { + out.writeString(delete); + } + + out.writeVInt(diffs.size()); + for (Map.Entry> entry : diffs.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + } + + +} diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index dc264b8aa4cc1..81da147b6516e 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -83,12 +83,22 @@ public class ClusterBlocks extends AbstractClusterStatePart { } } + @Override + public String partType() { + return TYPE; + } + public static class Factory extends AbstractFactory { @Override public ClusterBlocks readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readClusterBlocks(in); } + + @Override + public String partType() { + return TYPE; + } } public ImmutableSet global() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 008935ec026a4..5ba084f07c4f9 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -21,6 +21,9 @@ import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.NamedClusterStatePart; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +41,51 @@ /** * */ -public class AliasMetaData { +public class AliasMetaData extends AbstractClusterStatePart implements NamedClusterStatePart, IndexClusterStatePart { + + public static final String TYPE = "alias"; + + public static final Factory FACTORY = new Factory(); + + @Override + public void writeTo(StreamOutput out) throws IOException { + Builder.writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(this, builder, params); + return builder; + } + + @Override + public String key() { + return alias; + } + + @Override + public String partType() { + return TYPE; + } + + + @Override + public AliasMetaData mergeWith(AliasMetaData second) { + return null; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public AliasMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in); + } + + @Override + public String partType() { + return TYPE; + } + } private final String alias; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java index 6cd899994d890..02646f3e0c80d 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java @@ -55,6 +55,11 @@ public int hashCode() { return entries.hashCode(); } + @Override + public String partType() { + return TYPE; + } + public static class Entry { private final State state; private final String benchmarkId; @@ -200,6 +205,11 @@ public BenchmarkMetaData readFrom(StreamInput in, LocalContext context) throws I } return new BenchmarkMetaData(entries); } + + @Override + public String partType() { + return TYPE; + } } public boolean contains(String benchmarkId) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java new file mode 100644 index 0000000000000..691182796fdd8 --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +/** +*/ +public interface IndexClusterStatePart extends ClusterStatePart { + T mergeWith(T second); +} diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 43e0cf3c631f2..b89e7c57ac5f5 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -22,12 +22,9 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.ImmutableMap; -import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; -import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.LocalContext; -import org.elasticsearch.cluster.MapItemClusterStatePart; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; @@ -47,14 +44,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -65,64 +60,37 @@ /** * */ -public class IndexMetaData extends AbstractClusterStatePart implements MapItemClusterStatePart { +public class IndexMetaData extends NamedCompositeClusterStatePart implements NamedClusterStatePart { public static String TYPE = "index_metadata"; public static Factory FACTORY = new Factory(); - public interface Custom { + public static final String SETTINGS_TYPE = "settings"; - String type(); + public static final ClusterStateSettingsPart.Factory SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(SETTINGS_TYPE); - interface Factory { + public static final String MAPPINGS_TYPE = "mappings"; - String type(); + public static final MapClusterStatePart.Factory MAPPINGS_FACTORY = new MapClusterStatePart.Factory<>(MAPPINGS_TYPE, MappingMetaData.FACTORY, API_GATEWAY_SNAPSHOT); - T readFrom(StreamInput in) throws IOException; + public static final String ALIASES_TYPE = "aliases"; - void writeTo(T customIndexMetaData, StreamOutput out) throws IOException; - - T fromMap(Map map) throws IOException; - - T fromXContent(XContentParser parser) throws IOException; - - void toXContent(T customIndexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException; - - /** - * Merges from first to second, with first being more important, i.e., if something exists in first and second, - * first will prevail. - */ - T merge(T first, T second); - } - } - - public static Map customFactories = new HashMap<>(); + public static final MapClusterStatePart.Factory ALIASES_FACTORY = new MapClusterStatePart.Factory<>(ALIASES_TYPE, AliasMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { + FACTORY.registerFactory(SETTINGS_TYPE, SETTINGS_FACTORY); + FACTORY.registerFactory(MAPPINGS_TYPE, MAPPINGS_FACTORY); + FACTORY.registerFactory(ALIASES_TYPE, ALIASES_FACTORY); // register non plugin custom metadata - registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); - } - - /** - * Register a custom index meta data factory. Make sure to call it from a static block. - */ - public static void registerFactory(String type, Custom.Factory factory) { - customFactories.put(type, factory); + FACTORY.registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); } - @Nullable - public static Custom.Factory lookupFactory(String type) { - return customFactories.get(type); + @Override + public String partType() { + return TYPE; } - public static Custom.Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { - Custom.Factory factory = customFactories.get(type); - if (factory == null) { - throw new ElasticsearchIllegalArgumentException("No custom index metadata factoy registered for type [" + type + "]"); - } - return factory; - } public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA)); public static final ClusterBlock INDEX_READ_BLOCK = new ClusterBlock(7, "index read (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.READ)); @@ -189,11 +157,8 @@ public static State fromString(String state) { private final ImmutableOpenMap aliases; private final Settings settings; - private final ImmutableOpenMap mappings; - private final ImmutableOpenMap customs; - private transient final int totalNumberOfShards; private final DiscoveryNodeFilters requireFilters; @@ -204,17 +169,17 @@ public static State fromString(String state) { private final HashFunction routingHashFunction; private final boolean useTypeForRouting; - private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { - Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) != null, "must specify numberOfShards for index [" + index + "]"); - Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) != null, "must specify numberOfReplicas for index [" + index + "]"); + private IndexMetaData(String index, long version, State state, ImmutableOpenMap parts) { + super(parts); this.index = index; this.version = version; this.state = state; - this.settings = settings; - this.mappings = mappings; - this.customs = customs; + this.settings = parts.containsKey(SETTINGS_TYPE) ? ((ClusterStateSettingsPart)get(SETTINGS_TYPE)).getSettings() : ImmutableSettings.EMPTY; this.totalNumberOfShards = numberOfShards() * (numberOfReplicas() + 1); - this.aliases = aliases; + Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) != null, "must specify numberOfShards for index [" + index + "]"); + Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) != null, "must specify numberOfReplicas for index [" + index + "]"); + this.mappings = parts.containsKey(MAPPINGS_TYPE) ? ((MapClusterStatePart)get(MAPPINGS_TYPE)).parts() : ImmutableOpenMap.of(); + this.aliases = parts.containsKey(ALIASES_TYPE) ? ((MapClusterStatePart)get(ALIASES_TYPE)).parts() : ImmutableOpenMap.of(); ImmutableMap requireMap = settings.getByPrefix("index.routing.allocation.require.").getAsMap(); if (requireMap.isEmpty()) { @@ -404,17 +369,17 @@ public MappingMetaData mappingOrDefault(String mappingType) { return mappings.get(MapperService.DEFAULT_MAPPING); } - public ImmutableOpenMap customs() { - return this.customs; + public ImmutableOpenMap customs() { + return this.parts; } - public ImmutableOpenMap getCustoms() { - return this.customs; + public ImmutableOpenMap getCustoms() { + return this.parts; } @SuppressWarnings("unchecked") - public T custom(String type) { - return (T) customs.get(type); + public T custom(String type) { + return (T) parts.get(type); } @Nullable @@ -443,23 +408,11 @@ public boolean equals(Object o) { IndexMetaData that = (IndexMetaData) o; - if (!aliases.equals(that.aliases)) { - return false; - } - if (!index.equals(that.index)) { - return false; - } - if (!mappings.equals(that.mappings)) { - return false; - } - if (!settings.equals(that.settings)) { - return false; - } if (state != that.state) { return false; } - if (!customs.equals(that.customs)) { + if (!parts.equals(that.parts)) { return false; } @@ -470,13 +423,29 @@ public boolean equals(Object o) { public int hashCode() { int result = index.hashCode(); result = 31 * result + state.hashCode(); - result = 31 * result + aliases.hashCode(); - result = 31 * result + settings.hashCode(); - result = 31 * result + mappings.hashCode(); - result = 31 * result + customs.hashCode(); + result = 31 * result + parts.hashCode(); return result; } + @Override + protected void valuesPartWriteTo(StreamOutput out) throws IOException { + out.writeVLong(version); + out.writeByte(state.id()); + } + + @Override + protected void valuesPartToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("version", version); + builder.field("state", state.toString().toLowerCase(Locale.ENGLISH)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // TODO: switch to generic toXContent + Builder.toXContent(this, builder, params); + return builder; + } + public static Builder builder(String index) { return new Builder(index); } @@ -485,7 +454,7 @@ public static Builder builder(IndexMetaData indexMetaData) { return new Builder(indexMetaData); } - public static class Builder { + public static class Builder extends NamedCompositeClusterStatePart.Builder { private String index; private State state = State.OPEN; @@ -493,13 +462,11 @@ public static class Builder { private Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS; private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; - private final ImmutableOpenMap.Builder customs; public Builder(String index) { this.index = index; this.mappings = ImmutableOpenMap.builder(); this.aliases = ImmutableOpenMap.builder(); - this.customs = ImmutableOpenMap.builder(); } public Builder(IndexMetaData indexMetaData) { @@ -509,7 +476,10 @@ public Builder(IndexMetaData indexMetaData) { this.settings = indexMetaData.settings(); this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); - this.customs = ImmutableOpenMap.builder(indexMetaData.customs); + parts.putAll(indexMetaData.parts()); + parts.remove(SETTINGS_TYPE); + parts.remove(MAPPINGS_TYPE); + parts.remove(ALIASES_TYPE); } public String index() { @@ -521,6 +491,10 @@ public Builder index(String index) { return this; } + public String getKey() { + return index; + } + public Builder numberOfShards(int numberOfShards) { settings = settingsBuilder().put(settings).put(SETTING_NUMBER_OF_SHARDS, numberOfShards).build(); return this; @@ -604,18 +578,18 @@ public Builder removeAllAliases() { return this; } - public Builder putCustom(String type, Custom customIndexMetaData) { - this.customs.put(type, customIndexMetaData); + public Builder putCustom(String type, IndexClusterStatePart customIndexMetaData) { + this.parts.put(type, customIndexMetaData); return this; } public Builder removeCustom(String type) { - this.customs.remove(type); + this.parts.remove(type); return this; } - public Custom getCustom(String type) { - return this.customs.get(type); + public ClusterStatePart getCustom(String type) { + return this.parts.get(type); } public long version() { @@ -639,7 +613,40 @@ public IndexMetaData build() { } } - return new IndexMetaData(index, version, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); + return new IndexMetaData(index, version, state, buildParts(tmpSettings, mappings.build(), tmpAliases.build(), parts.build())); + } + + @Override + public void parseValuePart(XContentParser parser, String currentFieldName, LocalContext context) throws IOException { + if ("state".equals(currentFieldName)) { + state = State.fromString(parser.text()); + } else if ("version".equals(currentFieldName)) { + version = parser.longValue(); + } + } + + @Override + public void readValuePartsFrom(StreamInput in, LocalContext context) throws IOException { + version = in.readVLong(); + state = State.fromId(in.readByte()); + } + + @Override + public void writeValuePartsTo(StreamOutput out) throws IOException { + out.writeVLong(version); + out.writeByte(state.id()); + } + + private static ImmutableOpenMap buildParts(Settings settings, + ImmutableOpenMap mappings, + ImmutableOpenMap aliases, + ImmutableOpenMap parts) { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + builder.put(SETTINGS_TYPE, SETTINGS_FACTORY.fromSettings(settings)); + builder.put(MAPPINGS_TYPE, MAPPINGS_FACTORY.fromOpenMap(mappings)); + builder.put(ALIASES_TYPE, ALIASES_FACTORY.fromOpenMap(aliases)); + builder.putAll(parts); + return builder.build(); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -648,63 +655,33 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build builder.field("version", indexMetaData.version()); builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); - boolean binary = params.paramAsBoolean("binary", false); - - builder.startObject("settings"); - Settings settings = indexMetaData.settings(); - settings.toXContent(builder, params); - for (Map.Entry entry : indexMetaData.settings().getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - if (params.paramAsBoolean("reduce_mappings", false)) { builder.startObject("mappings"); for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - byte[] mappingSource = cursor.value.source().uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor.key); - } - builder.field(cursor.key); - builder.map(mapping); + cursor.value.toXContent(builder, params); } builder.endObject(); } else { builder.startArray("mappings"); for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - if (binary) { - builder.value(cursor.value.source().compressed()); - } else { - byte[] data = cursor.value.source().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrdered(); - parser.close(); - builder.map(mapping); - } + cursor.value.toXContent(builder, params); } builder.endArray(); } - for (ObjectObjectCursor cursor : indexMetaData.customs()) { - builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); - lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - builder.endObject(); - } - - builder.startObject("aliases"); - for (ObjectCursor cursor : indexMetaData.aliases().values()) { - AliasMetaData.Builder.toXContent(cursor.value, builder, params); + for (ObjectObjectCursor cursor : indexMetaData.parts()) { + if (!cursor.key.equals(MAPPINGS_TYPE)) { + builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); + cursor.value.toXContent(builder, params); + builder.endObject(); + } } - builder.endObject(); - builder.endObject(); } public static IndexMetaData fromXContent(XContentParser parser) throws IOException { + // TODO : switch to NamedCompositeClusterStatePart.AbstractFactory parser if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -737,12 +714,12 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } else { // check if its a custom index metadata - Custom.Factory factory = lookupFactory(currentFieldName); + ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); if (factory == null) { //TODO warn parser.skipChildren(); } else { - builder.putCustom(factory.type(), factory.fromXContent(parser)); + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } } else if (token == XContentParser.Token.START_ARRAY) { @@ -771,47 +748,11 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } public static IndexMetaData readFrom(StreamInput in) throws IOException { - Builder builder = new Builder(in.readString()); - builder.version(in.readLong()); - builder.state(State.fromId(in.readByte())); - builder.settings(readSettingsFromStream(in)); - int mappingsSize = in.readVInt(); - for (int i = 0; i < mappingsSize; i++) { - MappingMetaData mappingMd = MappingMetaData.readFrom(in); - builder.putMapping(mappingMd); - } - int aliasesSize = in.readVInt(); - for (int i = 0; i < aliasesSize; i++) { - AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in); - builder.putAlias(aliasMd); - } - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in); - builder.putCustom(type, customIndexMetaData); - } - return builder.build(); + return FACTORY.readFrom(in, null); } public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException { - out.writeString(indexMetaData.index()); - out.writeLong(indexMetaData.version()); - out.writeByte(indexMetaData.state().id()); - writeSettingsToStream(indexMetaData.settings(), out); - out.writeVInt(indexMetaData.mappings().size()); - for (ObjectCursor cursor : indexMetaData.mappings().values()) { - MappingMetaData.writeTo(cursor.value, out); - } - out.writeVInt(indexMetaData.aliases().size()); - for (ObjectCursor cursor : indexMetaData.aliases().values()) { - AliasMetaData.Builder.writeTo(cursor.value, out); - } - out.writeVInt(indexMetaData.customs().size()); - for (ObjectObjectCursor cursor : indexMetaData.customs()) { - out.writeString(cursor.key); - lookupFactorySafe(cursor.key).writeTo(cursor.value, out); - } + indexMetaData.writeTo(out); } } @@ -820,28 +761,27 @@ public String key() { return index; } - @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Builder.toXContent(this, builder, params); - return builder; - } + public static class Factory extends NamedCompositeClusterStatePart.AbstractFactory { - public static class Factory extends AbstractClusterStatePart.AbstractFactory { + @Override + public NamedCompositeClusterStatePart.Builder builder(String key) { + return new Builder(key); + } @Override - public IndexMetaData readFrom(StreamInput in, LocalContext context) throws IOException { - return Builder.readFrom(in); + public NamedCompositeClusterStatePart.Builder builder(IndexMetaData part) { + return new Builder(part); } @Override public IndexMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { return Builder.fromXContent(parser); } + + @Override + public String partType() { + return TYPE; + } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 5e4c88a2bb27b..6ebfcf3ff8a5e 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -21,11 +21,9 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.Sets; -import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; -import org.elasticsearch.cluster.MapItemClusterStatePart; +import org.elasticsearch.cluster.NamedClusterStatePart; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedString; @@ -47,7 +45,7 @@ /** * */ -public class IndexTemplateMetaData extends AbstractClusterStatePart implements MapItemClusterStatePart { +public class IndexTemplateMetaData extends AbstractClusterStatePart implements NamedClusterStatePart{ public static String TYPE = "template"; @@ -66,10 +64,10 @@ public class IndexTemplateMetaData extends AbstractClusterStatePart implements M private final ImmutableOpenMap aliases; - private final ImmutableOpenMap customs; + private final ImmutableOpenMap customs; public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap mappings, - ImmutableOpenMap aliases, ImmutableOpenMap customs) { + ImmutableOpenMap aliases, ImmutableOpenMap customs) { this.name = name; this.order = order; this.template = template; @@ -127,16 +125,21 @@ public ImmutableOpenMap getAliases() { return this.aliases; } - public ImmutableOpenMap customs() { + public ImmutableOpenMap customs() { return this.customs; } - public ImmutableOpenMap getCustoms() { + public ImmutableOpenMap getCustoms() { return this.customs; } + @Override + public String partType() { + return TYPE; + } + @SuppressWarnings("unchecked") - public T custom(String type) { + public T custom(String type) { return (T) customs.get(type); } @@ -177,9 +180,10 @@ public EnumSet context() { public static class Builder { - private static final Set VALID_FIELDS = Sets.newHashSet("template", "order", "mappings", "settings"); + private static final Set VALID_FIELDS = Sets.newHashSet("template", "order"); static { - VALID_FIELDS.addAll(IndexMetaData.customFactories.keySet()); + VALID_FIELDS.addAll(IndexMetaData.FACTORY.availableFactories()); + VALID_FIELDS.remove("aliases"); } private String name; @@ -194,7 +198,7 @@ public static class Builder { private final ImmutableOpenMap.Builder aliases; - private final ImmutableOpenMap.Builder customs; + private final ImmutableOpenMap.Builder customs; public Builder(String name) { this.name = name; @@ -263,7 +267,7 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) { return this; } - public Builder putCustom(String type, IndexMetaData.Custom customIndexMetaData) { + public Builder putCustom(String type, IndexClusterStatePart customIndexMetaData) { this.customs.put(type, customIndexMetaData); return this; } @@ -273,7 +277,7 @@ public Builder removeCustom(String type) { return this; } - public IndexMetaData.Custom getCustom(String type) { + public IndexClusterStatePart getCustom(String type) { return this.customs.get(type); } @@ -317,9 +321,9 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont builder.endArray(); } - for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { + for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); - IndexMetaData.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); + cursor.value.toXContent(builder, params); builder.endObject(); } @@ -367,12 +371,12 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t } } else { // check if its a custom index metadata - IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(currentFieldName); + IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactory(currentFieldName); if (factory == null) { //TODO warn parser.skipChildren(); } else { - builder.putCustom(factory.type(), factory.fromXContent(parser)); + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } } else if (token == XContentParser.Token.START_ARRAY) { @@ -437,7 +441,7 @@ public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupFactorySafe(type).readFrom(in); + IndexClusterStatePart customIndexMetaData = IndexMetaData.FACTORY.lookupFactorySafe(type).readFrom(in, null); builder.putCustom(type, customIndexMetaData); } return builder.build(); @@ -458,9 +462,9 @@ public static void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOu AliasMetaData.Builder.writeTo(cursor.value, out); } out.writeVInt(indexTemplateMetaData.customs().size()); - for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { + for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { out.writeString(cursor.key); - IndexMetaData.lookupFactorySafe(cursor.key).writeTo(cursor.value, out); + cursor.value.writeTo(out); } } } @@ -492,6 +496,11 @@ public IndexTemplateMetaData readFrom(StreamInput in, LocalContext context) thro public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { return Builder.fromXContent(parser, parser.currentName()); } + + @Override + public String partType() { + return TYPE; + } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 0c7535394beae..f16554448ef2a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -20,8 +20,10 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.ElasticsearchIllegalStateException; -import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.NamedClusterStatePart; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedString; @@ -29,10 +31,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; @@ -46,7 +45,68 @@ /** * Mapping configuration for a type. */ -public class MappingMetaData { +public class MappingMetaData extends AbstractClusterStatePart implements NamedClusterStatePart, IndexClusterStatePart { + + public static final String TYPE = "mapping"; + + public static final Factory FACTORY = new Factory(); + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeTo(this, out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if(params.paramAsBoolean("reduce_mappings", false)) { + byte[] mappingSource = source().uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(type)) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(type); + } + builder.field(type); + builder.map(mapping); + } else if (params.paramAsBoolean("binary", false)) { + builder.value(source().compressed()); + } else { + byte[] data = source().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrdered(); + parser.close(); + builder.map(mapping); + } + return builder; + } + + @Override + public String key() { + return type; + } + + @Override + public MappingMetaData mergeWith(MappingMetaData second) { + return null; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public MappingMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + return MappingMetaData.readFrom(in); + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public MappingMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return super.fromXContent(parser, context); + } + } public static class Id { @@ -394,6 +454,11 @@ public String type() { return this.type; } + @Override + public String partType() { + return TYPE; + } + public CompressedString source() { return this.source; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 37f4844f00275..cafe77601b934 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -34,14 +34,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.HppcMaps; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.Index; import org.elasticsearch.indices.IndexClosedException; @@ -68,7 +65,7 @@ public class MetaData extends CompositeClusterStatePart implements Ite public static final String ALL = "_all"; - public static class Factory extends AbstractCompositeClusterStatePartFactory { + public static class Factory extends AbstractCompositeFactory { @Override public MetaData readFrom(StreamInput in, LocalContext context) throws IOException { @@ -79,6 +76,11 @@ public MetaData readFrom(StreamInput in, LocalContext context) throws IOExceptio public MetaData fromParts(long version, String uuid, ImmutableOpenMap.Builder parts) { return new MetaData(version, uuid, parts.build()); } + + @Override + public String partType() { + return TYPE; + } } @Override @@ -91,33 +93,38 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return super.toXContent(builder, new DelegatingBooleanParams("reduce_mappings", true, params)); } + @Override + public String partType() { + return TYPE; + } + public static final String PERSISTENT_SETTINGS_TYPE = "settings"; - public static final ClusterStateSettingsPart.Factory PERSISTENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(API_GATEWAY_SNAPSHOT); + public static final ClusterStateSettingsPart.Factory PERSISTENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(PERSISTENT_SETTINGS_TYPE, API_GATEWAY_SNAPSHOT); public static final String TRANSIENT_SETTINGS_TYPE = "transient_settings"; - public static final ClusterStateSettingsPart.Factory TRANSIENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(API); + public static final ClusterStateSettingsPart.Factory TRANSIENT_SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(TRANSIENT_SETTINGS_TYPE, API); public static final String INDICES_TYPE = "indices"; - public static final MapClusterStatePart.Factory INDICES_FACTORY = new MapClusterStatePart.Factory<>(IndexMetaData.FACTORY); + public static final MapClusterStatePart.Factory INDICES_FACTORY = new MapClusterStatePart.Factory<>(INDICES_TYPE, IndexMetaData.FACTORY); public static final String TEMPLATES_TYPE = "templates"; - public static final MapClusterStatePart.Factory TEMPLATES_FACTORY = new MapClusterStatePart.Factory<>(IndexTemplateMetaData.FACTORY, API_GATEWAY_SNAPSHOT); + public static final MapClusterStatePart.Factory TEMPLATES_FACTORY = new MapClusterStatePart.Factory<>(TEMPLATES_TYPE, IndexTemplateMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { - registerFactory(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY); - registerFactory(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY); - registerFactory(INDICES_TYPE, INDICES_FACTORY); - registerFactory(TEMPLATES_TYPE, TEMPLATES_FACTORY); + FACTORY.registerFactory(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY); + FACTORY.registerFactory(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY); + FACTORY.registerFactory(INDICES_TYPE, INDICES_FACTORY); + FACTORY.registerFactory(TEMPLATES_TYPE, TEMPLATES_FACTORY); // register non plugin custom metadata - registerFactory(RepositoriesMetaData.TYPE, RepositoriesMetaData.FACTORY); - registerFactory(SnapshotMetaData.TYPE, SnapshotMetaData.FACTORY); - registerFactory(RestoreMetaData.TYPE, RestoreMetaData.FACTORY); - registerFactory(BenchmarkMetaData.TYPE, BenchmarkMetaData.FACTORY); + FACTORY.registerFactory(RepositoriesMetaData.TYPE, RepositoriesMetaData.FACTORY); + FACTORY.registerFactory(SnapshotMetaData.TYPE, SnapshotMetaData.FACTORY); + FACTORY.registerFactory(RestoreMetaData.TYPE, RestoreMetaData.FACTORY); + FACTORY.registerFactory(BenchmarkMetaData.TYPE, BenchmarkMetaData.FACTORY); } public static final String SETTING_READ_ONLY = "cluster.blocks.read_only"; @@ -1312,8 +1319,8 @@ private static ImmutableOpenMap buildParts(Settings tr ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); builder.put(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY.fromSettings(transientSettings)); builder.put(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY.fromSettings(persistentSettings)); - builder.put(INDICES_TYPE, INDICES_FACTORY.fromMap(indices)); - builder.put(TEMPLATES_TYPE, TEMPLATES_FACTORY.fromMap(templates)); + builder.put(INDICES_TYPE, INDICES_FACTORY.fromOpenMap(indices)); + builder.put(TEMPLATES_TYPE, TEMPLATES_FACTORY.fromOpenMap(templates)); builder.putAll(customs); return builder.build(); } @@ -1379,7 +1386,7 @@ public static MetaData readFrom(StreamInput in) throws IOException { for (int i = 0; i < customSize; i++) { String type = in.readString(); // TODO: context - ClusterStatePart customIndexMetaData = lookupFactorySafe(type).readFrom(in, null); + ClusterStatePart customIndexMetaData = FACTORY.lookupFactorySafe(type).readFrom(in, null); builder.putCustom(type, customIndexMetaData); } return builder.build(); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 4edbe17f16bb1..af36e6a712f0e 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -247,7 +247,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { // find templates, highest order are better matching List templates = findTemplates(request, currentState, indexTemplateFilter); - Map customs = Maps.newHashMap(); + Map customs = Maps.newHashMap(); // add the request mapping Map> mappings = Maps.newHashMap(); @@ -260,7 +260,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { mappings.put(entry.getKey(), parseMapping(entry.getValue())); } - for (Map.Entry entry : request.customs().entrySet()) { + for (Map.Entry entry : request.customs().entrySet()) { customs.put(entry.getKey(), entry.getValue()); } @@ -275,14 +275,14 @@ public ClusterState execute(ClusterState currentState) throws Exception { } } // handle custom - for (ObjectObjectCursor cursor : template.customs()) { + for (ObjectObjectCursor cursor : template.customs()) { String type = cursor.key; - IndexMetaData.Custom custom = cursor.value; - IndexMetaData.Custom existing = customs.get(type); + IndexClusterStatePart custom = cursor.value; + IndexClusterStatePart existing = customs.get(type); if (existing == null) { customs.put(type, custom); } else { - IndexMetaData.Custom merged = IndexMetaData.lookupFactorySafe(type).merge(existing, custom); + IndexClusterStatePart merged = existing.mergeWith(custom); customs.put(type, merged); } } @@ -439,7 +439,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { indexMetaDataBuilder.putAlias(aliasMetaData); } - for (Map.Entry customEntry : customs.entrySet()) { + for (Map.Entry customEntry : customs.entrySet()) { indexMetaDataBuilder.putCustom(customEntry.getKey(), customEntry.getValue()); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index 9aeaf8a54d0ef..c938958671c77 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -146,7 +146,7 @@ public void putTemplate(final PutRequest request, final PutListener listener) { .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build(); templateBuilder.putAlias(aliasMetaData); } - for (Map.Entry entry : request.customs.entrySet()) { + for (Map.Entry entry : request.customs.entrySet()) { templateBuilder.putCustom(entry.getKey(), entry.getValue()); } } catch (Throwable e) { @@ -238,7 +238,7 @@ public static class PutRequest { Settings settings = ImmutableSettings.Builder.EMPTY_SETTINGS; Map mappings = Maps.newHashMap(); List aliases = Lists.newArrayList(); - Map customs = Maps.newHashMap(); + Map customs = Maps.newHashMap(); TimeValue masterTimeout = MasterNodeOperationRequest.DEFAULT_MASTER_NODE_TIMEOUT; @@ -277,7 +277,7 @@ public PutRequest aliases(Set aliases) { return this; } - public PutRequest customs(Map customs) { + public PutRequest customs(Map customs) { this.customs.putAll(customs); return this; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index 73547e65be59f..e871f4ab72edc 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -119,6 +119,11 @@ public static void toXContent(RepositoryMetaData repository, XContentBuilder bui builder.endObject(); } + @Override + public String partType() { + return TYPE; + } + @Override public EnumSet context() { return ClusterStatePart.API_GATEWAY; @@ -186,6 +191,12 @@ public RepositoriesMetaData fromXContent(XContentParser parser, LocalContext con } return new RepositoriesMetaData(repository.toArray(new RepositoryMetaData[repository.size()])); } + + @Override + public String partType() { + return TYPE; + } + } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java index 33aad9ec51579..4da59476d55c3 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java @@ -105,6 +105,11 @@ public int hashCode() { return entries.hashCode(); } + @Override + public String partType() { + return TYPE; + } + /** * Restore metadata */ @@ -497,6 +502,11 @@ public RestoreMetaData readFrom(StreamInput in, LocalContext context) throws IOE } return new RestoreMetaData(entries); } + + @Override + public String partType() { + return TYPE; + } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java index 747703eb35196..94568a7d2fc5b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java @@ -62,6 +62,11 @@ public int hashCode() { return entries.hashCode(); } + @Override + public String partType() { + return TYPE; + } + public static class Entry { private final State state; private final SnapshotId snapshotId; @@ -438,6 +443,10 @@ public SnapshotMetaData readFrom(StreamInput in, LocalContext context) throws IO return new SnapshotMetaData(entries); } + @Override + public String partType() { + return TYPE; + } } diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 1bdace3b93963..2a679c9f62ccc 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -71,6 +71,11 @@ public static class Factory extends AbstractFactory { public DiscoveryNodes readFrom(StreamInput in, LocalContext context) throws IOException { return Builder.readFrom(in, context.getLocalNode()); } + + @Override + public String partType() { + return TYPE; + } } private DiscoveryNodes(ImmutableOpenMap nodes, ImmutableOpenMap dataNodes, ImmutableOpenMap masterNodes, String masterNodeId, String localNodeId, Version minNodeVersion, Version minNonClientNodeVersion) { @@ -505,6 +510,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @Override + public String partType() { + return TYPE; + } + public static class Delta { private final String localNodeId; diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 9c92ac83c0364..acaec1da069eb 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -94,6 +94,11 @@ public class IndexRoutingTable extends AbstractClusterStatePart implements Itera this.allActiveShards = allActiveShards.build(); } + @Override + public String partType() { + return TYPE; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(index(), XContentBuilder.FieldCaseConversion.NONE); @@ -137,6 +142,11 @@ public IndexRoutingTable readFrom(StreamInput in, LocalContext context) throws I return Builder.readFrom(in); } + @Override + public String partType() { + return TYPE; + } + } /** * Return the index id diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 8af05d31116e7..1f16e50c4723d 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -67,6 +67,11 @@ public class RoutingTable extends AbstractClusterStatePart implements Iterable { @@ -134,7 +139,11 @@ public Diff readDiffFrom(StreamInput in, LocalContext context) thr return new NoDiff<>(); } } - + + @Override + public String partType() { + return TYPE; + } } private static class RoutingTableDiff implements Diff { diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 78afffb1fa375..7ab8b4a38fdc6 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -314,6 +314,7 @@ private void publish(LocalDiscovery[] members, ClusterState clusterState, final ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(new BytesStreamInput(clusterStateDiffBytes, false), discovery.localNode); try { newNodeSpecificClusterState = diff.apply(discovery.clusterService.state()); + logger.debug("sending diff cluster state version with size {} to [{}]", clusterStateDiffBytes.length, discovery.localNode.getName()); } catch (IncompatibleClusterStateVersionException ex) { logger.debug("incompatible cluster state version - resending complete cluster state", ex); } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java index dc800f370620e..d2653bc1745c5 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java @@ -147,7 +147,7 @@ private void writeWarmers(ImmutableList warmers, XCo builder.startObject(Fields.WARMERS); if (warmers != null) { for (IndexWarmersMetaData.Entry warmer : warmers) { - IndexWarmersMetaData.FACTORY.toXContent(warmer, builder, params); + IndexWarmersMetaData.toXContent(warmer, builder, params); } } builder.endObject(); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/get/RestGetWarmerAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/get/RestGetWarmerAction.java index 7023eecedd44b..be83ccbe4b52f 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/get/RestGetWarmerAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/get/RestGetWarmerAction.java @@ -72,7 +72,7 @@ public RestResponse buildResponse(GetWarmersResponse response, XContentBuilder b builder.startObject(entry.key, XContentBuilder.FieldCaseConversion.NONE); builder.startObject(IndexWarmersMetaData.TYPE, XContentBuilder.FieldCaseConversion.NONE); for (IndexWarmersMetaData.Entry warmerEntry : entry.value) { - IndexWarmersMetaData.FACTORY.toXContent(warmerEntry, builder, request); + IndexWarmersMetaData.toXContent(warmerEntry, builder, request); } builder.endObject(); builder.endObject(); diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index de56f823eacf5..5f42d622a3948 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -21,8 +21,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.metadata.IndexClusterStatePart; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -38,12 +39,82 @@ /** */ -public class IndexWarmersMetaData implements IndexMetaData.Custom { +public class IndexWarmersMetaData extends AbstractClusterStatePart implements IndexClusterStatePart { public static final String TYPE = "warmers"; public static final Factory FACTORY = new Factory(); + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(entries.size()); + for (Entry entry : entries) { + out.writeString(entry.name()); + out.writeStringArray(entry.types()); + if (entry.source() == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeBytesReference(entry.source()); + } + out.writeOptionalBoolean(entry.queryCache()); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + //No need, IndexMetaData already writes it + //builder.startObject(TYPE, XContentBuilder.FieldCaseConversion.NONE); + for (Entry entry : entries) { + toXContent(entry, builder, params); + } + //No need, IndexMetaData already writes it + //builder.endObject(); + return builder; + } + + public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + boolean binary = params.paramAsBoolean("binary", false); + builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("types", entry.types()); + if (entry.queryCache() != null) { + builder.field("queryCache", entry.queryCache()); + } + builder.field("source"); + if (binary) { + builder.value(entry.source()); + } else { + Map mapping = XContentFactory.xContent(entry.source()).createParser(entry.source()).mapOrderedAndClose(); + builder.map(mapping); + } + builder.endObject(); + } + + @Override + public IndexWarmersMetaData mergeWith(IndexWarmersMetaData second) { + List entries = Lists.newArrayList(); + entries.addAll(entries); + for (Entry secondEntry : second.entries()) { + boolean found = false; + for (Entry firstEntry : entries) { + if (firstEntry.name().equals(secondEntry.name())) { + found = true; + break; + } + } + if (!found) { + entries.add(secondEntry); + } + } + return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); + } + + @Override + public String partType() { + return TYPE; + } + + public static class Entry { private final String name; private final String[] types; @@ -87,20 +158,10 @@ public ImmutableList entries() { return this.entries; } - @Override - public String type() { - return TYPE; - } - - public static class Factory implements IndexMetaData.Custom.Factory { + public static class Factory extends AbstractClusterStatePart.AbstractFactory { @Override - public String type() { - return TYPE; - } - - @Override - public IndexWarmersMetaData readFrom(StreamInput in) throws IOException { + public IndexWarmersMetaData readFrom(StreamInput in, LocalContext context) throws IOException { Entry[] entries = new Entry[in.readVInt()]; for (int i = 0; i < entries.length; i++) { String name = in.readString(); @@ -117,37 +178,7 @@ public IndexWarmersMetaData readFrom(StreamInput in) throws IOException { } @Override - public void writeTo(IndexWarmersMetaData warmers, StreamOutput out) throws IOException { - out.writeVInt(warmers.entries().size()); - for (Entry entry : warmers.entries()) { - out.writeString(entry.name()); - out.writeStringArray(entry.types()); - if (entry.source() == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeBytesReference(entry.source()); - } - out.writeOptionalBoolean(entry.queryCache()); - } - } - - @Override - public IndexWarmersMetaData fromMap(Map map) throws IOException { - // if it starts with the type, remove it - if (map.size() == 1 && map.containsKey(TYPE)) { - map = (Map) map.values().iterator().next(); - } - XContentBuilder builder = XContentFactory.smileBuilder().map(map); - try (XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.bytes())) { - // move to START_OBJECT - parser.nextToken(); - return fromXContent(parser); - } - } - - @Override - public IndexWarmersMetaData fromXContent(XContentParser parser) throws IOException { + public IndexWarmersMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { // we get here after we are at warmers token String currentFieldName = null; XContentParser.Token token; @@ -191,50 +222,8 @@ public IndexWarmersMetaData fromXContent(XContentParser parser) throws IOExcepti } @Override - public void toXContent(IndexWarmersMetaData warmers, XContentBuilder builder, ToXContent.Params params) throws IOException { - //No need, IndexMetaData already writes it - //builder.startObject(TYPE, XContentBuilder.FieldCaseConversion.NONE); - for (Entry entry : warmers.entries()) { - toXContent(entry, builder, params); - } - //No need, IndexMetaData already writes it - //builder.endObject(); - } - - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - boolean binary = params.paramAsBoolean("binary", false); - builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("types", entry.types()); - if (entry.queryCache() != null) { - builder.field("queryCache", entry.queryCache()); - } - builder.field("source"); - if (binary) { - builder.value(entry.source()); - } else { - Map mapping = XContentFactory.xContent(entry.source()).createParser(entry.source()).mapOrderedAndClose(); - builder.map(mapping); - } - builder.endObject(); - } - - @Override - public IndexWarmersMetaData merge(IndexWarmersMetaData first, IndexWarmersMetaData second) { - List entries = Lists.newArrayList(); - entries.addAll(first.entries()); - for (Entry secondEntry : second.entries()) { - boolean found = false; - for (Entry firstEntry : first.entries()) { - if (firstEntry.name().equals(secondEntry.name())) { - found = true; - break; - } - } - if (!found) { - entries.add(secondEntry); - } - } - return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); + public String partType() { + return TYPE; } } } diff --git a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java index deb955ed51cad..57591cc86f7c7 100644 --- a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerResponse; import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersResponse; import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.ImmutableSettings; @@ -43,6 +44,7 @@ import org.elasticsearch.search.warmer.IndexWarmerMissingException; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.hamcrest.Matchers; import org.junit.Test; @@ -341,6 +343,8 @@ public void testEagerLoading() throws Exception { } } + @Test + @TestLogging("discovery.zen.publish:DEBUG") public void testQueryCacheOnWarmer() { createIndex("test"); ensureGreen(); @@ -351,16 +355,18 @@ public void testQueryCacheOnWarmer() { .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery())) .get()); - client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + logger.info("index first time"); + IndexResponse indexResponse = client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet(); + logger.info("indexed " + indexResponse.getId()); assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0l)); logger.info("register warmer with query cache, validate caching happened"); assertAcked(client().admin().indices().preparePutWarmer("warmer_1") .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setQueryCache(true)) .get()); - - // index again, to make sure it gets refreshed - client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + logger.info("index again, to make sure it gets refreshed"); + client().prepareIndex("test", "type1", "1").setSource("field2", "value1").setRefresh(true).execute().actionGet(); + logger.info("checking stats"); assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); client().admin().indices().prepareClearCache().setQueryCache(true).get(); // clean the cache @@ -373,8 +379,8 @@ public void testQueryCacheOnWarmer() { .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery())) .get()); - // index again, to make sure it gets refreshed - client().prepareIndex("test", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + logger.info("index again, to make sure it gets refreshed"); + client().prepareIndex("test", "type1", "1").setSource("field3", "value1").setRefresh(true).execute().actionGet(); assertThat(client().admin().indices().prepareStats("test").setQueryCache(true).get().getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0l)); } } From 188da911c9052028b77880142c4825a6ab2cdcd0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 5 Jan 2015 14:36:55 -0500 Subject: [PATCH 12/20] Restore commented out snapshot/restore tests --- .../DedicatedClusterSnapshotRestoreTests.java | 678 +++++++++--------- 1 file changed, 354 insertions(+), 324 deletions(-) diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index b67f2bfba5bb4..d598c2f543b44 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -37,9 +37,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; @@ -115,108 +113,108 @@ public void restorePersistentSettingsTest() throws Exception { .getMetaData().persistentSettings().get(ThreadPool.THREADPOOL_GROUP + "dummy.value"), equalTo(settingValue)); } -// @Test -// public void restoreCustomMetadata() throws Exception { -// Path tempDir = newTempDirPath(); -// -// logger.info("--> start node"); -// internalCluster().startNode(settingsBuilder().put("gateway.type", "local")); -// Client client = client(); -// createIndex("test-idx"); -// ensureYellow(); -// logger.info("--> add custom persistent metadata"); -// updateClusterState(new ClusterStateUpdater() { -// @Override -// public ClusterState execute(ClusterState currentState) throws Exception { -// ClusterState.Builder builder = ClusterState.builder(currentState); -// MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); -// metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s")); -// metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns")); -// metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw")); -// metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw")); -// metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi")); -// builder.metaData(metadataBuilder); -// return builder.build(); -// } -// }); -// -// logger.info("--> create repository"); -// PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") -// .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); -// assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); -// -// logger.info("--> start snapshot"); -// CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); -// assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0)); -// assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); -// assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); -// -// logger.info("--> change custom persistent metadata"); -// updateClusterState(new ClusterStateUpdater() { -// @Override -// public ClusterState execute(ClusterState currentState) throws Exception { -// ClusterState.Builder builder = ClusterState.builder(currentState); -// MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); -// if (randomBoolean()) { -// metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s")); -// } else { -// metadataBuilder.removeCustom(SnapshottableMetadata.TYPE); -// } -// metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns")); -// if (randomBoolean()) { -// metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw")); -// } else { -// metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE); -// } -// metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw")); -// metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE); -// builder.metaData(metadataBuilder); -// return builder.build(); -// } -// }); -// -// logger.info("--> delete repository"); -// assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo")); -// -// logger.info("--> create repository"); -// putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2") -// .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); -// assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); -// -// logger.info("--> restore snapshot"); -// client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet(); -// -// logger.info("--> make sure old repository wasn't restored"); -// assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class); -// assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); -// -// logger.info("--> check that custom persistent metadata was restored"); -// ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); -// logger.info("Cluster state: {}", clusterState); -// MetaData metaData = clusterState.getMetaData(); -// assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); -// assertThat(((NonSnapshottableMetadata) metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns")); -// assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); -// assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); -// -// logger.info("--> restart all nodes"); -// internalCluster().fullRestart(); -// ensureYellow(); -// -// logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); -// clusterState = client().admin().cluster().prepareState().get().getState(); -// logger.info("Cluster state: {}", clusterState); -// metaData = clusterState.getMetaData(); -// assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); -// assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue()); -// assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); -// assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); -// // Shouldn't be returned as part of API response -// assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue()); -// // But should still be in state -// metaData = internalCluster().getInstance(ClusterService.class).state().metaData(); -// assertThat(((SnapshotableGatewayNoApiMetadata) metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi")); -// } + @Test + public void restoreCustomMetadata() throws Exception { + Path tempDir = newTempDirPath(); + + logger.info("--> start node"); + internalCluster().startNode(settingsBuilder().put("gateway.type", "local")); + Client client = client(); + createIndex("test-idx"); + ensureYellow(); + logger.info("--> add custom persistent metadata"); + updateClusterState(new ClusterStateUpdater() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState.Builder builder = ClusterState.builder(currentState); + MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); + metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s")); + metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns")); + metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw")); + metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw")); + metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi")); + builder.metaData(metadataBuilder); + return builder.build(); + } + }); + + logger.info("--> create repository"); + PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") + .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + logger.info("--> start snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); + assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards())); + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + + logger.info("--> change custom persistent metadata"); + updateClusterState(new ClusterStateUpdater() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState.Builder builder = ClusterState.builder(currentState); + MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); + if (randomBoolean()) { + metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s")); + } else { + metadataBuilder.removeCustom(SnapshottableMetadata.TYPE); + } + metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns")); + if (randomBoolean()) { + metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw")); + } else { + metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE); + } + metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw")); + metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE); + builder.metaData(metadataBuilder); + return builder.build(); + } + }); + + logger.info("--> delete repository"); + assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo")); + + logger.info("--> create repository"); + putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2") + .setType("fs").setSettings(ImmutableSettings.settingsBuilder().put("location", tempDir)).execute().actionGet(); + assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); + + logger.info("--> restore snapshot"); + client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet(); + + logger.info("--> make sure old repository wasn't restored"); + assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class); + assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); + + logger.info("--> check that custom persistent metadata was restored"); + ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + logger.info("Cluster state: {}", clusterState); + MetaData metaData = clusterState.getMetaData(); + assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); + assertThat(((NonSnapshottableMetadata) metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns")); + assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); + assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); + + logger.info("--> restart all nodes"); + internalCluster().fullRestart(); + ensureYellow(); + + logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); + clusterState = client().admin().cluster().prepareState().get().getState(); + logger.info("Cluster state: {}", clusterState); + metaData = clusterState.getMetaData(); + assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); + assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue()); + assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw")); + assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw")); + // Shouldn't be returned as part of API response + assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue()); + // But should still be in state + metaData = internalCluster().getInstance(ClusterService.class).state().metaData(); + assertThat(((SnapshotableGatewayNoApiMetadata) metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi")); + } private void updateClusterState(final ClusterStateUpdater updater) throws InterruptedException { final CountDownLatch countDownLatch = new CountDownLatch(1); @@ -686,223 +684,255 @@ private void createTestIndex(String name) { )); } -// public static abstract class TestCustomMetaData implements MetaData.Custom { -// private final String data; -// -// protected TestCustomMetaData(String data) { -// this.data = data; -// } -// -// public String getData() { -// return data; -// } -// -// @Override -// public boolean equals(Object o) { -// if (this == o) return true; -// if (o == null || getClass() != o.getClass()) return false; -// -// TestCustomMetaData that = (TestCustomMetaData) o; -// -// if (!data.equals(that.data)) return false; -// -// return true; -// } -// -// @Override -// public int hashCode() { -// return data.hashCode(); -// } -// -// public static abstract class TestCustomMetaDataFactory extends MetaData.Custom.Factory { -// -// protected abstract TestCustomMetaData newTestCustomMetaData(String data); -// -// @Override -// public T readFrom(StreamInput in) throws IOException { -// return (T) newTestCustomMetaData(in.readString()); -// } -// -// @Override -// public void writeTo(T metadata, StreamOutput out) throws IOException { -// out.writeString(metadata.getData()); -// } -// -// @Override -// public T fromXContent(XContentParser parser) throws IOException { -// XContentParser.Token token; -// String data = null; -// while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { -// if (token == XContentParser.Token.FIELD_NAME) { -// String currentFieldName = parser.currentName(); -// if ("data".equals(currentFieldName)) { -// if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { -// throw new ElasticsearchParseException("failed to parse snapshottable metadata, invalid data type"); -// } -// data = parser.text(); -// } else { -// throw new ElasticsearchParseException("failed to parse snapshottable metadata, unknown field [" + currentFieldName + "]"); -// } -// } else { -// throw new ElasticsearchParseException("failed to parse snapshottable metadata"); -// } -// } -// if (data == null) { -// throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found"); -// } -// return (T) newTestCustomMetaData(data); -// } -// -// @Override -// public void toXContent(T metadata, XContentBuilder builder, ToXContent.Params params) throws IOException { -// builder.field("data", metadata.getData()); -// } -// } -// } -// -// static { -// MetaData.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY); -// MetaData.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY); -// MetaData.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY); -// MetaData.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY); -// MetaData.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY); -// } -// -// public static class SnapshottableMetadata extends TestCustomMetaData { -// public static final String TYPE = "test_snapshottable"; -// -// public static final Factory FACTORY = new Factory(); -// -// public SnapshottableMetadata(String data) { -// super(data); -// } -// -// private static class Factory extends TestCustomMetaDataFactory { -// -// @Override -// public String type() { -// return TYPE; -// } -// -// @Override -// protected TestCustomMetaData newTestCustomMetaData(String data) { -// return new SnapshottableMetadata(data); -// } -// -// @Override -// public EnumSet context() { -// return MetaData.API_AND_SNAPSHOT; -// } -// } -// } -// -// public static class NonSnapshottableMetadata extends TestCustomMetaData { -// public static final String TYPE = "test_non_snapshottable"; -// -// public static final Factory FACTORY = new Factory(); -// -// public NonSnapshottableMetadata(String data) { -// super(data); -// } -// -// private static class Factory extends TestCustomMetaDataFactory { -// -// @Override -// public String type() { -// return TYPE; -// } -// -// @Override -// protected NonSnapshottableMetadata newTestCustomMetaData(String data) { -// return new NonSnapshottableMetadata(data); -// } -// } -// } -// -// public static class SnapshottableGatewayMetadata extends TestCustomMetaData { -// public static final String TYPE = "test_snapshottable_gateway"; -// -// public static final Factory FACTORY = new Factory(); -// -// public SnapshottableGatewayMetadata(String data) { -// super(data); -// } -// -// private static class Factory extends TestCustomMetaDataFactory { -// -// @Override -// public String type() { -// return TYPE; -// } -// -// @Override -// protected TestCustomMetaData newTestCustomMetaData(String data) { -// return new SnapshottableGatewayMetadata(data); -// } -// -// @Override -// public EnumSet context() { -// return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.SNAPSHOT, MetaData.XContentContext.GATEWAY); -// } -// } -// } -// -// public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData { -// public static final String TYPE = "test_non_snapshottable_gateway"; -// -// public static final Factory FACTORY = new Factory(); -// -// public NonSnapshottableGatewayMetadata(String data) { -// super(data); -// } -// -// private static class Factory extends TestCustomMetaDataFactory { -// -// @Override -// public String type() { -// return TYPE; -// } -// -// @Override -// protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) { -// return new NonSnapshottableGatewayMetadata(data); -// } -// -// @Override -// public EnumSet context() { -// return MetaData.API_AND_GATEWAY; -// } -// -// } -// } -// -// public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData { -// public static final String TYPE = "test_snapshottable_gateway_no_api"; -// -// public static final Factory FACTORY = new Factory(); -// -// public SnapshotableGatewayNoApiMetadata(String data) { -// super(data); -// } -// -// private static class Factory extends TestCustomMetaDataFactory { -// -// @Override -// public String type() { -// return TYPE; -// } -// -// @Override -// protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) { -// return new SnapshotableGatewayNoApiMetadata(data); -// } -// -// @Override -// public EnumSet context() { -// return EnumSet.of(MetaData.XContentContext.GATEWAY, MetaData.XContentContext.SNAPSHOT); -// } -// -// } -// } -// + public static abstract class TestCustomMetaData extends AbstractClusterStatePart { + private final String data; + + protected TestCustomMetaData(String data) { + this.data = data; + } + + public String getData() { + return data; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TestCustomMetaData that = (TestCustomMetaData) o; + + if (!data.equals(that.data)) return false; + + return true; + } + + @Override + public int hashCode() { + return data.hashCode(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(getData()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field("data",getData()); + return builder; + } + + public static abstract class TestCustomMetaDataFactory extends AbstractClusterStatePart.AbstractFactory { + + protected abstract TestCustomMetaData newTestCustomMetaData(String data); + + @Override + public T readFrom(StreamInput in, LocalContext context) throws IOException { + return (T) newTestCustomMetaData(in.readString()); + } + + + @Override + public T fromXContent(XContentParser parser, LocalContext context) throws IOException { + XContentParser.Token token; + String data = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if ("data".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { + throw new ElasticsearchParseException("failed to parse snapshottable metadata, invalid data type"); + } + data = parser.text(); + } else { + throw new ElasticsearchParseException("failed to parse snapshottable metadata, unknown field [" + currentFieldName + "]"); + } + } else { + throw new ElasticsearchParseException("failed to parse snapshottable metadata"); + } + } + if (data == null) { + throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found"); + } + return (T) newTestCustomMetaData(data); + } + + } + } + + static { + MetaData.FACTORY.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY); + MetaData.FACTORY.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY); + MetaData.FACTORY.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY); + MetaData.FACTORY.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY); + MetaData.FACTORY.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY); + } + + public static class SnapshottableMetadata extends TestCustomMetaData { + public static final String TYPE = "test_snapshottable"; + + public static final Factory FACTORY = new Factory(); + + public SnapshottableMetadata(String data) { + super(data); + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public EnumSet context() { + return MetaData.API_SNAPSHOT; + } + + private static class Factory extends TestCustomMetaDataFactory { + + @Override + public String partType() { + return TYPE; + } + + @Override + protected TestCustomMetaData newTestCustomMetaData(String data) { + return new SnapshottableMetadata(data); + } + } + } + + public static class NonSnapshottableMetadata extends TestCustomMetaData { + public static final String TYPE = "test_non_snapshottable"; + + public static final Factory FACTORY = new Factory(); + + public NonSnapshottableMetadata(String data) { + super(data); + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public EnumSet context() { + return MetaData.API; + } + + private static class Factory extends TestCustomMetaDataFactory { + + @Override + public String partType() { + return TYPE; + } + + @Override + protected NonSnapshottableMetadata newTestCustomMetaData(String data) { + return new NonSnapshottableMetadata(data); + } + } + } + + public static class SnapshottableGatewayMetadata extends TestCustomMetaData { + public static final String TYPE = "test_snapshottable_gateway"; + + public static final Factory FACTORY = new Factory(); + + public SnapshottableGatewayMetadata(String data) { + super(data); + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public EnumSet context() { + return MetaData.API_GATEWAY_SNAPSHOT; + } + + private static class Factory extends TestCustomMetaDataFactory { + + @Override + public String partType() { + return TYPE; + } + + @Override + protected TestCustomMetaData newTestCustomMetaData(String data) { + return new SnapshottableGatewayMetadata(data); + } + } + } + + public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData { + public static final String TYPE = "test_non_snapshottable_gateway"; + + public static final Factory FACTORY = new Factory(); + + public NonSnapshottableGatewayMetadata(String data) { + super(data); + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public EnumSet context() { + return MetaData.API_GATEWAY; + } + + private static class Factory extends TestCustomMetaDataFactory { + + @Override + public String partType() { + return TYPE; + } + + @Override + protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) { + return new NonSnapshottableGatewayMetadata(data); + } + } + } + + public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData { + public static final String TYPE = "test_snapshottable_gateway_no_api"; + + public static final Factory FACTORY = new Factory(); + + public SnapshotableGatewayNoApiMetadata(String data) { + super(data); + } + + + @Override + public String partType() { + return TYPE; + } + + @Override + public EnumSet context() { + return MetaData.GATEWAY_SNAPSHOT; + } + + private static class Factory extends TestCustomMetaDataFactory { + + @Override + public String partType() { + return TYPE; + } + + @Override + protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) { + return new SnapshotableGatewayNoApiMetadata(data); + } + } + } + } From afab5d27285c891d2cf3123141fa35bac94ba756 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 6 Jan 2015 13:31:34 -0500 Subject: [PATCH 13/20] Add some javadocs --- .../get/TransportGetSettingsAction.java | 2 +- .../cluster/AbstractClusterStatePart.java | 1 + .../cluster/ClusterStatePart.java | 22 ++++++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 022acd51b0282..fd01607e95451 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -86,7 +86,7 @@ protected void masterOperation(GetSettingsRequest request, ClusterState state, A continue; } - Settings settings = settingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.settings()); + Settings settings = SettingsFilter.filterSettings(settingsFilter.getPatterns(), indexMetaData.settings()); if (!CollectionUtils.isEmpty(request.names())) { ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder(); for (Map.Entry entry : settings.getAsMap().entrySet()) { diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 39e989288d00b..63b79373cb3a8 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -33,6 +33,7 @@ import java.util.Map; /** + * Basic implementation of cluster state part that send entire part as a difference if part got changed. */ public abstract class AbstractClusterStatePart implements ClusterStatePart { diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index c8f75cf6aff9f..3a8a4f79018ce 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -30,19 +30,26 @@ import java.util.Map; /** + * Part of the cluster state + * + * Each cluster state part is serializable in both binary and json formats and can calculate differences with the + * previous version of the same part. */ public interface ClusterStatePart extends ToXContent { public static final String CONTEXT_MODE_PARAM = "context_mode"; + /** + * Represents the context in which this part should be serialized as JSON + */ public enum XContentContext { - /* Custom metadata should be returns as part of API call */ + /* A part should be returned as part of API call */ API, - /* Custom metadata should be stored as part of the persistent cluster state */ + /* A part should be stored by gateway as a persistent cluster state */ GATEWAY, - /* Custom metadata should be stored as part of a snapshot */ + /* A part should be stored as part of a snapshot */ SNAPSHOT; } @@ -54,10 +61,19 @@ public enum XContentContext { public static EnumSet GATEWAY = EnumSet.of(XContentContext.GATEWAY); public static EnumSet GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.GATEWAY, XContentContext.SNAPSHOT); + /** + * Writes part to output stream + */ void writeTo(StreamOutput out) throws IOException; + /** + * Returns a set of contexts in which this part should be serialized as JSON + */ EnumSet context(); + /** + * Returns part type + */ String partType(); interface Factory { From eb8a4e71dbd27abc79138903e7b0b6f339b5dfc8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 6 Jan 2015 17:03:45 -0500 Subject: [PATCH 14/20] Move write and toXContent methods to Factories --- .../state/TransportClusterStateAction.java | 5 +- .../indices/create/CreateIndexRequest.java | 7 +- .../template/put/PutIndexTemplateRequest.java | 7 +- .../cluster/AbstractClusterStatePart.java | 104 ++++----- .../elasticsearch/cluster/ClusterName.java | 48 ++-- .../elasticsearch/cluster/ClusterState.java | 31 ++- .../cluster/ClusterStatePart.java | 56 +++-- .../cluster/ClusterStateSettingsPart.java | 49 ++--- .../cluster/CompositeClusterStatePart.java | 150 ++++++------- .../cluster/MapClusterStatePart.java | 135 ++++++------ .../NamedCompositeClusterStatePart.java | 205 +++++++++--------- .../cluster/block/ClusterBlocks.java | 65 +++--- .../cluster/metadata/AliasMetaData.java | 29 +-- .../cluster/metadata/BenchmarkMetaData.java | 45 ++-- .../metadata/IndexClusterStatePart.java | 5 + .../cluster/metadata/IndexMetaData.java | 93 ++++---- .../metadata/IndexTemplateMetaData.java | 85 ++++---- .../cluster/metadata/MappingMetaData.java | 61 +++--- .../cluster/metadata/MetaData.java | 37 ++-- .../metadata/RepositoriesMetaData.java | 43 ++-- .../cluster/metadata/RestoreMetaData.java | 146 ++++++------- .../cluster/metadata/SnapshotMetaData.java | 69 +++--- .../cluster/node/DiscoveryNodes.java | 51 ++--- .../cluster/routing/IndexRoutingTable.java | 47 ++-- .../cluster/routing/RoutingTable.java | 136 +++++------- .../publish/PublishClusterStateAction.java | 4 +- .../search/warmer/IndexWarmersMetaData.java | 56 +++-- .../snapshots/RestoreService.java | 2 +- .../DedicatedClusterSnapshotRestoreTests.java | 101 ++++----- 29 files changed, 888 insertions(+), 984 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index a5c2a85a43870..edf64f884e951 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart.Factory; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -125,8 +126,10 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS } // Filter our metadata that shouldn't be returned by API + // TODO: we probably shouldn't do that for(ObjectObjectCursor cursor : currentState.metaData().customs()) { - if(!cursor.value.context().contains(MetaData.XContentContext.API)) { + Factory factory = MetaData.FACTORY.lookupFactorySafe(cursor.key); + if(!factory.context().contains(ClusterStatePart.XContentContext.API)) { mdBuilder.removeCustom(cursor.key); } } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 79313a8869e5f..090c12830c50a 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -481,8 +481,11 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactorySafe(entry.getKey()); + if (factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(entry.getKey()); + factory.writeTo(entry.getValue(), out); + } } out.writeVInt(aliases.size()); for (Alias alias : aliases) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index de1144bf5a9e7..2407bb04b9ce0 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -469,8 +469,11 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeVInt(customs.size()); for (Map.Entry entry : customs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactorySafe(entry.getKey()); + if(factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(entry.getKey()); + factory.writeTo(entry.getValue(), out); + } } out.writeVInt(aliases.size()); for (Alias alias : aliases) { diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 63b79373cb3a8..23ae846838431 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -19,14 +19,12 @@ package org.elasticsearch.cluster; +import org.apache.commons.lang3.builder.Diff; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.*; import java.io.IOException; import java.util.EnumSet; @@ -37,91 +35,85 @@ */ public abstract class AbstractClusterStatePart implements ClusterStatePart { - @Override - public EnumSet context() { - return API; - } - - public static class CompleteDiff implements Diff { - private T part; - - public CompleteDiff(T part) { - this.part = part; - } + protected static abstract class AbstractFactory implements ClusterStatePart.Factory { @Override - public T apply(T state) { - return part; - } - - public void writeTo(StreamOutput out) throws IOException{ - out.writeBoolean(true); - part.writeTo(out); + public Version addedIn() { + return Version.V_2_0_0; } - } - - protected static class NoDiff implements Diff { - public NoDiff() { + @Override + public EnumSet context() { + return API; } @Override - public T apply(T part) { - return part; + public T fromXContent(XContentParser parser, LocalContext context) throws IOException { + throw new UnsupportedOperationException("Not implemented yet"); } @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(false); + public T fromMap(Map map, LocalContext context) throws IOException { + // if it starts with the type, remove it + if (map.size() == 1 && map.containsKey(partType())) { + map = (Map) map.values().iterator().next(); + } + XContentBuilder builder = XContentFactory.smileBuilder().map(map); + try (XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.bytes())) { + // move to START_OBJECT + parser.nextToken(); + return fromXContent(parser, context); + } } - } - - protected static abstract class AbstractFactory implements ClusterStatePart.Factory { @Override public Diff diff(@Nullable T before, T after) { - assert after != null; if (after.equals(before)) { - return new NoDiff<>(); + return new FullClusterStatePartDiff(null); } else { - return new CompleteDiff<>(after); + return new FullClusterStatePartDiff(after); } + } @Override public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { - if(in.readBoolean()) { - T part = readFrom(in, context); - return new CompleteDiff(part); + if (in.readBoolean()) { + return new FullClusterStatePartDiff(readFrom(in, context)); } else { - return new NoDiff(); + return new FullClusterStatePartDiff(null); } } @Override - public Version addedIn() { - return null; + public void writeDiffsTo(Diff diff, StreamOutput out) throws IOException { + FullClusterStatePartDiff fullDiff = (FullClusterStatePartDiff) diff; + if (fullDiff.part != null) { + out.writeBoolean(true); + writeTo(fullDiff.part, out); + } else { + out.writeBoolean(false); + } } + } - @Override - public T fromXContent(XContentParser parser, LocalContext context) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); + + private static class FullClusterStatePartDiff implements Diff { + + private T part; + + FullClusterStatePartDiff(@Nullable T part) { + this.part = part; } @Override - public T fromMap(Map map, LocalContext context) throws IOException { - // if it starts with the type, remove it - if (map.size() == 1 && map.containsKey(partType())) { - map = (Map) map.values().iterator().next(); - } - XContentBuilder builder = XContentFactory.smileBuilder().map(map); - try (XContentParser parser = XContentFactory.xContent(XContentType.SMILE).createParser(builder.bytes())) { - // move to START_OBJECT - parser.nextToken(); - return fromXContent(parser, context); + public T apply(T part) { + if (this.part != null) { + return this.part; + } else { + return part; } } - } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterName.java b/src/main/java/org/elasticsearch/cluster/ClusterName.java index 9031cb056a61b..0d229eb579826 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; @@ -40,6 +41,29 @@ public class ClusterName extends AbstractClusterStatePart implements Streamable public static final ClusterName DEFAULT = new ClusterName("elasticsearch".intern()); + public static class Factory extends AbstractFactory { + + @Override + public ClusterName readFrom(StreamInput in, LocalContext context) throws IOException { + return readClusterName(in); + } + + @Override + public void writeTo(ClusterName part, StreamOutput out) throws IOException { + part.writeTo(out); + } + + @Override + public void toXContent(ClusterName clusterName, XContentBuilder builder, Params params) throws IOException { + builder.field("cluster_name", clusterName); + } + + @Override + public String partType() { + return TYPE; + } + } + private String value; public static ClusterName clusterNameFromSettings(Settings settings) { @@ -86,11 +110,6 @@ public boolean equals(Object o) { return true; } - @Override - public String partType() { - return TYPE; - } - @Override public int hashCode() { return value != null ? value.hashCode() : 0; @@ -100,23 +119,4 @@ public int hashCode() { public String toString() { return "Cluster [" + value + "]"; } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("cluster_name", value); - return builder; - } - - public static class Factory extends AbstractFactory { - - @Override - public ClusterName readFrom(StreamInput in, LocalContext context) throws IOException { - return readClusterName(in); - } - - @Override - public String partType() { - return TYPE; - } - } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 597f6e68f8826..811f54d4cce1f 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -34,6 +34,8 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -41,11 +43,12 @@ import java.util.*; import static com.google.common.collect.Sets.newHashSet; +import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; /** * */ -public class ClusterState extends CompositeClusterStatePart { +public class ClusterState extends CompositeClusterStatePart implements ToXContent { public static final String TYPE = "cluster"; @@ -86,11 +89,6 @@ public ClusterState apply(ClusterState previous) throws IncompatibleClusterState return newState; } - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(version); - diff.writeTo(out); - } - public long version() { return version; } @@ -290,8 +288,6 @@ public String toString() { } } - - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { EnumSet metrics = Metric.parseString(params.param("metric", "_all"), true); @@ -311,7 +307,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for(ObjectObjectCursor partIter : parts) { if (metricStrings.contains(partIter.key)) { builder.startObject(partIter.key); - partIter.value.toXContent(builder, params); + FACTORY.lookupFactorySafe(partIter.key).toXContent(partIter.value, builder, params); builder.endObject(); } } @@ -341,11 +337,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public String partType() { - return TYPE; - } - public static Builder builder(ClusterName clusterName) { return new Builder(clusterName); } @@ -474,7 +465,7 @@ public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throw } public static void writeTo(ClusterState state, StreamOutput out) throws IOException { - state.writeTo(out); + FACTORY.writeTo(state, out); } /** @@ -486,18 +477,25 @@ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode loca } public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { + + // TODO: Do we need this version long version = in.readVLong(); LocalContext localContext = new LocalContext(localNode); return new ClusterStateDiff(version, FACTORY.readDiffFrom(in, localContext)); } + public static void writeDiffTo(ClusterStateDiff diff, StreamOutput out) throws IOException { + out.writeVLong(diff.version); + FACTORY.writeDiffsTo(diff.diff, out); + } + public static ClusterStateDiff diff(ClusterState before, ClusterState after) { return new ClusterStateDiff(after.version(), ClusterState.FACTORY.diff(before, after) ); } public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws IOException { BytesStreamOutput os = new BytesStreamOutput(); - diff(before, after).writeTo(os); + writeDiffTo(diff(before, after), os); return os.bytes().toBytes(); } @@ -505,6 +503,7 @@ public static ClusterState fromDiffBytes(ClusterState before, byte[] data, Disco ClusterStateDiff diff = readDiffFrom(new BytesStreamInput(data, false), localNode); return diff.apply(before); } + } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index 3a8a4f79018ce..acbbf162f48f9 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -20,9 +20,11 @@ package org.elasticsearch.cluster; import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; @@ -35,7 +37,7 @@ * Each cluster state part is serializable in both binary and json formats and can calculate differences with the * previous version of the same part. */ -public interface ClusterStatePart extends ToXContent { +public interface ClusterStatePart { public static final String CONTEXT_MODE_PARAM = "context_mode"; @@ -61,43 +63,59 @@ public enum XContentContext { public static EnumSet GATEWAY = EnumSet.of(XContentContext.GATEWAY); public static EnumSet GATEWAY_SNAPSHOT = EnumSet.of(XContentContext.GATEWAY, XContentContext.SNAPSHOT); - /** - * Writes part to output stream - */ - void writeTo(StreamOutput out) throws IOException; + interface Factory { - /** - * Returns a set of contexts in which this part should be serialized as JSON - */ - EnumSet context(); + /** + * Reads a part from input stream + */ + T readFrom(StreamInput in, LocalContext context) throws IOException; - /** - * Returns part type - */ - String partType(); + /** + * Writes the part to output stream + */ + void writeTo(T part, StreamOutput out) throws IOException; - interface Factory { + /** + * Reads a part from XContent stream + */ + T fromXContent(XContentParser parser, LocalContext context) throws IOException; + + /** + * Writes a part from XContent stream + */ + void toXContent(T part, XContentBuilder builder, ToXContent.Params params) throws IOException; + + /** + * Reads a part from map + */ + T fromMap(Map map, LocalContext context) throws IOException; Diff diff(T before, T after); Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException; - T readFrom(StreamInput in, LocalContext context) throws IOException; - - T fromXContent(XContentParser parser, LocalContext context) throws IOException; + void writeDiffsTo(Diff diff, StreamOutput out) throws IOException; - T fromMap(Map map, LocalContext context) throws IOException; + /** + * Returns part type + */ String partType(); + /** + * Returns the version of Elasticsearch where this part was first added + */ Version addedIn(); + /** + * Returns a set of contexts in which this part should be serialized as JSON + */ + EnumSet context(); } interface Diff { T apply(T part); - void writeTo(StreamOutput out) throws IOException; } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index 61ce77ec8d6e6..fa29a02434797 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -40,25 +41,10 @@ public class ClusterStateSettingsPart extends AbstractClusterStatePart implement private final String type; private final Settings settings; - private final EnumSet xContentContext; - public ClusterStateSettingsPart(String type, Settings settings, EnumSet xContentContext) { + public ClusterStateSettingsPart(String type, Settings settings) { this.type = type; this.settings = settings; - this.xContentContext = xContentContext; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - writeSettingsToStream(settings, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - for (Map.Entry entry : settings.getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - return builder; } public Settings getSettings() { @@ -66,8 +52,8 @@ public Settings getSettings() { } @Override - public EnumSet context() { - return xContentContext; + public String partType() { + return type; } @Override @@ -75,11 +61,6 @@ public ClusterStateSettingsPart mergeWith(ClusterStateSettingsPart second) { return second; } - @Override - public String partType() { - return type; - } - public static class Factory extends AbstractClusterStatePart.AbstractFactory { private final String type; private final EnumSet xContentContext; @@ -96,19 +77,35 @@ public Factory(String type, EnumSet xContentContext) { @Override public ClusterStateSettingsPart readFrom(StreamInput in, LocalContext context) throws IOException { - return new ClusterStateSettingsPart(type, readSettingsFromStream(in), xContentContext); + return new ClusterStateSettingsPart(type, readSettingsFromStream(in)); } + @Override + public void writeTo(ClusterStateSettingsPart clusterStateSettingsPart, StreamOutput out) throws IOException { + writeSettingsToStream(clusterStateSettingsPart.settings, out); + } @Override public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext context) throws IOException { Settings settings = ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())).build(); - return new ClusterStateSettingsPart(type, settings, xContentContext); + return new ClusterStateSettingsPart(type, settings); + } + + + @Override + public void toXContent(ClusterStateSettingsPart clusterStateSettingsPart, XContentBuilder builder, Params params) throws IOException { + for (Map.Entry entry : clusterStateSettingsPart.settings.getAsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } } public ClusterStateSettingsPart fromSettings(Settings settings) { - return new ClusterStateSettingsPart(type, settings, xContentContext); + return new ClusterStateSettingsPart(type, settings); } + @Override + public EnumSet context() { + return xContentContext; + } @Override public String partType() { diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index afb2ef8c1923b..ca42666203b5a 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -20,11 +20,14 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.commons.lang3.builder.Diff; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -75,32 +78,6 @@ public ImmutableOpenMap parts() { return parts; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(version); - out.writeString(uuid); - out.writeVInt(parts().size()); - for (ObjectObjectCursor cursor : parts()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); - builder.field("version", version); - builder.field("uuid", uuid); - for (ObjectObjectCursor partIter : parts) { - if (partIter.value.context().contains(context)) { - builder.startObject(partIter.key); - partIter.value.toXContent(builder, params); - builder.endObject(); - } - } - return builder; - } - public T get(String type) { return (T) parts.get(type); } @@ -160,6 +137,34 @@ public T readFrom(StreamInput in, LocalContext context) throws IOException { public abstract T fromParts(long version, String uuid, ImmutableOpenMap.Builder parts); + @Override + public void writeTo(T part, StreamOutput out) throws IOException { + out.writeVLong(part.version); + out.writeString(part.uuid); + out.writeVInt(part.parts().size()); + ImmutableOpenMap parts = part.parts; + for (ObjectObjectCursor cursor : parts) { + out.writeString(cursor.key); + lookupFactorySafe(cursor.key).writeTo(cursor.value, out); + } + } + + @Override + public void toXContent(T part, XContentBuilder builder, Params params) throws IOException { + XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); + builder.field("version", part.version); + builder.field("uuid", part.uuid); + ImmutableOpenMap parts = part.parts; + for (ObjectObjectCursor partIter : parts) { + Factory factory = lookupFactorySafe(partIter.key); + if (factory.context().contains(context)) { + builder.startObject(partIter.key); + factory.toXContent(partIter.value, builder, params); + builder.endObject(); + } + } + } + @Override public Diff diff(@Nullable T before, T after) { assert after != null; @@ -168,20 +173,16 @@ public Diff diff(@Nullable T before, T after) { if (before != null) { ImmutableOpenMap beforeParts = before.parts(); ImmutableOpenMap afterParts = after.parts(); - if (before.equals(after)) { - return new NoDiff<>(); - } else { - for (ObjectObjectCursor partIter : beforeParts) { - if (!afterParts.containsKey(partIter.key)) { - deletes.add(partIter.key); - } + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); } - for (ObjectObjectCursor partIter : afterParts) { - ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); - ClusterStatePart beforePart = beforeParts.get(partIter.key); - if (!partIter.value.equals(beforePart)) { - diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); - } + } + for (ObjectObjectCursor partIter : afterParts) { + ClusterStatePart.Factory factory = lookupFactorySafe(partIter.key); + ClusterStatePart beforePart = beforeParts.get(partIter.key); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); } } } else { @@ -196,27 +197,44 @@ public Diff diff(@Nullable T before, T after) { @Override public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { - if (in.readBoolean()) { - long version = in.readVLong(); - String previousUuid = in.readString(); - String uuid = in.readString(); - int deletesSize = in.readVInt(); - List deletes = new ArrayList<>(); - for (int i = 0; i < deletesSize; i++) { - deletes.add(in.readString()); - } + long version = in.readVLong(); + String previousUuid = in.readString(); + String uuid = in.readString(); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } - int diffsSize = in.readVInt(); - Map> diffs = newHashMap(); - for (int i = 0; i < diffsSize; i++) { - String key = in.readString(); - diffs.put(key, lookupFactorySafe(key).readDiffFrom(in, context)); - } - return new CompositeDiff<>(this, version, previousUuid, uuid, deletes, diffs); + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String key = in.readString(); + diffs.put(key, lookupFactorySafe(key).readDiffFrom(in, context)); + } + return new CompositeDiff<>(this, version, previousUuid, uuid, deletes, diffs); + } - } else { - return new NoDiff(); + @Override + public void writeDiffsTo(Diff diff, StreamOutput out) throws IOException { + CompositeDiff compositeDiff = (CompositeDiff) diff; + out.writeVLong(compositeDiff.version); + out.writeString(compositeDiff.previousUuid); + out.writeString(compositeDiff.uuid); + out.writeVInt(compositeDiff.deletes.size()); + for (String delete : compositeDiff.deletes) { + out.writeString(delete); + } + + out.writeVInt(compositeDiff.diffs.size()); + for (Map.Entry> entry : compositeDiff.diffs.entrySet()) { + Factory factory = lookupFactory(entry.getKey()); + if (factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(entry.getKey()); + factory.writeDiffsTo(entry.getValue(), out); + } } + } @Override @@ -284,24 +302,6 @@ public T apply(T part) { } return factory.fromParts(version, uuid, parts); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(true); // We have diffs - out.writeVLong(version); - out.writeString(previousUuid); - out.writeString(uuid); - out.writeVInt(deletes.size()); - for (String delete : deletes) { - out.writeString(delete); - } - - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } - } } diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index c2729a253a02c..04b568f648cd9 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -43,30 +44,12 @@ */ public class MapClusterStatePart extends AbstractClusterStatePart implements IndexClusterStatePart< MapClusterStatePart> { - private final ImmutableOpenMap parts; - private final EnumSet xContentContext; private final String type; + private final ImmutableOpenMap parts; - public MapClusterStatePart(String type, ImmutableOpenMap parts, EnumSet xContentContext) { - this.parts = parts; + public MapClusterStatePart(String type, ImmutableOpenMap parts) { this.type = type; - this.xContentContext = xContentContext; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(parts.size()); - for (ObjectObjectCursor part : parts) { - part.value.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - for (ObjectCursor part : parts.values()) { - part.value.toXContent(builder, params); - } - return builder; + this.parts = parts; } public ImmutableOpenMap parts() { @@ -77,11 +60,6 @@ public T get(String type) { return parts.get(type); } - @Override - public EnumSet context() { - return xContentContext; - } - @Override public MapClusterStatePart mergeWith(MapClusterStatePart second) { //TODO: Implement @@ -119,7 +97,22 @@ public MapClusterStatePart readFrom(StreamInput in, LocalContext context) thr T part = factory.readFrom(in, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(type, builder.build(), xContentContext); + return new MapClusterStatePart<>(type, builder.build()); + } + + @Override + public void writeTo(MapClusterStatePart part, StreamOutput out) throws IOException { + out.writeVInt(part.parts.size()); + for (ObjectObjectCursor cursor : part.parts) { + factory.writeTo(cursor.value, out); + } + } + + @Override + public void toXContent(MapClusterStatePart part, XContentBuilder builder, Params params) throws IOException { + for (ObjectCursor cursor : part.parts.values()) { + factory.toXContent(cursor.value, builder, params); + } } @Override @@ -129,7 +122,7 @@ public MapClusterStatePart fromXContent(XContentParser parser, LocalContext c T part = factory.fromXContent(parser, context); builder.put(part.key(), part); } - return new MapClusterStatePart<>(type, builder.build(), xContentContext); + return new MapClusterStatePart<>(type, builder.build()); } @Override @@ -140,19 +133,15 @@ public Diff> diff(@Nullable MapClusterStatePart before if (before != null) { ImmutableOpenMap beforeParts = before.parts(); ImmutableOpenMap afterParts = after.parts(); - if (before.equals(after)) { - return new NoDiff<>(); - } else { - for (ObjectObjectCursor partIter : beforeParts) { - if (!afterParts.containsKey(partIter.key)) { - deletes.add(partIter.key); - } + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); } - for (ObjectObjectCursor partIter : afterParts) { - T beforePart = beforeParts.get(partIter.key); - if (!partIter.value.equals(beforePart)) { - diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); - } + } + for (ObjectObjectCursor partIter : afterParts) { + T beforePart = beforeParts.get(partIter.key); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); } } } else { @@ -166,28 +155,29 @@ public Diff> diff(@Nullable MapClusterStatePart before @Override public Diff> readDiffFrom(StreamInput in, LocalContext context) throws IOException { - if (in.readBoolean()) { - int deletesSize = in.readVInt(); - List deletes = new ArrayList<>(); - for (int i = 0; i < deletesSize; i++) { - deletes.add(in.readString()); - } - - int diffsSize = in.readVInt(); - Map> diffs = newHashMap(); - for (int i = 0; i < diffsSize; i++) { - String key = in.readString(); - diffs.put(key, factory.readDiffFrom(in, context)); - } - return new MapDiff<>(type, deletes, diffs, xContentContext); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } - } else { - return new NoDiff<>(); + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String key = in.readString(); + diffs.put(key, factory.readDiffFrom(in, context)); } + return new MapDiff<>(type, deletes, diffs, xContentContext); } public MapClusterStatePart fromOpenMap(ImmutableOpenMap map) { - return new MapClusterStatePart<>(type, map, xContentContext); + return new MapClusterStatePart<>(type, map); + } + + + @Override + public EnumSet context() { + return xContentContext; } @@ -196,6 +186,20 @@ public String partType() { return type; } + @Override + public void writeDiffsTo(Diff> diff, StreamOutput out) throws IOException { + MapDiff mapDiff = (MapDiff)diff; + out.writeVInt(mapDiff.deletes.size()); + for (String delete : mapDiff.deletes) { + out.writeString(delete); + } + + out.writeVInt(mapDiff.diffs.size()); + for (Map.Entry> entry : mapDiff.diffs.entrySet()) { + out.writeString(entry.getKey()); + factory.writeDiffsTo(entry.getValue(), out); + } + } } private static class MapDiff implements Diff> { @@ -229,22 +233,7 @@ public MapClusterStatePart apply(MapClusterStatePart part) { parts.put(entry.getKey(), entry.getValue().apply(null)); } } - return new MapClusterStatePart(type, parts.build(), xContentContext); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(true); // We have diffs - out.writeVInt(deletes.size()); - for (String delete : deletes) { - out.writeString(delete); - } - - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } + return new MapClusterStatePart(type, parts.build()); } } diff --git a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java index b857875371d3a..6a679c5b444f2 100644 --- a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -39,7 +40,7 @@ *

* Only one instance of each type can be present in the composite part. The key of the map is the part's type. */ -public abstract class NamedCompositeClusterStatePart extends AbstractClusterStatePart implements NamedClusterStatePart{ +public abstract class NamedCompositeClusterStatePart extends AbstractClusterStatePart implements NamedClusterStatePart { protected final ImmutableOpenMap parts; @@ -68,34 +69,6 @@ public ImmutableOpenMap parts() { return parts; } - protected abstract void valuesPartWriteTo(StreamOutput out) throws IOException; - protected abstract void valuesPartToXContent(XContentBuilder builder, Params params) throws IOException; - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(key()); - valuesPartWriteTo(out); - out.writeVInt(parts().size()); - for (ObjectObjectCursor cursor : parts()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); - valuesPartToXContent(builder, params); - for (ObjectObjectCursor partIter : parts) { - if (partIter.value.context().contains(context)) { - builder.startObject(partIter.key); - partIter.value.toXContent(builder, params); - builder.endObject(); - } - } - return builder; - } - public T get(String type) { return (T) parts.get(type); } @@ -131,6 +104,7 @@ public void remove(String type) { public static abstract class AbstractFactory> extends AbstractClusterStatePart.AbstractFactory { private final Map> partFactories = new HashMap<>(); + /** * Register a custom index meta data factory. Make sure to call it from a static block. */ @@ -144,7 +118,7 @@ public Set availableFactories() { @Nullable public Factory lookupFactory(String type) { - return (Factory)partFactories.get(type); + return (Factory) partFactories.get(type); } public Factory lookupFactorySafe(String type) throws ElasticsearchIllegalArgumentException { @@ -173,62 +147,18 @@ public T readFrom(StreamInput in, LocalContext context) throws IOException { return builder.build(); } - @Override - public Diff diff(@Nullable T before, T after) { - assert after != null; - Map> diffs = newHashMap(); - List deletes = newArrayList(); - if (before != null) { - ImmutableOpenMap beforeParts = before.parts(); - ImmutableOpenMap afterParts = after.parts(); - if (before.equals(after)) { - return new NoDiff<>(); - } else { - for (ObjectObjectCursor partIter : beforeParts) { - if (!afterParts.containsKey(partIter.key)) { - deletes.add(partIter.key); - } - } - for (ObjectObjectCursor partIter : afterParts) { - Factory factory = lookupFactorySafe(partIter.key); - E beforePart = beforeParts.get(partIter.key); - if (!partIter.value.equals(beforePart)) { - diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); - } - } - } - } else { - ImmutableOpenMap afterParts = after.parts(); - for (ObjectObjectCursor partIter : afterParts) { - Factory factory = lookupFactorySafe(partIter.key); - diffs.put(partIter.key, factory.diff(null, partIter.value)); - } - } - return new CompositeDiff<>(builder(after), deletes, diffs); - } @Override - public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { - if (in.readBoolean()) { - String key = in.readString(); - Builder builder = builder(key); - builder.readValuePartsFrom(in, context); - int deletesSize = in.readVInt(); - List deletes = new ArrayList<>(); - for (int i = 0; i < deletesSize; i++) { - deletes.add(in.readString()); + public void writeTo(T part, StreamOutput out) throws IOException { + out.writeString(part.key()); + valuesPartWriteTo(part, out); + out.writeVInt(part.parts().size()); + for (ObjectObjectCursor cursor : part.parts()) { + Factory factory = lookupFactorySafe(cursor.key); + if (factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(cursor.key); + factory.writeTo(cursor.value, out); } - - int diffsSize = in.readVInt(); - Map> diffs = newHashMap(); - for (int i = 0; i < diffsSize; i++) { - String partKey = in.readString(); - diffs.put(partKey, lookupFactorySafe(partKey).readDiffFrom(in, context)); - } - return new CompositeDiff<>(builder, deletes, diffs); - - } else { - return new NoDiff(); } } @@ -257,15 +187,98 @@ public T fromXContent(XContentParser parser, LocalContext context) throws IOExce } } else if (token.isValue()) { builder.parseValuePart(parser, currentFieldName, context); -// if ("version".equals(currentFieldName)) { -// version = parser.longValue(); -// } else if ("uuid".equals(currentFieldName)) { -// uuid = parser.text(); -// } } } return builder.build(); } + + protected abstract void valuesPartWriteTo(T part, StreamOutput out) throws IOException; + + protected abstract void valuesPartToXContent(T part, XContentBuilder builder, Params params) throws IOException; + + @Override + public void toXContent(T part, XContentBuilder builder, Params params) throws IOException { + XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); + valuesPartToXContent(part, builder, params); + for (ObjectObjectCursor cursor : part.parts) { + Factory factory = lookupFactorySafe(cursor.key); + if (factory.context().contains(context)) { + builder.startObject(cursor.key); + factory.toXContent(cursor.value, builder, params); + builder.endObject(); + } + } + } + + @Override + public Diff diff(@Nullable T before, T after) { + assert after != null; + Map> diffs = newHashMap(); + List deletes = newArrayList(); + if (before != null) { + ImmutableOpenMap beforeParts = before.parts(); + ImmutableOpenMap afterParts = after.parts(); + for (ObjectObjectCursor partIter : beforeParts) { + if (!afterParts.containsKey(partIter.key)) { + deletes.add(partIter.key); + } + } + for (ObjectObjectCursor partIter : afterParts) { + Factory factory = lookupFactorySafe(partIter.key); + E beforePart = beforeParts.get(partIter.key); + if (!partIter.value.equals(beforePart)) { + diffs.put(partIter.key, factory.diff(beforePart, partIter.value)); + } + } + } else { + ImmutableOpenMap afterParts = after.parts(); + for (ObjectObjectCursor partIter : afterParts) { + Factory factory = lookupFactorySafe(partIter.key); + diffs.put(partIter.key, factory.diff(null, partIter.value)); + } + } + return new CompositeDiff<>(builder(after), deletes, diffs); + } + + @Override + public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { + String key = in.readString(); + Builder builder = builder(key); + builder.readValuePartsFrom(in, context); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } + + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String partKey = in.readString(); + diffs.put(partKey, lookupFactorySafe(partKey).readDiffFrom(in, context)); + } + return new CompositeDiff<>(builder, deletes, diffs); + } + + @Override + public void writeDiffsTo(Diff diff, StreamOutput out) throws IOException { + CompositeDiff compositeDiff = (CompositeDiff) diff; + out.writeString(compositeDiff.builder.getKey()); + compositeDiff.builder.writeValuePartsTo(out); + out.writeVInt(compositeDiff.deletes.size()); + for (String delete : compositeDiff.deletes) { + out.writeString(delete); + } + + out.writeVInt(compositeDiff.diffs.size()); + for (Map.Entry> entry : compositeDiff.diffs.entrySet()) { + Factory factory = lookupFactorySafe(entry.getKey()); + if (factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(entry.getKey()); + factory.writeDiffsTo(entry.getValue(), out); + } + } + } } private static class CompositeDiff> implements Diff { @@ -299,22 +312,6 @@ public T apply(T part) { return builder.build(); } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(true); // We have diffs - out.writeString(builder.getKey()); - builder.writeValuePartsTo(out); - out.writeVInt(deletes.size()); - for (String delete : deletes) { - out.writeString(delete); - } - - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } - } } diff --git a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index 81da147b6516e..d7ee0719a2df2 100644 --- a/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.RestStatus; @@ -83,11 +84,6 @@ public class ClusterBlocks extends AbstractClusterStatePart { } } - @Override - public String partType() { - return TYPE; - } - public static class Factory extends AbstractFactory { @Override @@ -95,6 +91,35 @@ public ClusterBlocks readFrom(StreamInput in, LocalContext context) throws IOExc return Builder.readClusterBlocks(in); } + @Override + public void writeTo(ClusterBlocks clusterBlocks, StreamOutput out) throws IOException { + Builder.writeClusterBlocks(clusterBlocks, out); + } + + @Override + public void toXContent(ClusterBlocks clusterBlocks, XContentBuilder builder, Params params) throws IOException { + if (!clusterBlocks.global().isEmpty()) { + builder.startObject("global"); + for (ClusterBlock block : clusterBlocks.global()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + + if (!clusterBlocks.indices().isEmpty()) { + builder.startObject("indices"); + for (Map.Entry> entry : clusterBlocks.indices().entrySet()) { + builder.startObject(entry.getKey()); + for (ClusterBlock block : entry.getValue()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + builder.endObject(); + } + } + + @Override public String partType() { return TYPE; @@ -228,36 +253,6 @@ public ClusterBlockException indicesBlockedException(ClusterBlockLevel level, St return new ClusterBlockException(builder.build()); } - @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeClusterBlocks(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (!global().isEmpty()) { - builder.startObject("global"); - for (ClusterBlock block : global()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - - if (!indices().isEmpty()) { - builder.startObject("indices"); - for (Map.Entry> entry : indices().entrySet()) { - builder.startObject(entry.getKey()); - for (ClusterBlock block : entry.getValue()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - builder.endObject(); - } - - return builder; - } - static class ImmutableLevelHolder { static final ImmutableLevelHolder EMPTY = new ImmutableLevelHolder(ImmutableSet.of(), ImmutableMap.>of()); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 5ba084f07c4f9..2a7f7ede260a6 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -48,14 +49,8 @@ public class AliasMetaData extends AbstractClusterStatePart implements NamedClus public static final Factory FACTORY = new Factory(); @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Builder.toXContent(this, builder, params); - return builder; + public String partType() { + return TYPE; } @Override @@ -63,12 +58,6 @@ public String key() { return alias; } - @Override - public String partType() { - return TYPE; - } - - @Override public AliasMetaData mergeWith(AliasMetaData second) { return null; @@ -81,10 +70,22 @@ public AliasMetaData readFrom(StreamInput in, LocalContext context) throws IOExc return Builder.readFrom(in); } + @Override + public void writeTo(AliasMetaData aliasMetaData, StreamOutput out) throws IOException { + Builder.writeTo(aliasMetaData, out); + } + + @Override + public void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(aliasMetaData, builder, params); + } + @Override public String partType() { return TYPE; } + + } private final String alias; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java index 02646f3e0c80d..b61de01bace83 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java @@ -55,11 +55,6 @@ public int hashCode() { return entries.hashCode(); } - @Override - public String partType() { - return TYPE; - } - public static class Entry { private final State state; private final String benchmarkId; @@ -161,27 +156,7 @@ public ImmutableList entries() { return this.entries; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(entries.size()); - for (Entry entry : entries) { - out.writeString(entry.benchmarkId()); - out.writeByte(entry.state().id()); - out.writeStringArray(entry.nodes()); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray("benchmarks"); - for (Entry entry : entries) { - toXContent(entry, builder, params); - } - builder.endArray(); - return builder; - } - - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field("id", entry.benchmarkId()); builder.field("state", entry.state()); @@ -206,6 +181,24 @@ public BenchmarkMetaData readFrom(StreamInput in, LocalContext context) throws I return new BenchmarkMetaData(entries); } + @Override + public void writeTo(BenchmarkMetaData benchmarkMetaData, StreamOutput out) throws IOException { + out.writeVInt(benchmarkMetaData.entries.size()); + for (Entry entry : benchmarkMetaData.entries) { + out.writeString(entry.benchmarkId()); + out.writeByte(entry.state().id()); + out.writeStringArray(entry.nodes()); + } + } + + @Override + public void toXContent(BenchmarkMetaData benchmarkMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray("benchmarks"); + for (Entry entry : benchmarkMetaData.entries) { + BenchmarkMetaData.toXContent(entry, builder, params); + } + builder.endArray(); + } @Override public String partType() { return TYPE; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java index 691182796fdd8..b3d4db29f8561 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexClusterStatePart.java @@ -30,4 +30,9 @@ */ public interface IndexClusterStatePart extends ClusterStatePart { T mergeWith(T second); + + /** + * Returns part type + */ + String partType(); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b89e7c57ac5f5..178edbc25118f 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -86,11 +86,46 @@ public class IndexMetaData extends NamedCompositeClusterStatePart { + + @Override + public NamedCompositeClusterStatePart.Builder builder(String key) { + return new Builder(key); + } + + @Override + public NamedCompositeClusterStatePart.Builder builder(IndexMetaData part) { + return new Builder(part); + } + + @Override + public IndexMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return Builder.fromXContent(parser); + } + + @Override + protected void valuesPartWriteTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException { + out.writeVLong(indexMetaData.version); + out.writeByte(indexMetaData.state.id()); + } + + @Override + protected void valuesPartToXContent(IndexMetaData indexMetaData, XContentBuilder builder, Params params) throws IOException { + builder.field("version", indexMetaData.version); + builder.field("state", indexMetaData.state.toString().toLowerCase(Locale.ENGLISH)); + } + @Override + public void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, Params params) throws IOException { + // TODO: switch to generic toXContent + Builder.toXContent(indexMetaData, builder, params); + } + + @Override + public String partType() { + return TYPE; + } + } public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(5, "index read-only (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA)); public static final ClusterBlock INDEX_READ_BLOCK = new ClusterBlock(7, "index read (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.READ)); @@ -427,25 +462,6 @@ public int hashCode() { return result; } - @Override - protected void valuesPartWriteTo(StreamOutput out) throws IOException { - out.writeVLong(version); - out.writeByte(state.id()); - } - - @Override - protected void valuesPartToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("version", version); - builder.field("state", state.toString().toLowerCase(Locale.ENGLISH)); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - // TODO: switch to generic toXContent - Builder.toXContent(this, builder, params); - return builder; - } - public static Builder builder(String index) { return new Builder(index); } @@ -658,13 +674,13 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build if (params.paramAsBoolean("reduce_mappings", false)) { builder.startObject("mappings"); for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - cursor.value.toXContent(builder, params); + MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); } builder.endObject(); } else { builder.startArray("mappings"); for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - cursor.value.toXContent(builder, params); + MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); } builder.endArray(); } @@ -672,7 +688,7 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build for (ObjectObjectCursor cursor : indexMetaData.parts()) { if (!cursor.key.equals(MAPPINGS_TYPE)) { builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); - cursor.value.toXContent(builder, params); + FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); builder.endObject(); } } @@ -752,7 +768,7 @@ public static IndexMetaData readFrom(StreamInput in) throws IOException { } public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException { - indexMetaData.writeTo(out); + FACTORY.writeTo(indexMetaData, out); } } @@ -761,27 +777,4 @@ public String key() { return index; } - public static class Factory extends NamedCompositeClusterStatePart.AbstractFactory { - - @Override - public NamedCompositeClusterStatePart.Builder builder(String key) { - return new Builder(key); - } - - @Override - public NamedCompositeClusterStatePart.Builder builder(IndexMetaData part) { - return new Builder(part); - } - - @Override - public IndexMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { - return Builder.fromXContent(parser); - } - - @Override - public String partType() { - return TYPE; - } - } - } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 6ebfcf3ff8a5e..5444c0482fc52 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -51,6 +52,41 @@ public class IndexTemplateMetaData extends AbstractClusterStatePart implements N public static Factory FACTORY = new Factory(); + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public IndexTemplateMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + return Builder.readFrom(in); + } + + @Override + public void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOutput out) throws IOException { + Builder.writeTo(indexTemplateMetaData, out); + } + + @Override + public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return Builder.fromXContent(parser, parser.currentName()); + } + + @Override + public void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(indexTemplateMetaData, builder, params); + } + + @Override + public EnumSet context() { + return API_GATEWAY_SNAPSHOT; + } + + + @Override + public String partType() { + return TYPE; + } + } + + private final String name; private final int order; @@ -133,11 +169,6 @@ public ImmutableOpenMap getCustoms() { return this.customs; } - @Override - public String partType() { - return TYPE; - } - @SuppressWarnings("unchecked") public T custom(String type) { return (T) customs.get(type); @@ -173,11 +204,6 @@ public int hashCode() { return result; } - @Override - public EnumSet context() { - return API_GATEWAY_SNAPSHOT; - } - public static class Builder { private static final Set VALID_FIELDS = Sets.newHashSet("template", "order"); @@ -323,7 +349,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); - cursor.value.toXContent(builder, params); + IndexMetaData.FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); builder.endObject(); } @@ -463,8 +489,11 @@ public static void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOu } out.writeVInt(indexTemplateMetaData.customs().size()); for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); + IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactorySafe(cursor.key); + if (factory.addedIn().onOrAfter(out.getVersion())) { + out.writeString(cursor.key); + factory.writeTo(cursor.value, out); + } } } } @@ -474,34 +503,4 @@ public String key() { return name; } - @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Builder.toXContent(this, builder, params); - return builder; - } - - public static class Factory extends AbstractClusterStatePart.AbstractFactory { - - @Override - public IndexTemplateMetaData readFrom(StreamInput in, LocalContext context) throws IOException { - return Builder.readFrom(in); - } - - @Override - public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { - return Builder.fromXContent(parser, parser.currentName()); - } - - @Override - public String partType() { - return TYPE; - } - } - - } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index f16554448ef2a..a6ef5af11ecdb 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.NamedClusterStatePart; import org.elasticsearch.common.Nullable; @@ -52,32 +53,8 @@ public class MappingMetaData extends AbstractClusterStatePart implements NamedCl public static final Factory FACTORY = new Factory(); @Override - public void writeTo(StreamOutput out) throws IOException { - writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if(params.paramAsBoolean("reduce_mappings", false)) { - byte[] mappingSource = source().uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(type)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(type); - } - builder.field(type); - builder.map(mapping); - } else if (params.paramAsBoolean("binary", false)) { - builder.value(source().compressed()); - } else { - byte[] data = source().uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrdered(); - parser.close(); - builder.map(mapping); - } - return builder; + public String partType() { + return TYPE; } @Override @@ -97,14 +74,37 @@ public MappingMetaData readFrom(StreamInput in, LocalContext context) throws IOE return MappingMetaData.readFrom(in); } + @Override + public void writeTo(MappingMetaData part, StreamOutput out) throws IOException { + MappingMetaData.writeTo(part, out); + } + @Override public String partType() { return TYPE; } @Override - public MappingMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { - return super.fromXContent(parser, context); + public void toXContent(MappingMetaData part, XContentBuilder builder, ToXContent.Params params) throws IOException { + if(params.paramAsBoolean("reduce_mappings", false)) { + byte[] mappingSource = part.source().uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(partType())) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(partType()); + } + builder.field(partType()); + builder.map(mapping); + } else if (params.paramAsBoolean("binary", false)) { + builder.value(part.source().compressed()); + } else { + byte[] data = part.source().uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrdered(); + parser.close(); + builder.map(mapping); + } } } @@ -454,11 +454,6 @@ public String type() { return this.type; } - @Override - public String partType() { - return TYPE; - } - public CompressedString source() { return this.source; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index cafe77601b934..51035aaea2c43 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -77,27 +77,17 @@ public MetaData fromParts(long version, String uuid, ImmutableOpenMap.Builder cursor : metaData1.parts()) { - if (cursor.value.context().contains(XContentContext.GATEWAY)) { + if (FACTORY.lookupFactory(cursor.key).context().contains(XContentContext.GATEWAY)) { if (!cursor.value.equals(metaData2.get(cursor.key))) return false; customCount1++; } } int customCount2 = 0; for (ObjectObjectCursor cursor : metaData2.parts()) { - if (cursor.value.context().contains(XContentContext.GATEWAY)) { + if (FACTORY.lookupFactory(cursor.key).context().contains(XContentContext.GATEWAY)) { customCount2++; } } @@ -1332,7 +1322,7 @@ public MetaData build() { public static String toXContent(MetaData metaData) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject(); - metaData.toXContent(builder, ToXContent.EMPTY_PARAMS); + FACTORY.toXContent(metaData, builder, ToXContent.EMPTY_PARAMS); toXContent(metaData, builder, ToXContent.EMPTY_PARAMS); builder.endObject(); return builder.string(); @@ -1340,7 +1330,7 @@ public static String toXContent(MetaData metaData) throws IOException { public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject("meta-data"); - metaData.toXContent(builder, params); + FACTORY.toXContent(metaData, builder, params); builder.endObject(); } @@ -1405,12 +1395,15 @@ public static void writeTo(MetaData metaData, StreamOutput out) throws IOExcepti for (ObjectCursor cursor : metaData.templates.values()) { IndexTemplateMetaData.Builder.writeTo(cursor.value, out); } - //TODO: Hack - generalize the toXContent + //TODO: Hack - generalize out.writeVInt(metaData.parts().size() - 4); for (ObjectObjectCursor cursor : metaData.parts()) { if (!cursor.key.equals(TRANSIENT_SETTINGS_TYPE) && !cursor.key.equals(PERSISTENT_SETTINGS_TYPE) && !cursor.key.equals(INDICES_TYPE) && !cursor.key.equals(TEMPLATES_TYPE)) { - out.writeString(cursor.key); - cursor.value.writeTo(out); + ClusterStatePart.Factory factory = FACTORY.lookupFactorySafe(cursor.key); + if (factory.addedIn().onOrAfter(out.getVersion())){ + out.writeString(cursor.key); + factory.writeTo(cursor.value, out); + } } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index e871f4ab72edc..77888d2971341 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -83,22 +84,6 @@ public RepositoryMetaData repository(String name) { return null; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(repositories.size()); - for (RepositoryMetaData repository : repositories) { - repository.writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - for (RepositoryMetaData repository : repositories()) { - toXContent(repository, builder, params); - } - return builder; - } - /** * Serializes information about a single repository * @@ -119,16 +104,6 @@ public static void toXContent(RepositoryMetaData repository, XContentBuilder bui builder.endObject(); } - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return ClusterStatePart.API_GATEWAY; - } - /** * Repository metadata factory */ @@ -146,6 +121,14 @@ public RepositoriesMetaData readFrom(StreamInput in, LocalContext context) throw return new RepositoriesMetaData(repository); } + @Override + public void writeTo(RepositoriesMetaData repositoriesMetaData, StreamOutput out) throws IOException { + out.writeVInt(repositoriesMetaData.repositories.size()); + for (RepositoryMetaData repository : repositoriesMetaData.repositories) { + repository.writeTo(out); + } + } + /** * {@inheritDoc} */ @@ -192,6 +175,14 @@ public RepositoriesMetaData fromXContent(XContentParser parser, LocalContext con return new RepositoriesMetaData(repository.toArray(new RepositoryMetaData[repository.size()])); } + @Override + public void toXContent(RepositoriesMetaData repositoriesMetaData, XContentBuilder builder, Params params) throws IOException { + for (RepositoryMetaData repository : repositoriesMetaData.repositories()) { + RepositoriesMetaData.toXContent(repository, builder, params); + } + } + + @Override public String partType() { return TYPE; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java index 4da59476d55c3..3bbeba3f24fe0 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java @@ -43,6 +43,75 @@ public class RestoreMetaData extends AbstractClusterStatePart { public static final Factory FACTORY = new Factory(); + /** + * Restore metadata factory + */ + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + /** + * {@inheritDoc} + */ + @Override + public RestoreMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + Entry[] entries = new Entry[in.readVInt()]; + for (int i = 0; i < entries.length; i++) { + SnapshotId snapshotId = SnapshotId.readSnapshotId(in); + State state = State.fromValue(in.readByte()); + int indices = in.readVInt(); + ImmutableList.Builder indexBuilder = ImmutableList.builder(); + for (int j = 0; j < indices; j++) { + indexBuilder.add(in.readString()); + } + ImmutableMap.Builder builder = ImmutableMap.builder(); + int shards = in.readVInt(); + for (int j = 0; j < shards; j++) { + ShardId shardId = ShardId.readShardId(in); + ShardRestoreStatus shardState = ShardRestoreStatus.readShardRestoreStatus(in); + builder.put(shardId, shardState); + } + entries[i] = new Entry(snapshotId, state, indexBuilder.build(), builder.build()); + } + return new RestoreMetaData(entries); + } + + /** + * {@inheritDoc} + */ + @Override + public void writeTo(RestoreMetaData restoreMetaData, StreamOutput out) throws IOException { + out.writeVInt(restoreMetaData.entries.size()); + for (Entry entry : restoreMetaData.entries) { + entry.snapshotId().writeTo(out); + out.writeByte(entry.state().value()); + out.writeVInt(entry.indices().size()); + for (String index : entry.indices()) { + out.writeString(index); + } + out.writeVInt(entry.shards().size()); + for (Map.Entry shardEntry : entry.shards().entrySet()) { + shardEntry.getKey().writeTo(out); + shardEntry.getValue().writeTo(out); + } + } + } + /** + * {@inheritDoc} + */ + @Override + public void toXContent(RestoreMetaData restoreMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray("snapshots"); + for (Entry entry : restoreMetaData.entries) { + RestoreMetaData.toXContent(entry, builder, params); + } + builder.endArray(); + } + + + @Override + public String partType() { + return TYPE; + } + } + private final ImmutableList entries; /** @@ -105,11 +174,6 @@ public int hashCode() { return entries.hashCode(); } - @Override - public String partType() { - return TYPE; - } - /** * Restore metadata */ @@ -401,40 +465,6 @@ public static State fromValue(byte value) { } } - /** - * {@inheritDoc} - */ - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(entries.size()); - for (Entry entry : entries) { - entry.snapshotId().writeTo(out); - out.writeByte(entry.state().value()); - out.writeVInt(entry.indices().size()); - for (String index : entry.indices()) { - out.writeString(index); - } - out.writeVInt(entry.shards().size()); - for (Map.Entry shardEntry : entry.shards().entrySet()) { - shardEntry.getKey().writeTo(out); - shardEntry.getValue().writeTo(out); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray("snapshots"); - for (Entry entry : entries) { - toXContent(entry, builder, params); - } - builder.endArray(); - return builder; - } - /** * Serializes single restore operation * @@ -443,7 +473,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par * @param params serialization parameters * @throws IOException */ - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field("snapshot", entry.snapshotId().getSnapshot()); builder.field("repository", entry.snapshotId().getRepository()); @@ -473,41 +503,5 @@ public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params p builder.endArray(); builder.endObject(); } - /** - * Restore metadata factory - */ - public static class Factory extends AbstractClusterStatePart.AbstractFactory { - /** - * {@inheritDoc} - */ - @Override - public RestoreMetaData readFrom(StreamInput in, LocalContext context) throws IOException { - Entry[] entries = new Entry[in.readVInt()]; - for (int i = 0; i < entries.length; i++) { - SnapshotId snapshotId = SnapshotId.readSnapshotId(in); - State state = State.fromValue(in.readByte()); - int indices = in.readVInt(); - ImmutableList.Builder indexBuilder = ImmutableList.builder(); - for (int j = 0; j < indices; j++) { - indexBuilder.add(in.readString()); - } - ImmutableMap.Builder builder = ImmutableMap.builder(); - int shards = in.readVInt(); - for (int j = 0; j < shards; j++) { - ShardId shardId = ShardId.readShardId(in); - ShardRestoreStatus shardState = ShardRestoreStatus.readShardRestoreStatus(in); - builder.put(shardId, shardState); - } - entries[i] = new Entry(snapshotId, state, indexBuilder.build(), builder.build()); - } - return new RestoreMetaData(entries); - } - - @Override - public String partType() { - return TYPE; - } - } - } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java index 94568a7d2fc5b..26af875212099 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java @@ -62,11 +62,6 @@ public int hashCode() { return entries.hashCode(); } - @Override - public String partType() { - return TYPE; - } - public static class Entry { private final State state; private final SnapshotId snapshotId; @@ -337,27 +332,6 @@ public Entry snapshot(SnapshotId snapshotId) { return null; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(entries.size()); - for (Entry entry : entries) { - entry.snapshotId().writeTo(out); - out.writeBoolean(entry.includeGlobalState()); - out.writeByte(entry.state().value()); - out.writeVInt(entry.indices().size()); - for (String index : entry.indices()) { - out.writeString(index); - } - out.writeLong(entry.startTime()); - out.writeVInt(entry.shards().size()); - for (Map.Entry shardEntry : entry.shards().entrySet()) { - shardEntry.getKey().writeTo(out); - out.writeOptionalString(shardEntry.getValue().nodeId()); - out.writeByte(shardEntry.getValue().state().value()); - } - } - } - static final class Fields { static final XContentBuilderString REPOSITORY = new XContentBuilderString("repository"); static final XContentBuilderString SNAPSHOTS = new XContentBuilderString("snapshots"); @@ -373,17 +347,7 @@ static final class Fields { static final XContentBuilderString NODE = new XContentBuilderString("node"); } - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startArray(Fields.SNAPSHOTS); - for (Entry entry : entries) { - toXContent(entry, builder, params); - } - builder.endArray(); - return builder; - } - - public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository()); builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot()); @@ -443,6 +407,37 @@ public SnapshotMetaData readFrom(StreamInput in, LocalContext context) throws IO return new SnapshotMetaData(entries); } + @Override + public void writeTo(SnapshotMetaData snapshotMetaData, StreamOutput out) throws IOException { + out.writeVInt(snapshotMetaData.entries.size()); + for (Entry entry : snapshotMetaData.entries) { + entry.snapshotId().writeTo(out); + out.writeBoolean(entry.includeGlobalState()); + out.writeByte(entry.state().value()); + out.writeVInt(entry.indices().size()); + for (String index : entry.indices()) { + out.writeString(index); + } + out.writeLong(entry.startTime()); + out.writeVInt(entry.shards().size()); + for (Map.Entry shardEntry : entry.shards().entrySet()) { + shardEntry.getKey().writeTo(out); + out.writeOptionalString(shardEntry.getValue().nodeId()); + out.writeByte(shardEntry.getValue().state().value()); + } + } + } + + @Override + public void toXContent(SnapshotMetaData snapshotMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startArray(Fields.SNAPSHOTS); + for (Entry entry : snapshotMetaData.entries) { + SnapshotMetaData.toXContent(entry, builder, params); + } + builder.endArray(); + } + + @Override public String partType() { return TYPE; diff --git a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 2a679c9f62ccc..748aa1fc2bb1f 100644 --- a/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; @@ -72,6 +73,28 @@ public DiscoveryNodes readFrom(StreamInput in, LocalContext context) throws IOEx return Builder.readFrom(in, context.getLocalNode()); } + @Override + public void writeTo(DiscoveryNodes discoveryNodes, StreamOutput out) throws IOException { + DiscoveryNodes.Builder.writeTo(discoveryNodes, out); + } + + @Override + public void toXContent(DiscoveryNodes discoveryNodes, XContentBuilder builder, Params params) throws IOException { + for (DiscoveryNode node : discoveryNodes) { + builder.startObject(node.id(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("name", node.name()); + builder.field("transport_address", node.address().toString()); + + builder.startObject("attributes"); + for (Map.Entry attr : node.attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue()); + } + builder.endObject(); + + builder.endObject(); + } + } + @Override public String partType() { return TYPE; @@ -487,34 +510,6 @@ public Delta emptyDelta() { return new Delta(null, null, localNodeId, DiscoveryNode.EMPTY_LIST, DiscoveryNode.EMPTY_LIST); } - @Override - public void writeTo(StreamOutput out) throws IOException { - DiscoveryNodes.Builder.writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - for (DiscoveryNode node : this) { - builder.startObject(node.id(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("name", node.name()); - builder.field("transport_address", node.address().toString()); - - builder.startObject("attributes"); - for (Map.Entry attr : node.attributes().entrySet()) { - builder.field(attr.getKey(), attr.getValue()); - } - builder.endObject(); - - builder.endObject(); - } - return builder; - } - - @Override - public String partType() { - return TYPE; - } - public static class Delta { private final String localNodeId; diff --git a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index acaec1da069eb..f317d85882eb0 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; @@ -94,27 +95,6 @@ public class IndexRoutingTable extends AbstractClusterStatePart implements Itera this.allActiveShards = allActiveShards.build(); } - @Override - public String partType() { - return TYPE; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(index(), XContentBuilder.FieldCaseConversion.NONE); - builder.startObject("shards"); - for (ObjectCursor indexShardRoutingTable : shards.values()) { - builder.startArray(Integer.toString(indexShardRoutingTable.value.shardId().id())); - for (ShardRouting shardRouting : indexShardRoutingTable.value) { - shardRouting.toXContent(builder, params); - } - builder.endArray(); - } - builder.endObject(); - builder.endObject(); - return builder; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -142,6 +122,26 @@ public IndexRoutingTable readFrom(StreamInput in, LocalContext context) throws I return Builder.readFrom(in); } + @Override + public void writeTo(IndexRoutingTable indexRoutingTable, StreamOutput out) throws IOException { + Builder.writeTo(indexRoutingTable, out); + } + + @Override + public void toXContent(IndexRoutingTable indexRoutingTable, XContentBuilder builder, Params params) throws IOException { + builder.startObject(indexRoutingTable.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject("shards"); + for (ObjectCursor indexShardRoutingTable : indexRoutingTable.shards.values()) { + builder.startArray(Integer.toString(indexShardRoutingTable.value.shardId().id())); + for (ShardRouting shardRouting : indexShardRoutingTable.value) { + shardRouting.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); + } + @Override public String partType() { return TYPE; @@ -393,11 +393,6 @@ public static Builder builder(String index) { return new Builder(index); } - @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeTo(this, out); - } - public static class Builder { private final String index; diff --git a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 1f16e50c4723d..27e24750fdf1d 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -23,12 +23,14 @@ import com.google.common.collect.*; import org.elasticsearch.cluster.AbstractClusterStatePart; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStatePart; import org.elasticsearch.cluster.LocalContext; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; @@ -67,12 +69,6 @@ public class RoutingTable extends AbstractClusterStatePart implements Iterable { @Override @@ -81,8 +77,27 @@ public RoutingTable readFrom(StreamInput in, LocalContext context) throws IOExce } @Override - public RoutingTable fromXContent(XContentParser parser, LocalContext context) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); + public void writeTo(RoutingTable routingTable, StreamOutput out) throws IOException { + Builder.writeTo(routingTable, out); + } + + @Override + public void toXContent(RoutingTable routingTable, XContentBuilder builder, Params params) throws IOException { + builder.startObject("indices"); + for (IndexRoutingTable indexRoutingTable : routingTable) { + builder.startObject(indexRoutingTable.index(), XContentBuilder.FieldCaseConversion.NONE); + builder.startObject("shards"); + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); + for (ShardRouting shardRouting : indexShardRoutingTable) { + shardRouting.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); + } + builder.endObject(); } @Override @@ -93,19 +108,15 @@ public Diff diff(@Nullable RoutingTable before, RoutingTable after if (before != null) { ImmutableMap beforeParts = before.indicesRouting; ImmutableMap afterParts = after.indicesRouting; - if (before.equals(after)) { - return new NoDiff<>(); - } else { - for (String key : beforeParts.keySet()) { - if (!afterParts.containsKey(key)) { - deletes.add(key); - } + for (String key : beforeParts.keySet()) { + if (!afterParts.containsKey(key)) { + deletes.add(key); } - for (Map.Entry part : afterParts.entrySet()) { - IndexRoutingTable beforePart = beforeParts.get(part.getKey()); - if (!part.getValue().equals(beforePart)) { - diffs.put(part.getKey(), IndexRoutingTable.FACTORY.diff(beforePart, part.getValue())); - } + } + for (Map.Entry part : afterParts.entrySet()) { + IndexRoutingTable beforePart = beforeParts.get(part.getKey()); + if (!part.getValue().equals(beforePart)) { + diffs.put(part.getKey(), IndexRoutingTable.FACTORY.diff(beforePart, part.getValue())); } } } else { @@ -119,25 +130,37 @@ public Diff diff(@Nullable RoutingTable before, RoutingTable after @Override public Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException { - if (in.readBoolean()) { - long version = in.readVLong(); - int deletesSize = in.readVInt(); - List deletes = new ArrayList<>(); - for (int i = 0; i < deletesSize; i++) { - deletes.add(in.readString()); - } + long version = in.readVLong(); + int deletesSize = in.readVInt(); + List deletes = new ArrayList<>(); + for (int i = 0; i < deletesSize; i++) { + deletes.add(in.readString()); + } - int diffsSize = in.readVInt(); - Map> diffs = newHashMap(); - for (int i = 0; i < diffsSize; i++) { - String key = in.readString(); - diffs.put(key, IndexRoutingTable.FACTORY.readDiffFrom(in, context)); - } - return new RoutingTableDiff(version, deletes, diffs); + int diffsSize = in.readVInt(); + Map> diffs = newHashMap(); + for (int i = 0; i < diffsSize; i++) { + String key = in.readString(); + diffs.put(key, IndexRoutingTable.FACTORY.readDiffFrom(in, context)); + } + return new RoutingTableDiff(version, deletes, diffs); + } - } else { - return new NoDiff<>(); + @Override + public void writeDiffsTo(Diff diff, StreamOutput out) throws IOException { + RoutingTableDiff routingTableDiff = (RoutingTableDiff) diff; + out.writeVLong(routingTableDiff.version); + out.writeVInt(routingTableDiff.deletes.size()); + for (String delete : routingTableDiff.deletes) { + out.writeString(delete); + } + + out.writeVInt(routingTableDiff.diffs.size()); + for (Map.Entry> entry : routingTableDiff.diffs.entrySet()) { + out.writeString(entry.getKey()); + IndexRoutingTable.FACTORY.writeDiffsTo(entry.getValue(), out); } + } @Override @@ -171,22 +194,6 @@ public RoutingTable apply(RoutingTable part) { } return new RoutingTable(version, parts); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeBoolean(true); // We have diffs - out.writeVLong(version); - out.writeVInt(deletes.size()); - for (String delete : deletes) { - out.writeString(delete); - } - - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } - } } /** * Returns the version of the {@link RoutingTable}. @@ -425,31 +432,6 @@ public static Builder builder(RoutingTable routingTable) { return new Builder(routingTable); } - @Override - public void writeTo(StreamOutput out) throws IOException { - Builder.writeTo(this, out); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("indices"); - for (IndexRoutingTable indexRoutingTable : this) { - builder.startObject(indexRoutingTable.index(), XContentBuilder.FieldCaseConversion.NONE); - builder.startObject("shards"); - for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { - builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id())); - for (ShardRouting shardRouting : indexShardRoutingTable) { - shardRouting.toXContent(builder, params); - } - builder.endArray(); - } - builder.endObject(); - builder.endObject(); - } - builder.endObject(); - return builder; - } - public static class Builder { private long version; diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index ec6a27b077fe5..58e8345dbd291 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterState.ClusterStateDiff; import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; @@ -295,7 +296,8 @@ private BytesReference serializeDiffClusterState(ClusterState clusterState, Vers StreamOutput stream = new HandlesStreamOutput(CompressorFactory.defaultCompressor().streamOutput(bStream)); stream.setVersion(nodeVersion); stream.writeBoolean(false); - ClusterState.Builder.diff(lastProcessedClusterState, clusterState).writeTo(stream); + ClusterStateDiff diff = ClusterState.Builder.diff(lastProcessedClusterState, clusterState); + ClusterState.Builder.writeDiffTo(diff, stream); stream.close(); return bStream.bytes(); } diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 5f42d622a3948..fd1d7a0bccb81 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -45,34 +45,6 @@ public class IndexWarmersMetaData extends AbstractClusterStatePart implements In public static final Factory FACTORY = new Factory(); - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(entries.size()); - for (Entry entry : entries) { - out.writeString(entry.name()); - out.writeStringArray(entry.types()); - if (entry.source() == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeBytesReference(entry.source()); - } - out.writeOptionalBoolean(entry.queryCache()); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - //No need, IndexMetaData already writes it - //builder.startObject(TYPE, XContentBuilder.FieldCaseConversion.NONE); - for (Entry entry : entries) { - toXContent(entry, builder, params); - } - //No need, IndexMetaData already writes it - //builder.endObject(); - return builder; - } - public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { boolean binary = params.paramAsBoolean("binary", false); builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); @@ -114,7 +86,6 @@ public String partType() { return TYPE; } - public static class Entry { private final String name; private final String[] types; @@ -177,6 +148,22 @@ public IndexWarmersMetaData readFrom(StreamInput in, LocalContext context) throw return new IndexWarmersMetaData(entries); } + @Override + public void writeTo(IndexWarmersMetaData indexWarmersMetaData, StreamOutput out) throws IOException { + out.writeVInt(indexWarmersMetaData.entries.size()); + for (Entry entry : indexWarmersMetaData.entries) { + out.writeString(entry.name()); + out.writeStringArray(entry.types()); + if (entry.source() == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeBytesReference(entry.source()); + } + out.writeOptionalBoolean(entry.queryCache()); + } + } + @Override public IndexWarmersMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { // we get here after we are at warmers token @@ -221,6 +208,17 @@ public IndexWarmersMetaData fromXContent(XContentParser parser, LocalContext con return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); } + @Override + public void toXContent(IndexWarmersMetaData indexWarmersMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + //No need, IndexMetaData already writes it + //builder.startObject(TYPE, XContentBuilder.FieldCaseConversion.NONE); + for (Entry entry : indexWarmersMetaData.entries) { + IndexWarmersMetaData.toXContent(entry, builder, params); + } + //No need, IndexMetaData already writes it + //builder.endObject(); + } + @Override public String partType() { return TYPE; diff --git a/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c42363a56a315..445a9603dce06 100644 --- a/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -283,7 +283,7 @@ private void validateExistingIndex(IndexMetaData currentIndexMetaData, IndexMeta private void restoreGlobalStateIfRequested(MetaData.Builder mdBuilder) { if (request.includeGlobalState()) { for (ObjectObjectCursor cursor : metaData.parts()) { - if (cursor.value.context().contains(ClusterStatePart.XContentContext.SNAPSHOT)) { + if (MetaData.FACTORY.lookupFactory(cursor.key).context().contains(ClusterStatePart.XContentContext.SNAPSHOT)) { if (cursor.key.equals(IndexTemplateMetaData.TYPE)) { // Merge templates instead of replacing them for (ObjectCursor cursor2 : metaData.templates().values()) { diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index d598c2f543b44..29997b7dec157 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -712,17 +712,6 @@ public int hashCode() { return data.hashCode(); } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(getData()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.field("data",getData()); - return builder; - } - public static abstract class TestCustomMetaDataFactory extends AbstractClusterStatePart.AbstractFactory { protected abstract TestCustomMetaData newTestCustomMetaData(String data); @@ -733,6 +722,16 @@ public T readFrom(StreamInput in, LocalContext context) throws IOException { } + @Override + public void writeTo(T part, StreamOutput out) throws IOException { + out.writeString(part.getData()); + } + + @Override + public void toXContent(T part, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field("data", part.getData()); + } + @Override public T fromXContent(XContentParser parser, LocalContext context) throws IOException { XContentParser.Token token; @@ -778,16 +777,6 @@ public SnapshottableMetadata(String data) { super(data); } - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return MetaData.API_SNAPSHOT; - } - private static class Factory extends TestCustomMetaDataFactory { @Override @@ -799,6 +788,12 @@ public String partType() { protected TestCustomMetaData newTestCustomMetaData(String data) { return new SnapshottableMetadata(data); } + + @Override + public EnumSet context() { + return MetaData.API_SNAPSHOT; + } + } } @@ -811,16 +806,6 @@ public NonSnapshottableMetadata(String data) { super(data); } - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return MetaData.API; - } - private static class Factory extends TestCustomMetaDataFactory { @Override @@ -832,6 +817,11 @@ public String partType() { protected NonSnapshottableMetadata newTestCustomMetaData(String data) { return new NonSnapshottableMetadata(data); } + + @Override + public EnumSet context() { + return MetaData.API; + } } } @@ -844,16 +834,6 @@ public SnapshottableGatewayMetadata(String data) { super(data); } - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return MetaData.API_GATEWAY_SNAPSHOT; - } - private static class Factory extends TestCustomMetaDataFactory { @Override @@ -865,6 +845,12 @@ public String partType() { protected TestCustomMetaData newTestCustomMetaData(String data) { return new SnapshottableGatewayMetadata(data); } + + @Override + public EnumSet context() { + return MetaData.API_GATEWAY_SNAPSHOT; + } + } } @@ -877,16 +863,6 @@ public NonSnapshottableGatewayMetadata(String data) { super(data); } - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return MetaData.API_GATEWAY; - } - private static class Factory extends TestCustomMetaDataFactory { @Override @@ -898,6 +874,12 @@ public String partType() { protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) { return new NonSnapshottableGatewayMetadata(data); } + + @Override + public EnumSet context() { + return MetaData.API_GATEWAY; + } + } } @@ -910,17 +892,6 @@ public SnapshotableGatewayNoApiMetadata(String data) { super(data); } - - @Override - public String partType() { - return TYPE; - } - - @Override - public EnumSet context() { - return MetaData.GATEWAY_SNAPSHOT; - } - private static class Factory extends TestCustomMetaDataFactory { @Override @@ -932,6 +903,12 @@ public String partType() { protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) { return new SnapshotableGatewayNoApiMetadata(data); } + + @Override + public EnumSet context() { + return MetaData.GATEWAY_SNAPSHOT; + } + } } From 66b9dfdffcd15c304e7d3db8727de5f94e80deee Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 6 Jan 2015 21:16:29 -0500 Subject: [PATCH 15/20] Clean up --- .../state/TransportClusterStateAction.java | 1 - .../delete/TransportDeleteWarmerAction.java | 4 +- .../warmer/put/TransportPutWarmerAction.java | 2 +- .../cluster/AbstractClusterStatePart.java | 3 +- .../elasticsearch/cluster/ClusterState.java | 145 ++++++++---------- .../cluster/ClusterStatePart.java | 21 ++- .../cluster/ClusterStateSettingsPart.java | 6 +- .../cluster/CompositeClusterStatePart.java | 3 +- .../cluster/metadata/AliasMetaData.java | 4 + .../cluster/metadata/IndexMetaData.java | 36 +---- .../cluster/metadata/MetaData.java | 2 +- .../discovery/local/LocalDiscovery.java | 1 - .../reroute/RestClusterRerouteAction.java | 4 +- .../cluster/state/RestClusterStateAction.java | 14 +- .../elasticsearch/search/SearchService.java | 2 +- .../warmer/GatewayIndicesWarmerTests.java | 8 +- .../warmer/SimpleIndicesWarmerTests.java | 4 +- 17 files changed, 116 insertions(+), 144 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index edf64f884e951..06d3195864af1 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -126,7 +126,6 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS } // Filter our metadata that shouldn't be returned by API - // TODO: we probably shouldn't do that for(ObjectObjectCursor cursor : currentState.metaData().customs()) { Factory factory = MetaData.FACTORY.lookupFactorySafe(cursor.key); if(!factory.context().contains(ClusterStatePart.XContentContext.API)) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/warmer/delete/TransportDeleteWarmerAction.java b/src/main/java/org/elasticsearch/action/admin/indices/warmer/delete/TransportDeleteWarmerAction.java index 862295820373e..9d05de9e159e1 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/warmer/delete/TransportDeleteWarmerAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/warmer/delete/TransportDeleteWarmerAction.java @@ -102,7 +102,7 @@ public ClusterState execute(ClusterState currentState) { if (indexMetaData == null) { throw new IndexMissingException(new Index(index)); } - IndexWarmersMetaData warmers = indexMetaData.custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmers = indexMetaData.get(IndexWarmersMetaData.TYPE); if (warmers != null) { List entries = Lists.newArrayList(); for (IndexWarmersMetaData.Entry entry : warmers.entries()) { @@ -138,7 +138,7 @@ public ClusterState execute(ClusterState currentState) { if (indexMetaData == null) { throw new IndexMissingException(new Index(index)); } - IndexWarmersMetaData warmers = indexMetaData.custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmers = indexMetaData.get(IndexWarmersMetaData.TYPE); if (warmers != null) { for (IndexWarmersMetaData.Entry entry : warmers.entries()) { for (String warmer : request.names()) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java b/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java index 5ccd1f0653e04..d91c7a20b46ac 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java @@ -128,7 +128,7 @@ public ClusterState execute(ClusterState currentState) { if (indexMetaData == null) { throw new IndexMissingException(new Index(index)); } - IndexWarmersMetaData warmers = indexMetaData.custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmers = indexMetaData.get(IndexWarmersMetaData.TYPE); if (warmers == null) { logger.info("[{}] putting warmer [{}]", index, request.name()); warmers = new IndexWarmersMetaData(new IndexWarmersMetaData.Entry(request.name(), request.searchRequest().types(), request.searchRequest().queryCache(), source)); diff --git a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java index 23ae846838431..9a874718f9faf 100644 --- a/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/AbstractClusterStatePart.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster; -import org.apache.commons.lang3.builder.Diff; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; @@ -31,7 +30,7 @@ import java.util.Map; /** - * Basic implementation of cluster state part that send entire part as a difference if part got changed. + * Basic implementation of cluster state part that sends entire part as a difference if the part got changed. */ public abstract class AbstractClusterStatePart implements ClusterStatePart { diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 811f54d4cce1f..8ac9fb8f1644b 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -20,7 +20,7 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.ElasticsearchIllegalArgumentException; +import com.google.common.collect.ImmutableSet; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -35,21 +35,33 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import java.io.IOException; -import java.util.*; - -import static com.google.common.collect.Sets.newHashSet; -import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; /** * */ public class ClusterState extends CompositeClusterStatePart implements ToXContent { + public static enum ClusterStateStatus { + UNKNOWN((byte) 0), + RECEIVED((byte) 1), + BEING_APPLIED((byte) 2), + APPLIED((byte) 3); + + private final byte id; + + ClusterStateStatus(byte id) { + this.id = id; + } + + public byte id() { + return this.id; + } + } + public static final String TYPE = "cluster"; public static final Factory FACTORY = new Factory(); @@ -85,8 +97,7 @@ public ClusterStateDiff(long version, ClusterState.Diff diff) { } public ClusterState apply(ClusterState previous) throws IncompatibleClusterStateVersionException { - ClusterState newState = diff.apply(previous); - return newState; + return diff.apply(previous); } public long version() { @@ -94,23 +105,6 @@ public long version() { } } - public static enum ClusterStateStatus { - UNKNOWN((byte) 0), - RECEIVED((byte) 1), - BEING_APPLIED((byte) 2), - APPLIED((byte) 3); - - private final byte id; - - ClusterStateStatus(byte id) { - this.id = id; - } - - public byte id() { - return this.id; - } - } - public static final long UNKNOWN_VERSION = -1; private final RoutingTable routingTable; @@ -122,7 +116,7 @@ public byte id() { private final ClusterBlocks blocks; private final ClusterName clusterName; - + // built on demand private volatile RoutingNodes routingNodes; @@ -138,7 +132,6 @@ public ClusterState(long version, String uuid, ImmutableOpenMap valueToEnum; - - static { - valueToEnum = new HashMap<>(); - for (Metric metric : Metric.values()) { - valueToEnum.put(metric.value, metric); - } - } - - private final String value; - - private Metric(String value) { - this.value = value; - } - - public static EnumSet parseString(String param, boolean ignoreUnknown) { - String[] metrics = Strings.splitStringByCommaToArray(param); - EnumSet result = EnumSet.noneOf(Metric.class); - for (String metric : metrics) { - if ("_all".equals(metric)) { - result = EnumSet.allOf(Metric.class); - break; - } - Metric m = valueToEnum.get(metric); - if (m == null) { - if (!ignoreUnknown) { - throw new ElasticsearchIllegalArgumentException("Unknown metric [" + metric + "]"); + public static class Metrics { + + public static final String VERSION = "version"; + public static final String MASTER_NODE = "master_node"; + public static final String NODES = DiscoveryNodes.TYPE; + public static final String ROUTING_TABLE = RoutingTable.TYPE; + public static final String METADATA = MetaData.TYPE; + public static final String BLOCKS = ClusterBlocks.TYPE; + + + private final ImmutableSet includes; + private final ImmutableSet excludes; + + public Metrics(String params) { + if (params.equals("_all")) { + this.includes = ImmutableSet.of(); + this.excludes = ImmutableSet.of(); + } else { + ImmutableSet.Builder includes = ImmutableSet.builder(); + ImmutableSet.Builder excludes = ImmutableSet.builder(); + String[] metrics = Strings.splitStringByCommaToArray(params); + for (String metric : metrics) { + if (metric.startsWith("+")) { + includes.add(metric.substring(1)); + } else if (metric.startsWith("-")) { + excludes.add(metric.substring(1)); + } else { + includes.add(metric); } - } else { - result.add(m); } + this.includes = includes.build(); + this.excludes = excludes.build(); } - return result; } - @Override - public String toString() { - return value; + public boolean matches(String metric) { + if (includes.isEmpty()) { + return !excludes.contains(metric); + } + return includes.contains(metric) && !excludes.contains(metric); } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - EnumSet metrics = Metric.parseString(params.param("metric", "_all"), true); - Set metricStrings = newHashSet(); - for (Metric metric : metrics) { - metricStrings.add(metric.value); - } + Metrics metrics = new Metrics(params.param("metric", "_all")); - if (metrics.contains(Metric.VERSION)) { + if (metrics.matches(Metrics.VERSION)) { builder.field("version", version); } - if (metrics.contains(Metric.MASTER_NODE)) { + if (metrics.matches(Metrics.MASTER_NODE)) { builder.field("master_node", nodes().masterNodeId()); } for(ObjectObjectCursor partIter : parts) { - if (metricStrings.contains(partIter.key)) { + if (metrics.matches(partIter.key)) { builder.startObject(partIter.key); FACTORY.lookupFactorySafe(partIter.key).toXContent(partIter.value, builder, params); builder.endObject(); @@ -313,7 +295,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // routing nodes - if (metrics.contains(Metric.ROUTING_TABLE)) { + if (metrics.matches(Metrics.ROUTING_TABLE)) { builder.startObject("routing_nodes"); builder.startArray("unassigned"); for (ShardRouting shardRouting : readOnlyRoutingNodes().unassigned()) { @@ -477,8 +459,6 @@ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode loca } public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { - - // TODO: Do we need this version long version = in.readVLong(); LocalContext localContext = new LocalContext(localNode); return new ClusterStateDiff(version, FACTORY.readDiffFrom(in, localContext)); @@ -499,11 +479,6 @@ public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws return os.bytes().toBytes(); } - public static ClusterState fromDiffBytes(ClusterState before, byte[] data, DiscoveryNode localNode) throws IOException { - ClusterStateDiff diff = readDiffFrom(new BytesStreamInput(data, false), localNode); - return diff.apply(before); - } - } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index acbbf162f48f9..7b7d867529a9c 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -90,10 +90,23 @@ interface Factory { */ T fromMap(Map map, LocalContext context) throws IOException; - Diff diff(T before, T after); + /** + * Returns differences between before state of the part and after state of the part + * + * If the part didn't exists the before part can be null + */ + Diff diff(@Nullable T before, T after); + /** + * Reads the Diff object from an input stream + * + * The context contains information that might be necessary to deserialize the part such as local node. + */ Diff readDiffFrom(StreamInput in, LocalContext context) throws IOException; + /** + * Writes the Diff object to an output stream + */ void writeDiffsTo(Diff diff, StreamOutput out) throws IOException; @@ -113,8 +126,14 @@ interface Factory { EnumSet context(); } + /** + * Differences between two versions of the same part + */ interface Diff { + /** + * Returns the new version of the part after applying differences + */ T apply(T part); } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index fa29a02434797..ed6135c3ec65a 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -36,13 +36,16 @@ import static org.elasticsearch.common.settings.ImmutableSettings.writeSettingsToStream; /** + * Wrapper for settings as a cluster state part. + * + * Used to represent persistent, transient and index settings in the cluster metadata. */ public class ClusterStateSettingsPart extends AbstractClusterStatePart implements IndexClusterStatePart { private final String type; private final Settings settings; - public ClusterStateSettingsPart(String type, Settings settings) { + private ClusterStateSettingsPart(String type, Settings settings) { this.type = type; this.settings = settings; } @@ -90,7 +93,6 @@ public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext return new ClusterStateSettingsPart(type, settings); } - @Override public void toXContent(ClusterStateSettingsPart clusterStateSettingsPart, XContentBuilder builder, Params params) throws IOException { for (Map.Entry entry : clusterStateSettingsPart.settings.getAsMap().entrySet()) { diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index ca42666203b5a..cd0cfc469f8fb 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -98,7 +98,7 @@ public String uuid() { return uuid; } - public static abstract class AbstractCompositeFactory extends AbstractClusterStatePart.AbstractFactory { + protected static abstract class AbstractCompositeFactory extends AbstractClusterStatePart.AbstractFactory { private final Map partFactories = new HashMap<>(); /** @@ -234,7 +234,6 @@ public void writeDiffsTo(Diff diff, StreamOutput out) throws IOException { factory.writeDiffsTo(entry.getValue(), out); } } - } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 2a7f7ede260a6..f974d1d32478d 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -86,6 +86,10 @@ public String partType() { } + @Override + public AliasMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + return AliasMetaData.Builder.fromXContent(parser); + } } private final String alias; diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 178edbc25118f..ce4ea9be5a538 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilder.FieldCaseConversion; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MapperService; @@ -404,19 +405,6 @@ public MappingMetaData mappingOrDefault(String mappingType) { return mappings.get(MapperService.DEFAULT_MAPPING); } - public ImmutableOpenMap customs() { - return this.parts; - } - - public ImmutableOpenMap getCustoms() { - return this.parts; - } - - @SuppressWarnings("unchecked") - public T custom(String type) { - return (T) parts.get(type); - } - @Nullable public DiscoveryNodeFilters requireFilters() { return requireFilters; @@ -667,10 +655,9 @@ private static ImmutableOpenMap buildParts(Settin public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); + FACTORY.valuesPartToXContent(indexMetaData, builder, params); - builder.field("version", indexMetaData.version()); - builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); - + // TODO: we can make this generic but we need to move startObject inside Factory.toXContent if (params.paramAsBoolean("reduce_mappings", false)) { builder.startObject("mappings"); for (ObjectObjectCursor cursor : indexMetaData.mappings()) { @@ -687,7 +674,8 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build for (ObjectObjectCursor cursor : indexMetaData.parts()) { if (!cursor.key.equals(MAPPINGS_TYPE)) { - builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); + builder.field(cursor.key, FieldCaseConversion.NONE); + builder.startObject(); FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); builder.endObject(); } @@ -712,9 +700,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("settings".equals(currentFieldName)) { - builder.settings(ImmutableSettings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered()))); - } else if ("mappings".equals(currentFieldName)) { + if ("mappings".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -724,10 +710,6 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti builder.putMapping(new MappingMetaData(mappingType, mappingSource)); } } - } else if ("aliases".equals(currentFieldName)) { - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); - } } else { // check if its a custom index metadata ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); @@ -753,11 +735,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } } else if (token.isValue()) { - if ("state".equals(currentFieldName)) { - builder.state(State.fromString(parser.text())); - } else if ("version".equals(currentFieldName)) { - builder.version(parser.longValue()); - } + builder.parseValuePart(parser, currentFieldName, null); } } return builder.build(); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 51035aaea2c43..fac4e9e86884b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -401,7 +401,7 @@ public ImmutableOpenMap> findW Iterable intersection = HppcMaps.intersection(ObjectOpenHashSet.from(concreteIndices), indices.keys()); for (String index : intersection) { IndexMetaData indexMetaData = indices.get(index); - IndexWarmersMetaData indexWarmersMetaData = indexMetaData.custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData indexWarmersMetaData = indexMetaData.get(IndexWarmersMetaData.TYPE); if (indexWarmersMetaData == null || indexWarmersMetaData.entries().isEmpty()) { continue; } diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 7ab8b4a38fdc6..d44e3b0393ec1 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -310,7 +310,6 @@ private void publish(LocalDiscovery[] members, ClusterState clusterState, final if (clusterStateDiffBytes == null) { clusterStateDiffBytes = Builder.toDiffBytes(lastProcessedClusterState, clusterState); } -// nodeSpecificClusterState = ClusterState.Builder.fromDiffBytes(lastProcessedClusterState, clusterStateDiffBytes, discovery.localNode); ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(new BytesStreamInput(clusterStateDiffBytes, false), discovery.localNode); try { newNodeSpecificClusterState = diff.apply(discovery.clusterService.state()); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java index 5000f29e81ed7..be8ddfdc7394b 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterState.Metrics; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -42,7 +42,7 @@ public class RestClusterRerouteAction extends BaseRestHandler { private final SettingsFilter settingsFilter; - private static String DEFAULT_METRICS = Strings.arrayToCommaDelimitedString(EnumSet.complementOf(EnumSet.of(ClusterState.Metric.METADATA)).toArray()); + private static String DEFAULT_METRICS = "-" + Metrics.METADATA; // Returns everything except metadata @Inject public RestClusterRerouteAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) { diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java index f7042bdd0bb63..102e386971a40 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterState.Metrics; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -33,8 +33,6 @@ import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestBuilderListener; -import java.util.EnumSet; - /** * @@ -67,12 +65,12 @@ public void handleRequest(final RestRequest request, final RestChannel channel, } if (request.hasParam("metric")) { - EnumSet metrics = ClusterState.Metric.parseString(request.param("metric"), true); + Metrics metrics = new Metrics(request.param("metric")); // do not ask for what we do not need. - clusterStateRequest.nodes(metrics.contains(ClusterState.Metric.NODES) || metrics.contains(ClusterState.Metric.MASTER_NODE)); - clusterStateRequest.routingTable(metrics.contains(ClusterState.Metric.ROUTING_TABLE)); - clusterStateRequest.metaData(metrics.contains(ClusterState.Metric.METADATA)); - clusterStateRequest.blocks(metrics.contains(ClusterState.Metric.BLOCKS)); + clusterStateRequest.nodes(metrics.matches(Metrics.NODES) || metrics.matches(Metrics.MASTER_NODE)); + clusterStateRequest.routingTable(metrics.matches(Metrics.ROUTING_TABLE)); + clusterStateRequest.metaData(metrics.matches(Metrics.METADATA)); + clusterStateRequest.blocks(metrics.matches(Metrics.BLOCKS)); } client.admin().cluster().state(clusterStateRequest, new RestBuilderListener(channel) { diff --git a/src/main/java/org/elasticsearch/search/SearchService.java b/src/main/java/org/elasticsearch/search/SearchService.java index 2af704bb479c0..cfb731915c12a 100644 --- a/src/main/java/org/elasticsearch/search/SearchService.java +++ b/src/main/java/org/elasticsearch/search/SearchService.java @@ -961,7 +961,7 @@ public TerminationHandle warmTopReader(IndexShard indexShard, IndexMetaData inde } public TerminationHandle internalWarm(final IndexShard indexShard, final IndexMetaData indexMetaData, final IndicesWarmer.WarmerContext warmerContext, ThreadPool threadPool, final boolean top) { - IndexWarmersMetaData custom = indexMetaData.custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData custom = indexMetaData.get(IndexWarmersMetaData.TYPE); if (custom == null) { return TerminationHandle.NO_WAIT; } diff --git a/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java b/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java index 2b35c2a2f5844..378fc8effc34c 100644 --- a/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java @@ -80,7 +80,7 @@ public void testStatePersistence() throws Exception { logger.info("--> verify warmers are registered in cluster state"); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(2)); @@ -100,7 +100,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { logger.info("--> verify warmers are recovered"); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - IndexWarmersMetaData recoveredWarmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData recoveredWarmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(recoveredWarmersMetaData.entries().size(), equalTo(warmersMetaData.entries().size())); for (int i = 0; i < warmersMetaData.entries().size(); i++) { assertThat(recoveredWarmersMetaData.entries().get(i).name(), equalTo(warmersMetaData.entries().get(i).name())); @@ -122,7 +122,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { logger.info("--> verify warmers (delete) are registered in cluster state"); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + warmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(1)); @@ -138,7 +138,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { logger.info("--> verify warmers are recovered"); clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - recoveredWarmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + recoveredWarmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(recoveredWarmersMetaData.entries().size(), equalTo(warmersMetaData.entries().size())); for (int i = 0; i < warmersMetaData.entries().size(); i++) { assertThat(recoveredWarmersMetaData.entries().get(i).name(), equalTo(warmersMetaData.entries().get(i).name())); diff --git a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java index 57591cc86f7c7..7c6b3d92b3321 100644 --- a/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java @@ -131,7 +131,7 @@ public void templateWarmer() { ensureGreen(); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(1)); @@ -156,7 +156,7 @@ public void createIndexWarmer() { "}")); ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); - IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData warmersMetaData = clusterState.metaData().index("test").get(IndexWarmersMetaData.TYPE); assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(1)); From a000e812b1127cb84ca06f475c88feb616a00c48 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 6 Jan 2015 22:13:52 -0500 Subject: [PATCH 16/20] Generalize handling of mappings in IndexMetaData --- .../cluster/ClusterStateSettingsPart.java | 2 + .../cluster/CompositeClusterStatePart.java | 3 +- .../cluster/MapClusterStatePart.java | 2 + .../cluster/metadata/BenchmarkMetaData.java | 2 + .../cluster/metadata/IndexMetaData.java | 78 +++---------- .../metadata/IndexTemplateMetaData.java | 3 +- .../cluster/metadata/MappingsMetaData.java | 104 ++++++++++++++++++ .../metadata/RepositoriesMetaData.java | 2 + .../cluster/metadata/RestoreMetaData.java | 2 + .../search/warmer/IndexWarmersMetaData.java | 6 +- .../DedicatedClusterSnapshotRestoreTests.java | 2 + 11 files changed, 137 insertions(+), 69 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index ed6135c3ec65a..7f2c8e4e97a2b 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -95,9 +95,11 @@ public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext @Override public void toXContent(ClusterStateSettingsPart clusterStateSettingsPart, XContentBuilder builder, Params params) throws IOException { + builder.startObject(); for (Map.Entry entry : clusterStateSettingsPart.settings.getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } + builder.endObject(); } public ClusterStateSettingsPart fromSettings(Settings settings) { diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index cd0cfc469f8fb..855b485a741c4 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -158,9 +158,8 @@ public void toXContent(T part, XContentBuilder builder, Params params) throws IO for (ObjectObjectCursor partIter : parts) { Factory factory = lookupFactorySafe(partIter.key); if (factory.context().contains(context)) { - builder.startObject(partIter.key); + builder.field(partIter.key); factory.toXContent(partIter.value, builder, params); - builder.endObject(); } } } diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index 04b568f648cd9..b30fd466bf37c 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -110,9 +110,11 @@ public void writeTo(MapClusterStatePart part, StreamOutput out) throws IOExce @Override public void toXContent(MapClusterStatePart part, XContentBuilder builder, Params params) throws IOException { + builder.startObject(); for (ObjectCursor cursor : part.parts.values()) { factory.toXContent(cursor.value, builder, params); } + builder.endObject(); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java index b61de01bace83..e4cb374162e94 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/BenchmarkMetaData.java @@ -193,11 +193,13 @@ public void writeTo(BenchmarkMetaData benchmarkMetaData, StreamOutput out) throw @Override public void toXContent(BenchmarkMetaData benchmarkMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); builder.startArray("benchmarks"); for (Entry entry : benchmarkMetaData.entries) { BenchmarkMetaData.toXContent(entry, builder, params); } builder.endArray(); + builder.endObject(); } @Override public String partType() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index ce4ea9be5a538..e90ed7c78c31b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -71,17 +71,13 @@ public class IndexMetaData extends NamedCompositeClusterStatePart MAPPINGS_FACTORY = new MapClusterStatePart.Factory<>(MAPPINGS_TYPE, MappingMetaData.FACTORY, API_GATEWAY_SNAPSHOT); - public static final String ALIASES_TYPE = "aliases"; public static final MapClusterStatePart.Factory ALIASES_FACTORY = new MapClusterStatePart.Factory<>(ALIASES_TYPE, AliasMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { FACTORY.registerFactory(SETTINGS_TYPE, SETTINGS_FACTORY); - FACTORY.registerFactory(MAPPINGS_TYPE, MAPPINGS_FACTORY); + FACTORY.registerFactory(MappingsMetaData.TYPE, MappingsMetaData.FACTORY); FACTORY.registerFactory(ALIASES_TYPE, ALIASES_FACTORY); // register non plugin custom metadata FACTORY.registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); @@ -214,7 +210,7 @@ private IndexMetaData(String index, long version, State state, ImmutableOpenMap< this.totalNumberOfShards = numberOfShards() * (numberOfReplicas() + 1); Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) != null, "must specify numberOfShards for index [" + index + "]"); Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) != null, "must specify numberOfReplicas for index [" + index + "]"); - this.mappings = parts.containsKey(MAPPINGS_TYPE) ? ((MapClusterStatePart)get(MAPPINGS_TYPE)).parts() : ImmutableOpenMap.of(); + this.mappings = parts.containsKey(MappingsMetaData.TYPE) ? ((MapClusterStatePart)get(MappingsMetaData.TYPE)).parts() : ImmutableOpenMap.of(); this.aliases = parts.containsKey(ALIASES_TYPE) ? ((MapClusterStatePart)get(ALIASES_TYPE)).parts() : ImmutableOpenMap.of(); ImmutableMap requireMap = settings.getByPrefix("index.routing.allocation.require.").getAsMap(); @@ -482,7 +478,7 @@ public Builder(IndexMetaData indexMetaData) { this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); parts.putAll(indexMetaData.parts()); parts.remove(SETTINGS_TYPE); - parts.remove(MAPPINGS_TYPE); + parts.remove(MappingsMetaData.TYPE); parts.remove(ALIASES_TYPE); } @@ -647,7 +643,7 @@ private static ImmutableOpenMap buildParts(Settin ImmutableOpenMap parts) { ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); builder.put(SETTINGS_TYPE, SETTINGS_FACTORY.fromSettings(settings)); - builder.put(MAPPINGS_TYPE, MAPPINGS_FACTORY.fromOpenMap(mappings)); + builder.put(MappingsMetaData.TYPE, MappingsMetaData.FACTORY.fromOpenMap(mappings)); builder.put(ALIASES_TYPE, ALIASES_FACTORY.fromOpenMap(aliases)); builder.putAll(parts); return builder.build(); @@ -656,31 +652,10 @@ private static ImmutableOpenMap buildParts(Settin public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); FACTORY.valuesPartToXContent(indexMetaData, builder, params); - - // TODO: we can make this generic but we need to move startObject inside Factory.toXContent - if (params.paramAsBoolean("reduce_mappings", false)) { - builder.startObject("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); - } - builder.endObject(); - } else { - builder.startArray("mappings"); - for (ObjectObjectCursor cursor : indexMetaData.mappings()) { - MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); - } - builder.endArray(); - } - for (ObjectObjectCursor cursor : indexMetaData.parts()) { - if (!cursor.key.equals(MAPPINGS_TYPE)) { - builder.field(cursor.key, FieldCaseConversion.NONE); - builder.startObject(); - FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - builder.endObject(); - } + builder.field(cursor.key, FieldCaseConversion.NONE); + FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); } - builder.endObject(); } @@ -700,39 +675,20 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("mappings".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - String mappingType = currentFieldName; - Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); - builder.putMapping(new MappingMetaData(mappingType, mappingSource)); - } - } + ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn + parser.skipChildren(); } else { - // check if its a custom index metadata - ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); - if (factory == null) { - //TODO warn - parser.skipChildren(); - } else { - builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); - } + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } else if (token == XContentParser.Token.START_ARRAY) { - if ("mappings".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); - } else { - Map mapping = parser.mapOrdered(); - if (mapping.size() == 1) { - String mappingType = mapping.keySet().iterator().next(); - builder.putMapping(new MappingMetaData(mappingType, mapping)); - } - } - } + ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn + parser.skipChildren(); + } else { + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } else if (token.isValue()) { builder.parseValuePart(parser, currentFieldName, null); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 5444c0482fc52..b9b27b1c026b5 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -348,9 +348,8 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont } for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { - builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE); + builder.field(cursor.key, XContentBuilder.FieldCaseConversion.NONE); IndexMetaData.FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - builder.endObject(); } builder.startObject("aliases"); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java new file mode 100644 index 0000000000000..ca1213da7d3fe --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.metadata; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.cluster.ClusterStatePart; +import org.elasticsearch.cluster.ClusterStatePart.Factory; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.cluster.MapClusterStatePart; +import org.elasticsearch.cluster.NamedClusterStatePart; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilder.FieldCaseConversion; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +public class MappingsMetaData extends MapClusterStatePart { + + public static String TYPE = "mappings"; + + public static Factory FACTORY = new Factory(); + + public MappingsMetaData(ImmutableOpenMap parts) { + super(TYPE, parts); + } + + public static class Factory extends MapClusterStatePart.Factory { + + public Factory() { + super(MappingMetaData.TYPE, MappingMetaData.FACTORY); + } + + @Override + public void toXContent(MapClusterStatePart part, XContentBuilder builder, Params params) throws IOException { + if (params.paramAsBoolean("reduce_mappings", false)) { + builder.startObject(); + for (ObjectObjectCursor cursor : part.parts()) { + MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); + } + builder.endObject(); + } else { + builder.startArray(); + for (ObjectObjectCursor cursor : part.parts()) { + MappingMetaData.FACTORY.toXContent(cursor.value, builder, params); + } + builder.endArray(); + } + } + + @Override + public MapClusterStatePart fromXContent(XContentParser parser, LocalContext context) throws IOException { + XContentParser.Token token = parser.currentToken(); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + String currentFieldName = null; + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + String mappingType = currentFieldName; + Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); + builder.put(mappingType, new MappingMetaData(mappingType, mappingSource)); + } + } + } else if (token == XContentParser.Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + MappingMetaData mappingMetaData = new MappingMetaData(new CompressedString(parser.binaryValue())); + builder.put(mappingMetaData.type(), mappingMetaData); + } else { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + builder.put(mappingType, new MappingMetaData(mappingType, mapping)); + } + } + } + } + return new MappingsMetaData(builder.build()); + } + } +} diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index 77888d2971341..e2fe04a503e4a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -177,9 +177,11 @@ public RepositoriesMetaData fromXContent(XContentParser parser, LocalContext con @Override public void toXContent(RepositoriesMetaData repositoriesMetaData, XContentBuilder builder, Params params) throws IOException { + builder.startObject(); for (RepositoryMetaData repository : repositoriesMetaData.repositories()) { RepositoriesMetaData.toXContent(repository, builder, params); } + builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java index 3bbeba3f24fe0..fed5fb75dc09d 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RestoreMetaData.java @@ -98,11 +98,13 @@ public void writeTo(RestoreMetaData restoreMetaData, StreamOutput out) throws IO */ @Override public void toXContent(RestoreMetaData restoreMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); builder.startArray("snapshots"); for (Entry entry : restoreMetaData.entries) { RestoreMetaData.toXContent(entry, builder, params); } builder.endArray(); + builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index fd1d7a0bccb81..07a38003db39e 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -210,13 +210,11 @@ public IndexWarmersMetaData fromXContent(XContentParser parser, LocalContext con @Override public void toXContent(IndexWarmersMetaData indexWarmersMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - //No need, IndexMetaData already writes it - //builder.startObject(TYPE, XContentBuilder.FieldCaseConversion.NONE); + builder.startObject(); for (Entry entry : indexWarmersMetaData.entries) { IndexWarmersMetaData.toXContent(entry, builder, params); } - //No need, IndexMetaData already writes it - //builder.endObject(); + builder.endObject(); } @Override diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index 29997b7dec157..06dc07851c441 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -729,7 +729,9 @@ public void writeTo(T part, StreamOutput out) throws IOException { @Override public void toXContent(T part, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); builder.field("data", part.getData()); + builder.endObject(); } @Override From c02ed4ab634ac77874a05be4a91d18a3294c4151 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 7 Jan 2015 09:25:03 -0500 Subject: [PATCH 17/20] Generalize XContent serialization in IndexMetaData --- .../NamedCompositeClusterStatePart.java | 11 ++-- .../cluster/metadata/IndexMetaData.java | 55 +------------------ .../cluster/metadata/SnapshotMetaData.java | 2 + .../metadata/ToAndFromJsonMetaDataTests.java | 2 +- 4 files changed, 11 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java index 6a679c5b444f2..f8c84f17bc3fa 100644 --- a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java @@ -164,7 +164,6 @@ public void writeTo(T part, StreamOutput out) throws IOException { @Override public T fromXContent(XContentParser parser, LocalContext context) throws IOException { - XContentParser.Token token; if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -172,11 +171,12 @@ public T fromXContent(XContentParser parser, LocalContext context) throws IOExce parser.nextToken(); } Builder builder = builder(parser.currentName()); - String currentFieldName = parser.currentName(); + String currentFieldName = null; + XContentParser.Token token = parser.nextToken(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { + } else if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) { // check if its a custom index metadata Factory factory = lookupFactory(currentFieldName); if (factory == null) { @@ -199,15 +199,16 @@ public T fromXContent(XContentParser parser, LocalContext context) throws IOExce @Override public void toXContent(T part, XContentBuilder builder, Params params) throws IOException { XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, XContentContext.API.toString())); + builder.startObject(part.key(), XContentBuilder.FieldCaseConversion.NONE); valuesPartToXContent(part, builder, params); for (ObjectObjectCursor cursor : part.parts) { Factory factory = lookupFactorySafe(cursor.key); if (factory.context().contains(context)) { - builder.startObject(cursor.key); + builder.field(cursor.key); factory.toXContent(cursor.value, builder, params); - builder.endObject(); } } + builder.endObject(); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index e90ed7c78c31b..b08a5c9fe67f5 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -95,11 +95,6 @@ public NamedCompositeClusterStatePart.Builder buildParts(Settin } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); - FACTORY.valuesPartToXContent(indexMetaData, builder, params); - for (ObjectObjectCursor cursor : indexMetaData.parts()) { - builder.field(cursor.key, FieldCaseConversion.NONE); - FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - } - builder.endObject(); + FACTORY.toXContent(indexMetaData, builder, params); } public static IndexMetaData fromXContent(XContentParser parser) throws IOException { - // TODO : switch to NamedCompositeClusterStatePart.AbstractFactory parser - if (parser.currentToken() == null) { // fresh parser? move to the first token - parser.nextToken(); - } - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { // on a start object move to next token - parser.nextToken(); - } - Builder builder = new Builder(parser.currentName()); - - String currentFieldName = null; - XContentParser.Token token = parser.nextToken(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); - if (factory == null) { - //TODO warn - parser.skipChildren(); - } else { - builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); - } - } else if (token == XContentParser.Token.START_ARRAY) { - ClusterStatePart.Factory factory = FACTORY.lookupFactory(currentFieldName); - if (factory == null) { - //TODO warn - parser.skipChildren(); - } else { - builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); - } - } else if (token.isValue()) { - builder.parseValuePart(parser, currentFieldName, null); - } - } - return builder.build(); + return FACTORY.fromXContent(parser, null); } public static IndexMetaData readFrom(StreamInput in) throws IOException { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java index 26af875212099..f3513a7752b7a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/SnapshotMetaData.java @@ -430,11 +430,13 @@ public void writeTo(SnapshotMetaData snapshotMetaData, StreamOutput out) throws @Override public void toXContent(SnapshotMetaData snapshotMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); builder.startArray(Fields.SNAPSHOTS); for (Entry entry : snapshotMetaData.entries) { SnapshotMetaData.toXContent(entry, builder, params); } builder.endArray(); + builder.endObject(); } diff --git a/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index d270402e10d0a..2cad0fd63566b 100644 --- a/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -146,7 +146,7 @@ public void testSimpleJsonFromAndTo() throws IOException { .build(); String metaDataSource = MetaData.Builder.toXContent(metaData); -// System.out.println("ToJson: " + metaDataSource); + logger.trace("ToJson: [{}]", metaDataSource); MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource)); From 4f7b5ed1772af35720cdcfeebfb6ee82573ee998 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 7 Jan 2015 12:07:41 -0500 Subject: [PATCH 18/20] Convert IndexTemplateMetaData to NamedCompositeClusterStatePart --- .../get/TransportGetIndexTemplatesAction.java | 2 +- .../cluster/ClusterStateSettingsPart.java | 4 +- .../metadata/CompressedMappingMetaData.java | 164 +++++++++++ .../metadata/IndexTemplateMetaData.java | 267 +++++++----------- .../metadata/MetaDataCreateIndexService.java | 31 +- .../search/warmer/IndexWarmersMetaData.java | 6 + .../warmer/GatewayIndicesWarmerTests.java | 4 +- 7 files changed, 292 insertions(+), 186 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index 32ed8cb213b49..e7fbbc368f2d6 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -85,7 +85,7 @@ protected void masterOperation(GetIndexTemplatesRequest request, ClusterState st results.add(entry.value); } } - } else if (state.metaData().templates().containsKey(name)) { + } else if (state.metaData().templates().containsKey(name)) { results.add(state.metaData().templates().get(name)); } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index 7f2c8e4e97a2b..a80f85b858dc3 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -96,9 +96,7 @@ public ClusterStateSettingsPart fromXContent(XContentParser parser, LocalContext @Override public void toXContent(ClusterStateSettingsPart clusterStateSettingsPart, XContentBuilder builder, Params params) throws IOException { builder.startObject(); - for (Map.Entry entry : clusterStateSettingsPart.settings.getAsMap().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } + clusterStateSettingsPart.settings.toXContent(builder, params); builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java new file mode 100644 index 0000000000000..0a01fdbb2cc8a --- /dev/null +++ b/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.metadata; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.cluster.AbstractClusterStatePart; +import org.elasticsearch.cluster.LocalContext; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.XContentParser.Token; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Map; + +/** + */ +public class CompressedMappingMetaData extends AbstractClusterStatePart implements IndexClusterStatePart { + + public static final String TYPE = "mappings"; + + public static final Factory FACTORY = new Factory(); + + @Override + public CompressedMappingMetaData mergeWith(CompressedMappingMetaData second) { + throw new UnsupportedOperationException("merge should occur in MetaDataCreateIndexService"); + } + + @Override + public String partType() { + return TYPE; + } + + public static class Factory extends AbstractClusterStatePart.AbstractFactory { + + @Override + public CompressedMappingMetaData readFrom(StreamInput in, LocalContext context) throws IOException { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + int mappingsSize = in.readVInt(); + for (int i = 0; i < mappingsSize; i++) { + builder.put(in.readString(), CompressedString.readCompressedString(in)); + } + return new CompressedMappingMetaData(builder.build()); + } + + @Override + public void writeTo(CompressedMappingMetaData mappings, StreamOutput out) throws IOException { + out.writeVInt(mappings.mappings().size()); + for (ObjectObjectCursor cursor : mappings.mappings()) { + out.writeString(cursor.key); + cursor.value.writeTo(out); + } + } + + @Override + public String partType() { + return TYPE; + } + + @Override + public void toXContent(CompressedMappingMetaData mappings, XContentBuilder builder, ToXContent.Params params) throws IOException { + if (params.paramAsBoolean("reduce_mappings", false)) { + builder.startObject(); + for (ObjectObjectCursor cursor : mappings.mappings()) { + byte[] mappingSource = cursor.value.uncompressed(); + XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); + Map mapping = parser.map(); + if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { + // the type name is the root value, reduce it + mapping = (Map) mapping.get(cursor.key); + } + builder.field(cursor.key); + builder.map(mapping); + } + builder.endObject(); + } else { + builder.startArray(); + for (ObjectObjectCursor cursor : mappings.mappings()) { + byte[] data = cursor.value.uncompressed(); + XContentParser parser = XContentFactory.xContent(data).createParser(data); + Map mapping = parser.mapOrderedAndClose(); + builder.map(mapping); + } + builder.endArray(); + } + } + + @Override + public CompressedMappingMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; + if (token == Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + String mappingType = currentFieldName; + Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); + builder.put(mappingType, new CompressedString(XContentFactory.jsonBuilder().map(mappingSource).string())); + } + } + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + Map mapping = parser.mapOrdered(); + if (mapping.size() == 1) { + String mappingType = mapping.keySet().iterator().next(); + String mappingSource = XContentFactory.jsonBuilder().map(mapping).string(); + + if (mappingSource == null) { + // crap, no mapping source, warn? + } else { + builder.put(mappingType, new CompressedString(mappingSource)); + } + } + } + } + return new CompressedMappingMetaData(builder.build()); + } + + public CompressedMappingMetaData fromOpenMap(ImmutableOpenMap map) { + return new CompressedMappingMetaData(map); + } + + @Override + public EnumSet context() { + return API_GATEWAY_SNAPSHOT; + } + } + + private ImmutableOpenMap mappings; + + public CompressedMappingMetaData(ImmutableOpenMap mappings) { + this.mappings = mappings; + } + + public ImmutableOpenMap getMappings() { + return mappings; + } + + public ImmutableOpenMap mappings() { + return mappings; + } +} diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index b9b27b1c026b5..15960b2905c32 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -18,14 +18,9 @@ */ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.Sets; -import org.elasticsearch.cluster.AbstractClusterStatePart; -import org.elasticsearch.cluster.LocalContext; -import org.elasticsearch.cluster.NamedClusterStatePart; +import org.elasticsearch.cluster.*; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -35,8 +30,8 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.warmer.IndexWarmersMetaData; import java.io.IOException; import java.util.EnumSet; @@ -46,32 +41,61 @@ /** * */ -public class IndexTemplateMetaData extends AbstractClusterStatePart implements NamedClusterStatePart{ +public class IndexTemplateMetaData extends NamedCompositeClusterStatePart implements NamedClusterStatePart { public static String TYPE = "template"; public static Factory FACTORY = new Factory(); - public static class Factory extends AbstractClusterStatePart.AbstractFactory { + public static final String SETTINGS_TYPE = "settings"; + + public static final ClusterStateSettingsPart.Factory SETTINGS_FACTORY = new ClusterStateSettingsPart.Factory(SETTINGS_TYPE, API_GATEWAY_SNAPSHOT); + + public static final String ALIASES_TYPE = "aliases"; + + public static final MapClusterStatePart.Factory ALIASES_FACTORY = new MapClusterStatePart.Factory<>(ALIASES_TYPE, AliasMetaData.FACTORY, API_GATEWAY_SNAPSHOT); + + static { + FACTORY.registerFactory(SETTINGS_TYPE, SETTINGS_FACTORY); + FACTORY.registerFactory(CompressedMappingMetaData.TYPE, CompressedMappingMetaData.FACTORY); + FACTORY.registerFactory(ALIASES_TYPE, ALIASES_FACTORY); + // register non plugin custom metadata + FACTORY.registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); + } + + public static class Factory extends NamedCompositeClusterStatePart.AbstractFactory { @Override - public IndexTemplateMetaData readFrom(StreamInput in, LocalContext context) throws IOException { - return Builder.readFrom(in); + public NamedCompositeClusterStatePart.Builder builder(String key) { + return new Builder(key); } @Override - public void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOutput out) throws IOException { - Builder.writeTo(indexTemplateMetaData, out); + public NamedCompositeClusterStatePart.Builder builder(IndexTemplateMetaData part) { + return new Builder(part); } @Override - public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { - return Builder.fromXContent(parser, parser.currentName()); + protected void valuesPartWriteTo(IndexTemplateMetaData indexTemplateMetaData, StreamOutput out) throws IOException { + out.writeInt(indexTemplateMetaData.order()); + out.writeString(indexTemplateMetaData.template()); } @Override - public void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, Params params) throws IOException { - Builder.toXContent(indexTemplateMetaData, builder, params); + protected void valuesPartToXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, Params params) throws IOException { + builder.field("order", indexTemplateMetaData.order()); + builder.field("template", indexTemplateMetaData.template()); + } + + @Override + public IndexTemplateMetaData fromXContent(XContentParser parser, LocalContext context) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { // on a start object move to next token + parser.nextToken(); + } + return Builder.fromXContent(parser, parser.currentName()); } @Override @@ -100,17 +124,14 @@ public String partType() { private final ImmutableOpenMap aliases; - private final ImmutableOpenMap customs; - - public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap mappings, - ImmutableOpenMap aliases, ImmutableOpenMap customs) { + public IndexTemplateMetaData(String name, int order, String template, ImmutableOpenMap parts) { + super(parts); this.name = name; this.order = order; this.template = template; - this.settings = settings; - this.mappings = mappings; - this.aliases = aliases; - this.customs = customs; + this.settings = parts.containsKey(SETTINGS_TYPE) ? ((ClusterStateSettingsPart)get(SETTINGS_TYPE)).getSettings() : ImmutableSettings.EMPTY; + this.mappings = parts.containsKey(CompressedMappingMetaData.TYPE) ? ((CompressedMappingMetaData)get(CompressedMappingMetaData.TYPE)).mappings() : ImmutableOpenMap.of(); + this.aliases = parts.containsKey(ALIASES_TYPE) ? ((MapClusterStatePart)get(ALIASES_TYPE)).parts() : ImmutableOpenMap.of(); } public String name() { @@ -161,19 +182,6 @@ public ImmutableOpenMap getAliases() { return this.aliases; } - public ImmutableOpenMap customs() { - return this.customs; - } - - public ImmutableOpenMap getCustoms() { - return this.customs; - } - - @SuppressWarnings("unchecked") - public T custom(String type) { - return (T) customs.get(type); - } - public static Builder builder(String name) { return new Builder(name); } @@ -204,7 +212,7 @@ public int hashCode() { return result; } - public static class Builder { + public static class Builder extends NamedCompositeClusterStatePart.Builder { private static final Set VALID_FIELDS = Sets.newHashSet("template", "order"); static { @@ -224,13 +232,10 @@ public static class Builder { private final ImmutableOpenMap.Builder aliases; - private final ImmutableOpenMap.Builder customs; - public Builder(String name) { this.name = name; mappings = ImmutableOpenMap.builder(); aliases = ImmutableOpenMap.builder(); - customs = ImmutableOpenMap.builder(); } public Builder(IndexTemplateMetaData indexTemplateMetaData) { @@ -238,10 +243,12 @@ public Builder(IndexTemplateMetaData indexTemplateMetaData) { order(indexTemplateMetaData.order()); template(indexTemplateMetaData.template()); settings(indexTemplateMetaData.settings()); - mappings = ImmutableOpenMap.builder(indexTemplateMetaData.mappings()); aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases()); - customs = ImmutableOpenMap.builder(indexTemplateMetaData.customs()); + parts.putAll(indexTemplateMetaData.parts()); + parts.remove(SETTINGS_TYPE); + parts.remove(CompressedMappingMetaData.TYPE); + parts.remove(ALIASES_TYPE); } public Builder order(int order) { @@ -258,6 +265,10 @@ public String template() { return template; } + public String getKey() { + return name; + } + public Builder settings(Settings.Builder settings) { this.settings = settings.build(); return this; @@ -294,71 +305,59 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) { } public Builder putCustom(String type, IndexClusterStatePart customIndexMetaData) { - this.customs.put(type, customIndexMetaData); + this.parts.put(type, customIndexMetaData); return this; } public Builder removeCustom(String type) { - this.customs.remove(type); + this.parts.remove(type); return this; } public IndexClusterStatePart getCustom(String type) { - return this.customs.get(type); + return this.parts.get(type); } public IndexTemplateMetaData build() { - return new IndexTemplateMetaData(name, order, template, settings, mappings.build(), aliases.build(), customs.build()); + return new IndexTemplateMetaData(name, order, template, buildParts(settings, mappings.build(), aliases.build(), parts.build())); } - @SuppressWarnings("unchecked") - public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(indexTemplateMetaData.name(), XContentBuilder.FieldCaseConversion.NONE); - - builder.field("order", indexTemplateMetaData.order()); - builder.field("template", indexTemplateMetaData.template()); - - builder.startObject("settings"); - indexTemplateMetaData.settings().toXContent(builder, params); - builder.endObject(); - - if (params.paramAsBoolean("reduce_mappings", false)) { - builder.startObject("mappings"); - for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { - byte[] mappingSource = cursor.value.uncompressed(); - XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); - Map mapping = parser.map(); - if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { - // the type name is the root value, reduce it - mapping = (Map) mapping.get(cursor.key); - } - builder.field(cursor.key); - builder.map(mapping); - } - builder.endObject(); - } else { - builder.startArray("mappings"); - for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { - byte[] data = cursor.value.uncompressed(); - XContentParser parser = XContentFactory.xContent(data).createParser(data); - Map mapping = parser.mapOrderedAndClose(); - builder.map(mapping); - } - builder.endArray(); + @Override + public void parseValuePart(XContentParser parser, String currentFieldName, LocalContext context) throws IOException { + if ("template".equals(currentFieldName)) { + template(parser.text()); + } else if ("order".equals(currentFieldName)) { + order(parser.intValue()); } + } - for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { - builder.field(cursor.key, XContentBuilder.FieldCaseConversion.NONE); - IndexMetaData.FACTORY.lookupFactorySafe(cursor.key).toXContent(cursor.value, builder, params); - } + @Override + public void readValuePartsFrom(StreamInput in, LocalContext context) throws IOException { + order(in.readInt()); + template(in.readString()); + } - builder.startObject("aliases"); - for (ObjectCursor cursor : indexTemplateMetaData.aliases().values()) { - AliasMetaData.Builder.toXContent(cursor.value, builder, params); - } - builder.endObject(); + @Override + public void writeValuePartsTo(StreamOutput out) throws IOException { + out.writeInt(order); + out.writeString(template); + } + + private static ImmutableOpenMap buildParts(Settings settings, + ImmutableOpenMap mappings, + ImmutableOpenMap aliases, + ImmutableOpenMap parts) { + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + builder.put(SETTINGS_TYPE, SETTINGS_FACTORY.fromSettings(settings)); + builder.put(CompressedMappingMetaData.TYPE, CompressedMappingMetaData.FACTORY.fromOpenMap(mappings)); + builder.put(ALIASES_TYPE, ALIASES_FACTORY.fromOpenMap(aliases)); + builder.putAll(parts); + return builder.build(); + } - builder.endObject(); + @SuppressWarnings("unchecked") + public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + FACTORY.toXContent(indexTemplateMetaData, builder, params); } public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException { @@ -380,22 +379,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t } } builder.settings(templateSettingsBuilder.build()); - } else if ("mappings".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - String mappingType = currentFieldName; - Map mappingSource = MapBuilder.newMapBuilder().put(mappingType, parser.mapOrdered()).map(); - builder.putMapping(mappingType, XContentFactory.jsonBuilder().map(mappingSource).string()); - } - } - } else if ("aliases".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); - } } else { - // check if its a custom index metadata IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactory(currentFieldName); if (factory == null) { //TODO warn @@ -405,27 +389,15 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t } } } else if (token == XContentParser.Token.START_ARRAY) { - if ("mappings".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - Map mapping = parser.mapOrdered(); - if (mapping.size() == 1) { - String mappingType = mapping.keySet().iterator().next(); - String mappingSource = XContentFactory.jsonBuilder().map(mapping).string(); - - if (mappingSource == null) { - // crap, no mapping source, warn? - } else { - builder.putMapping(mappingType, mappingSource); - } - } - } + IndexClusterStatePart.Factory factory = IndexTemplateMetaData.FACTORY.lookupFactory(currentFieldName); + if (factory == null) { + //TODO warn + parser.skipChildren(); + } else { + builder.putCustom(currentFieldName, factory.fromXContent(parser, null)); } } else if (token.isValue()) { - if ("template".equals(currentFieldName)) { - builder.template(parser.text()); - } else if ("order".equals(currentFieldName)) { - builder.order(parser.intValue()); - } + builder.parseValuePart(parser, currentFieldName, null); } } return builder.build(); @@ -450,50 +422,11 @@ private static String skipTemplateName(XContentParser parser) throws IOException } public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException { - Builder builder = new Builder(in.readString()); - builder.order(in.readInt()); - builder.template(in.readString()); - builder.settings(ImmutableSettings.readSettingsFromStream(in)); - int mappingsSize = in.readVInt(); - for (int i = 0; i < mappingsSize; i++) { - builder.putMapping(in.readString(), CompressedString.readCompressedString(in)); - } - int aliasesSize = in.readVInt(); - for (int i = 0; i < aliasesSize; i++) { - AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in); - builder.putAlias(aliasMd); - } - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - IndexClusterStatePart customIndexMetaData = IndexMetaData.FACTORY.lookupFactorySafe(type).readFrom(in, null); - builder.putCustom(type, customIndexMetaData); - } - return builder.build(); + return FACTORY.readFrom(in, null); } public static void writeTo(IndexTemplateMetaData indexTemplateMetaData, StreamOutput out) throws IOException { - out.writeString(indexTemplateMetaData.name()); - out.writeInt(indexTemplateMetaData.order()); - out.writeString(indexTemplateMetaData.template()); - ImmutableSettings.writeSettingsToStream(indexTemplateMetaData.settings(), out); - out.writeVInt(indexTemplateMetaData.mappings().size()); - for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { - out.writeString(cursor.key); - cursor.value.writeTo(out); - } - out.writeVInt(indexTemplateMetaData.aliases().size()); - for (ObjectCursor cursor : indexTemplateMetaData.aliases().values()) { - AliasMetaData.Builder.writeTo(cursor.value, out); - } - out.writeVInt(indexTemplateMetaData.customs().size()); - for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { - IndexClusterStatePart.Factory factory = IndexMetaData.FACTORY.lookupFactorySafe(cursor.key); - if (factory.addedIn().onOrAfter(out.getVersion())) { - out.writeString(cursor.key); - factory.writeTo(cursor.value, out); - } - } + FACTORY.writeTo(indexTemplateMetaData, out); } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index af36e6a712f0e..d2424fbdde805 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -89,7 +89,7 @@ /** * Service responsible for submitting create index requests */ -public class MetaDataCreateIndexService extends AbstractComponent { + public class MetaDataCreateIndexService extends AbstractComponent { public final static int MAX_INDEX_NAME_BYTES = 255; private static final DefaultIndexTemplateFilter DEFAULT_INDEX_TEMPLATE_FILTER = new DefaultIndexTemplateFilter(); @@ -274,18 +274,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { mappings.put(cursor.key, parseMapping(cursor.value.string())); } } - // handle custom - for (ObjectObjectCursor cursor : template.customs()) { - String type = cursor.key; - IndexClusterStatePart custom = cursor.value; - IndexClusterStatePart existing = customs.get(type); - if (existing == null) { - customs.put(type, custom); - } else { - IndexClusterStatePart merged = existing.mergeWith(custom); - customs.put(type, merged); - } - } //handle aliases for (ObjectObjectCursor cursor : template.aliases()) { AliasMetaData aliasMetaData = cursor.value; @@ -308,6 +296,23 @@ public ClusterState execute(ClusterState currentState) throws Exception { aliasValidator.validateAliasMetaData(aliasMetaData, request.index(), currentState.metaData()); templatesAliases.put(aliasMetaData.alias(), aliasMetaData); } + // handle custom + for (ObjectObjectCursor cursor : template.parts()) { + String type = cursor.key; + if (type.equals(ALIASES_TYPE) || type.equals(CompressedMappingMetaData.TYPE) || type.equals(SETTINGS_TYPE)) { + // We do special processing for aliases, settings and mappings - skipping + continue; + } + IndexClusterStatePart custom = cursor.value; + IndexClusterStatePart existing = customs.get(type); + if (existing == null) { + customs.put(type, custom); + } else { + IndexClusterStatePart merged = existing.mergeWith(custom); + customs.put(type, merged); + } + } + } // now add config level mappings diff --git a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 07a38003db39e..6afc573dcc4b1 100644 --- a/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -221,5 +222,10 @@ public void toXContent(IndexWarmersMetaData indexWarmersMetaData, XContentBuilde public String partType() { return TYPE; } + + @Override + public EnumSet context() { + return API_GATEWAY_SNAPSHOT; + } } } diff --git a/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java b/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java index 378fc8effc34c..1c2383cddbe96 100644 --- a/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java +++ b/src/test/java/org/elasticsearch/indices/warmer/GatewayIndicesWarmerTests.java @@ -84,7 +84,7 @@ public void testStatePersistence() throws Exception { assertThat(warmersMetaData, Matchers.notNullValue()); assertThat(warmersMetaData.entries().size(), equalTo(2)); - IndexWarmersMetaData templateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData templateWarmers = clusterState.metaData().templates().get("template_1").get(IndexWarmersMetaData.TYPE); assertThat(templateWarmers, Matchers.notNullValue()); assertThat(templateWarmers.entries().size(), equalTo(1)); @@ -108,7 +108,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { } logger.info("--> verify warmers in template are recovered"); - IndexWarmersMetaData recoveredTemplateWarmers = clusterState.metaData().templates().get("template_1").custom(IndexWarmersMetaData.TYPE); + IndexWarmersMetaData recoveredTemplateWarmers = clusterState.metaData().templates().get("template_1").get(IndexWarmersMetaData.TYPE); assertThat(recoveredTemplateWarmers.entries().size(), equalTo(templateWarmers.entries().size())); for (int i = 0; i < templateWarmers.entries().size(); i++) { assertThat(recoveredTemplateWarmers.entries().get(i).name(), equalTo(templateWarmers.entries().get(i).name())); From 447fce473050461c1051ddcccb840f4e7c30e035 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 8 Jan 2015 15:09:04 -0500 Subject: [PATCH 19/20] Add javadocs and general cleanup --- .../get/TransportGetIndexTemplatesAction.java | 2 +- .../org/elasticsearch/cluster/ClusterState.java | 11 ++++++----- .../elasticsearch/cluster/ClusterStatePart.java | 2 +- .../cluster/ClusterStateSettingsPart.java | 2 +- .../cluster/CompositeClusterStatePart.java | 14 +++++++++----- ...IncompatibleClusterStateVersionException.java | 5 +---- .../cluster/MapClusterStatePart.java | 7 ++++--- .../cluster/NamedClusterStatePart.java | 4 ++-- .../cluster/NamedCompositeClusterStatePart.java | 12 +++++++----- .../metadata/CompressedMappingMetaData.java | 3 ++- .../cluster/metadata/IndexMetaData.java | 11 ++++++----- .../cluster/metadata/IndexTemplateMetaData.java | 8 ++++---- .../cluster/metadata/MappingMetaData.java | 2 +- .../cluster/metadata/MappingsMetaData.java | 2 +- .../elasticsearch/cluster/metadata/MetaData.java | 16 ++++++++-------- .../settings/ClusterDynamicSettingsModule.java | 2 +- .../common/xcontent/ToXContent.java | 6 ++++++ .../discovery/DiscoverySettings.java | 13 +++++++++++++ .../discovery/zen/ZenDiscovery.java | 2 +- .../zen/publish/PublishClusterStateAction.java | 16 +++++----------- .../DedicatedClusterSnapshotRestoreTests.java | 10 +++++----- 21 files changed, 85 insertions(+), 65 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index e7fbbc368f2d6..32ed8cb213b49 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -85,7 +85,7 @@ protected void masterOperation(GetIndexTemplatesRequest request, ClusterState st results.add(entry.value); } } - } else if (state.metaData().templates().containsKey(name)) { + } else if (state.metaData().templates().containsKey(name)) { results.add(state.metaData().templates().get(name)); } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 8ac9fb8f1644b..9346a1bf0e3d0 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -80,11 +80,11 @@ public String partType() { } static { - FACTORY.registerFactory(ClusterName.TYPE, ClusterName.FACTORY); - FACTORY.registerFactory(DiscoveryNodes.TYPE, DiscoveryNodes.FACTORY); - FACTORY.registerFactory(ClusterBlocks.TYPE, ClusterBlocks.FACTORY); - FACTORY.registerFactory(RoutingTable.TYPE, RoutingTable.FACTORY); - FACTORY.registerFactory(MetaData.TYPE, MetaData.FACTORY); + FACTORY.registerFactory(ClusterName.FACTORY); + FACTORY.registerFactory(DiscoveryNodes.FACTORY); + FACTORY.registerFactory(ClusterBlocks.FACTORY); + FACTORY.registerFactory(RoutingTable.FACTORY); + FACTORY.registerFactory(MetaData.FACTORY); } public static class ClusterStateDiff { @@ -192,6 +192,7 @@ public ClusterBlocks getBlocks() { public ClusterName getClusterName() { return clusterName(); } + public ClusterName clusterName() { return this.clusterName; } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java index 7b7d867529a9c..6102a2b3d599b 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStatePart.java @@ -134,7 +134,7 @@ interface Diff { /** * Returns the new version of the part after applying differences */ - T apply(T part); + T apply(T part) throws IncompatibleClusterStateVersionException; } } diff --git a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java index a80f85b858dc3..8265a0f84803d 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterStateSettingsPart.java @@ -61,7 +61,7 @@ public String partType() { @Override public ClusterStateSettingsPart mergeWith(ClusterStateSettingsPart second) { - return second; + throw new UnsupportedOperationException("custom merge should occur in MetaDataCreateIndexService"); } public static class Factory extends AbstractClusterStatePart.AbstractFactory { diff --git a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java index 855b485a741c4..19e5bcc27095b 100644 --- a/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/CompositeClusterStatePart.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.builder.Diff; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -36,14 +37,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Maps.newHashMap; /** - * Represents a map of cluster state parts with different types. + * Base class for cluster state parts that consist of several other cluster state parts with different types. *

- * Only one instance of each type can be present in the composite part. The key of the map is the part's type. + * Only one instance of each type can be present in the composite part. Internally all parts are stored in a map. + * The key of the map is the part's type. */ public abstract class CompositeClusterStatePart extends AbstractClusterStatePart { @@ -100,12 +103,13 @@ public String uuid() { protected static abstract class AbstractCompositeFactory extends AbstractClusterStatePart.AbstractFactory { - private final Map partFactories = new HashMap<>(); + private final Map partFactories = new ConcurrentHashMap<>(); + /** * Register a custom index meta data factory. Make sure to call it from a static block. */ - public void registerFactory(String type, ClusterStatePart.Factory factory) { - partFactories.put(type, factory); + public void registerFactory(ClusterStatePart.Factory factory) { + partFactories.put(factory.partType(), factory); } @Nullable diff --git a/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java b/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java index 6a602e0102ff5..2e75e27ca46fc 100644 --- a/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java +++ b/src/main/java/org/elasticsearch/cluster/IncompatibleClusterStateVersionException.java @@ -22,13 +22,10 @@ import org.elasticsearch.ElasticsearchException; /** + * Thrown by {@link org.elasticsearch.cluster.ClusterStatePart.Diff#apply(ClusterStatePart)} method */ public class IncompatibleClusterStateVersionException extends ElasticsearchException { public IncompatibleClusterStateVersionException(String msg) { super(msg); } - - public IncompatibleClusterStateVersionException(String msg, Throwable cause) { - super(msg, cause); - } } diff --git a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java index b30fd466bf37c..cd67d8bd014d1 100644 --- a/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/MapClusterStatePart.java @@ -40,7 +40,9 @@ import static com.google.common.collect.Maps.newHashMap; /** - * Represents a map of cluster state parts with the same type. + * A cluster part that consists of a map of cluster state parts + * + * All cluster state part have to have the same type and implement {@link org.elasticsearch.cluster.NamedClusterStatePart} */ public class MapClusterStatePart extends AbstractClusterStatePart implements IndexClusterStatePart< MapClusterStatePart> { @@ -62,8 +64,7 @@ public T get(String type) { @Override public MapClusterStatePart mergeWith(MapClusterStatePart second) { - //TODO: Implement - return null; + throw new UnsupportedOperationException("custom merge should occur in MetaDataCreateIndexService"); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java index 3c6feadfb1a1e..0d39cfb070611 100644 --- a/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/NamedClusterStatePart.java @@ -20,9 +20,9 @@ package org.elasticsearch.cluster; /** - * All cluster state parts that can be put into a map should implement this interface + * All cluster state parts that can be put into a {@link org.elasticsearch.cluster.MapClusterStatePart} should + * implement this interface */ public interface NamedClusterStatePart extends ClusterStatePart { String key(); - } diff --git a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java index f8c84f17bc3fa..d9a89f63c6d4d 100644 --- a/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java +++ b/src/main/java/org/elasticsearch/cluster/NamedCompositeClusterStatePart.java @@ -22,6 +22,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -31,12 +32,13 @@ import java.io.IOException; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Maps.newHashMap; /** - * Represents a map of cluster state parts with different types. + * Base class for the composite cluster state part that can also be a part of MapClusterState part (IndexMetaData and IndexTemplateMetaData for example) *

* Only one instance of each type can be present in the composite part. The key of the map is the part's type. */ @@ -103,13 +105,13 @@ public void remove(String type) { public static abstract class AbstractFactory> extends AbstractClusterStatePart.AbstractFactory { - private final Map> partFactories = new HashMap<>(); + private final Map> partFactories = new ConcurrentHashMap<>(); /** - * Register a custom index meta data factory. Make sure to call it from a static block. + * Register a custom index meta data factory. */ - public void registerFactory(String type, Factory factory) { - partFactories.put(type, factory); + public void registerFactory(Factory factory) { + partFactories.put(factory.partType(), factory); } public Set availableFactories() { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java index 0a01fdbb2cc8a..0b644f89e3548 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/CompressedMappingMetaData.java @@ -34,6 +34,7 @@ import java.util.Map; /** + * Wrapper class for a map of compressed mappings that stored as a cluster state part */ public class CompressedMappingMetaData extends AbstractClusterStatePart implements IndexClusterStatePart { @@ -43,7 +44,7 @@ public class CompressedMappingMetaData extends AbstractClusterStatePart implemen @Override public CompressedMappingMetaData mergeWith(CompressedMappingMetaData second) { - throw new UnsupportedOperationException("merge should occur in MetaDataCreateIndexService"); + throw new UnsupportedOperationException("custom merge should occur in MetaDataCreateIndexService"); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b08a5c9fe67f5..9b425102e371b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -76,11 +76,12 @@ public class IndexMetaData extends NamedCompositeClusterStatePart ALIASES_FACTORY = new MapClusterStatePart.Factory<>(ALIASES_TYPE, AliasMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { - FACTORY.registerFactory(SETTINGS_TYPE, SETTINGS_FACTORY); - FACTORY.registerFactory(MappingsMetaData.TYPE, MappingsMetaData.FACTORY); - FACTORY.registerFactory(ALIASES_TYPE, ALIASES_FACTORY); - // register non plugin custom metadata - FACTORY.registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); + FACTORY.registerFactory(SETTINGS_FACTORY); + FACTORY.registerFactory(MappingsMetaData.FACTORY); + FACTORY.registerFactory(ALIASES_FACTORY); + + // register non plugin custom index metadata + FACTORY.registerFactory(IndexWarmersMetaData.FACTORY); } public static class Factory extends NamedCompositeClusterStatePart.AbstractFactory { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 15960b2905c32..e9e16a31996a9 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -56,11 +56,11 @@ public class IndexTemplateMetaData extends NamedCompositeClusterStatePart ALIASES_FACTORY = new MapClusterStatePart.Factory<>(ALIASES_TYPE, AliasMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { - FACTORY.registerFactory(SETTINGS_TYPE, SETTINGS_FACTORY); - FACTORY.registerFactory(CompressedMappingMetaData.TYPE, CompressedMappingMetaData.FACTORY); - FACTORY.registerFactory(ALIASES_TYPE, ALIASES_FACTORY); + FACTORY.registerFactory(SETTINGS_FACTORY); + FACTORY.registerFactory(CompressedMappingMetaData.FACTORY); + FACTORY.registerFactory(ALIASES_FACTORY); // register non plugin custom metadata - FACTORY.registerFactory(IndexWarmersMetaData.TYPE, IndexWarmersMetaData.FACTORY); + FACTORY.registerFactory(IndexWarmersMetaData.FACTORY); } public static class Factory extends NamedCompositeClusterStatePart.AbstractFactory { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index a6ef5af11ecdb..659f62c9cd7b3 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -64,7 +64,7 @@ public String key() { @Override public MappingMetaData mergeWith(MappingMetaData second) { - return null; + throw new UnsupportedOperationException("custom merge should occur in MetaDataCreateIndexService"); } public static class Factory extends AbstractClusterStatePart.AbstractFactory { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java index ca1213da7d3fe..71158c16d6279 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingsMetaData.java @@ -49,7 +49,7 @@ public MappingsMetaData(ImmutableOpenMap parts) { public static class Factory extends MapClusterStatePart.Factory { public Factory() { - super(MappingMetaData.TYPE, MappingMetaData.FACTORY); + super(TYPE, MappingMetaData.FACTORY); } @Override diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index fac4e9e86884b..c92b70591e496 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -105,16 +105,16 @@ public String partType() { public static final MapClusterStatePart.Factory TEMPLATES_FACTORY = new MapClusterStatePart.Factory<>(TEMPLATES_TYPE, IndexTemplateMetaData.FACTORY, API_GATEWAY_SNAPSHOT); static { - FACTORY.registerFactory(TRANSIENT_SETTINGS_TYPE, TRANSIENT_SETTINGS_FACTORY); - FACTORY.registerFactory(PERSISTENT_SETTINGS_TYPE, PERSISTENT_SETTINGS_FACTORY); - FACTORY.registerFactory(INDICES_TYPE, INDICES_FACTORY); - FACTORY.registerFactory(TEMPLATES_TYPE, TEMPLATES_FACTORY); + FACTORY.registerFactory(TRANSIENT_SETTINGS_FACTORY); + FACTORY.registerFactory(PERSISTENT_SETTINGS_FACTORY); + FACTORY.registerFactory(INDICES_FACTORY); + FACTORY.registerFactory(TEMPLATES_FACTORY); // register non plugin custom metadata - FACTORY.registerFactory(RepositoriesMetaData.TYPE, RepositoriesMetaData.FACTORY); - FACTORY.registerFactory(SnapshotMetaData.TYPE, SnapshotMetaData.FACTORY); - FACTORY.registerFactory(RestoreMetaData.TYPE, RestoreMetaData.FACTORY); - FACTORY.registerFactory(BenchmarkMetaData.TYPE, BenchmarkMetaData.FACTORY); + FACTORY.registerFactory(RepositoriesMetaData.FACTORY); + FACTORY.registerFactory(SnapshotMetaData.FACTORY); + FACTORY.registerFactory(RestoreMetaData.FACTORY); + FACTORY.registerFactory(BenchmarkMetaData.FACTORY); } public static final String SETTING_READ_ONLY = "cluster.blocks.read_only"; diff --git a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java index cb4aca624ea1c..c3a1a1a31b1e6 100644 --- a/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/cluster/settings/ClusterDynamicSettingsModule.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; -import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.indices.cache.filter.IndicesFilterCache; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -91,6 +90,7 @@ public ClusterDynamicSettingsModule() { clusterDynamicSettings.addDynamicSetting(SnapshotInProgressAllocationDecider.CLUSTER_ROUTING_ALLOCATION_SNAPSHOT_RELOCATION_ENABLED); clusterDynamicSettings.addDynamicSetting(DestructiveOperations.REQUIRES_NAME); clusterDynamicSettings.addDynamicSetting(DiscoverySettings.PUBLISH_TIMEOUT, Validator.TIME_NON_NEGATIVE); + clusterDynamicSettings.addDynamicSetting(DiscoverySettings.PUBLISH_DIFF_ENABLE, Validator.BOOLEAN); clusterDynamicSettings.addDynamicSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, Validator.MEMORY_SIZE); clusterDynamicSettings.addDynamicSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, Validator.MEMORY_SIZE); clusterDynamicSettings.addDynamicSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, Validator.NON_NEGATIVE_DOUBLE); diff --git a/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java index a892ab8be2373..d7a7e5fe1e030 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java +++ b/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java @@ -146,6 +146,9 @@ public Boolean paramAsBooleanOptional(String key, Boolean defaultValue) { } } + /** + * Delegating params that adds a single string param to existing params + */ public static class DelegatingStringParams implements Params { private final String key; @@ -190,6 +193,9 @@ public Boolean paramAsBooleanOptional(String key, Boolean defaultValue) { } } + /** + * Delegating params that adds a single boolean param to existing params + */ public static class DelegatingBooleanParams implements Params { private final String key; diff --git a/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java b/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java index b8d48b16129a9..3ef877f326141 100644 --- a/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java +++ b/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java @@ -38,16 +38,19 @@ public class DiscoverySettings extends AbstractComponent { public static final String PUBLISH_TIMEOUT = "discovery.zen.publish_timeout"; public static final String NO_MASTER_BLOCK = "discovery.zen.no_master_block"; + public static final String PUBLISH_DIFF_ENABLE = "discovery.zen.publish_diffs.enable"; public static final TimeValue DEFAULT_PUBLISH_TIMEOUT = TimeValue.timeValueSeconds(30); public static final String DEFAULT_NO_MASTER_BLOCK = "write"; public final static int NO_MASTER_BLOCK_ID = 2; + public final static boolean DEFAULT_PUBLISH_DIFF_ENABLE = true; public final static ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL); public final static ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, false, RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA)); private volatile ClusterBlock noMasterBlock; private volatile TimeValue publishTimeout = DEFAULT_PUBLISH_TIMEOUT; + private volatile boolean publishDiffs = DEFAULT_PUBLISH_DIFF_ENABLE; @Inject public DiscoverySettings(Settings settings, NodeSettingsService nodeSettingsService) { @@ -55,6 +58,7 @@ public DiscoverySettings(Settings settings, NodeSettingsService nodeSettingsServ nodeSettingsService.addListener(new ApplySettings()); this.noMasterBlock = parseNoMasterBlock(settings.get(NO_MASTER_BLOCK, DEFAULT_NO_MASTER_BLOCK)); this.publishTimeout = settings.getAsTime(PUBLISH_TIMEOUT, publishTimeout); + this.publishDiffs = settings.getAsBoolean(PUBLISH_DIFF_ENABLE, DEFAULT_PUBLISH_DIFF_ENABLE); } /** @@ -68,6 +72,8 @@ public ClusterBlock getNoMasterBlock() { return noMasterBlock; } + public boolean getPublishDiffs() { return publishDiffs;} + private class ApplySettings implements NodeSettingsService.Listener { @Override public void onRefreshSettings(Settings settings) { @@ -85,6 +91,13 @@ public void onRefreshSettings(Settings settings) { noMasterBlock = newNoMasterBlock; } } + Boolean newPublishDiffs = settings.getAsBoolean(PUBLISH_DIFF_ENABLE, null); + if (newPublishDiffs != null) { + if (newPublishDiffs != publishDiffs) { + logger.info("updating [{}] from [{}] to [{}]", PUBLISH_DIFF_ENABLE, publishDiffs, newPublishDiffs); + publishDiffs = newPublishDiffs; + } + } } } diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index 0be1d6c089bfa..1e35a24404427 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -192,7 +192,7 @@ public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threa this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterName); this.nodesFD.addListener(new NodeFaultDetectionListener()); - this.publishClusterState = new PublishClusterStateAction(settings, transportService, this, new NewClusterStateListener(), discoverySettings, clusterName); + this.publishClusterState = new PublishClusterStateAction(settings, transportService, this, new NewClusterStateListener(), discoverySettings); this.pingService.setPingContextProvider(this); this.membership = new MembershipAction(settings, clusterService, transportService, this, new MembershipListener()); diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 58e8345dbd291..fc1f29729d649 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -19,15 +19,12 @@ package org.elasticsearch.discovery.zen.publish; -import com.google.common.collect.Maps; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState.ClusterStateDiff; import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; @@ -46,7 +43,6 @@ import java.util.Iterator; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -75,19 +71,17 @@ static interface NewStateProcessed { private final DiscoveryNodesProvider nodesProvider; private final NewClusterStateListener listener; private final DiscoverySettings discoverySettings; - private final ClusterName clusterName; private final Map lastFullVersionSent = newHashMap(); private final Lock versionMapLock = new ReentrantLock(); private volatile ClusterState lastProcessedClusterState; public PublishClusterStateAction(Settings settings, TransportService transportService, DiscoveryNodesProvider nodesProvider, - NewClusterStateListener listener, DiscoverySettings discoverySettings, ClusterName clusterName) { + NewClusterStateListener listener, DiscoverySettings discoverySettings) { super(settings); this.transportService = transportService; this.nodesProvider = nodesProvider; this.listener = listener; this.discoverySettings = discoverySettings; - this.clusterName = clusterName; transportService.registerHandler(ACTION_NAME, new PublishClusterStateRequestHandler()); lastProcessedClusterState = null; } @@ -109,7 +103,8 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR final AtomicBoolean timedOutWaitingForNodes = new AtomicBoolean(false); final TimeValue publishTimeout = discoverySettings.getPublishTimeout(); - final boolean sendFullVersion = lastProcessedClusterState == null || clusterState.version() - lastProcessedClusterState.version() != 1; + final boolean sendFullVersion = !discoverySettings.getPublishDiffs() || + lastProcessedClusterState == null || clusterState.version() - lastProcessedClusterState.version() != 1; // Remove nodes that are no longer part of the cluster versionMapLock.lock(); @@ -275,8 +270,7 @@ public void handleException(TransportException exp) { publishResponseHandler.onFailure(node, t); } } else { - logger.debug("skipping sending version {} to node {}, node already received version {}", clusterStateToSend.version(), node, lastVersionSent); - // TODO: Should this be a failure since we are not really sure if this node got the cluster state or not at this moment + logger.debug("skipping sending diffs for the version {} to the node {}, the node already was sent the full version {}", clusterStateToSend.version(), node, lastVersionSent); publishResponseHandler.onResponse(node); } } @@ -324,7 +318,7 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); logger.debug("received full cluster state version {} with size {}", clusterState.version(), request.bytes().length()); } else { - if(lastProcessedClusterState == null) { + if (lastProcessedClusterState == null) { logger.debug("received diff cluster state version {} but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); } diff --git a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java index 06dc07851c441..2655b4b6b2ce6 100644 --- a/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java @@ -763,11 +763,11 @@ public T fromXContent(XContentParser parser, LocalContext context) throws IOExce } static { - MetaData.FACTORY.registerFactory(SnapshottableMetadata.TYPE, SnapshottableMetadata.FACTORY); - MetaData.FACTORY.registerFactory(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.FACTORY); - MetaData.FACTORY.registerFactory(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.FACTORY); - MetaData.FACTORY.registerFactory(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.FACTORY); - MetaData.FACTORY.registerFactory(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.FACTORY); + MetaData.FACTORY.registerFactory(SnapshottableMetadata.FACTORY); + MetaData.FACTORY.registerFactory(NonSnapshottableMetadata.FACTORY); + MetaData.FACTORY.registerFactory(SnapshottableGatewayMetadata.FACTORY); + MetaData.FACTORY.registerFactory(NonSnapshottableGatewayMetadata.FACTORY); + MetaData.FACTORY.registerFactory(SnapshotableGatewayNoApiMetadata.FACTORY); } public static class SnapshottableMetadata extends TestCustomMetaData { From 04161516fac1da82cf8ecb26e709c6c4c3a5994a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 15 Jan 2015 22:19:54 -0500 Subject: [PATCH 20/20] Switch from versions to uuid for resend logic Changes after the first round of review by @bleskes --- .../elasticsearch/cluster/ClusterState.java | 27 ++- .../service/InternalClusterService.java | 2 +- .../elasticsearch/discovery/Discovery.java | 4 +- .../discovery/DiscoveryService.java | 6 +- .../discovery/local/LocalDiscovery.java | 15 +- .../discovery/zen/ZenDiscovery.java | 8 +- .../publish/PublishClusterStateAction.java | 178 +++++++----------- 7 files changed, 105 insertions(+), 135 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 9346a1bf0e3d0..dca1ee53b9d87 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -88,11 +88,13 @@ public String partType() { } public static class ClusterStateDiff { - private long version; + private String previousUuid; + private String uuid; private ClusterState.Diff diff; - public ClusterStateDiff(long version, ClusterState.Diff diff) { - this.version = version; + public ClusterStateDiff(String previousUuid, String uuid, ClusterState.Diff diff) { + this.previousUuid = previousUuid; + this.uuid = uuid; this.diff = diff; } @@ -100,9 +102,14 @@ public ClusterState apply(ClusterState previous) throws IncompatibleClusterState return diff.apply(previous); } - public long version() { - return version; + public String uuid() { + return uuid; } + + public String previousUuid() { + return previousUuid; + } + } public static final long UNKNOWN_VERSION = -1; @@ -460,18 +467,20 @@ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode loca } public static ClusterStateDiff readDiffFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { - long version = in.readVLong(); + String previousUuid = in.readString(); + String uuid = in.readString(); LocalContext localContext = new LocalContext(localNode); - return new ClusterStateDiff(version, FACTORY.readDiffFrom(in, localContext)); + return new ClusterStateDiff(previousUuid, uuid, FACTORY.readDiffFrom(in, localContext)); } public static void writeDiffTo(ClusterStateDiff diff, StreamOutput out) throws IOException { - out.writeVLong(diff.version); + out.writeString(diff.previousUuid); + out.writeString(diff.uuid); FACTORY.writeDiffsTo(diff.diff, out); } public static ClusterStateDiff diff(ClusterState before, ClusterState after) { - return new ClusterStateDiff(after.version(), ClusterState.FACTORY.diff(before, after) ); + return new ClusterStateDiff(before.uuid(), after.uuid(), ClusterState.FACTORY.diff(before, after) ); } public static byte[] toDiffBytes(ClusterState before, ClusterState after) throws IOException { diff --git a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index c21d9b0b17c9e..50afad946acdf 100644 --- a/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -434,7 +434,7 @@ public void run() { // we don't want to notify if (newClusterState.nodes().localNodeMaster()) { logger.debug("publishing cluster state version {}", newClusterState.version()); - discoveryService.publish(newClusterState, ackListener); + discoveryService.publish(clusterChangedEvent, ackListener); } // update the current cluster state diff --git a/src/main/java/org/elasticsearch/discovery/Discovery.java b/src/main/java/org/elasticsearch/discovery/Discovery.java index dfd51e6348f07..9ce93d3517daf 100644 --- a/src/main/java/org/elasticsearch/discovery/Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/Discovery.java @@ -19,7 +19,7 @@ package org.elasticsearch.discovery; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.Nullable; @@ -59,7 +59,7 @@ public interface Discovery extends LifecycleComponent { * The {@link AckListener} allows to keep track of the ack received from nodes, and verify whether * they updated their own cluster state or not. */ - void publish(ClusterState clusterState, AckListener ackListener); + void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener); public static interface AckListener { void onNodeAck(DiscoveryNode node, @Nullable Throwable t); diff --git a/src/main/java/org/elasticsearch/discovery/DiscoveryService.java b/src/main/java/org/elasticsearch/discovery/DiscoveryService.java index f73f2bbb5939b..8c8ee99950938 100644 --- a/src/main/java/org/elasticsearch/discovery/DiscoveryService.java +++ b/src/main/java/org/elasticsearch/discovery/DiscoveryService.java @@ -21,7 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; @@ -132,9 +132,9 @@ public String nodeDescription() { * The {@link org.elasticsearch.discovery.Discovery.AckListener} allows to acknowledge the publish * event based on the response gotten from all nodes */ - public void publish(ClusterState clusterState, Discovery.AckListener ackListener) { + public void publish(ClusterChangedEvent clusterChangedEvent, Discovery.AckListener ackListener) { if (lifecycle.started()) { - discovery.publish(clusterState, ackListener); + discovery.publish(clusterChangedEvent, ackListener); } } diff --git a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index d44e3b0393ec1..5961f657992e1 100644 --- a/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -76,8 +76,6 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem private static final ConcurrentMap clusterGroups = ConcurrentCollections.newConcurrentMap(); - private volatile ClusterState lastProcessedClusterState; - @Inject public LocalDiscovery(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService, DiscoveryNodeService discoveryNodeService, Version version, DiscoverySettings discoverySettings) { @@ -275,13 +273,13 @@ public String nodeDescription() { return clusterName.value() + "/" + localNode.id(); } - public void publish(ClusterState clusterState, final Discovery.AckListener ackListener) { + public void publish(ClusterChangedEvent clusterChangedEvent, final Discovery.AckListener ackListener) { if (!master) { throw new ElasticsearchIllegalStateException("Shouldn't publish state when not master"); } LocalDiscovery[] members = members(); if (members.length > 0) { - publish(members, clusterState, new AckClusterStatePublishResponseHandler(members.length - 1, ackListener)); + publish(members, clusterChangedEvent, new AckClusterStatePublishResponseHandler(members.length - 1, ackListener)); } } @@ -294,21 +292,22 @@ private LocalDiscovery[] members() { return members.toArray(new LocalDiscovery[members.size()]); } - private void publish(LocalDiscovery[] members, ClusterState clusterState, final ClusterStatePublishResponseHandler publishResponseHandler) { + private void publish(LocalDiscovery[] members, ClusterChangedEvent clusterChangedEvent, final ClusterStatePublishResponseHandler publishResponseHandler) { try { byte[] clusterStateBytes = null; byte[] clusterStateDiffBytes = null; + ClusterState clusterState = clusterChangedEvent.state(); for (final LocalDiscovery discovery : members) { if (discovery.master) { continue; } ClusterState newNodeSpecificClusterState = null; // we do the marshaling intentionally, to check it works well... - if (lastProcessedClusterState != null && lastProcessedClusterState.nodes().nodeExists(discovery.localNode.id())) { + if (clusterChangedEvent.previousState() != null && clusterChangedEvent.previousState().nodes().nodeExists(discovery.localNode.id())) { if (clusterStateDiffBytes == null) { - clusterStateDiffBytes = Builder.toDiffBytes(lastProcessedClusterState, clusterState); + clusterStateDiffBytes = Builder.toDiffBytes(clusterChangedEvent.previousState(), clusterState); } ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(new BytesStreamInput(clusterStateDiffBytes, false), discovery.localNode); try { @@ -386,8 +385,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS Thread.currentThread().interrupt(); } } - lastProcessedClusterState = clusterState; - } catch (Exception e) { // failure to marshal or un-marshal throw new ElasticsearchIllegalStateException("Cluster state failed to serialize", e); diff --git a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index 1e35a24404427..8cd6fb63c545e 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -334,12 +334,12 @@ public boolean nodeHasJoinedClusterOnce() { @Override - public void publish(ClusterState clusterState, AckListener ackListener) { - if (!clusterState.getNodes().localNodeMaster()) { + public void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener) { + if (!clusterChangedEvent.state().getNodes().localNodeMaster()) { throw new ElasticsearchIllegalStateException("Shouldn't publish state when not master"); } - nodesFD.updateNodesAndPing(clusterState); - publishClusterState.publish(clusterState, ackListener); + nodesFD.updateNodesAndPing(clusterChangedEvent.state()); + publishClusterState.publish(clusterChangedEvent, ackListener); } /** diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index fc1f29729d649..4fedb5d4ad14a 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.discovery.zen.publish; import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState.ClusterStateDiff; import org.elasticsearch.cluster.IncompatibleClusterStateVersionException; @@ -40,7 +41,6 @@ import org.elasticsearch.transport.*; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; @@ -71,9 +71,6 @@ static interface NewStateProcessed { private final DiscoveryNodesProvider nodesProvider; private final NewClusterStateListener listener; private final DiscoverySettings discoverySettings; - private final Map lastFullVersionSent = newHashMap(); - private final Lock versionMapLock = new ReentrantLock(); - private volatile ClusterState lastProcessedClusterState; public PublishClusterStateAction(Settings settings, TransportService transportService, DiscoveryNodesProvider nodesProvider, NewClusterStateListener listener, DiscoverySettings discoverySettings) { @@ -83,18 +80,17 @@ public PublishClusterStateAction(Settings settings, TransportService transportSe this.listener = listener; this.discoverySettings = discoverySettings; transportService.registerHandler(ACTION_NAME, new PublishClusterStateRequestHandler()); - lastProcessedClusterState = null; } public void close() { transportService.removeHandler(ACTION_NAME); } - public void publish(ClusterState clusterState, final Discovery.AckListener ackListener) { - publish(clusterState, new AckClusterStatePublishResponseHandler(clusterState.nodes().size() - 1, ackListener)); + public void publish(ClusterChangedEvent clusterChangedEvent, final Discovery.AckListener ackListener) { + publish(clusterChangedEvent, new AckClusterStatePublishResponseHandler(clusterChangedEvent.state().nodes().size() - 1, ackListener)); } - private void publish(final ClusterState clusterState, final ClusterStatePublishResponseHandler publishResponseHandler) { + private void publish(final ClusterChangedEvent clusterChangedEvent, final ClusterStatePublishResponseHandler publishResponseHandler) { DiscoveryNode localNode = nodesProvider.nodes().localNode(); @@ -103,21 +99,8 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR final AtomicBoolean timedOutWaitingForNodes = new AtomicBoolean(false); final TimeValue publishTimeout = discoverySettings.getPublishTimeout(); - final boolean sendFullVersion = !discoverySettings.getPublishDiffs() || - lastProcessedClusterState == null || clusterState.version() - lastProcessedClusterState.version() != 1; - - // Remove nodes that are no longer part of the cluster - versionMapLock.lock(); - try { - Iterator iterator = lastFullVersionSent.keySet().iterator(); - while (iterator.hasNext()) { - if (!clusterState.nodes().nodeExists(iterator.next())) { - iterator.remove(); - } - } - } finally { - versionMapLock.unlock(); - } + final boolean sendFullVersion = !discoverySettings.getPublishDiffs() || clusterChangedEvent.previousState() == null; + final ClusterState clusterState = clusterChangedEvent.state(); for (final DiscoveryNode node : clusterState.nodes()) { if (node.equals(localNode)) { @@ -126,14 +109,8 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR // try and serialize the cluster state once (or per version), so we don't serialize it // per node when we send it over the wire, compress it while we are at it... BytesReference bytes; - boolean newlyAddedNode = !lastProcessedClusterState.nodes().nodeExists(node.id()); + boolean newlyAddedNode = !clusterChangedEvent.previousState().nodes().nodeExists(node.id()); if (sendFullVersion || newlyAddedNode) { - versionMapLock.lock(); - try { - lastFullVersionSent.put(node.getId(), clusterState.version()); - } finally { - versionMapLock.unlock(); - } bytes = serializedStates.get(node.version()); if (bytes == null) { try { @@ -149,7 +126,7 @@ private void publish(final ClusterState clusterState, final ClusterStatePublishR bytes = serializedDiffs.get(node.version()); if (bytes == null) { try { - bytes = serializeDiffClusterState(clusterState, node.version()); + bytes = serializeDiffClusterState(clusterChangedEvent, node.version()); serializedDiffs.put(node.version(), bytes); } catch (Throwable e) { logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); @@ -192,7 +169,6 @@ public void handleException(TransportException exp) { publishResponseHandler.onFailure(node, t); } } - lastProcessedClusterState = clusterState; if (publishTimeout.millis() > 0) { // only wait if the publish timeout is configured... try { @@ -209,69 +185,41 @@ public void handleException(TransportException exp) { private void resendFullClusterState(final ClusterState clusterState, final DiscoveryNode node, final AtomicBoolean timedOutWaitingForNodes, final ClusterStatePublishResponseHandler publishResponseHandler) { final TimeValue publishTimeout = discoverySettings.getPublishTimeout(); - final ClusterState currentClusterState = lastProcessedClusterState; - final ClusterState clusterStateToSend; - if (currentClusterState == null || currentClusterState.version() < clusterState.version()) { - clusterStateToSend = clusterState; - } else { - clusterStateToSend = currentClusterState; - } - boolean shouldSendAnUpdate = true; - final Long lastVersionSent; - versionMapLock.lock(); + BytesReference bytes; try { - lastVersionSent = lastFullVersionSent.get(node.getId()); - if (lastVersionSent != null && clusterStateToSend.version() <= lastVersionSent) { - //we already sent a newer full version of cluster state to this node - no reason to send it again - shouldSendAnUpdate = false; - } else { - lastFullVersionSent.put(node.getId(), clusterStateToSend.version()); - } - } finally { - versionMapLock.unlock(); + bytes = serializeFullClusterState(clusterState, node.version()); + } catch (Throwable e) { + logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); + publishResponseHandler.onFailure(node, e); + return; } - - if (shouldSendAnUpdate) { - - BytesReference bytes; - try { - bytes = serializeFullClusterState(clusterStateToSend, node.version()); - } catch (Throwable e) { - logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node); - publishResponseHandler.onFailure(node, e); - return; - } - try { - TransportRequestOptions options = TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withCompress(false); - // no need to put a timeout on the options here, because we want the response to eventually be received - // and not log an error if it arrives after the timeout - transportService.sendRequest(node, ACTION_NAME, - new BytesTransportRequest(bytes, node.version()), - options, // no need to compress, we already compressed the bytes - - new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { - - @Override - public void handleResponse(TransportResponse.Empty response) { - if (timedOutWaitingForNodes.get()) { - logger.debug("node {} responded for cluster state [{}] (took longer than [{}])", node, clusterState.version(), publishTimeout); - } - publishResponseHandler.onResponse(node); - } - - @Override - public void handleException(TransportException exp) { - logger.debug("failed to send cluster state to {}", exp, node); - publishResponseHandler.onFailure(node, exp); + try { + TransportRequestOptions options = TransportRequestOptions.options().withType(TransportRequestOptions.Type.STATE).withCompress(false); + // no need to put a timeout on the options here, because we want the response to eventually be received + // and not log an error if it arrives after the timeout + transportService.sendRequest(node, ACTION_NAME, + new BytesTransportRequest(bytes, node.version()), + options, // no need to compress, we already compressed the bytes + + new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + + @Override + public void handleResponse(TransportResponse.Empty response) { + if (timedOutWaitingForNodes.get()) { + logger.debug("node {} responded for cluster state [{}] (took longer than [{}])", node, clusterState.version(), publishTimeout); } - }); - } catch (Throwable t) { - logger.debug("error sending cluster state to {}", t, node); - publishResponseHandler.onFailure(node, t); - } - } else { - logger.debug("skipping sending diffs for the version {} to the node {}, the node already was sent the full version {}", clusterStateToSend.version(), node, lastVersionSent); - publishResponseHandler.onResponse(node); + publishResponseHandler.onResponse(node); + } + + @Override + public void handleException(TransportException exp) { + logger.debug("failed to send cluster state to {}", exp, node); + publishResponseHandler.onFailure(node, exp); + } + }); + } catch (Throwable t) { + logger.debug("error sending cluster state to {}", t, node); + publishResponseHandler.onFailure(node, t); } } @@ -285,18 +233,21 @@ private BytesReference serializeFullClusterState(ClusterState clusterState, Vers return bStream.bytes(); } - private BytesReference serializeDiffClusterState(ClusterState clusterState, Version nodeVersion) throws IOException { + private BytesReference serializeDiffClusterState(ClusterChangedEvent clusterChangedEvent, Version nodeVersion) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); StreamOutput stream = new HandlesStreamOutput(CompressorFactory.defaultCompressor().streamOutput(bStream)); stream.setVersion(nodeVersion); stream.writeBoolean(false); - ClusterStateDiff diff = ClusterState.Builder.diff(lastProcessedClusterState, clusterState); + ClusterStateDiff diff = ClusterState.Builder.diff(clusterChangedEvent.previousState(), clusterChangedEvent.state()); ClusterState.Builder.writeDiffTo(diff, stream); stream.close(); return bStream.bytes(); } private class PublishClusterStateRequestHandler extends BaseTransportRequestHandler { + private volatile ClusterState lastProcessedClusterState; + + private final Lock lock = new ReentrantLock(); @Override public BytesTransportRequest newInstance() { @@ -313,27 +264,40 @@ public void messageReceived(BytesTransportRequest request, final TransportChanne in = CachedStreamInput.cachedHandles(request.bytes().streamInput()); } in.setVersion(request.version()); - final ClusterState clusterState; + ClusterState clusterState = null; + ClusterState.ClusterStateDiff diff = null; + if (in.readBoolean()) { clusterState = ClusterState.Builder.readFrom(in, nodesProvider.nodes().localNode()); logger.debug("received full cluster state version {} with size {}", clusterState.version(), request.bytes().length()); } else { - if (lastProcessedClusterState == null) { - logger.debug("received diff cluster state version {} but don't have any local cluster state - requesting full state"); - throw new IncompatibleClusterStateVersionException("have no local cluster state"); - } - ClusterState.ClusterStateDiff diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode()); - if (lastProcessedClusterState.version() >= diff.version()) { - logger.debug("got diffs for obsolete version {}, current version {}, ignoring the diff", diff.version(), lastProcessedClusterState.version()); - return; + diff = ClusterState.Builder.readDiffFrom(in, nodesProvider.nodes().localNode()); + } + + lock.lock(); + try { + if (clusterState == null) { + if (lastProcessedClusterState == null) { + logger.debug("received diffs for {}..{} but don't have any local cluster state - requesting full state", diff.previousUuid(), diff.uuid()); + throw new IncompatibleClusterStateVersionException("have no local cluster state"); + } else if (lastProcessedClusterState.uuid().equals(diff.uuid())) { + logger.debug("received diffs {}..{}, already at {} and version {} ignoring the diff, reprocessing the cluster state", diff.previousUuid(), diff.uuid(), lastProcessedClusterState.uuid(), lastProcessedClusterState.version()); + } else if (!lastProcessedClusterState.uuid().equals(diff.previousUuid())) { + logger.debug("received diffs {}..{}, which are incompatible with current uuid {} version {}, requesting full state", diff.previousUuid(), diff.uuid(), lastProcessedClusterState.uuid(), lastProcessedClusterState.version()); + throw new IncompatibleClusterStateVersionException("Expected diffs for version " + lastProcessedClusterState.version() + " with uuid " + lastProcessedClusterState.uuid() + " got uuid " + diff.previousUuid()); + } else { + clusterState = diff.apply(lastProcessedClusterState); + logger.debug("received diff cluster diffs {}..{} with version {}, diff size {}", diff.previousUuid(), diff.uuid(), clusterState.version(), request.bytes().length()); + lastProcessedClusterState = clusterState; + } + } else { + lastProcessedClusterState = clusterState; } - clusterState = diff.apply(lastProcessedClusterState); - logger.debug("received diff cluster state version {} with size {}", clusterState.version(), request.bytes().length()); + } finally { + lock.unlock(); } - lastProcessedClusterState = clusterState; clusterState.status(ClusterState.ClusterStateStatus.RECEIVED); -// logger.debug("received cluster state version {}", clusterState.version()); listener.onNewClusterState(clusterState, new NewClusterStateListener.NewStateProcessed() { @Override public void onNewClusterStateProcessed() {