Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove PROTO-serialization from IndexMetaData.Custom #32749

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
3f5d717
Remove PROTO-serialization from IndexMetaData.Custom (II)
imotov Aug 8, 2018
b8239f5
Remove automatic xcontent registry discovery
imotov Aug 14, 2018
9986710
Address @dakrone's review comments
imotov Aug 20, 2018
d85bd10
Merge remote-tracking branch 'elastic/master' into remove-proto-from-…
imotov Aug 20, 2018
290afc4
Merge remote-tracking branch 'origin/master' into remove-proto-from-i…
dakrone Aug 27, 2018
c1fc780
WIP
dakrone Aug 28, 2018
4a95041
Merge remote-tracking branch 'origin/master' into remove-proto-from-i…
dakrone Aug 28, 2018
df0f86a
Remove more vestiges
dakrone Aug 28, 2018
f4831f2
Fix some things
dakrone Aug 28, 2018
5a471f2
Fix a test
dakrone Aug 28, 2018
08d3a50
AwaitsFix a test until we decide
dakrone Aug 28, 2018
b8c9628
Remove most of the REST-facing custom metadata parts
dakrone Aug 28, 2018
db11af1
More removals
dakrone Aug 28, 2018
ea49b8c
Merge remote-tracking branch 'origin/master' into remove-proto-from-i…
dakrone Aug 29, 2018
cd46e81
Implement diffs for DiffableStringMap
dakrone Aug 29, 2018
af115fd
Fix NativeRolesStoreTests
dakrone Aug 29, 2018
cdb21f9
Fix serialization for pre-7.x versions
dakrone Aug 29, 2018
65c8165
Simplify DiffableStringMap
dakrone Aug 30, 2018
2643723
Return an immutable map from `getCustomData(String)`
dakrone Aug 30, 2018
3d15e60
Remove newline from SPI file
dakrone Aug 30, 2018
bf50d45
Remove logging from test
dakrone Aug 30, 2018
6523d40
Add random diffing and serialization test
dakrone Aug 30, 2018
81be184
Make receiving custom metadata throw real exception
dakrone Aug 30, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -141,4 +141,15 @@ public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentPa
return categoryClass.cast(entry.parser.parse(parser, context));
}

/**
* Returns true if the given named object is supported by the current stream
*/
public <T> boolean supportsNamedObject(Class<T> categoryClass, String name) {
Map<String, Entry> parsers = registry.get(categoryClass);
if (parsers == null || registry.isEmpty()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the registry.isEmpty() check is superfluous here

return false;
}
return parsers.containsKey(name);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedObjectNotFoundException;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -58,9 +59,9 @@
import java.util.Set;

import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;

/**
* A request to create an index. Best created with {@link org.elasticsearch.client.Requests#createIndexRequest(String)}.
Expand Down Expand Up @@ -348,6 +349,14 @@ public CreateIndexRequest source(XContentBuilder source) {
return source(BytesReference.bytes(source), source.contentType());
}


/**
* Sets the settings and mappings as a single source.
*/
public CreateIndexRequest source(XContentBuilder source, NamedXContentRegistry xContentRegistry) {
return source(BytesReference.bytes(source), source.contentType(), xContentRegistry);
}

/**
* Sets the settings and mappings as a single source.
*/
Expand All @@ -374,8 +383,25 @@ public CreateIndexRequest source(BytesReference source, XContentType xContentTyp
/**
* Sets the settings and mappings as a single source.
*/
@SuppressWarnings("unchecked")
public CreateIndexRequest source(BytesReference source, XContentType xContentType, NamedXContentRegistry xContentRegistry) {
Objects.requireNonNull(xContentType);
source(XContentHelper.convertToMap(source, false, xContentType).v2(), LoggingDeprecationHandler.INSTANCE, xContentRegistry);
return this;
}

/**
* Sets the settings and mappings as a single source.
*/
public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler deprecationHandler) {
return source(source, deprecationHandler, null);
}


/**
* Sets the settings, mappings and customs as a single source, using specified registry to parse customs
*/
@SuppressWarnings("unchecked")
public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler deprecationHandler, NamedXContentRegistry customsRegistry) {
for (Map.Entry<String, ?> entry : source.entrySet()) {
String name = entry.getKey();
if (SETTINGS.match(name, deprecationHandler)) {
Expand All @@ -389,15 +415,20 @@ public CreateIndexRequest source(Map<String, ?> source, DeprecationHandler depre
aliases((Map<String, Object>) entry.getValue());
} else {
// maybe custom?
IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
if (proto != null) {
if (entry.getValue() instanceof Map) {
if (customsRegistry == null) {
throw new ElasticsearchParseException("unknown key [{}] for create index, and custom registry is not specified",
name);
}
try {
customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
customs.put(name, IndexMetaData.parseCustom(name, (Map<String, Object>) entry.getValue(), deprecationHandler,
customsRegistry));
} catch (IOException e) {
throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", e, name);
} catch (NamedObjectNotFoundException e) {
throw new ElasticsearchParseException("unknown key [{}] for create index", e, name);
}
} else {
// found a key which is neither custom defined nor one of the supported ones
throw new ElasticsearchParseException("unknown key [{}] for create index", name);
}
}
Expand All @@ -417,7 +448,7 @@ public Set<Alias> aliases() {
* Adds custom metadata to the index to be created.
*/
public CreateIndexRequest custom(IndexMetaData.Custom custom) {
customs.put(custom.type(), custom);
customs.put(custom.getWriteableName(), custom);
return this;
}

Expand Down Expand Up @@ -476,9 +507,8 @@ public void readFrom(StreamInput in) throws IOException {
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readString();
IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
customs.put(type, customIndexMetaData);
IndexMetaData.Custom customIndexMetaData = in.readNamedWriteable(IndexMetaData.Custom.class);;
customs.put(customIndexMetaData.getWriteableName(), customIndexMetaData);
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
Expand All @@ -502,9 +532,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(entry.getValue());
}
out.writeVInt(customs.size());
for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
out.writeString(entry.getKey());
entry.getValue().writeTo(out);
for (IndexMetaData.Custom custom : customs.values()) {
out.writeNamedWriteable(custom);
}
out.writeVInt(aliases.size());
for (Alias alias : aliases) {
Expand Down Expand Up @@ -548,4 +577,5 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
}
return builder;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,14 @@ public CreateIndexRequestBuilder addAlias(Alias alias) {
return this;
}

/**
* Sets custom metadata on index creation
*/
public CreateIndexRequestBuilder setCustom(IndexMetaData.Custom custom) {
request.custom(custom);
return this;
}

/**
* Sets the settings and mappings as a single source.
*/
Expand Down Expand Up @@ -224,14 +232,6 @@ public CreateIndexRequestBuilder setSource(Map<String, ?> source) {
return this;
}

/**
* Adds custom metadata to the index to be created.
*/
public CreateIndexRequestBuilder addCustom(IndexMetaData.Custom custom) {
request.custom(custom);
return this;
}

/**
* Sets the settings and mappings as a single source.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedObjectNotFoundException;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand All @@ -61,9 +62,9 @@
import java.util.stream.Collectors;

import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;

/**
* A request to create an index template.
Expand Down Expand Up @@ -308,8 +309,24 @@ public PutIndexTemplateRequest source(XContentBuilder templateBuilder) {
/**
* The template source definition.
*/
public PutIndexTemplateRequest source(XContentBuilder templateBuilder, NamedXContentRegistry customsRegistry) {
try {
return source(BytesReference.bytes(templateBuilder), templateBuilder.contentType(), customsRegistry);
} catch (Exception e) {
throw new IllegalArgumentException("Failed to build json for template request", e);
}
}

@SuppressWarnings("unchecked")
public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
return source(templateSource, null);
}

/**
* The template source definition.
*/
@SuppressWarnings("unchecked")
public PutIndexTemplateRequest source(Map<String, Object> templateSource, NamedXContentRegistry customsRegistry) {
Map<String, Object> source = templateSource;
for (Map.Entry<String, Object> entry : source.entrySet()) {
String name = entry.getKey();
Expand Down Expand Up @@ -354,13 +371,21 @@ public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
aliases((Map<String, Object>) entry.getValue());
} else {
// maybe custom?
IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
if (proto != null) {
if (entry.getValue() instanceof Map) {
if (customsRegistry == null) {
throw new ElasticsearchParseException("unknown key [{}] for create index, and custom registry is not specified",
name);
}
try {
customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
customs.put(name, IndexMetaData.parseCustom(name, (Map<String, Object>) entry.getValue(),
LoggingDeprecationHandler.INSTANCE, customsRegistry));
} catch (IOException e) {
throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", e, name);
} catch (NamedObjectNotFoundException e) {
throw new ElasticsearchParseException("unknown key [{}] in the template", e, name);
}
} else {
throw new ElasticsearchParseException("unknown key [{}] in the template ", name);
}
}
}
Expand Down Expand Up @@ -388,6 +413,13 @@ public PutIndexTemplateRequest source(byte[] source, int offset, int length, XCo
return source(new BytesArray(source, offset, length), xContentType);
}

/**
* The template source definition that should be parsed using specified customs registry.
*/
public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType, NamedXContentRegistry customsRegistry) {
return source(XContentHelper.convertToMap(source, true, xContentType).v2(), customsRegistry);
}

/**
* The template source definition.
*/
Expand All @@ -396,7 +428,7 @@ public PutIndexTemplateRequest source(BytesReference source, XContentType xConte
}

public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) {
customs.put(custom.type(), custom);
customs.put(custom.getWriteableName(), custom);
return this;
}

Expand Down Expand Up @@ -501,9 +533,8 @@ public void readFrom(StreamInput in) throws IOException {
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readString();
IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
customs.put(type, customIndexMetaData);
IndexMetaData.Custom customIndexMetaData = in.readNamedWriteable(IndexMetaData.Custom.class);
customs.put(customIndexMetaData.getWriteableName(), customIndexMetaData);
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
Expand Down Expand Up @@ -531,9 +562,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(entry.getValue());
}
out.writeVInt(customs.size());
for (Map.Entry<String, IndexMetaData.Custom> entry : customs.entrySet()) {
out.writeString(entry.getKey());
entry.getValue().writeTo(out);
for (IndexMetaData.Custom custom : customs.values()) {
out.writeNamedWriteable(custom);
}
out.writeVInt(aliases.size());
for (Alias alias : aliases) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
Expand Down Expand Up @@ -210,6 +211,16 @@ public PutIndexTemplateRequestBuilder addMapping(String type, Map<String, Object
return this;
}

/**
* Sets custom metadata object.
*
* @param custom The custom index metadata object
*/
public PutIndexTemplateRequestBuilder setCustom(IndexMetaData.Custom custom) {
request.custom(custom);
return this;
}

/**
* The template source definition.
*/
Expand Down
Loading