From 173087541153c382fa7b513643f405c8b6629d04 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 19 Jun 2018 11:09:38 +0200 Subject: [PATCH 01/16] Add get field mappings to High Level REST API Client Relates to #27205 --- .../elasticsearch/client/IndicesClient.java | 31 +++++ .../client/RequestConverters.java | 22 ++++ .../elasticsearch/client/IndicesClientIT.java | 38 ++++++ .../client/RequestConvertersTests.java | 48 +++++++ .../IndicesClientDocumentationIT.java | 110 ++++++++++++++++ .../indices/get_field_mappings.asciidoc | 77 +++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../common/xcontent/XContentBuilder.java | 6 + .../mapping/get/GetFieldMappingsResponse.java | 121 +++++++++++++++++- .../get/GetFieldMappingsResponseTests.java | 51 +++++++- .../common/xcontent/BaseXContentTestCase.java | 2 +- 11 files changed, 503 insertions(+), 5 deletions(-) create mode 100644 docs/java-rest/high-level/indices/get_field_mappings.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 5f85b18091d72..d4f0060bfcc1f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -37,6 +37,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -186,6 +188,35 @@ public void getMappingsAsync(GetMappingsRequest getMappingsRequest, RequestOptio GetMappingsResponse::fromXContent, listener, emptySet()); } + /** + * Retrieves the field mappings on an index or indices using the Get Field Mapping API. + * See + * Get Field Mapping API on elastic.co + * @param getFieldMappingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetFieldMappingsResponse getFieldMappings(GetFieldMappingsRequest getFieldMappingsRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMappings, options, + GetFieldMappingsResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously retrieves the field mappings on an index on indices using the Get Field Mapping API. + * See + * Get Field Mapping API on elastic.co + * @param getFieldMappingsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getFieldMappingsAsync(GetFieldMappingsRequest getFieldMappingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMappings, options, + GetFieldMappingsResponse::fromXContent, listener, emptySet()); + } + /** * Updates aliases using the Index Aliases API. * See diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 93bf6a1a19881..b4c4225fa8997 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -47,6 +47,7 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; @@ -225,6 +226,20 @@ static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOExcep return request; } + static Request getFieldMappings(GetFieldMappingsRequest getFieldMappingsRequest) throws IOException { + String[] indices = getFieldMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.indices(); + String[] types = getFieldMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.types(); + String[] fields = getFieldMappingsRequest.fields() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.fields(); + + Request request = new Request(HttpGet.METHOD_NAME, endpoint(indices, "_mapping", types, "field", fields)); + + Params parameters = new Params(request); + parameters.withIndicesOptions(getFieldMappingsRequest.indicesOptions()); + parameters.withIncludeDefaults(getFieldMappingsRequest.includeDefaults()); + parameters.withLocal(getFieldMappingsRequest.local()); + return request; + } + static Request refresh(RefreshRequest refreshRequest) { String[] indices = refreshRequest.indices() == null ? Strings.EMPTY_ARRAY : refreshRequest.indices(); Request request = new Request(HttpPost.METHOD_NAME, endpoint(indices, "_refresh")); @@ -908,6 +923,13 @@ static String endpoint(String[] indices, String endpoint, String[] suffixes) { .addCommaSeparatedPathParts(suffixes).build(); } + static String endpoint(String[] indices, String endpoint, String[] suffixes, String extraEndpoint, String[] extraSuffixes) { + return new EndpointBuilder().addCommaSeparatedPathParts(indices) + .addPathPartAsIs(endpoint).addCommaSeparatedPathParts(suffixes) + .addPathPartAsIs(extraEndpoint).addCommaSeparatedPathParts(extraSuffixes) + .build(); + } + static String endpoint(String[] indices, String endpoint, String type) { return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 82ac161f5afe0..46428cfe9e055 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -42,6 +42,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -71,6 +73,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -373,6 +376,41 @@ public void testGetMapping() throws IOException { assertThat(mappings, equalTo(expected)); } + public void testGetFieldMapping() throws IOException { + String indexName = "test"; + createIndex(indexName, Settings.EMPTY); + + PutMappingRequest putMappingRequest = new PutMappingRequest(indexName); + putMappingRequest.type("_doc"); + XContentBuilder mappingBuilder = JsonXContent.contentBuilder(); + mappingBuilder.startObject().startObject("properties").startObject("field"); + mappingBuilder.field("type", "text"); + mappingBuilder.endObject().endObject().endObject(); + putMappingRequest.source(mappingBuilder); + + PutMappingResponse putMappingResponse = + execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); + assertTrue(putMappingResponse.isAcknowledged()); + + GetFieldMappingsRequest getFieldMappingsRequest = new GetFieldMappingsRequest() + .indices(indexName) + .types("_doc") + .fields("field"); + + GetFieldMappingsResponse getFieldMappingsResponse = + execute(getFieldMappingsRequest, + highLevelClient().indices()::getFieldMappings, + highLevelClient().indices()::getFieldMappingsAsync); + + final Map fieldMappingMap = + getFieldMappingsResponse.mappings().get(indexName).get("_doc"); + + final GetFieldMappingsResponse.FieldMappingMetaData metaData = + new GetFieldMappingsResponse.FieldMappingMetaData("field", + new BytesArray("{\"field\":{\"type\":\"text\"}}")); + assertThat(fieldMappingMap, equalTo(Collections.singletonMap("field", metaData))); + } + public void testDeleteIndex() throws IOException { { // Delete index if exists diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index aa8221f30991e..6b6a2dd7af481 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; @@ -452,6 +453,53 @@ public void testGetMapping() throws IOException { assertThat(HttpGet.METHOD_NAME, equalTo(request.getMethod())); } + public void testGetFieldMapping() throws IOException { + GetFieldMappingsRequest getFieldMappingsRequest = new GetFieldMappingsRequest(); + + String[] indices = Strings.EMPTY_ARRAY; + if (randomBoolean()) { + indices = randomIndicesNames(0, 5); + getFieldMappingsRequest.indices(indices); + } else if (randomBoolean()) { + getFieldMappingsRequest.indices((String[]) null); + } + + String type = null; + if (randomBoolean()) { + type = randomAlphaOfLengthBetween(3, 10); + getFieldMappingsRequest.types(type); + } else if (randomBoolean()) { + getFieldMappingsRequest.types((String[]) null); + } + + String[] fields = new String[randomIntBetween(1, 5)]; + for(int i = 0; i < fields.length; i++) { + fields[i] = randomAlphaOfLengthBetween(3, 10); + } + getFieldMappingsRequest.fields(fields); + + Map expectedParams = new HashMap<>(); + + setRandomIndicesOptions(getFieldMappingsRequest::indicesOptions, getFieldMappingsRequest::indicesOptions, expectedParams); + + Request request = RequestConverters.getFieldMappings(getFieldMappingsRequest); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + String index = String.join(",", indices); + if (Strings.hasLength(index)) { + endpoint.add(index); + } + endpoint.add("_mapping"); + if (type != null) { + endpoint.add(type); + } + endpoint.add("field"); + endpoint.add(String.join(",", fields)); + assertThat(endpoint.toString(), equalTo(request.getEndpoint())); + + assertThat(expectedParams, equalTo(request.getParameters())); + assertThat(HttpGet.METHOD_NAME, equalTo(request.getMethod())); + } + public void testDeleteIndex() { String[] indices = randomIndicesNames(0, 5); DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indices); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 7bd6b16cecc99..b66ba2dc88b2d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -41,6 +41,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -86,6 +88,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -699,6 +702,113 @@ public void onFailure(Exception e) { } } + public void testGetFieldMapping() throws IOException, InterruptedException { + RestHighLevelClient client = highLevelClient(); + + { + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); + assertTrue(createIndexResponse.isAcknowledged()); + PutMappingRequest request = new PutMappingRequest("twitter"); + request.type("tweet"); + request.source( + "{\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"timestamp\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + "}", // <1> + XContentType.JSON); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); + assertTrue(putMappingResponse.isAcknowledged()); + } + + // tag::get-field-mapping-request + GetFieldMappingsRequest request = new GetFieldMappingsRequest(); // <1> + request.indices("twitter"); // <2> + request.types("tweet"); // <3> + request.fields("message", "timestamp"); // <4> + // end::get-field-mapping-request + + // tag::get-field-mapping-request-indicesOptions + request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::get-field-mapping-request-indicesOptions + + { + + // tag::get-field-mapping-execute + GetFieldMappingsResponse response = + client.indices().getFieldMappings(request, RequestOptions.DEFAULT); + // end::get-field-mapping-execute + + // tag::get-field-mapping-response + final Map>> mappings = + response.mappings();// <1> + final Map typeMappings = + mappings.get("twitter").get("tweet"); // <2> + final GetFieldMappingsResponse.FieldMappingMetaData metaData = + typeMappings.get("message");// <3> + + final String fullName = metaData.fullName();// <4> + final Map source = metaData.sourceAsMap(); // <5> + + // end::get-field-mapping-response + + assertThat(fullName, equalTo("message")); + assertThat(source, equalTo(Collections.singletonMap("message", Collections.singletonMap("type", "text")))); + } + + { + // tag::get-field-mapping-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(GetFieldMappingsResponse putMappingResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-field-mapping-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + final ActionListener latchListener = new LatchedActionListener<>(listener, latch); + listener = ActionListener.wrap(r -> { + final Map>> mappings = + r.mappings(); + final Map typeMappings = + mappings.get("twitter").get("tweet"); + final GetFieldMappingsResponse.FieldMappingMetaData metaData1 = typeMappings.get("message"); + + final String fullName = metaData1.fullName(); + final Map source = metaData1.sourceAsMap(); + + assertThat(fullName, equalTo("message")); + assertThat(source, equalTo(Collections.singletonMap("message", Collections.singletonMap("type", "text")))); + latchListener.onResponse(r); + }, e -> { + latchListener.onFailure(e); + fail("should not fail"); + }); + + // tag::get-field-mapping-execute-async + client.indices().getFieldMappingsAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::get-field-mapping-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + + } + + public void testOpenIndex() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/docs/java-rest/high-level/indices/get_field_mappings.asciidoc b/docs/java-rest/high-level/indices/get_field_mappings.asciidoc new file mode 100644 index 0000000000000..28f6a60d72018 --- /dev/null +++ b/docs/java-rest/high-level/indices/get_field_mappings.asciidoc @@ -0,0 +1,77 @@ +[[java-rest-high-get-field-mappings]] +=== Get Field Mappings API + +[[java-rest-high-get-field-mappings-request]] +==== Get Field Mappings Request + +A `GetFieldMappingsRequest` can have an optional list of indices, optional list of types and the list of fields: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-request] +-------------------------------------------------- +<1> An empty request +<2> Setting the indices to fetch mapping for +<3> The types to be returned +<4> The fields to be returned + +==== Optional arguments +The following arguments can also optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-request-indicesOptions] +-------------------------------------------------- +<1> Options for expanding indices names + +[[java-rest-high-get-field-mappings-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-execute] +-------------------------------------------------- + +[[java-rest-high-get-field-mapping-async]] +==== Asynchronous Execution + +The asynchronous execution of a get mappings request requires both the +`GetFieldMappingsRequest` instance and an `ActionListener` instance to be passed to +the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-execute-async] +-------------------------------------------------- +<1> The `GetFieldMappingsRequest` to execute and the `ActionListener` to use when the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method if +the execution successfully completed or using the `onFailure` method if it +failed. + +A typical listener for `GetMappingsResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-get-field-mapping-response]] +==== Get Field Mappings Response + +The returned `GetFieldMappingsResponse` allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-response] +-------------------------------------------------- +<1> Returning all requested indices fields' mappings +<2> Retrieving the mappings for a particular index and type +<3> Getting the mappings metadata for the `message` field +<4> Getting the full name of the field +<5> Getting the mapping source of the field + diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index b33c2421b06d3..fa266f1bb53c0 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -76,6 +76,7 @@ Index Management:: Mapping Management:: * <> +* <> Alias Management:: * <> @@ -97,6 +98,7 @@ include::indices/force_merge.asciidoc[] include::indices/rollover.asciidoc[] include::indices/put_mapping.asciidoc[] include::indices/get_mappings.asciidoc[] +include::indices/get_field_mappings.asciidoc[] include::indices/update_aliases.asciidoc[] include::indices/exists_alias.asciidoc[] include::indices/get_alias.asciidoc[] diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index eae5e48a557f3..987380c7c7d42 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.ParseField; + import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.Flushable; @@ -253,6 +255,10 @@ public XContentBuilder startObject() throws IOException { return this; } + public XContentBuilder startObject(ParseField field) throws IOException { + return startObject(field.getPreferredName()); + } + public XContentBuilder startObject(String name) throws IOException { return field(name).startObject(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index d837c1cbd199b..f5db52c39eb3b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -20,13 +20,17 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; @@ -34,13 +38,20 @@ import java.io.InputStream; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName; /** Response object for {@link GetFieldMappingsRequest} API */ public class GetFieldMappingsResponse extends ActionResponse implements ToXContentFragment { + private static final ParseField MAPPINGS = new ParseField("mappings"); + private Map>> mappings = emptyMap(); GetFieldMappingsResponse(Map>> mappings) { @@ -77,7 +88,7 @@ public FieldMappingMetaData fieldMappings(String index, String type, String fiel public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (Map.Entry>> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); - builder.startObject("mappings"); + builder.startObject(MAPPINGS); for (Map.Entry> typeEntry : indexEntry.getValue().entrySet()) { builder.startObject(typeEntry.getKey()); for (Map.Entry fieldEntry : typeEntry.getValue().entrySet()) { @@ -93,9 +104,73 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static GetFieldMappingsResponse fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + Map>> mappings = new HashMap<>(); + if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + String index = parser.currentName(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + ensureFieldName(parser, parser.nextToken(), MAPPINGS.getPreferredName()); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + + Map> mapping = new HashMap<>(); + parser.nextToken(); + while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + final String typeName = parser.currentName(); + Map typeMapping = new HashMap<>(); + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + final String fieldName = parser.currentName(); + FieldMappingMetaData fieldMappingMetaData = FieldMappingMetaData.fromXContent(parser); + typeMapping.put(fieldName, fieldMappingMetaData); + parser.nextToken(); + } + } + } + + mapping.put(typeName, typeMapping); + parser.nextToken(); + } + + mappings.put(index, mapping); + parser.nextToken(); + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); + + parser.nextToken(); + } + return new GetFieldMappingsResponse(mappings); + } + + public static class FieldMappingMetaData implements ToXContentFragment { public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY); + private static final ParseField FULL_NAME = new ParseField("full_name"); + private static final ParseField MAPPING = new ParseField("mapping"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("field_mapping_meta_data", true, + a -> new FieldMappingMetaData((String)a[0], (BytesReference)a[1]) + ); + + static { + PARSER.declareField(optionalConstructorArg(), + (p, c) -> p.text(), FULL_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), + (p, c) -> { + Map source = p.map(); + XContentBuilder builder = jsonBuilder(); + builder.map(source); + final BytesReference bytes = BytesReference.bytes(builder); + return bytes; + }, MAPPING, ObjectParser.ValueType.OBJECT); + } + private String fullName; private BytesReference source; @@ -104,6 +179,10 @@ public FieldMappingMetaData(String fullName, BytesReference source) { this.source = source; } + public static FieldMappingMetaData fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + public String fullName() { return fullName; } @@ -134,6 +213,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } return builder; } + + @Override + public String toString() { + return "FieldMappingMetaData{fullName='" + fullName + '\'' + ", source=" + source + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof FieldMappingMetaData)) return false; + FieldMappingMetaData that = (FieldMappingMetaData) o; + return Objects.equals(fullName, that.fullName) && + Objects.equals(source, that.source); + } + + @Override + public int hashCode() { + return Objects.hash(fullName, source); + } } @Override @@ -178,4 +276,25 @@ public void writeTo(StreamOutput out) throws IOException { } } } + + @Override + public String toString() { + return "GetFieldMappingsResponse{" + + "mappings=" + mappings + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GetFieldMappingsResponse)) return false; + GetFieldMappingsResponse that = (GetFieldMappingsResponse) o; + return Objects.equals(mappings, that.mappings); + } + + @Override + public int hashCode() { + return Objects.hash(mappings); + } + } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 4dc396323c048..fe954a5ebe389 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -23,16 +23,18 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; -public class GetFieldMappingsResponseTests extends ESTestCase { +public class GetFieldMappingsResponseTests extends AbstractStreamableXContentTestCase { - public void testSerialization() throws IOException { + public void testManualSerialization() throws IOException { Map>> mappings = new HashMap<>(); FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}")); mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData))); @@ -49,4 +51,47 @@ public void testSerialization() throws IOException { } } } + + @Override + protected GetFieldMappingsResponse doParseInstance(XContentParser parser) throws IOException { + return GetFieldMappingsResponse.fromXContent(parser); + } + + @Override + protected GetFieldMappingsResponse createBlankInstance() { + return new GetFieldMappingsResponse(); + } + + @Override + protected GetFieldMappingsResponse createTestInstance() { + return new GetFieldMappingsResponse(randomMapping()); + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return s -> true; + } + + private Map>> randomMapping() { + Map>> mappings = new HashMap<>(); + + int indices = randomInt(10); + for(int i = 0; i < indices; i++) { + final Map> doctypesMappings = new HashMap<>(); + int doctypes = randomInt(10); + for(int j = 0; j < doctypes; j++) { + Map fieldMappings = new HashMap<>(); + int fields = randomInt(10); + for(int k = 0; k < fields; k++) { + final String mapping = randomBoolean() ? "{\"type\":\"string\"}" : "{\"type\":\"keyword\"}"; + FieldMappingMetaData metaData = + new FieldMappingMetaData("my field", new BytesArray(mapping)); + fieldMappings.put("field" + k, metaData); + } + doctypesMappings.put("doctype" + j, fieldMappings); + } + mappings.put("index" + i, doctypesMappings); + } + return mappings; + } } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index 86e55c1ab6a91..dc106a0f0afd2 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -94,7 +94,7 @@ public void testStartEndObject() throws IOException { expectObjectException(() -> Strings.toString(builder().endObject())); expectValueException(() -> builder().startObject("foo").endObject()); - expectNonNullFieldException(() -> builder().startObject().startObject(null)); + expectNonNullFieldException(() -> builder().startObject().startObject((String)null)); assertResult("{}", () -> builder().startObject().endObject()); assertResult("{'foo':{}}", () -> builder().startObject().startObject("foo").endObject().endObject()); From b350c339100a466cffae0372f549cb2d04ef9070 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 19 Jun 2018 17:30:31 +0200 Subject: [PATCH 02/16] don't introduce ParseField in XContentBuilder so far --- .../org/elasticsearch/common/xcontent/XContentBuilder.java | 6 ------ .../admin/indices/mapping/get/GetFieldMappingsResponse.java | 2 +- .../elasticsearch/common/xcontent/BaseXContentTestCase.java | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 987380c7c7d42..eae5e48a557f3 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -19,8 +19,6 @@ package org.elasticsearch.common.xcontent; -import org.elasticsearch.common.ParseField; - import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.Flushable; @@ -255,10 +253,6 @@ public XContentBuilder startObject() throws IOException { return this; } - public XContentBuilder startObject(ParseField field) throws IOException { - return startObject(field.getPreferredName()); - } - public XContentBuilder startObject(String name) throws IOException { return field(name).startObject(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index f5db52c39eb3b..fccc9514aff9a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -88,7 +88,7 @@ public FieldMappingMetaData fieldMappings(String index, String type, String fiel public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (Map.Entry>> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); - builder.startObject(MAPPINGS); + builder.startObject(MAPPINGS.getPreferredName()); for (Map.Entry> typeEntry : indexEntry.getValue().entrySet()) { builder.startObject(typeEntry.getKey()); for (Map.Entry fieldEntry : typeEntry.getValue().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index dc106a0f0afd2..86e55c1ab6a91 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -94,7 +94,7 @@ public void testStartEndObject() throws IOException { expectObjectException(() -> Strings.toString(builder().endObject())); expectValueException(() -> builder().startObject("foo").endObject()); - expectNonNullFieldException(() -> builder().startObject().startObject((String)null)); + expectNonNullFieldException(() -> builder().startObject().startObject(null)); assertResult("{}", () -> builder().startObject().endObject()); assertResult("{'foo':{}}", () -> builder().startObject().startObject("foo").endObject().endObject()); From 4cb66ae60df1629347470e06fdc004dd199a9083 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 19 Jun 2018 17:44:49 +0200 Subject: [PATCH 03/16] don't introduce complex endpoint method; improve fields randomization - nulls are welcome --- .../elasticsearch/client/RequestConverters.java | 14 ++++++-------- .../client/RequestConvertersTests.java | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index b4c4225fa8997..5beabe1da46e2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -231,7 +231,12 @@ static Request getFieldMappings(GetFieldMappingsRequest getFieldMappingsRequest) String[] types = getFieldMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.types(); String[] fields = getFieldMappingsRequest.fields() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.fields(); - Request request = new Request(HttpGet.METHOD_NAME, endpoint(indices, "_mapping", types, "field", fields)); + String endpoint = new EndpointBuilder().addCommaSeparatedPathParts(indices) + .addPathPartAsIs("_mapping").addCommaSeparatedPathParts(types) + .addPathPartAsIs("field").addCommaSeparatedPathParts(fields) + .build(); + + Request request = new Request(HttpGet.METHOD_NAME, endpoint); Params parameters = new Params(request); parameters.withIndicesOptions(getFieldMappingsRequest.indicesOptions()); @@ -923,13 +928,6 @@ static String endpoint(String[] indices, String endpoint, String[] suffixes) { .addCommaSeparatedPathParts(suffixes).build(); } - static String endpoint(String[] indices, String endpoint, String[] suffixes, String extraEndpoint, String[] extraSuffixes) { - return new EndpointBuilder().addCommaSeparatedPathParts(indices) - .addPathPartAsIs(endpoint).addCommaSeparatedPathParts(suffixes) - .addPathPartAsIs(extraEndpoint).addCommaSeparatedPathParts(extraSuffixes) - .build(); - } - static String endpoint(String[] indices, String endpoint, String type) { return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 6b6a2dd7af481..09347dca5f331 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -472,11 +472,16 @@ public void testGetFieldMapping() throws IOException { getFieldMappingsRequest.types((String[]) null); } - String[] fields = new String[randomIntBetween(1, 5)]; - for(int i = 0; i < fields.length; i++) { - fields[i] = randomAlphaOfLengthBetween(3, 10); + String[] fields = null; + if (randomBoolean()) { + fields = new String[randomIntBetween(1, 5)]; + for (int i = 0; i < fields.length; i++) { + fields[i] = randomAlphaOfLengthBetween(3, 10); + } + getFieldMappingsRequest.fields(fields); + } else if (randomBoolean()) { + getFieldMappingsRequest.fields((String[]) null); } - getFieldMappingsRequest.fields(fields); Map expectedParams = new HashMap<>(); @@ -493,7 +498,9 @@ public void testGetFieldMapping() throws IOException { endpoint.add(type); } endpoint.add("field"); - endpoint.add(String.join(",", fields)); + if (fields != null) { + endpoint.add(String.join(",", fields)); + } assertThat(endpoint.toString(), equalTo(request.getEndpoint())); assertThat(expectedParams, equalTo(request.getParameters())); From 207a8d59daf6d8e216b8e76283115e6f23244aee Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 20 Jun 2018 08:56:14 +0200 Subject: [PATCH 04/16] rename getFieldMappings to getFieldMapping for consistency with REST API SPEC --- .../java/org/elasticsearch/client/IndicesClient.java | 12 ++++++------ .../org/elasticsearch/client/RequestConverters.java | 2 +- .../org/elasticsearch/client/IndicesClientIT.java | 4 ++-- .../elasticsearch/client/RequestConvertersTests.java | 2 +- .../documentation/IndicesClientDocumentationIT.java | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index d4f0060bfcc1f..c6fa62377d59c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -197,9 +197,9 @@ public void getMappingsAsync(GetMappingsRequest getMappingsRequest, RequestOptio * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetFieldMappingsResponse getFieldMappings(GetFieldMappingsRequest getFieldMappingsRequest, - RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMappings, options, + public GetFieldMappingsResponse getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMapping, options, GetFieldMappingsResponse::fromXContent, emptySet()); } @@ -211,9 +211,9 @@ public GetFieldMappingsResponse getFieldMappings(GetFieldMappingsRequest getFiel * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ - public void getFieldMappingsAsync(GetFieldMappingsRequest getFieldMappingsRequest, RequestOptions options, - ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMappings, options, + public void getFieldMappingAsync(GetFieldMappingsRequest getFieldMappingsRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMapping, options, GetFieldMappingsResponse::fromXContent, listener, emptySet()); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 5beabe1da46e2..0cec22a5d260c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -226,7 +226,7 @@ static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOExcep return request; } - static Request getFieldMappings(GetFieldMappingsRequest getFieldMappingsRequest) throws IOException { + static Request getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest) throws IOException { String[] indices = getFieldMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.indices(); String[] types = getFieldMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.types(); String[] fields = getFieldMappingsRequest.fields() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.fields(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 46428cfe9e055..421a597beeca2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -399,8 +399,8 @@ public void testGetFieldMapping() throws IOException { GetFieldMappingsResponse getFieldMappingsResponse = execute(getFieldMappingsRequest, - highLevelClient().indices()::getFieldMappings, - highLevelClient().indices()::getFieldMappingsAsync); + highLevelClient().indices()::getFieldMapping, + highLevelClient().indices()::getFieldMappingAsync); final Map fieldMappingMap = getFieldMappingsResponse.mappings().get(indexName).get("_doc"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 09347dca5f331..644fc9c3594f6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -487,7 +487,7 @@ public void testGetFieldMapping() throws IOException { setRandomIndicesOptions(getFieldMappingsRequest::indicesOptions, getFieldMappingsRequest::indicesOptions, expectedParams); - Request request = RequestConverters.getFieldMappings(getFieldMappingsRequest); + Request request = RequestConverters.getFieldMapping(getFieldMappingsRequest); StringJoiner endpoint = new StringJoiner("/", "/", ""); String index = String.join(",", indices); if (Strings.hasLength(index)) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index b66ba2dc88b2d..e9deb646c3247 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -741,7 +741,7 @@ public void testGetFieldMapping() throws IOException, InterruptedException { // tag::get-field-mapping-execute GetFieldMappingsResponse response = - client.indices().getFieldMappings(request, RequestOptions.DEFAULT); + client.indices().getFieldMapping(request, RequestOptions.DEFAULT); // end::get-field-mapping-execute // tag::get-field-mapping-response @@ -799,7 +799,7 @@ public void onFailure(Exception e) { }); // tag::get-field-mapping-execute-async - client.indices().getFieldMappingsAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.indices().getFieldMappingAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-field-mapping-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); From f095c57363e8b67e1a6e8137b9ce4afd8a2f5fee Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 20 Jun 2018 08:57:05 +0200 Subject: [PATCH 05/16] simplification of GetFieldMappingsResponse.fromXContent + allow random fields at some level in test --- .../mapping/get/GetFieldMappingsResponse.java | 100 +++++++----------- .../get/GetFieldMappingsResponseTests.java | 19 +++- 2 files changed, 58 insertions(+), 61 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index fccc9514aff9a..38ecd00528d06 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -21,12 +21,11 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -42,10 +41,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName; /** Response object for {@link GetFieldMappingsRequest} API */ public class GetFieldMappingsResponse extends ActionResponse implements ToXContentFragment { @@ -105,72 +101,60 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static GetFieldMappingsResponse fromXContent(XContentParser parser) throws IOException { - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); - Map>> mappings = new HashMap<>(); - if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { - while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - String index = parser.currentName(); - - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); - ensureFieldName(parser, parser.nextToken(), MAPPINGS.getPreferredName()); - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); - - Map> mapping = new HashMap<>(); - parser.nextToken(); - while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - final String typeName = parser.currentName(); - Map typeMapping = new HashMap<>(); - if (parser.nextToken() == XContentParser.Token.START_OBJECT) { - if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { - while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - final String fieldName = parser.currentName(); - FieldMappingMetaData fieldMappingMetaData = FieldMappingMetaData.fromXContent(parser); - typeMapping.put(fieldName, fieldMappingMetaData); - parser.nextToken(); - } + final Map>> mappings = new HashMap<>(); + final Map map = parser.map(); + + for (Map.Entry entry : map.entrySet()) { + final Object value = entry.getValue(); + + Map> typeMappings = new HashMap<>(); + mappings.put(entry.getKey(), typeMappings); + + if (value instanceof Map) { + final Object o = ((Map) value).get(MAPPINGS.getPreferredName()); + if (!(o instanceof Map)) { + throw new ParsingException(parser.getTokenLocation(), "Nested " + MAPPINGS.getPreferredName() + " is not found"); + } + Map map1 = (Map) o; + for (Map.Entry typeObjectEntry : map1.entrySet()) { + Map fieldsMapping = new HashMap<>(); + typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); + + final Object o1 = typeObjectEntry.getValue(); + if (!(o1 instanceof Map)) { + throw new ParsingException(parser.getTokenLocation(), "Nested type mapping is not found"); + } + Map map2 = (Map) o1; + for (Map.Entry e : map2.entrySet()) { + final Object o2 = e.getValue(); + if (!(o2 instanceof Map)) { + throw new ParsingException(parser.getTokenLocation(), "Nested field mapping is not found"); } + Map map3 = (Map) o2; + + String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); + XContentBuilder builder = jsonBuilder(); + final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); + builder.map(values); + final BytesReference source = BytesReference.bytes(builder); + + FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); + fieldsMapping.put(e.getKey(), metaData); } - mapping.put(typeName, typeMapping); - parser.nextToken(); } - - mappings.put(index, mapping); - parser.nextToken(); - parser.nextToken(); } - ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); - - parser.nextToken(); } + return new GetFieldMappingsResponse(mappings); } - public static class FieldMappingMetaData implements ToXContentFragment { public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY); private static final ParseField FULL_NAME = new ParseField("full_name"); private static final ParseField MAPPING = new ParseField("mapping"); - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("field_mapping_meta_data", true, - a -> new FieldMappingMetaData((String)a[0], (BytesReference)a[1]) - ); - - static { - PARSER.declareField(optionalConstructorArg(), - (p, c) -> p.text(), FULL_NAME, ObjectParser.ValueType.STRING); - PARSER.declareField(optionalConstructorArg(), - (p, c) -> { - Map source = p.map(); - XContentBuilder builder = jsonBuilder(); - builder.map(source); - final BytesReference bytes = BytesReference.bytes(builder); - return bytes; - }, MAPPING, ObjectParser.ValueType.OBJECT); - } - private String fullName; private BytesReference source; @@ -179,10 +163,6 @@ public FieldMappingMetaData(String fullName, BytesReference source) { this.source = source; } - public static FieldMappingMetaData fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - public String fullName() { return fullName; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index fe954a5ebe389..a4896234885ec 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -67,9 +67,26 @@ protected GetFieldMappingsResponse createTestInstance() { return new GetFieldMappingsResponse(randomMapping()); } + static int charCount(final String s, char ch) { + int count = 0; + int pos = 0; + while (pos >= 0) { + pos = s.indexOf(ch, pos); + if (pos >= 0) { + count++; + pos++; + } + } + return count; + } + @Override protected Predicate getRandomFieldsExcludeFilter() { - return s -> true; + // allow random fields at the level of index and index.mappings.doctype.field + // otherwise random field could be evaluated as index name or type name + return s -> { + final int c = s.length() > 0 ? charCount(s, '.') : -1; + return c != 0 && c != 3;}; } private Map>> randomMapping() { From ce482f13221e6251882a094e8b95e70782741753 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 20 Jun 2018 08:59:47 +0200 Subject: [PATCH 06/16] add missed doc section for get-field-mapping-request-local (cherry picked from commit 38b7bd4) --- .../elasticsearch/client/RequestConvertersTests.java | 8 ++++++++ .../documentation/IndicesClientDocumentationIT.java | 4 ++++ .../high-level/indices/get_field_mappings.asciidoc | 11 ++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 644fc9c3594f6..33cb8d08215df 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -487,6 +487,14 @@ public void testGetFieldMapping() throws IOException { setRandomIndicesOptions(getFieldMappingsRequest::indicesOptions, getFieldMappingsRequest::indicesOptions, expectedParams); + if (randomBoolean()) { + boolean local = randomBoolean(); + getFieldMappingsRequest.local(local); + if (local) { + expectedParams.put("local", String.valueOf(local)); + } + } + Request request = RequestConverters.getFieldMapping(getFieldMappingsRequest); StringJoiner endpoint = new StringJoiner("/", "/", ""); String index = String.join(",", indices); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index e9deb646c3247..21d8213e11d4c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -737,6 +737,10 @@ public void testGetFieldMapping() throws IOException, InterruptedException { request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> // end::get-field-mapping-request-indicesOptions + // tag::get-field-mapping-request-local + request.local(true); // <1> + // end::get-field-mapping-request-local + { // tag::get-field-mapping-execute diff --git a/docs/java-rest/high-level/indices/get_field_mappings.asciidoc b/docs/java-rest/high-level/indices/get_field_mappings.asciidoc index 28f6a60d72018..3f5ff5aec6449 100644 --- a/docs/java-rest/high-level/indices/get_field_mappings.asciidoc +++ b/docs/java-rest/high-level/indices/get_field_mappings.asciidoc @@ -22,7 +22,16 @@ The following arguments can also optionally be provided: -------------------------------------------------- include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-request-indicesOptions] -------------------------------------------------- -<1> Options for expanding indices names +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and +how wildcard expressions are expanded + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[get-field-mapping-request-local] +-------------------------------------------------- +<1> The `local` flag (defaults to `false`) controls whether the aliases need +to be looked up in the local cluster state or in the cluster state held by +the elected master node [[java-rest-high-get-field-mappings-sync]] ==== Synchronous Execution From 2205664f61463e57cdadeae1c9f3d70bb5b6be6c Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 21 Jun 2018 13:06:51 +0200 Subject: [PATCH 07/16] clean up --- .../client/documentation/IndicesClientDocumentationIT.java | 4 ---- .../indices/mapping/get/GetFieldMappingsResponse.java | 7 +++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 21d8213e11d4c..4c5d000c0c236 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -758,11 +758,7 @@ public void testGetFieldMapping() throws IOException, InterruptedException { final String fullName = metaData.fullName();// <4> final Map source = metaData.sourceAsMap(); // <5> - // end::get-field-mapping-response - - assertThat(fullName, equalTo("message")); - assertThat(source, equalTo(Collections.singletonMap("message", Collections.singletonMap("type", "text")))); } { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 38ecd00528d06..fb11a04820dee 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -122,13 +122,16 @@ public static GetFieldMappingsResponse fromXContent(XContentParser parser) throw final Object o1 = typeObjectEntry.getValue(); if (!(o1 instanceof Map)) { - throw new ParsingException(parser.getTokenLocation(), "Nested type mapping is not found"); + throw new ParsingException(parser.getTokenLocation(), + "Nested type mapping at " + entry.getKey() + "." + typeObjectEntry.getKey() + " is not found"); } Map map2 = (Map) o1; for (Map.Entry e : map2.entrySet()) { final Object o2 = e.getValue(); if (!(o2 instanceof Map)) { - throw new ParsingException(parser.getTokenLocation(), "Nested field mapping is not found"); + throw new ParsingException(parser.getTokenLocation(), + "Nested field mapping at " + entry.getKey() + "." + typeObjectEntry.getKey() + "." + e.getKey() + + " is not found"); } Map map3 = (Map) o2; From 88192e8e7b42b83407976e006eafec64db2fe2a5 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 21 Jun 2018 13:09:10 +0200 Subject: [PATCH 08/16] use regexp in getRandomFieldsExcludeFilter --- .../get/GetFieldMappingsResponseTests.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index a4896234885ec..816a36fc72a00 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -67,26 +67,11 @@ protected GetFieldMappingsResponse createTestInstance() { return new GetFieldMappingsResponse(randomMapping()); } - static int charCount(final String s, char ch) { - int count = 0; - int pos = 0; - while (pos >= 0) { - pos = s.indexOf(ch, pos); - if (pos >= 0) { - count++; - pos++; - } - } - return count; - } - @Override protected Predicate getRandomFieldsExcludeFilter() { // allow random fields at the level of index and index.mappings.doctype.field // otherwise random field could be evaluated as index name or type name - return s -> { - final int c = s.length() > 0 ? charCount(s, '.') : -1; - return c != 0 && c != 3;}; + return s -> !(s.matches("[^.]+") || s.matches("[^.]+\\.[^.]+\\.[^.]+\\.[^.]+")); } private Map>> randomMapping() { From e38c145345b03fea150c58bd48289d12bb60a9ba Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 21 Jun 2018 14:43:45 +0200 Subject: [PATCH 09/16] add and use setRandomLocal(Consumer) --- .../client/RequestConvertersTests.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 33cb8d08215df..78df15fd4b3cd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -486,14 +486,7 @@ public void testGetFieldMapping() throws IOException { Map expectedParams = new HashMap<>(); setRandomIndicesOptions(getFieldMappingsRequest::indicesOptions, getFieldMappingsRequest::indicesOptions, expectedParams); - - if (randomBoolean()) { - boolean local = randomBoolean(); - getFieldMappingsRequest.local(local); - if (local) { - expectedParams.put("local", String.valueOf(local)); - } - } + setRandomLocal(getFieldMappingsRequest::local, expectedParams); Request request = RequestConverters.getFieldMapping(getFieldMappingsRequest); StringJoiner endpoint = new StringJoiner("/", "/", ""); @@ -2219,16 +2212,20 @@ private static void setRandomHumanReadable(GetIndexRequest request, Map request, Map expectedParams) { + private static void setRandomLocal(Consumer setter, Map expectedParams) { if (randomBoolean()) { boolean local = randomBoolean(); - request.local(local); + setter.accept(local); if (local) { expectedParams.put("local", String.valueOf(local)); } } } + private static void setRandomLocal(MasterNodeReadRequest request, Map expectedParams) { + setRandomLocal(request::local, expectedParams); + } + private static void setRandomTimeout(Consumer setter, TimeValue defaultTimeout, Map expectedParams) { if (randomBoolean()) { String timeout = randomTimeValue(); From a7403ed45470eb5b9f19f831048901ea5ec884fd Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 21 Jun 2018 14:44:44 +0200 Subject: [PATCH 10/16] rebuilt fromXContent --- .../mapping/get/GetFieldMappingsResponse.java | 95 +++++++++++-------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index fb11a04820dee..5ece6e972db33 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -42,12 +43,54 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; /** Response object for {@link GetFieldMappingsRequest} API */ public class GetFieldMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); + private static final ObjectParser PARSER = + new ObjectParser<>(MAPPINGS.getPreferredName(), true, Builder::new); + + static { + PARSER.declareField((p, b, index) -> { + final Map map = p.map(); + final Map> typeMappings = new HashMap<>(); + for (Map.Entry typeObjectEntry : map.entrySet()) { + final Map fieldsMapping = new HashMap<>(); + typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); + + final Object o1 = typeObjectEntry.getValue(); + if (!(o1 instanceof Map)) { + throw new ParsingException(p.getTokenLocation(), + "Nested type mapping at " + index + "." + typeObjectEntry.getKey() + " is not found"); + } + final Map map2 = (Map) o1; + for (Map.Entry e : map2.entrySet()) { + final Object o2 = e.getValue(); + if (!(o2 instanceof Map)) { + throw new ParsingException(p.getTokenLocation(), + "Nested field mapping at " + index + "." + typeObjectEntry.getKey() + "." + e.getKey() + + " is not found"); + } + final Map map3 = (Map) o2; + + final String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); + final XContentBuilder jsonBuilder = jsonBuilder(); + final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); + jsonBuilder.map(values); + final BytesReference source = BytesReference.bytes(jsonBuilder); + + final FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); + fieldsMapping.put(e.getKey(), metaData); + } + + } + b.mappings = typeMappings; + }, MAPPINGS, ObjectParser.ValueType.OBJECT); + } + private Map>> mappings = emptyMap(); GetFieldMappingsResponse(Map>> mappings) { @@ -100,52 +143,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private static class Builder { + Map> mappings; + } + public static GetFieldMappingsResponse fromXContent(XContentParser parser) throws IOException { final Map>> mappings = new HashMap<>(); - final Map map = parser.map(); - - for (Map.Entry entry : map.entrySet()) { - final Object value = entry.getValue(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); - Map> typeMappings = new HashMap<>(); - mappings.put(entry.getKey(), typeMappings); + if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { + while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + final String index = parser.currentName(); - if (value instanceof Map) { - final Object o = ((Map) value).get(MAPPINGS.getPreferredName()); - if (!(o instanceof Map)) { - throw new ParsingException(parser.getTokenLocation(), "Nested " + MAPPINGS.getPreferredName() + " is not found"); - } - Map map1 = (Map) o; - for (Map.Entry typeObjectEntry : map1.entrySet()) { - Map fieldsMapping = new HashMap<>(); - typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); - - final Object o1 = typeObjectEntry.getValue(); - if (!(o1 instanceof Map)) { - throw new ParsingException(parser.getTokenLocation(), - "Nested type mapping at " + entry.getKey() + "." + typeObjectEntry.getKey() + " is not found"); - } - Map map2 = (Map) o1; - for (Map.Entry e : map2.entrySet()) { - final Object o2 = e.getValue(); - if (!(o2 instanceof Map)) { - throw new ParsingException(parser.getTokenLocation(), - "Nested field mapping at " + entry.getKey() + "." + typeObjectEntry.getKey() + "." + e.getKey() - + " is not found"); - } - Map map3 = (Map) o2; - - String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); - XContentBuilder builder = jsonBuilder(); - final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); - builder.map(values); - final BytesReference source = BytesReference.bytes(builder); - - FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); - fieldsMapping.put(e.getKey(), metaData); - } + final Builder builder = PARSER.parse(parser, index); + mappings.put(index, builder.mappings); - } + parser.nextToken(); } } From a12c32e525d19c541c377304863608fdceb3c4ed Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 12:58:48 +0200 Subject: [PATCH 11/16] polish a bit randomFieldsExcludeFilter regexp --- .../indices/mapping/get/GetFieldMappingsResponseTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 816a36fc72a00..63108764293b9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -69,9 +69,10 @@ protected GetFieldMappingsResponse createTestInstance() { @Override protected Predicate getRandomFieldsExcludeFilter() { - // allow random fields at the level of index and index.mappings.doctype.field + // allow random fields at the level of `index` and `index.mappings.doctype.field` // otherwise random field could be evaluated as index name or type name - return s -> !(s.matches("[^.]+") || s.matches("[^.]+\\.[^.]+\\.[^.]+\\.[^.]+")); + return s -> false == (s.matches("(?[^.]+)") + || s.matches("(?[^.]+)\\.mappings\\.(?[^.]+)\\.(?[^.]+)")); } private Map>> randomMapping() { From 34247902af508324a0fc20ed2ee37d6ae18e2aeb Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 14:36:21 +0200 Subject: [PATCH 12/16] drop unnecessary assertThat-s --- .../client/documentation/IndicesClientDocumentationIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index ddfbf3713ec86..d90cfaccb698e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -793,9 +793,6 @@ public void onFailure(Exception e) { final String fullName = metaData1.fullName(); final Map source = metaData1.sourceAsMap(); - - assertThat(fullName, equalTo("message")); - assertThat(source, equalTo(Collections.singletonMap("message", Collections.singletonMap("type", "text")))); latchListener.onResponse(r); }, e -> { latchListener.onFailure(e); From 03b0396a7eb72ea74f7535b28b3069c6f330e677 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 14:58:44 +0200 Subject: [PATCH 13/16] drop intermediate builder from fromXContent process --- .../mapping/get/GetFieldMappingsResponse.java | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 5ece6e972db33..78a3d8ccbd83f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -50,47 +50,6 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte private static final ParseField MAPPINGS = new ParseField("mappings"); - private static final ObjectParser PARSER = - new ObjectParser<>(MAPPINGS.getPreferredName(), true, Builder::new); - - static { - PARSER.declareField((p, b, index) -> { - final Map map = p.map(); - final Map> typeMappings = new HashMap<>(); - for (Map.Entry typeObjectEntry : map.entrySet()) { - final Map fieldsMapping = new HashMap<>(); - typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); - - final Object o1 = typeObjectEntry.getValue(); - if (!(o1 instanceof Map)) { - throw new ParsingException(p.getTokenLocation(), - "Nested type mapping at " + index + "." + typeObjectEntry.getKey() + " is not found"); - } - final Map map2 = (Map) o1; - for (Map.Entry e : map2.entrySet()) { - final Object o2 = e.getValue(); - if (!(o2 instanceof Map)) { - throw new ParsingException(p.getTokenLocation(), - "Nested field mapping at " + index + "." + typeObjectEntry.getKey() + "." + e.getKey() - + " is not found"); - } - final Map map3 = (Map) o2; - - final String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); - final XContentBuilder jsonBuilder = jsonBuilder(); - final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); - jsonBuilder.map(values); - final BytesReference source = BytesReference.bytes(jsonBuilder); - - final FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); - fieldsMapping.put(e.getKey(), metaData); - } - - } - b.mappings = typeMappings; - }, MAPPINGS, ObjectParser.ValueType.OBJECT); - } - private Map>> mappings = emptyMap(); GetFieldMappingsResponse(Map>> mappings) { @@ -143,20 +102,61 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private static class Builder { - Map> mappings; + private static ObjectParser createParser(Map>> mappings) { + final ObjectParser>>, String> parser = + new ObjectParser<>(MAPPINGS.getPreferredName(), true, () -> mappings); + + parser.declareField((p, b, index) -> { + final Map map = p.map(); + final Map> typeMappings = new HashMap<>(); + for (Map.Entry typeObjectEntry : map.entrySet()) { + final Map fieldsMapping = new HashMap<>(); + typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); + + final Object o1 = typeObjectEntry.getValue(); + if (!(o1 instanceof Map)) { + throw new ParsingException(p.getTokenLocation(), + "Nested type mapping at " + index + "." + typeObjectEntry.getKey() + " is not found"); + } + final Map map2 = (Map) o1; + for (Map.Entry e : map2.entrySet()) { + final Object o2 = e.getValue(); + if (!(o2 instanceof Map)) { + throw new ParsingException(p.getTokenLocation(), + "Nested field mapping at " + index + "." + typeObjectEntry.getKey() + "." + e.getKey() + + " is not found"); + } + final Map map3 = (Map) o2; + + final String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); + final XContentBuilder jsonBuilder = jsonBuilder(); + final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); + jsonBuilder.map(values); + final BytesReference source = BytesReference.bytes(jsonBuilder); + + final FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); + fieldsMapping.put(e.getKey(), metaData); + } + + } + mappings.put(index, typeMappings); + }, MAPPINGS, ObjectParser.ValueType.OBJECT); + + return parser; + } public static GetFieldMappingsResponse fromXContent(XContentParser parser) throws IOException { - final Map>> mappings = new HashMap<>(); ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + final Map>> mappings = new HashMap<>(); + final ObjectParser objectParser = createParser(mappings); + if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { final String index = parser.currentName(); - final Builder builder = PARSER.parse(parser, index); - mappings.put(index, builder.mappings); + objectParser.parse(parser, index); parser.nextToken(); } From c991dbc04d3164b6267697d54f9eec9051ddbe97 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 17:22:03 +0200 Subject: [PATCH 14/16] unused import... --- .../client/documentation/IndicesClientDocumentationIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index d90cfaccb698e..95fa7f7185b5b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -92,7 +92,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; From fa77332dad143975d3c81ca465e69fb9d779adb2 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 17:28:11 +0200 Subject: [PATCH 15/16] another round of fromXContent simplification --- .../mapping/get/GetFieldMappingsResponse.java | 97 +++++++++---------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 78a3d8ccbd83f..a0d5de2133ac1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -21,11 +21,11 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -42,6 +42,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -50,6 +51,27 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte private static final ParseField MAPPINGS = new ParseField("mappings"); + private static final ObjectParser>, String> PARSER = + new ObjectParser<>(MAPPINGS.getPreferredName(), true, HashMap::new); + + static { + PARSER.declareField((p, typeMappings, index) -> { + while (p.nextToken() == XContentParser.Token.FIELD_NAME) { + final String typeName = p.currentName(); + final Map typeMapping = new HashMap<>(); + typeMappings.put(typeName, typeMapping); + + if (p.nextToken() == XContentParser.Token.START_OBJECT) { + while (p.nextToken() == XContentParser.Token.FIELD_NAME) { + final String fieldName = p.currentName(); + final FieldMappingMetaData fieldMappingMetaData = FieldMappingMetaData.fromXContent(p); + typeMapping.put(fieldName, fieldMappingMetaData); + } + } + } + }, MAPPINGS, ObjectParser.ValueType.OBJECT); + } + private Map>> mappings = emptyMap(); GetFieldMappingsResponse(Map>> mappings) { @@ -102,61 +124,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private static ObjectParser createParser(Map>> mappings) { - final ObjectParser>>, String> parser = - new ObjectParser<>(MAPPINGS.getPreferredName(), true, () -> mappings); - - parser.declareField((p, b, index) -> { - final Map map = p.map(); - final Map> typeMappings = new HashMap<>(); - for (Map.Entry typeObjectEntry : map.entrySet()) { - final Map fieldsMapping = new HashMap<>(); - typeMappings.put(typeObjectEntry.getKey(), fieldsMapping); - - final Object o1 = typeObjectEntry.getValue(); - if (!(o1 instanceof Map)) { - throw new ParsingException(p.getTokenLocation(), - "Nested type mapping at " + index + "." + typeObjectEntry.getKey() + " is not found"); - } - final Map map2 = (Map) o1; - for (Map.Entry e : map2.entrySet()) { - final Object o2 = e.getValue(); - if (!(o2 instanceof Map)) { - throw new ParsingException(p.getTokenLocation(), - "Nested field mapping at " + index + "." + typeObjectEntry.getKey() + "." + e.getKey() - + " is not found"); - } - final Map map3 = (Map) o2; - - final String fullName = (String) map3.get(FieldMappingMetaData.FULL_NAME.getPreferredName()); - final XContentBuilder jsonBuilder = jsonBuilder(); - final Map values = (Map) map3.get(FieldMappingMetaData.MAPPING.getPreferredName()); - jsonBuilder.map(values); - final BytesReference source = BytesReference.bytes(jsonBuilder); - - final FieldMappingMetaData metaData = new FieldMappingMetaData(fullName, source); - fieldsMapping.put(e.getKey(), metaData); - } - - } - mappings.put(index, typeMappings); - }, MAPPINGS, ObjectParser.ValueType.OBJECT); - - return parser; - - } - public static GetFieldMappingsResponse fromXContent(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); final Map>> mappings = new HashMap<>(); - final ObjectParser objectParser = createParser(mappings); - if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { while (parser.currentToken() == XContentParser.Token.FIELD_NAME) { final String index = parser.currentName(); - objectParser.parse(parser, index); + final Map> typeMappings = PARSER.parse(parser, index); + mappings.put(index, typeMappings); parser.nextToken(); } @@ -171,6 +148,22 @@ public static class FieldMappingMetaData implements ToXContentFragment { private static final ParseField FULL_NAME = new ParseField("full_name"); private static final ParseField MAPPING = new ParseField("mapping"); + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("field_mapping_meta_data", true, + a -> new FieldMappingMetaData((String)a[0], (BytesReference)a[1]) + ); + + static { + PARSER.declareField(optionalConstructorArg(), + (p, c) -> p.text(), FULL_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), + (p, c) -> { + final XContentBuilder jsonBuilder = jsonBuilder().copyCurrentStructure(p); + final BytesReference bytes = BytesReference.bytes(jsonBuilder); + return bytes; + }, MAPPING, ObjectParser.ValueType.OBJECT); + } + private String fullName; private BytesReference source; @@ -197,14 +190,18 @@ BytesReference getSource() { return source; } + public static FieldMappingMetaData fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("full_name", fullName); + builder.field(FULL_NAME.getPreferredName(), fullName); if (params.paramAsBoolean("pretty", false)) { builder.field("mapping", sourceAsMap()); } else { try (InputStream stream = source.streamInput()) { - builder.rawField("mapping", stream, XContentType.JSON); + builder.rawField(MAPPING.getPreferredName(), stream, XContentType.JSON); } } return builder; From 7d61d23d33e744e27dc8ca2dda43e7370c3679d9 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 22 Jun 2018 21:08:11 +0200 Subject: [PATCH 16/16] improve junk protection on parsing --- .../mapping/get/GetFieldMappingsResponse.java | 11 +++-- .../get/GetFieldMappingsResponseTests.java | 46 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index a0d5de2133ac1..81b9812d61c5f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -56,18 +56,23 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte static { PARSER.declareField((p, typeMappings, index) -> { - while (p.nextToken() == XContentParser.Token.FIELD_NAME) { + p.nextToken(); + while (p.currentToken() == XContentParser.Token.FIELD_NAME) { final String typeName = p.currentName(); - final Map typeMapping = new HashMap<>(); - typeMappings.put(typeName, typeMapping); if (p.nextToken() == XContentParser.Token.START_OBJECT) { + final Map typeMapping = new HashMap<>(); + typeMappings.put(typeName, typeMapping); + while (p.nextToken() == XContentParser.Token.FIELD_NAME) { final String fieldName = p.currentName(); final FieldMappingMetaData fieldMappingMetaData = FieldMappingMetaData.fromXContent(p); typeMapping.put(fieldName, fieldMappingMetaData); } + } else { + p.skipChildren(); } + p.nextToken(); } }, MAPPINGS, ObjectParser.ValueType.OBJECT); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 63108764293b9..b6e785a4d05be 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -23,7 +23,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.io.IOException; @@ -32,6 +34,8 @@ import java.util.Map; import java.util.function.Predicate; +import static org.hamcrest.CoreMatchers.equalTo; + public class GetFieldMappingsResponseTests extends AbstractStreamableXContentTestCase { public void testManualSerialization() throws IOException { @@ -52,6 +56,48 @@ public void testManualSerialization() throws IOException { } } + public void testManualJunkedJson() throws Exception { + // in fact random fields could be evaluated as proper mapping, while proper junk in this case is arrays and values + final String json = + "{\"index1\":{\"mappings\":" + + "{\"doctype0\":{\"field1\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}," + + "\"field0\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}}," + // junk here + + "\"junk1\": [\"field1\", {\"field2\":{}}]," + + "\"junk2\": [{\"field3\":{}}]," + + "\"junk3\": 42," + + "\"junk4\": \"Q\"," + + "\"doctype1\":{\"field1\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}," + + "\"field0\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}}}}," + + "\"index0\":{\"mappings\":" + + "{\"doctype0\":{\"field1\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}," + + "\"field0\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}}," + + "\"doctype1\":{\"field1\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}," + + "\"field0\":{\"full_name\":\"my field\",\"mapping\":{\"type\":\"keyword\"}}}}}}"; + + final XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), + LoggingDeprecationHandler.INSTANCE, json.getBytes("UTF-8")); + + final GetFieldMappingsResponse response = GetFieldMappingsResponse.fromXContent(parser); + + FieldMappingMetaData fieldMappingMetaData = + new FieldMappingMetaData("my field", new BytesArray("{\"type\":\"keyword\"}")); + Map fieldMapping = new HashMap<>(); + fieldMapping.put("field0", fieldMappingMetaData); + fieldMapping.put("field1", fieldMappingMetaData); + + Map> typeMapping = new HashMap<>(); + typeMapping.put("doctype0", fieldMapping); + typeMapping.put("doctype1", fieldMapping); + + Map>> mappings = new HashMap<>(); + mappings.put("index0", typeMapping); + mappings.put("index1", typeMapping); + + final Map>> responseMappings = response.mappings(); + assertThat(responseMappings, equalTo(mappings)); + } + @Override protected GetFieldMappingsResponse doParseInstance(XContentParser parser) throws IOException { return GetFieldMappingsResponse.fromXContent(parser);