From 7364161b0c8acd12898cae247bbf1c2c2d977b4c Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 3 May 2018 21:19:56 +0000 Subject: [PATCH 1/7] [Rollup] Allow customization of how RollupCaps are grouped This adds a `key` parameter to the GetRollupCaps API, which allows the user to specify how they would like the capabilities grouped at the top level. The default behavior is as before: grouping by index pattern. But if the user specifies `key: rollup_index`, it groups the capabilities by the concrete rollup index where the rollups are stored. --- .../en/rest-api/rollup/rollup-caps.asciidoc | 84 +++++- .../rollup/action/GetRollupCapsAction.java | 47 +++- .../action/TransportGetRollupCapsAction.java | 43 ++- .../rollup/rest/RestGetRollupCapsAction.java | 4 +- .../GetRollupCapsActionRequestTests.java | 134 +++++++++- .../api/xpack.rollup.get_rollup_caps.json | 6 + .../test/rollup/get_rollup_caps.yml | 246 ++++++++++++++++++ 7 files changed, 555 insertions(+), 9 deletions(-) diff --git a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc index 5a4dab69d937f..cdffa80d7f393 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc @@ -29,6 +29,13 @@ live? rollup job capabilities will be returned +==== Query String parameters + +`key`:: + (string) How the response should be keyed. `index_pattern`, which groups rollup capabilities by their index pattern. + Alternative option is `rollup_index`, which groups capabilities by the index the rollup is stored in. If omitted, + the default is `index_pattern` + ==== Request Body There is no request body for the Get Jobs API. @@ -177,4 +184,79 @@ GET _xpack/rollup/data/sensor-1 Why is this? The original rollup job was configured against a specific index pattern (`sensor-*`) not a concrete index (`sensor-1`). So while the index belongs to the pattern, the rollup job is only valid across the entirety of the pattern not just one of it's containing indices. So for that reason, the Rollup Capabilities API only returns information based -on the originally configured index name or pattern. \ No newline at end of file +on the originally configured index name or pattern. + +===== Changing key grouping + +It is possible to change how rollup capabilities are grouped. By default, they are grouped by their index pattern. Instead, +they can be grouped by the concrete index the rollups are stored in. This is sometimes a useful operation if you are managing +the rollup data itself. + +To change the grouping, you can specify a `key` parameter in the URL: + +[source,js] +-------------------------------------------------- +GET _xpack/rollup/data/sensor_rollup?key=rollup_index +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +Note how we are requesting the concrete rollup index name (`sensor_rollup`) instead of a pattern, and specifying `key=rollup_index`. +This will yield the following response: + +[source,js] +---- +{ + "sensor_rollup" : { <1> + "rollup_jobs" : [ + { + "job_id" : "sensor", + "rollup_index" : "sensor_rollup", + "index_pattern" : "sensor-*", + "fields" : { + "node" : [ + { + "agg" : "terms" + } + ], + "temperature" : [ + { + "agg" : "min" + }, + { + "agg" : "max" + }, + { + "agg" : "sum" + } + ], + "timestamp" : [ + { + "agg" : "date_histogram", + "time_zone" : "UTC", + "interval" : "1h", + "delay": "7d" + } + ], + "voltage" : [ + { + "agg" : "avg" + } + ] + } + } + ] + } +} +---- +// TESTRESPONSE +<1> The response is now grouped by the rollup index name at the top level + +You can also get the capabilities for all rollups using `_all`, grouped by the rollup index: + +[source,js] +-------------------------------------------------- +GET _xpack/rollup/data/_all?key=rollup_index +-------------------------------------------------- +// CONSOLE +// TEST[continued] \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java index 83e9c7f8a5b42..c5b5074c34472 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.rollup.action; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; @@ -25,6 +26,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -52,13 +54,15 @@ public Response newResponse() { public static class Request extends ActionRequest implements ToXContent { private String indexPattern; + private KeyOrder key = KeyOrder.INDEX_PATTERN; - public Request(String indexPattern) { + public Request(String indexPattern, String key) { if (Strings.isNullOrEmpty(indexPattern) || indexPattern.equals("*")) { this.indexPattern = MetaData.ALL; } else { this.indexPattern = indexPattern; } + this.key = KeyOrder.fromString(key); } public Request() {} @@ -67,16 +71,26 @@ public String getIndexPattern() { return indexPattern; } + public KeyOrder getKey() { + return key; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.indexPattern = in.readString(); + if (in.getVersion().onOrAfter(Version.V_6_4_0)) { + this.key = KeyOrder.fromStream(in); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(indexPattern); + if (out.getVersion().onOrAfter(Version.V_6_4_0)) { + key.writeTo(out); + } } @Override @@ -87,12 +101,13 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(RollupField.ID.getPreferredName(), indexPattern); + builder.field(KeyOrder.KEY.getPreferredName(), key); return builder; } @Override public int hashCode() { - return Objects.hash(indexPattern); + return Objects.hash(indexPattern, key); } @Override @@ -104,7 +119,8 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(indexPattern, other.indexPattern); + return Objects.equals(indexPattern, other.indexPattern) + && Objects.equals(key, other.key); } } @@ -173,4 +189,29 @@ public final String toString() { return Strings.toString(this); } } + + public enum KeyOrder implements Writeable { + INDEX_PATTERN, ROLLUP_INDEX; + + public static final ParseField KEY = new ParseField("key"); + + public static KeyOrder fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static KeyOrder fromStream(StreamInput in) throws IOException { + return in.readEnum(KeyOrder.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + KeyOrder state = this; + out.writeEnum(state); + } + + public String value() { + return name().toLowerCase(Locale.ROOT); + } + } + } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java index 8789fb75ad1a0..9e5f4b08bb24d 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; +import org.elasticsearch.xpack.core.rollup.job.RollupJob; import java.util.List; import java.util.Map; @@ -49,7 +50,14 @@ public TransportGetRollupCapsAction(Settings settings, @Override protected void doExecute(GetRollupCapsAction.Request request, ActionListener listener) { - Map allCaps = getCaps(request.getIndexPattern(), clusterService.state().getMetaData().indices()); + Map allCaps; + if (request.getKey().equals(GetRollupCapsAction.KeyOrder.INDEX_PATTERN)) { + allCaps = getCaps(request.getIndexPattern(), clusterService.state().getMetaData().indices()); + } else if (request.getKey().equals(GetRollupCapsAction.KeyOrder.ROLLUP_INDEX)) { + allCaps = getCapsByRollupIndex(request.getIndexPattern(), clusterService.state().getMetaData().indices()); + } else { + throw new IllegalArgumentException("Unknown key ordering: [" + request.getKey() + "]"); + } listener.onResponse(new GetRollupCapsAction.Response(allCaps)); } @@ -86,6 +94,37 @@ static Map getCaps(String indexPattern, ImmutableOpen return allCaps; } + static Map getCapsByRollupIndex(String rollupIndex, ImmutableOpenMap indices) { + Map allCaps = new TreeMap<>(); + if (rollupIndex.equals(MetaData.ALL)) { + for (ObjectObjectCursor entry : indices) { + + // Does this index have rollup metadata? + TransportGetRollupCapsAction.findRollupIndexCaps(entry.key, entry.value) + .ifPresent(cap -> { + cap.getJobCaps().forEach(jobCap -> { + // Do we already have an entry for this index? + RollableIndexCaps indexCaps = allCaps.get(jobCap.getRollupIndex()); + if (indexCaps == null) { + indexCaps = new RollableIndexCaps(jobCap.getRollupIndex()); + } + indexCaps.addJobCap(jobCap); + allCaps.put(jobCap.getRollupIndex(), indexCaps); + }); + }); + } + } else { + findRollupIndexCaps(rollupIndex, indices.get(rollupIndex)).ifPresent(cap -> { + RollableIndexCaps indexCaps = new RollableIndexCaps(rollupIndex); + cap.getJobCaps().forEach(jobCap -> { + indexCaps.addJobCap(jobCap); + allCaps.put(jobCap.getRollupIndex(), indexCaps); + }); + }); + } + return allCaps; + } + static Optional findRollupIndexCaps(String indexName, IndexMetaData indexMetaData) { if (indexMetaData == null) { return Optional.empty(); @@ -109,4 +148,4 @@ static Optional findRollupIndexCaps(String indexName, IndexMeta } return Optional.empty(); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java index dfda4d0e9e15b..f2b938613e5fc 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java @@ -19,6 +19,7 @@ public class RestGetRollupCapsAction extends BaseRestHandler { public static final ParseField ID = new ParseField("id"); + public static final ParseField KEY = new ParseField("key"); public RestGetRollupCapsAction(Settings settings, RestController controller) { super(settings); @@ -28,7 +29,8 @@ public RestGetRollupCapsAction(Settings settings, RestController controller) { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String id = restRequest.param(ID.getPreferredName()); - GetRollupCapsAction.Request request = new GetRollupCapsAction.Request(id); + String key = restRequest.param(KEY.getPreferredName(), "index_pattern"); + GetRollupCapsAction.Request request = new GetRollupCapsAction.Request(id, key); return channel -> client.execute(GetRollupCapsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java index e63650397abdf..d531be4d18ce2 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java @@ -31,10 +31,12 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< @Override protected GetRollupCapsAction.Request createTestInstance() { + String key = randomBoolean() ? GetRollupCapsAction.KeyOrder.INDEX_PATTERN.toString() + : GetRollupCapsAction.KeyOrder.ROLLUP_INDEX.toString(); if (randomBoolean()) { - return new GetRollupCapsAction.Request(MetaData.ALL); + return new GetRollupCapsAction.Request(MetaData.ALL, key); } - return new GetRollupCapsAction.Request(randomAlphaOfLengthBetween(1, 20)); + return new GetRollupCapsAction.Request(randomAlphaOfLengthBetween(1, 20), key); } @Override @@ -136,6 +138,12 @@ public void testNoIndices() { assertThat(caps.size(), equalTo(0)); } + public void testNoIndicesByRollup() { + ImmutableOpenMap indices = new ImmutableOpenMap.Builder().build(); + Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("foo", indices); + assertThat(caps.size(), equalTo(0)); + } + public void testAllIndices() throws IOException { int num = randomIntBetween(1,5); ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); @@ -166,6 +174,66 @@ public void testAllIndices() throws IOException { assertThat(caps.size(), equalTo(num * 5)); } + public void testAllIndicesByRollupSingleRollup() throws IOException { + int num = randomIntBetween(1,5); + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(num); + for (int i = 0; i < num; i++) { + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build()); + } + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("foo", meta); + } + + Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex(MetaData.ALL, indices.build()); + assertThat(caps.size(), equalTo(1)); + } + + public void testAllIndicesByRollupManyRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern(indexName) + .setRollupIndex("rollup_" + indexName).build()); + + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_" + indexName, meta); + } + + Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex(MetaData.ALL, indices.build()); + assertThat(caps.size(), equalTo(5)); + } + public void testOneIndex() throws IOException { int num = randomIntBetween(1,5); ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); @@ -198,6 +266,68 @@ public void testOneIndex() throws IOException { Map caps = TransportGetRollupCapsAction.getCaps(selectedIndexName, indices.build()); assertThat(caps.size(), equalTo(1)); } + + public void testOneIndexByRollupManyRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern("foo_" + indexName) + .setRollupIndex("rollup_" + indexName).build()); + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_" + indexName, meta); + } + + Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("rollup_1", indices.build()); + assertThat(caps.size(), equalTo(1)); + assertThat(caps.get("rollup_1").getIndexName(), equalTo("rollup_1")); + assertThat(caps.get("rollup_1").getJobCaps().size(), equalTo(1)); + } + + public void testOneIndexByRollupOneRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern("foo_" + indexName) + .setRollupIndex("rollup_foo").build()); + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_foo", meta); + } + + Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("rollup_foo", indices.build()); + assertThat(caps.size(), equalTo(1)); + assertThat(caps.get("rollup_foo").getIndexName(), equalTo("rollup_foo")); + assertThat(caps.get("rollup_foo").getJobCaps().size(), equalTo(1)); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json index 28edd044c3cfe..5bce163880af5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json @@ -11,6 +11,12 @@ "required": false, "description": "The ID of the index to check rollup capabilities on, or left blank for all jobs" } + }, + "params": { + "key": { + "type": "string", + "description": "How the response should be keyed at the top level" + } } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml index 050e49bc4b40f..6cac760cc80f8 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml @@ -22,6 +22,18 @@ setup: type: date value_field: type: integer + + - do: + indices.create: + index: foo3 + include_type_name: false + body: + mappings: + properties: + the_field: + type: date + value_field: + type: integer - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser @@ -229,3 +241,237 @@ setup: - agg: "max" - agg: "sum" +--- +"Verify one job caps by rollup index": + + - do: + xpack.rollup.get_rollup_caps: + id: "foo_rollup" + key: "rollup_index" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + +--- +"Verify two job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + xpack.rollup.get_rollup_caps: + id: "foo_rollup" + key: "rollup_index" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + - job_id: "foo2" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + + +--- +"Verify two different job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo2", + "rollup_index": "foo_rollup2", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + xpack.rollup.get_rollup_caps: + id: "foo_rollup" + key: "rollup_index" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + +--- +"Verify all job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo2", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo3 + body: > + { + "index_pattern": "foo3", + "rollup_index": "foo_rollup2", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + + - do: + xpack.rollup.get_rollup_caps: + id: "_all" + key: "rollup_index" + + - match: + $body: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + - job_id: "foo2" + rollup_index: "foo_rollup" + index_pattern: "foo2" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + foo_rollup2: + rollup_jobs: + - job_id: "foo3" + rollup_index: "foo_rollup2" + index_pattern: "foo3" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + + From e472d150fabd0f07c56792921184f73eed36aaf0 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 7 May 2018 19:32:43 +0000 Subject: [PATCH 2/7] Tweak version numbers These need to be 7.0 until the feature is backported, to keep bwc/mixed cluster tests passing. --- .../rest-api-spec/test/search.aggregation/80_typed_keys.yml | 2 +- .../xpack/core/rollup/action/GetRollupCapsAction.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml index 841d5cf611bab..afe1d6ed9fd9b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml @@ -196,7 +196,7 @@ setup: --- "Test typed keys parameter for date_histogram aggregation and max_bucket pipeline aggregation": - do: - search: + search:getr typed_keys: true body: size: 0 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java index c5b5074c34472..2b4fd55047cd9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java @@ -79,7 +79,7 @@ public KeyOrder getKey() { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.indexPattern = in.readString(); - if (in.getVersion().onOrAfter(Version.V_6_4_0)) { + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { this.key = KeyOrder.fromStream(in); } } @@ -88,7 +88,7 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(indexPattern); - if (out.getVersion().onOrAfter(Version.V_6_4_0)) { + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { key.writeTo(out); } } From e5b5f9c7e47d5543a7802115e8fa5455c6110e10 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 7 May 2018 19:36:09 +0000 Subject: [PATCH 3/7] [Docs] Update Changelog --- docs/CHANGELOG.asciidoc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index f9cb572eb81a4..b7e72bfda1b58 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -98,8 +98,12 @@ ones that the user is authorized to access in case field level security is enabl //[float] //=== New Features -//[float] -//=== Enhancements +[float] +=== Enhancements + +Rollup:: +* In addition to grouping by index pattern, the GetRollupCapabilities API now allows +grouping by the concrete index that rollups are physically stored in ({pull}30401[#30401]) [float] === Bug Fixes @@ -166,6 +170,10 @@ started or stopped. ({pull}30118[#30118]) Added put index template API to the high level rest client ({pull}30400[#30400]) +Rollup:: +* In addition to grouping by index pattern, the GetRollupCapabilities API now allows +grouping by the concrete index that rollups are physically stored in ({pull}30401[#30401]) + [float] === Bug Fixes From d25db3143cff835c162b540504776a18e97d04a7 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 26 Jun 2018 19:11:12 +0000 Subject: [PATCH 4/7] Move to new endpoint --- .../rollup/action/GetRollupCapsAction.java | 49 +-- .../action/GetRollupIndexCapsAction.java | 186 +++++++++ .../elasticsearch/xpack/rollup/Rollup.java | 33 +- .../action/TransportGetRollupCapsAction.java | 46 +-- .../TransportGetRollupIndexCapsAction.java | 75 ++++ .../rollup/rest/RestGetRollupCapsAction.java | 4 +- .../rest/RestGetRollupIndexCapsAction.java | 41 ++ .../GetRollupCapsActionRequestTests.java | 136 +------ .../GetRollupIndexCapsActionRequestTests.java | 184 +++++++++ .../api/xpack.rollup.get_rollup_caps.json | 6 - .../xpack.rollup.get_rollup_index_caps.json | 17 + .../test/rollup/get_rollup_caps.yml | 235 ------------ .../test/rollup/get_rollup_index_caps.yml | 363 ++++++++++++++++++ 13 files changed, 896 insertions(+), 479 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java create mode 100644 x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java create mode 100644 x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java create mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java index 99a8f339c1904..128874a6c8c87 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.core.rollup.action; -import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; @@ -26,7 +25,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -48,15 +46,13 @@ public Response newResponse() { public static class Request extends ActionRequest implements ToXContent { private String indexPattern; - private KeyOrder key = KeyOrder.INDEX_PATTERN; - public Request(String indexPattern, String key) { + public Request(String indexPattern) { if (Strings.isNullOrEmpty(indexPattern) || indexPattern.equals("*")) { this.indexPattern = MetaData.ALL; } else { this.indexPattern = indexPattern; } - this.key = KeyOrder.fromString(key); } public Request() {} @@ -65,26 +61,16 @@ public String getIndexPattern() { return indexPattern; } - public KeyOrder getKey() { - return key; - } - @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.indexPattern = in.readString(); - if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - this.key = KeyOrder.fromStream(in); - } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(indexPattern); - if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - key.writeTo(out); - } } @Override @@ -95,13 +81,12 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(RollupField.ID.getPreferredName(), indexPattern); - builder.field(KeyOrder.KEY.getPreferredName(), key); return builder; } @Override public int hashCode() { - return Objects.hash(indexPattern, key); + return Objects.hash(indexPattern); } @Override @@ -113,8 +98,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(indexPattern, other.indexPattern) - && Objects.equals(key, other.key); + return Objects.equals(indexPattern, other.indexPattern); } } @@ -155,7 +139,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); for (Map.Entry entry : jobs.entrySet()) { - entry.getValue().toXContent(builder, params); + entry.getValue().toXContent(builder, params); } builder.endObject(); return builder; @@ -183,29 +167,4 @@ public final String toString() { return Strings.toString(this); } } - - public enum KeyOrder implements Writeable { - INDEX_PATTERN, ROLLUP_INDEX; - - public static final ParseField KEY = new ParseField("key"); - - public static KeyOrder fromString(String name) { - return valueOf(name.trim().toUpperCase(Locale.ROOT)); - } - - public static KeyOrder fromStream(StreamInput in) throws IOException { - return in.readEnum(KeyOrder.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - KeyOrder state = this; - out.writeEnum(state); - } - - public String value() { - return name().toLowerCase(Locale.ROOT); - } - } - } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java new file mode 100644 index 0000000000000..9693bf42384c2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.rollup.action; + + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.rollup.RollupField; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +public class GetRollupIndexCapsAction extends Action { + + public static final GetRollupIndexCapsAction INSTANCE = new GetRollupIndexCapsAction(); + public static final String NAME = "indices:data/xpack/rollup/get_index/caps"; + public static final ParseField CONFIG = new ParseField("config"); + public static final ParseField STATUS = new ParseField("status"); + private static final ParseField INDICES_OPTIONS = new ParseField("indices_options"); + + private GetRollupIndexCapsAction() { + super(NAME); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + private String indexPattern; + private IndicesOptions options; + + public Request(String indexPattern) { + this(indexPattern, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); + } + + public Request(String indexPattern, IndicesOptions options) { + if (Strings.isNullOrEmpty(indexPattern) || indexPattern.equals("*")) { + this.indexPattern = MetaData.ALL; + } else { + this.indexPattern = indexPattern; + } + this.options = options; + } + + public Request() {} + + public String getIndexPattern() { + return indexPattern; + } + + public IndicesOptions getOptions() { + return options; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + this.indexPattern = in.readString(); + this.options = IndicesOptions.readIndicesOptions(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(indexPattern); + options.writeIndicesOptions(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(RollupField.ID.getPreferredName(), indexPattern); + builder.field(INDICES_OPTIONS.getPreferredName(), options); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern, options); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(indexPattern, other.indexPattern) + && Objects.equals(options, other.options); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, GetRollupIndexCapsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements Writeable, ToXContentObject { + + private Map jobs = Collections.emptyMap(); + + public Response() { + + } + + public Response(Map jobs) { + this.jobs = Objects.requireNonNull(jobs); + } + + Response(StreamInput in) throws IOException { + jobs = in.readMap(StreamInput::readString, RollableIndexCaps::new); + } + + public Map getJobs() { + return jobs; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(jobs, StreamOutput::writeString, (out1, value) -> value.writeTo(out1)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + for (Map.Entry entry : jobs.entrySet()) { + entry.getValue().toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobs); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(jobs, other.jobs); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } +} diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java index cc24a0b4ab944..546103df5dd46 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; +import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction; import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; @@ -47,6 +48,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.rollup.action.TransportDeleteRollupJobAction; import org.elasticsearch.xpack.rollup.action.TransportGetRollupCapsAction; +import org.elasticsearch.xpack.rollup.action.TransportGetRollupIndexCapsAction; import org.elasticsearch.xpack.rollup.action.TransportGetRollupJobAction; import org.elasticsearch.xpack.rollup.action.TransportPutRollupJobAction; import org.elasticsearch.xpack.rollup.action.TransportRollupSearchAction; @@ -55,6 +57,7 @@ import org.elasticsearch.xpack.rollup.job.RollupJobTask; import org.elasticsearch.xpack.rollup.rest.RestDeleteRollupJobAction; import org.elasticsearch.xpack.rollup.rest.RestGetRollupCapsAction; +import org.elasticsearch.xpack.rollup.rest.RestGetRollupIndexCapsAction; import org.elasticsearch.xpack.rollup.rest.RestGetRollupJobsAction; import org.elasticsearch.xpack.rollup.rest.RestPutRollupJobAction; import org.elasticsearch.xpack.rollup.rest.RestRollupSearchAction; @@ -136,13 +139,14 @@ public List getRestHandlers(Settings settings, RestController restC } return Arrays.asList( - new RestRollupSearchAction(settings, restController), - new RestPutRollupJobAction(settings, restController), - new RestStartRollupJobAction(settings, restController), - new RestStopRollupJobAction(settings, restController), - new RestDeleteRollupJobAction(settings, restController), - new RestGetRollupJobsAction(settings, restController), - new RestGetRollupCapsAction(settings, restController) + new RestRollupSearchAction(settings, restController), + new RestPutRollupJobAction(settings, restController), + new RestStartRollupJobAction(settings, restController), + new RestStopRollupJobAction(settings, restController), + new RestDeleteRollupJobAction(settings, restController), + new RestGetRollupJobsAction(settings, restController), + new RestGetRollupCapsAction(settings, restController), + new RestGetRollupIndexCapsAction(settings, restController) ); } @@ -153,13 +157,14 @@ public List getRestHandlers(Settings settings, RestController restC return emptyList(); } return Arrays.asList( - new ActionHandler<>(RollupSearchAction.INSTANCE, TransportRollupSearchAction.class), - new ActionHandler<>(PutRollupJobAction.INSTANCE, TransportPutRollupJobAction.class), - new ActionHandler<>(StartRollupJobAction.INSTANCE, TransportStartRollupAction.class), - new ActionHandler<>(StopRollupJobAction.INSTANCE, TransportStopRollupAction.class), - new ActionHandler<>(DeleteRollupJobAction.INSTANCE, TransportDeleteRollupJobAction.class), - new ActionHandler<>(GetRollupJobsAction.INSTANCE, TransportGetRollupJobAction.class), - new ActionHandler<>(GetRollupCapsAction.INSTANCE, TransportGetRollupCapsAction.class) + new ActionHandler<>(RollupSearchAction.INSTANCE, TransportRollupSearchAction.class), + new ActionHandler<>(PutRollupJobAction.INSTANCE, TransportPutRollupJobAction.class), + new ActionHandler<>(StartRollupJobAction.INSTANCE, TransportStartRollupAction.class), + new ActionHandler<>(StopRollupJobAction.INSTANCE, TransportStopRollupAction.class), + new ActionHandler<>(DeleteRollupJobAction.INSTANCE, TransportDeleteRollupJobAction.class), + new ActionHandler<>(GetRollupJobsAction.INSTANCE, TransportGetRollupJobAction.class), + new ActionHandler<>(GetRollupCapsAction.INSTANCE, TransportGetRollupCapsAction.class), + new ActionHandler<>(GetRollupIndexCapsAction.INSTANCE, TransportGetRollupIndexCapsAction.class) ); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java index 3150e045fb1a4..6d565e43b8644 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; -import org.elasticsearch.xpack.core.rollup.job.RollupJob; import java.util.List; import java.util.Map; @@ -45,15 +44,7 @@ public TransportGetRollupCapsAction(Settings settings, TransportService transpor @Override protected void doExecute(Task task, GetRollupCapsAction.Request request, ActionListener listener) { - - Map allCaps; - if (request.getKey().equals(GetRollupCapsAction.KeyOrder.INDEX_PATTERN)) { - allCaps = getCaps(request.getIndexPattern(), clusterService.state().getMetaData().indices()); - } else if (request.getKey().equals(GetRollupCapsAction.KeyOrder.ROLLUP_INDEX)) { - allCaps = getCapsByRollupIndex(request.getIndexPattern(), clusterService.state().getMetaData().indices()); - } else { - throw new IllegalArgumentException("Unknown key ordering: [" + request.getKey() + "]"); - } + Map allCaps = getCaps(request.getIndexPattern(), clusterService.state().getMetaData().indices()); listener.onResponse(new GetRollupCapsAction.Response(allCaps)); } @@ -75,7 +66,7 @@ static Map getCaps(String indexPattern, ImmutableOpen jobCaps.forEach(jobCap -> { String pattern = indexPattern.equals(MetaData.ALL) - ? jobCap.getIndexPattern() : indexPattern; + ? jobCap.getIndexPattern() : indexPattern; // Do we already have an entry for this index pattern? RollableIndexCaps indexCaps = allCaps.get(pattern); @@ -90,37 +81,6 @@ static Map getCaps(String indexPattern, ImmutableOpen return allCaps; } - static Map getCapsByRollupIndex(String rollupIndex, ImmutableOpenMap indices) { - Map allCaps = new TreeMap<>(); - if (rollupIndex.equals(MetaData.ALL)) { - for (ObjectObjectCursor entry : indices) { - - // Does this index have rollup metadata? - TransportGetRollupCapsAction.findRollupIndexCaps(entry.key, entry.value) - .ifPresent(cap -> { - cap.getJobCaps().forEach(jobCap -> { - // Do we already have an entry for this index? - RollableIndexCaps indexCaps = allCaps.get(jobCap.getRollupIndex()); - if (indexCaps == null) { - indexCaps = new RollableIndexCaps(jobCap.getRollupIndex()); - } - indexCaps.addJobCap(jobCap); - allCaps.put(jobCap.getRollupIndex(), indexCaps); - }); - }); - } - } else { - findRollupIndexCaps(rollupIndex, indices.get(rollupIndex)).ifPresent(cap -> { - RollableIndexCaps indexCaps = new RollableIndexCaps(rollupIndex); - cap.getJobCaps().forEach(jobCap -> { - indexCaps.addJobCap(jobCap); - allCaps.put(jobCap.getRollupIndex(), indexCaps); - }); - }); - } - return allCaps; - } - static Optional findRollupIndexCaps(String indexName, IndexMetaData indexMetaData) { if (indexMetaData == null) { return Optional.empty(); @@ -137,7 +97,7 @@ static Optional findRollupIndexCaps(String indexName, IndexMeta } RollupIndexCaps caps = RollupIndexCaps.parseMetadataXContent( - new BytesArray(rollupMapping.source().uncompressed()), indexName); + new BytesArray(rollupMapping.source().uncompressed()), indexName); if (caps.hasCaps()) { return Optional.of(caps); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java new file mode 100644 index 0000000000000..ea1a5ea7d9cf8 --- /dev/null +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.rollup.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; +import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Supplier; +import java.util.stream.StreamSupport; + +public class TransportGetRollupIndexCapsAction extends HandledTransportAction { + + private final ClusterService clusterService; + + @Inject + public TransportGetRollupIndexCapsAction(Settings settings, TransportService transportService, ClusterService clusterService, + ActionFilters actionFilters) { + super(settings, GetRollupIndexCapsAction.NAME, transportService, actionFilters, + (Supplier) GetRollupIndexCapsAction.Request::new); + this.clusterService = clusterService; + } + + @Override + protected void doExecute(Task task, GetRollupIndexCapsAction.Request request, ActionListener listener) { + + IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(clusterService.getSettings()); + String[] indices = resolver.concreteIndexNames(clusterService.state(), request.getOptions(), request.getIndexPattern()); + Map allCaps = getCapsByRollupIndex(Arrays.asList(indices), + clusterService.state().getMetaData().indices()); + listener.onResponse(new GetRollupIndexCapsAction.Response(allCaps)); + } + + static Map getCapsByRollupIndex(List resolvedIndexNames, ImmutableOpenMap indices) { + Map allCaps = new TreeMap<>(); + + StreamSupport.stream(indices.spliterator(), false) + .filter(entry -> resolvedIndexNames.contains(entry.key)) + .forEach(entry -> { + // Does this index have rollup metadata? + TransportGetRollupCapsAction.findRollupIndexCaps(entry.key, entry.value) + .ifPresent(cap -> { + cap.getJobCaps().forEach(jobCap -> { + // Do we already have an entry for this index? + RollableIndexCaps indexCaps = allCaps.get(jobCap.getRollupIndex()); + if (indexCaps == null) { + indexCaps = new RollableIndexCaps(jobCap.getRollupIndex()); + } + indexCaps.addJobCap(jobCap); + allCaps.put(jobCap.getRollupIndex(), indexCaps); + }); + }); + }); + + return allCaps; + } + +} diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java index f2b938613e5fc..dfda4d0e9e15b 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupCapsAction.java @@ -19,7 +19,6 @@ public class RestGetRollupCapsAction extends BaseRestHandler { public static final ParseField ID = new ParseField("id"); - public static final ParseField KEY = new ParseField("key"); public RestGetRollupCapsAction(Settings settings, RestController controller) { super(settings); @@ -29,8 +28,7 @@ public RestGetRollupCapsAction(Settings settings, RestController controller) { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String id = restRequest.param(ID.getPreferredName()); - String key = restRequest.param(KEY.getPreferredName(), "index_pattern"); - GetRollupCapsAction.Request request = new GetRollupCapsAction.Request(id, key); + GetRollupCapsAction.Request request = new GetRollupCapsAction.Request(id); return channel -> client.execute(GetRollupCapsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java new file mode 100644 index 0000000000000..4756191c168d0 --- /dev/null +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.rollup.rest; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; +import org.elasticsearch.xpack.rollup.Rollup; + +import java.io.IOException; + +public class RestGetRollupIndexCapsAction extends BaseRestHandler { + public static final ParseField ID = new ParseField("id"); + + public RestGetRollupIndexCapsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, "/{id}/_xpack/rollup/caps", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + String id = restRequest.param(ID.getPreferredName()); + IndicesOptions options = IndicesOptions.fromRequest(restRequest, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); + GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(id, options); + return channel -> client.execute(GetRollupIndexCapsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } + + @Override + public String getName() { + return "rollup_get_caps_action"; + } +} diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java index d531be4d18ce2..e3a45dbd66b87 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java @@ -11,11 +11,11 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.mockito.Mockito; import java.io.IOException; @@ -31,12 +31,10 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< @Override protected GetRollupCapsAction.Request createTestInstance() { - String key = randomBoolean() ? GetRollupCapsAction.KeyOrder.INDEX_PATTERN.toString() - : GetRollupCapsAction.KeyOrder.ROLLUP_INDEX.toString(); if (randomBoolean()) { - return new GetRollupCapsAction.Request(MetaData.ALL, key); + return new GetRollupCapsAction.Request(MetaData.ALL); } - return new GetRollupCapsAction.Request(randomAlphaOfLengthBetween(1, 20), key); + return new GetRollupCapsAction.Request(randomAlphaOfLengthBetween(1, 20)); } @Override @@ -138,12 +136,6 @@ public void testNoIndices() { assertThat(caps.size(), equalTo(0)); } - public void testNoIndicesByRollup() { - ImmutableOpenMap indices = new ImmutableOpenMap.Builder().build(); - Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("foo", indices); - assertThat(caps.size(), equalTo(0)); - } - public void testAllIndices() throws IOException { int num = randomIntBetween(1,5); ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); @@ -174,66 +166,6 @@ public void testAllIndices() throws IOException { assertThat(caps.size(), equalTo(num * 5)); } - public void testAllIndicesByRollupSingleRollup() throws IOException { - int num = randomIntBetween(1,5); - ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); - int indexCounter = 0; - for (int j = 0; j < 5; j++) { - - Map jobs = new HashMap<>(num); - for (int i = 0; i < num; i++) { - String jobName = randomAlphaOfLength(10); - String indexName = Integer.toString(indexCounter); - indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build()); - } - - MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); - - ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); - mappings.put(RollupField.TYPE_NAME, mappingMeta); - IndexMetaData meta = Mockito.mock(IndexMetaData.class); - Mockito.when(meta.getMappings()).thenReturn(mappings.build()); - indices.put("foo", meta); - } - - Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex(MetaData.ALL, indices.build()); - assertThat(caps.size(), equalTo(1)); - } - - public void testAllIndicesByRollupManyRollup() throws IOException { - ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); - int indexCounter = 0; - for (int j = 0; j < 5; j++) { - - Map jobs = new HashMap<>(1); - String jobName = randomAlphaOfLength(10); - String indexName = Integer.toString(indexCounter); - indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern(indexName) - .setRollupIndex("rollup_" + indexName).build()); - - - MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); - - ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); - mappings.put(RollupField.TYPE_NAME, mappingMeta); - IndexMetaData meta = Mockito.mock(IndexMetaData.class); - Mockito.when(meta.getMappings()).thenReturn(mappings.build()); - indices.put("rollup_" + indexName, meta); - } - - Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex(MetaData.ALL, indices.build()); - assertThat(caps.size(), equalTo(5)); - } - public void testOneIndex() throws IOException { int num = randomIntBetween(1,5); ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); @@ -266,68 +198,6 @@ public void testOneIndex() throws IOException { Map caps = TransportGetRollupCapsAction.getCaps(selectedIndexName, indices.build()); assertThat(caps.size(), equalTo(1)); } - - public void testOneIndexByRollupManyRollup() throws IOException { - ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); - int indexCounter = 0; - for (int j = 0; j < 5; j++) { - - Map jobs = new HashMap<>(1); - String jobName = randomAlphaOfLength(10); - String indexName = Integer.toString(indexCounter); - indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern("foo_" + indexName) - .setRollupIndex("rollup_" + indexName).build()); - - MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); - - ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); - mappings.put(RollupField.TYPE_NAME, mappingMeta); - IndexMetaData meta = Mockito.mock(IndexMetaData.class); - Mockito.when(meta.getMappings()).thenReturn(mappings.build()); - indices.put("rollup_" + indexName, meta); - } - - Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("rollup_1", indices.build()); - assertThat(caps.size(), equalTo(1)); - assertThat(caps.get("rollup_1").getIndexName(), equalTo("rollup_1")); - assertThat(caps.get("rollup_1").getJobCaps().size(), equalTo(1)); - } - - public void testOneIndexByRollupOneRollup() throws IOException { - ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); - int indexCounter = 0; - for (int j = 0; j < 5; j++) { - - Map jobs = new HashMap<>(1); - String jobName = randomAlphaOfLength(10); - String indexName = Integer.toString(indexCounter); - indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern("foo_" + indexName) - .setRollupIndex("rollup_foo").build()); - - MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, - Collections.singletonMap(RollupField.TYPE_NAME, - Collections.singletonMap("_meta", - Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); - - ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); - mappings.put(RollupField.TYPE_NAME, mappingMeta); - IndexMetaData meta = Mockito.mock(IndexMetaData.class); - Mockito.when(meta.getMappings()).thenReturn(mappings.build()); - indices.put("rollup_foo", meta); - } - - Map caps = TransportGetRollupCapsAction.getCapsByRollupIndex("rollup_foo", indices.build()); - assertThat(caps.size(), equalTo(1)); - assertThat(caps.get("rollup_foo").getIndexName(), equalTo("rollup_foo")); - assertThat(caps.get("rollup_foo").getJobCaps().size(), equalTo(1)); - } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java new file mode 100644 index 0000000000000..be78a50ca121d --- /dev/null +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.rollup.action; + + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.rollup.RollupField; +import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; +import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; +import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; +import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.elasticsearch.xpack.rollup.action.TransportGetRollupIndexCapsAction.getCapsByRollupIndex; +import static org.hamcrest.Matchers.equalTo; + + +public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected GetRollupIndexCapsAction.Request createTestInstance() { + if (randomBoolean()) { + return new GetRollupIndexCapsAction.Request(MetaData.ALL); + } + return new GetRollupIndexCapsAction.Request(randomAlphaOfLengthBetween(1, 20)); + } + + @Override + protected GetRollupIndexCapsAction.Request createBlankInstance() { + return new GetRollupIndexCapsAction.Request(); + } + + + public void testNoIndicesByRollup() { + ImmutableOpenMap indices = new ImmutableOpenMap.Builder().build(); + Map caps = getCapsByRollupIndex(Collections.singletonList("foo"), indices); + assertThat(caps.size(), equalTo(0)); + } + + public void testAllIndicesByRollupSingleRollup() throws IOException { + int num = randomIntBetween(1,5); + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(num); + for (int i = 0; i < num; i++) { + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build()); + } + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("foo", meta); + } + + Map caps = getCapsByRollupIndex(Collections.singletonList("foo"), + indices.build()); + assertThat(caps.size(), equalTo(1)); + } + + public void testAllIndicesByRollupManyRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern(indexName) + .setRollupIndex("rollup_" + indexName).build()); + + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_" + indexName, meta); + } + + Map caps = getCapsByRollupIndex(Arrays.asList(indices.keys().toArray(String.class)), indices.build()); + assertThat(caps.size(), equalTo(5)); + } + + + public void testOneIndexByRollupManyRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern("foo_" + indexName) + .setRollupIndex("rollup_" + indexName).build()); + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_" + indexName, meta); + } + + Map caps = getCapsByRollupIndex(Collections.singletonList("rollup_1"), indices.build()); + assertThat(caps.size(), equalTo(1)); + assertThat(caps.get("rollup_1").getIndexName(), equalTo("rollup_1")); + assertThat(caps.get("rollup_1").getJobCaps().size(), equalTo(1)); + } + + public void testOneIndexByRollupOneRollup() throws IOException { + ImmutableOpenMap.Builder indices = new ImmutableOpenMap.Builder<>(5); + int indexCounter = 0; + for (int j = 0; j < 5; j++) { + + Map jobs = new HashMap<>(1); + String jobName = randomAlphaOfLength(10); + String indexName = Integer.toString(indexCounter); + indexCounter += 1; + jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) + .setIndexPattern("foo_" + indexName) + .setRollupIndex("rollup_foo").build()); + + MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, + Collections.singletonMap(RollupField.TYPE_NAME, + Collections.singletonMap("_meta", + Collections.singletonMap(RollupField.ROLLUP_META, jobs)))); + + ImmutableOpenMap.Builder mappings = ImmutableOpenMap.builder(1); + mappings.put(RollupField.TYPE_NAME, mappingMeta); + IndexMetaData meta = Mockito.mock(IndexMetaData.class); + Mockito.when(meta.getMappings()).thenReturn(mappings.build()); + indices.put("rollup_foo", meta); + } + + Map caps = getCapsByRollupIndex(Collections.singletonList("rollup_foo"), indices.build()); + assertThat(caps.size(), equalTo(1)); + assertThat(caps.get("rollup_foo").getIndexName(), equalTo("rollup_foo")); + assertThat(caps.get("rollup_foo").getJobCaps().size(), equalTo(1)); + } +} + + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json index 5bce163880af5..28edd044c3cfe 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_caps.json @@ -11,12 +11,6 @@ "required": false, "description": "The ID of the index to check rollup capabilities on, or left blank for all jobs" } - }, - "params": { - "key": { - "type": "string", - "description": "How the response should be keyed at the top level" - } } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json new file mode 100644 index 0000000000000..e19d4771cdba3 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json @@ -0,0 +1,17 @@ +{ + "xpack.rollup.get_rollup_index_caps": { + "documentation": "", + "methods": [ "GET" ], + "url": { + "path": "/{index}/_xpack/rollup/caps}", + "paths": [ "/{index}/_xpack/rollup/caps}" ], + "parts": { + "index": { + "type": "string", + "required": false, + "description": "The rollup index or index pattern to obtain rollup capabilities from." + } + } + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml index 6cac760cc80f8..f8bb401a7721e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml @@ -240,238 +240,3 @@ setup: - agg: "min" - agg: "max" - agg: "sum" - ---- -"Verify one job caps by rollup index": - - - do: - xpack.rollup.get_rollup_caps: - id: "foo_rollup" - key: "rollup_index" - - - match: - foo_rollup: - rollup_jobs: - - job_id: "foo" - rollup_index: "foo_rollup" - index_pattern: "foo" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - ---- -"Verify two job caps by rollup index": - - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - xpack.rollup.put_job: - id: foo2 - body: > - { - "index_pattern": "foo", - "rollup_index": "foo_rollup", - "cron": "*/30 * * * * ?", - "page_size" :10, - "groups" : { - "date_histogram": { - "field": "the_field", - "interval": "1h" - } - }, - "metrics": [ - { - "field": "value_field", - "metrics": ["min", "max", "sum"] - } - ] - } - - do: - xpack.rollup.get_rollup_caps: - id: "foo_rollup" - key: "rollup_index" - - - match: - foo_rollup: - rollup_jobs: - - job_id: "foo" - rollup_index: "foo_rollup" - index_pattern: "foo" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - - job_id: "foo2" - rollup_index: "foo_rollup" - index_pattern: "foo" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - - ---- -"Verify two different job caps by rollup index": - - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - xpack.rollup.put_job: - id: foo2 - body: > - { - "index_pattern": "foo2", - "rollup_index": "foo_rollup2", - "cron": "*/30 * * * * ?", - "page_size" :10, - "groups" : { - "date_histogram": { - "field": "the_field", - "interval": "1h" - } - }, - "metrics": [ - { - "field": "value_field", - "metrics": ["min", "max", "sum"] - } - ] - } - - do: - xpack.rollup.get_rollup_caps: - id: "foo_rollup" - key: "rollup_index" - - - match: - foo_rollup: - rollup_jobs: - - job_id: "foo" - rollup_index: "foo_rollup" - index_pattern: "foo" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - ---- -"Verify all job caps by rollup index": - - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - xpack.rollup.put_job: - id: foo2 - body: > - { - "index_pattern": "foo2", - "rollup_index": "foo_rollup", - "cron": "*/30 * * * * ?", - "page_size" :10, - "groups" : { - "date_histogram": { - "field": "the_field", - "interval": "1h" - } - }, - "metrics": [ - { - "field": "value_field", - "metrics": ["min", "max", "sum"] - } - ] - } - - do: - headers: - Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - xpack.rollup.put_job: - id: foo3 - body: > - { - "index_pattern": "foo3", - "rollup_index": "foo_rollup2", - "cron": "*/30 * * * * ?", - "page_size" :10, - "groups" : { - "date_histogram": { - "field": "the_field", - "interval": "1h" - } - }, - "metrics": [ - { - "field": "value_field", - "metrics": ["min", "max", "sum"] - } - ] - } - - - do: - xpack.rollup.get_rollup_caps: - id: "_all" - key: "rollup_index" - - - match: - $body: - foo_rollup: - rollup_jobs: - - job_id: "foo" - rollup_index: "foo_rollup" - index_pattern: "foo" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - - job_id: "foo2" - rollup_index: "foo_rollup" - index_pattern: "foo2" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - foo_rollup2: - rollup_jobs: - - job_id: "foo3" - rollup_index: "foo_rollup2" - index_pattern: "foo3" - fields: - the_field: - - agg: "date_histogram" - interval: "1h" - time_zone: "UTC" - value_field: - - agg: "min" - - agg: "max" - - agg: "sum" - - diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml new file mode 100644 index 0000000000000..7a539edcc6741 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml @@ -0,0 +1,363 @@ +setup: + - do: + indices.create: + index: foo + include_type_name: false + body: + mappings: + properties: + the_field: + type: date + value_field: + type: integer + + - do: + indices.create: + index: foo2 + include_type_name: false + body: + mappings: + properties: + the_field: + type: date + value_field: + type: integer + + - do: + indices.create: + index: foo3 + include_type_name: false + body: + mappings: + properties: + the_field: + type: date + value_field: + type: integer + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo + body: > + { + "index_pattern": "foo", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + +--- +"Verify one job caps by rollup index": + + - do: + xpack.rollup.get_rollup_index_caps: + index: "foo_rollup" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + +--- +"Verify two job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + xpack.rollup.get_rollup_index_caps: + index: "foo_rollup" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + - job_id: "foo2" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + + +--- +"Verify two different job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo2", + "rollup_index": "foo_rollup2", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + xpack.rollup.get_rollup_index_caps: + index: "foo_rollup" + + - match: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + +--- +"Verify all job caps by rollup index": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo2", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo3 + body: > + { + "index_pattern": "foo3", + "rollup_index": "foo_rollup2", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + + - do: + xpack.rollup.get_rollup_index_caps: + index: "_all" + + - match: + $body: + foo_rollup: + rollup_jobs: + - job_id: "foo" + rollup_index: "foo_rollup" + index_pattern: "foo" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + - job_id: "foo2" + rollup_index: "foo_rollup" + index_pattern: "foo2" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + foo_rollup2: + rollup_jobs: + - job_id: "foo3" + rollup_index: "foo_rollup2" + index_pattern: "foo3" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + +--- +"Verify index pattern": + + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo2 + body: > + { + "index_pattern": "foo2", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo3 + body: > + { + "index_pattern": "foo3", + "rollup_index": "foo_rollup2", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum"] + } + ] + } + + - do: + xpack.rollup.get_rollup_index_caps: + index: "*_rollup2" + + - match: + $body: + foo_rollup2: + rollup_jobs: + - job_id: "foo3" + rollup_index: "foo_rollup2" + index_pattern: "foo3" + fields: + the_field: + - agg: "date_histogram" + interval: "1h" + time_zone: "UTC" + value_field: + - agg: "min" + - agg: "max" + - agg: "sum" + From 6ef7e095e52e865fae35eabef48449a0f54997e8 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 26 Jun 2018 21:48:59 +0000 Subject: [PATCH 5/7] Update docs --- .../en/rest-api/rollup/rollup-caps.asciidoc | 84 +-------- .../rollup/rollup-index-caps.asciidoc | 161 ++++++++++++++++++ 2 files changed, 162 insertions(+), 83 deletions(-) create mode 100644 x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc diff --git a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc index d518cbd297b68..270ad005144ac 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc @@ -31,13 +31,6 @@ live? rollup job capabilities will be returned -==== Query String parameters - -`key`:: - (string) How the response should be keyed. `index_pattern`, which groups rollup capabilities by their index pattern. - Alternative option is `rollup_index`, which groups capabilities by the index the rollup is stored in. If omitted, - the default is `index_pattern` - ==== Request Body There is no request body for the Get Jobs API. @@ -186,79 +179,4 @@ GET _xpack/rollup/data/sensor-1 Why is this? The original rollup job was configured against a specific index pattern (`sensor-*`) not a concrete index (`sensor-1`). So while the index belongs to the pattern, the rollup job is only valid across the entirety of the pattern not just one of it's containing indices. So for that reason, the Rollup Capabilities API only returns information based -on the originally configured index name or pattern. - -===== Changing key grouping - -It is possible to change how rollup capabilities are grouped. By default, they are grouped by their index pattern. Instead, -they can be grouped by the concrete index the rollups are stored in. This is sometimes a useful operation if you are managing -the rollup data itself. - -To change the grouping, you can specify a `key` parameter in the URL: - -[source,js] --------------------------------------------------- -GET _xpack/rollup/data/sensor_rollup?key=rollup_index --------------------------------------------------- -// CONSOLE -// TEST[continued] - -Note how we are requesting the concrete rollup index name (`sensor_rollup`) instead of a pattern, and specifying `key=rollup_index`. -This will yield the following response: - -[source,js] ----- -{ - "sensor_rollup" : { <1> - "rollup_jobs" : [ - { - "job_id" : "sensor", - "rollup_index" : "sensor_rollup", - "index_pattern" : "sensor-*", - "fields" : { - "node" : [ - { - "agg" : "terms" - } - ], - "temperature" : [ - { - "agg" : "min" - }, - { - "agg" : "max" - }, - { - "agg" : "sum" - } - ], - "timestamp" : [ - { - "agg" : "date_histogram", - "time_zone" : "UTC", - "interval" : "1h", - "delay": "7d" - } - ], - "voltage" : [ - { - "agg" : "avg" - } - ] - } - } - ] - } -} ----- -// TESTRESPONSE -<1> The response is now grouped by the rollup index name at the top level - -You can also get the capabilities for all rollups using `_all`, grouped by the rollup index: - -[source,js] --------------------------------------------------- -GET _xpack/rollup/data/_all?key=rollup_index --------------------------------------------------- -// CONSOLE -// TEST[continued] \ No newline at end of file +on the originally configured index name or pattern. \ No newline at end of file diff --git a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc new file mode 100644 index 0000000000000..840846f1a09db --- /dev/null +++ b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc @@ -0,0 +1,161 @@ +[role="xpack"] +[[rollup-get-rollup-index-caps]] +=== Get Rollup Index Capabilities +++++ +Get Rollup Index Caps +++++ + +experimental[] + +This API returns the rollup capabilities of all jobs inside of a rollup index (e.g. the index where rollup data is stored). +A single rollup index may store the data for multiple rollup jobs, and may have a variety of capabilities depending on those jobs. + +This API will allow you to determine: + +1. What jobs are stored in an index (or indices specified via a pattern)? +2. What target indices were rolled up, what fields were used in those rollups and what aggregations can be performed on each job? + +==== Request + +`GET {index}/_xpack/rollup/caps` + +//===== Description + +==== Path Parameters + +`index`:: + (string) Index or index-pattern of concrete rollup indices to check for capabilities. + + + +==== Request Body + +There is no request body for the Get Jobs API. + +==== Authorization + +You must have `monitor`, `monitor_rollup`, `manage` or `manage_rollup` cluster privileges to use this API. +For more information, see +{xpack-ref}/security-privileges.html[Security Privileges]. + +==== Examples + +Imagine we have an index named `sensor-1` full of raw data. We know that the data will grow over time, so there +will be a `sensor-2`, `sensor-3`, etc. Let's create a Rollup job, which stores it's data in `sensor_rollup`: + +[source,js] +-------------------------------------------------- +PUT _xpack/rollup/job/sensor +{ + "index_pattern": "sensor-*", + "rollup_index": "sensor_rollup", + "cron": "*/30 * * * * ?", + "page_size" :1000, + "groups" : { + "date_histogram": { + "field": "timestamp", + "interval": "1h", + "delay": "7d" + }, + "terms": { + "fields": ["node"] + } + }, + "metrics": [ + { + "field": "temperature", + "metrics": ["min", "max", "sum"] + }, + { + "field": "voltage", + "metrics": ["avg"] + } + ] +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:sensor_index] + +If at a later date, we'd like to determine what jobs and capabilities were stored in the `sensor_rollup` index, we can use the Get Rollup +Index API: + +[source,js] +-------------------------------------------------- +GET /sensor_rollup/_xpack/rollup/caps +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +Note how we are requesting the concrete rollup index name (`sensor_rollup`) as the first part of the URL. +This will yield the following response: + +[source,js] +---- +{ + "sensor_rollup" : { + "rollup_jobs" : [ + { + "job_id" : "sensor", + "rollup_index" : "sensor_rollup", + "index_pattern" : "sensor-*", + "fields" : { + "node" : [ + { + "agg" : "terms" + } + ], + "temperature" : [ + { + "agg" : "min" + }, + { + "agg" : "max" + }, + { + "agg" : "sum" + } + ], + "timestamp" : [ + { + "agg" : "date_histogram", + "time_zone" : "UTC", + "interval" : "1h", + "delay": "7d" + } + ], + "voltage" : [ + { + "agg" : "avg" + } + ] + } + } + ] + } +} +---- +// TESTRESPONSE + + +The response that is returned contains information that is similar to the original Rollup configuration, but formatted +differently. First, there are some house-keeping details: the Rollup job's ID, the index that holds the rolled data, +the index pattern that the job was targeting. + +Next it shows a list of fields that contain data eligible for rollup searches. Here we see four fields: `node`, `temperature`, +`timestamp` and `voltage`. Each of these fields list the aggregations that are possible. For example, you can use a min, max +or sum aggregation on the `temperature` field, but only a `date_histogram` on `timestamp`. + +Note that the `rollup_jobs` element is an array; there can be multiple, independent jobs configured for a single index +or index pattern. Each of these jobs may have different configurations, so the API returns a list of all the various +configurations available. + + +Like other APIs that interact with indices, you can specify index patterns instead of explicit indices: + +[source,js] +-------------------------------------------------- +GET /*_rollup/_xpack/rollup/caps +-------------------------------------------------- +// CONSOLE +// TEST[continued] + From 07cd4e688527bd557061a13fe2d90cb4fc94a806 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 27 Jun 2018 14:45:08 +0000 Subject: [PATCH 6/7] URL tweaks, doc fixes --- x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc | 6 +++--- .../xpack/rollup/rest/RestGetRollupIndexCapsAction.java | 8 ++++---- .../api/xpack.rollup.get_rollup_index_caps.json | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc index 840846f1a09db..4636d9775e9d3 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-index-caps.asciidoc @@ -17,7 +17,7 @@ This API will allow you to determine: ==== Request -`GET {index}/_xpack/rollup/caps` +`GET {index}/_xpack/rollup/data` //===== Description @@ -81,7 +81,7 @@ Index API: [source,js] -------------------------------------------------- -GET /sensor_rollup/_xpack/rollup/caps +GET /sensor_rollup/_xpack/rollup/data -------------------------------------------------- // CONSOLE // TEST[continued] @@ -154,7 +154,7 @@ Like other APIs that interact with indices, you can specify index patterns inste [source,js] -------------------------------------------------- -GET /*_rollup/_xpack/rollup/caps +GET /*_rollup/_xpack/rollup/data -------------------------------------------------- // CONSOLE // TEST[continued] diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java index 4756191c168d0..94b45c02fc211 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java @@ -19,18 +19,18 @@ import java.io.IOException; public class RestGetRollupIndexCapsAction extends BaseRestHandler { - public static final ParseField ID = new ParseField("id"); + public static final ParseField INDEX = new ParseField("index"); public RestGetRollupIndexCapsAction(Settings settings, RestController controller) { super(settings); - controller.registerHandler(RestRequest.Method.GET, "/{id}/_xpack/rollup/caps", this); + controller.registerHandler(RestRequest.Method.GET, "/{index}/_xpack/rollup/data", this); } @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - String id = restRequest.param(ID.getPreferredName()); + String index = restRequest.param(INDEX.getPreferredName()); IndicesOptions options = IndicesOptions.fromRequest(restRequest, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); - GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(id, options); + GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(index, options); return channel -> client.execute(GetRollupIndexCapsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json index e19d4771cdba3..0694f316beb72 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json @@ -3,8 +3,8 @@ "documentation": "", "methods": [ "GET" ], "url": { - "path": "/{index}/_xpack/rollup/caps}", - "paths": [ "/{index}/_xpack/rollup/caps}" ], + "path": "/{index}/_xpack/rollup/data", + "paths": [ "/{index}/_xpack/rollup/data" ], "parts": { "index": { "type": "string", From ee44f2d5ad124c434b77969ad528dd1dc18b3767 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 5 Jul 2018 14:12:12 -0400 Subject: [PATCH 7/7] Request must implement IndicesRequest --- .../action/GetRollupIndexCapsAction.java | 51 +++++++++++-------- .../TransportGetRollupIndexCapsAction.java | 16 +++--- .../rest/RestGetRollupIndexCapsAction.java | 5 +- .../GetRollupIndexCapsActionRequestTests.java | 13 ++--- .../xpack.rollup.get_rollup_index_caps.json | 2 +- 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java index 9693bf42384c2..4f95919c4986b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java @@ -11,9 +11,9 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.core.rollup.RollupField; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -32,7 +33,7 @@ public class GetRollupIndexCapsAction extends Action { public static final GetRollupIndexCapsAction INSTANCE = new GetRollupIndexCapsAction(); - public static final String NAME = "indices:data/xpack/rollup/get_index/caps"; + public static final String NAME = "indices:data/read/xpack/rollup/get/index/caps"; public static final ParseField CONFIG = new ParseField("config"); public static final ParseField STATUS = new ParseField("status"); private static final ParseField INDICES_OPTIONS = new ParseField("indices_options"); @@ -46,44 +47,52 @@ public Response newResponse() { return new Response(); } - public static class Request extends ActionRequest implements ToXContent { - private String indexPattern; + public static class Request extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { + private String[] indices; private IndicesOptions options; - public Request(String indexPattern) { - this(indexPattern, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); + public Request(String[] indices) { + this(indices, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); } - public Request(String indexPattern, IndicesOptions options) { - if (Strings.isNullOrEmpty(indexPattern) || indexPattern.equals("*")) { - this.indexPattern = MetaData.ALL; - } else { - this.indexPattern = indexPattern; - } + public Request(String[] indices, IndicesOptions options) { + this.indices = indices; this.options = options; } public Request() {} - public String getIndexPattern() { - return indexPattern; + @Override + public IndicesOptions indicesOptions() { + return options; } - public IndicesOptions getOptions() { - return options; + @Override + public String[] indices() { + return indices; + } + + @Override + public IndicesRequest indices(String... indices) { + Objects.requireNonNull(indices, "indices must not be null"); + for (String index : indices) { + Objects.requireNonNull(index, "index must not be null"); + } + this.indices = indices; + return this; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - this.indexPattern = in.readString(); + this.indices = in.readStringArray(); this.options = IndicesOptions.readIndicesOptions(in); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(indexPattern); + out.writeStringArray(indices); options.writeIndicesOptions(out); } @@ -94,14 +103,14 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RollupField.ID.getPreferredName(), indexPattern); + builder.array(RollupField.ID.getPreferredName(), indices); builder.field(INDICES_OPTIONS.getPreferredName(), options); return builder; } @Override public int hashCode() { - return Objects.hash(indexPattern, options); + return Objects.hash(Arrays.hashCode(indices), options); } @Override @@ -113,7 +122,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(indexPattern, other.indexPattern) + return Arrays.equals(indices, other.indices) && Objects.equals(options, other.options); } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java index ea1a5ea7d9cf8..5d81f4046ebd8 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java @@ -26,29 +26,33 @@ import java.util.function.Supplier; import java.util.stream.StreamSupport; -public class TransportGetRollupIndexCapsAction extends HandledTransportAction { +public class TransportGetRollupIndexCapsAction extends HandledTransportAction { private final ClusterService clusterService; @Inject - public TransportGetRollupIndexCapsAction(Settings settings, TransportService transportService, ClusterService clusterService, - ActionFilters actionFilters) { + public TransportGetRollupIndexCapsAction(Settings settings, TransportService transportService, + ClusterService clusterService, ActionFilters actionFilters) { super(settings, GetRollupIndexCapsAction.NAME, transportService, actionFilters, (Supplier) GetRollupIndexCapsAction.Request::new); this.clusterService = clusterService; } @Override - protected void doExecute(Task task, GetRollupIndexCapsAction.Request request, ActionListener listener) { + protected void doExecute(Task task, GetRollupIndexCapsAction.Request request, + ActionListener listener) { IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(clusterService.getSettings()); - String[] indices = resolver.concreteIndexNames(clusterService.state(), request.getOptions(), request.getIndexPattern()); + String[] indices = resolver.concreteIndexNames(clusterService.state(), + request.indicesOptions(), request.indices()); Map allCaps = getCapsByRollupIndex(Arrays.asList(indices), clusterService.state().getMetaData().indices()); listener.onResponse(new GetRollupIndexCapsAction.Response(allCaps)); } - static Map getCapsByRollupIndex(List resolvedIndexNames, ImmutableOpenMap indices) { + static Map getCapsByRollupIndex(List resolvedIndexNames, + ImmutableOpenMap indices) { Map allCaps = new TreeMap<>(); StreamSupport.stream(indices.spliterator(), false) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java index 94b45c02fc211..4f4336f11abf2 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestGetRollupIndexCapsAction.java @@ -14,9 +14,6 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; -import org.elasticsearch.xpack.rollup.Rollup; - -import java.io.IOException; public class RestGetRollupIndexCapsAction extends BaseRestHandler { public static final ParseField INDEX = new ParseField("index"); @@ -30,7 +27,7 @@ public RestGetRollupIndexCapsAction(Settings settings, RestController controller protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { String index = restRequest.param(INDEX.getPreferredName()); IndicesOptions options = IndicesOptions.fromRequest(restRequest, IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED); - GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(index, options); + GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(new String[]{index}, options); return channel -> client.execute(GetRollupIndexCapsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java index be78a50ca121d..2066d6649965f 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java @@ -7,28 +7,21 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.RollupField; -import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction; import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps; -import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.mockito.Mockito; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; import static org.elasticsearch.xpack.rollup.action.TransportGetRollupIndexCapsAction.getCapsByRollupIndex; import static org.hamcrest.Matchers.equalTo; @@ -39,9 +32,9 @@ public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTest @Override protected GetRollupIndexCapsAction.Request createTestInstance() { if (randomBoolean()) { - return new GetRollupIndexCapsAction.Request(MetaData.ALL); + return new GetRollupIndexCapsAction.Request(new String[]{MetaData.ALL}); } - return new GetRollupIndexCapsAction.Request(randomAlphaOfLengthBetween(1, 20)); + return new GetRollupIndexCapsAction.Request(new String[]{randomAlphaOfLengthBetween(1, 20)}); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json index 0694f316beb72..458311417d4ae 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.get_rollup_index_caps.json @@ -8,7 +8,7 @@ "parts": { "index": { "type": "string", - "required": false, + "required": true, "description": "The rollup index or index pattern to obtain rollup capabilities from." } }